From ff73d73c8c85c6ab79dac7fee83c043d5e131bf1 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 14 Nov 2026 15:28:42 -0800 Subject: [PATCH 1/3] fix: divide fair_unified's task share among active consumers CometFairMemoryPool limited each consumer that can spill to (task_share - unspillable) / num_spill, where num_spill counted every registered consumer that can spill. DataFusion registers each operator's consumer when it builds the stream tree, before any batch flows, so the operators above a blocking one counted while they held nothing. A sort below a window and four aggregates was held to a sixth of the task's share and spilled while most of it was unused. Count only the active consumers that can spill, plus the one asking. A consumer is active while it holds more than its low-water mark, the least it has held since its first reservation, because from DataFusion 56 an aggregate reserves its initial table when the plan starts (apache/datafusion#25900). The task's total is still checked against the task's share, so a consumer that starts later is refused until the others give memory back. A sort below an OrderedFinalAggregateStream, which #7096 held to half of the share, can now fill the share while the aggregate is idle. It gives part of that back before the aggregate's first request, which a test checks with the share set to the sort's own peak. The #6880 test of a PartialMerge over a sort-merge join used a 16 MiB pool because the sorts below the join got a fraction of it. They now get the whole share while they read, so it uses 2 and 8 MiB, like its sibling test. Closes #7628. --- .ai/skills/review-comet-memory-pr/SKILL.md | 42 +- .../contributor-guide/memory_management.md | 57 ++- .../user-guide/latest/migration-guide.md | 11 +- .../source/user-guide/latest/tuning/memory.md | 13 +- .../src/execution/memory_pools/fair_pool.rs | 411 ++++++++++++++---- .../execution/memory_pools/spill_replay.rs | 4 +- .../src/execution/memory_pools/testing.rs | 8 + .../src/execution/operators/iceberg_write.rs | 11 +- .../core/src/execution/planner/aggregate.rs | 8 +- .../comet/exec/CometAggregateSuite.scala | 3 +- 10 files changed, 429 insertions(+), 139 deletions(-) diff --git a/.ai/skills/review-comet-memory-pr/SKILL.md b/.ai/skills/review-comet-memory-pr/SKILL.md index ecc8b5aa14a..e0a04ab6132 100644 --- a/.ai/skills/review-comet-memory-pr/SKILL.md +++ b/.ai/skills/review-comet-memory-pr/SKILL.md @@ -103,24 +103,30 @@ configuration, from the inside out: - [ ] **`fair_unified` holds the consumers that can spill to a share of the task's share.** The task's share is `spark.memory.offHeap.size` divided by the tasks on the executor running native plans, which `RunningTasks` counts while each task holds its task-shared pool, or - `pool_size` if that is smaller. A consumer that can spill is - refused if what it holds plus the request exceeds what the consumers that cannot spill leave - of the task's share, divided by the registered consumers that can spill, or if the task's - total plus the request exceeds the task's share. A consumer that cannot spill is refused only - if the task's total plus the request exceeds `pool_size`, which binds before Spark's own - limit only while the deprecated fraction is below `1 / N`. With eight running tasks, a 16 GiB - pool and two consumers that can spill, each may hold 1 GiB, or 768 MiB while a merge holds - 512 MiB. The pool keeps a running total for each consumer id, so the sibling reservations - that `new_empty()`, `split()` and `take()` create count against one share. A PR that checks - `reservation.size()` instead lets an operator with several reservations, such as a sort's - streaming merge, take other consumers' shares. It also depends on when DataFusion updates the - size, which is after it calls `try_grow` but before it calls `shrink`. A PR that limits a - consumer that cannot spill to a share, or sizes the shares from the whole pool, brings back - #7088: one operator takes the task's share from Spark and a merge or hash join build fails - the task. If the PR changes any of these comparisons it changes when every query spills, so - it needs benchmark evidence, not reasoning. -- [ ] **The consumers that divide the task's share are every consumer that can spill in the - task**, across every native plan, because the pool is task-shared. + `pool_size` if that is smaller. A consumer that can spill is refused if what it holds plus + the request exceeds what the consumers that cannot spill leave of the task's share, divided + by the active consumers that can spill, itself included, or if the task's total plus + the request exceeds the task's share. A consumer that cannot spill is refused only if the + task's total plus the request exceeds `pool_size`, which binds before Spark's own limit only + while the deprecated fraction is below `1 / N`. With eight running tasks, a 16 GiB pool and + two active consumers that can spill, each may hold 1 GiB, or 768 MiB while a merge + holds 512 MiB. The pool keeps a running total for each consumer id, so the sibling + reservations that `new_empty()`, `split()` and `take()` create count against one share. A PR + that checks `reservation.size()` instead lets an operator with several reservations, such as + a sort's streaming merge, take other consumers' shares. It also depends on when DataFusion + updates the size, which is after it calls `try_grow` but before it calls `shrink`. A PR that + limits a consumer that cannot spill to a share, or sizes the shares from the whole pool, + brings back #7088: one operator takes the task's share from Spark and a merge or hash join + build fails the task. If the PR changes any of these comparisons it changes when every query + spills, so it needs benchmark evidence, not reasoning. +- [ ] **The consumers that divide the task's share are the active consumers in the task that can + spill**, across every native plan, because the pool is task-shared. A consumer is active + while it holds more than its low-water mark, the least it has held since its first + reservation. DataFusion registers every operator's consumer before any batch flows, and from + DataFusion 56 an aggregate also reserves its initial table then. A PR that divides by the + registered consumers, or counts a consumer for its first reservation, again holds a blocking + operator, such as a sort, to a fraction of the task's share while the operators above it have + not started (#7628). - [ ] **Task-shared pool lifetime.** `acquire_task_shared_pool` keeps a process-wide `HashMap>`. The returned `Arc` is the only lifetime handle, so there is no explicit release to forget and a `createPlan` that fails diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index c55d0b4378b..432f1419b12 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -356,7 +356,17 @@ limit = (task_share - unspillable) / num_spill ``` Here `unspillable` is the bytes held by consumers that cannot spill, and `num_spill` is the number -of registered consumers that can. +of active consumers that can spill, counting the one asking. A consumer is active while it holds +more than its low-water mark, the least it has held since its first reservation. DataFusion +registers each operator's consumer when it builds the stream tree, before any batch flows, so the +operators above a blocking one, such as a sort, are registered while it reads its input. From +DataFusion 56 an aggregate also reserves its initial table then +([apache/datafusion#25900](https://github.com/apache/datafusion/pull/25900)). Neither shows that +the operator has started work, so it takes no share until it is active, which leaves its share to +the operator doing the work +([issue #7628](https://github.com/apache/datafusion-comet/issues/7628)). An operator that emptied, +such as an aggregate that emitted early, is active again once it holds memory, and one that only +gives memory back, such as a sort emitting what it buffered, takes no share. Like Spark's own divisor, the number of running tasks follows tasks as they start and finish, so a task that runs alone has the whole pool. It is still not Spark's number or Spark's pool, and the @@ -382,17 +392,16 @@ Before it asks Spark for a consumer that can spill, the pool makes two checks, a request without calling Spark if either fails: - **The consumer against its share.** The share is what the consumers that cannot spill leave of - the task's share, divided by the number of registered consumers that can spill. What the - consumer already holds plus the request must not exceed it. That counts all of the consumer's - reservations, including the sibling reservations that `new_empty()`, `split()` and `take()` - create, so an operator that holds several reservations still gets one share. A sort's streaming - merge, for example, creates one for each batch it reads. The pool keeps a running total for each - consumer, because `reservation.size()` covers only one reservation. With eight running tasks, a - 16 GiB pool and two registered consumers that can spill, the task's share is 2 GiB and each - consumer's is 1 GiB, or 768 MiB while a merge holds 512 MiB. + the task's share, divided by the number of active consumers that can spill, this one + included. What the consumer already holds plus the request must not exceed it. That counts all + of the consumer's reservations, including the sibling reservations that `new_empty()`, `split()` + and `take()` create, so an operator that holds several reservations still gets one share. A + sort's streaming merge, for example, creates one for each batch it reads. The pool keeps a + running total for each consumer, because `reservation.size()` covers only one reservation. With + eight running tasks, a 16 GiB pool and two active consumers that can spill, the task's share is + 2 GiB and each consumer's is 1 GiB, or 768 MiB while a merge holds 512 MiB. - **The task's total against the task's share.** The shares alone do not bound the total, because - a consumer keeps what it reserved while fewer consumers were registered or fewer tasks were - running. + a consumer keeps what it reserved while fewer consumers were active or fewer tasks were running. A consumer that cannot spill takes what Spark grants. The pool refuses it only when the task's total plus the request would exceed `pool_size`, which Spark's own limit on the task is at least as @@ -404,21 +413,25 @@ as a workaround for [apache/datafusion#17334](https://github.com/apache/datafusi builds a spill path only for partially sorted input. Fully sorted, as in a `CometSortAggregateExec` over a native sort, a refusal fails the query, so the pool checks its requests only against the pool, as it does those of a consumer that cannot spill (`limited_to_its_share` in `fair_pool.rs`). -It still counts as a consumer that can spill when the pool divides the task's share, so a sort -below it stays at half of the share and leaves the aggregate the other half. The partially sorted -stream still spills when Spark refuses it. The exemption goes by the stream's name, so it also -covers a `PartialMerge` that the planner runs as a final aggregate because its input is sorted on -some of its grouping keys. That one is not a `CometSortAggregateExec` and its input is only -partially sorted, but it is likewise checked only against the pool and Spark's grant. A -`PartialMerge` over input sorted on every grouping key stays on the hash aggregate, since the final -aggregate has no spill path there and would only halve the limit of a sort below it. +It still counts as a consumer that can spill once it is active. A sort below it reads all of its +input before the aggregate asks for any, so the sort can take the whole share, and it gives part of +that back before the aggregate's first request: all of it if it spilled, and otherwise what +`ExternalSorter` reserves beyond its sorted runs and, if it merges them, their sort keys, because it +reserves about twice the bytes of each batch it buffers. The partially sorted stream still spills +when Spark refuses it. The exemption goes by the stream's name, so it also covers a `PartialMerge` +that the planner runs as a final aggregate because its input is sorted on some of its grouping +keys. That one is not a `CometSortAggregateExec` and its input is only partially sorted, but it is +likewise checked only against the pool and Spark's grant. A `PartialMerge` over input sorted on +every grouping key stays on the hash aggregate, since the final aggregate has no spill path there. Three details matter for tuning: - The pool is task-shared (see below), so the consumers that divide the task's share include those - of every native plan in the task, not just the plan making the request. Registering a consumer - that can spill lowers the share of every other one, but does not take back memory they already - hold. Registering one that cannot spill changes no share until it holds memory. + of every native plan in the task, not just the plan making the request. A consumer that can + spill lowers the share of every other one once it is active, not when it registers, but does not + take back memory they already hold, so a consumer that starts after another has filled the task's + share is refused until that one gives memory back. One that cannot spill changes the shares only + through the bytes it holds. - A consumer at its share is refused, and spills, even when the rest of the task's share is free. That keeps the memory for the task's other consumers, and is why `fair_unified` can spill earlier than `greedy_unified`. diff --git a/docs/source/user-guide/latest/migration-guide.md b/docs/source/user-guide/latest/migration-guide.md index 5b28b3372bf..0bcf33292bb 100644 --- a/docs/source/user-guide/latest/migration-guide.md +++ b/docs/source/user-guide/latest/migration-guide.md @@ -138,11 +138,12 @@ that cannot spill, such as the final merge of a sort, then failed the task ([#7088](https://github.com/apache/datafusion-comet/issues/7088)). Spark retried such tasks, and the retries usually succeeded. Comet 1.2.0 divides `spark.memory.offHeap.size` among the tasks running native plans, as Spark does, and within a task limits only the operators that can spill, each to an -even share of what the operators that cannot spill leave. So sorts, aggregations and the shuffle -writer can spill sooner when several of them run in one task, and the operators that cannot spill -are limited only by Spark, unless the deprecated `spark.comet.exec.memoryPool.fraction` is below -`1.0`. The native Iceberg fanout writer is one of the operators that can spill: it closes files -rather than take memory from the others, so it can write more files. +even share of what the operators that cannot spill leave, among those that are doing work. So sorts, +aggregations and the shuffle writer can spill sooner when several of them work at once in one task, +and the operators that cannot spill are limited only by Spark, unless the deprecated +`spark.comet.exec.memoryPool.fraction` is below `1.0`. The native Iceberg fanout writer is one of +the operators that can spill: it closes files rather than take memory from the others, so it can +write more files. An operator's limit can now be well below what Spark would grant its task, and a sort can fail where it used to run. A sort fails with "Not enough memory to continue external sort" instead of diff --git a/docs/source/user-guide/latest/tuning/memory.md b/docs/source/user-guide/latest/tuning/memory.md index 722670fe353..91525b04fcd 100644 --- a/docs/source/user-guide/latest/tuning/memory.md +++ b/docs/source/user-guide/latest/tuning/memory.md @@ -83,12 +83,13 @@ The `fair_unified` pool keeps memory for the operators that cannot spill, such a build side of a hash join. Spark limits each task to an even share of `spark.memory.offHeap.size` among the tasks using it, and the pool divides `spark.memory.offHeap.size` the same way among the tasks running native plans. Within a task, it limits each operator that can spill to an even share of what the operators that cannot spill leave of the -task's share, counting the operators of all of the task's native plans. The operators that cannot spill take what -Spark grants. So an operator that can spill sometimes spills even when there is memory to spare, in order to leave it -for the task's other operators. A limit that small can be less than one batch of wide rows, and a sort cannot spill a -batch it has not yet reserved, so it then fails with "Not enough memory to continue external sort". Give the -executors more off-heap memory, lower `spark.comet.batchSize`, or use `greedy_unified`, which leaves every limit to -Spark. +task's share, counting the operators of all of the task's native plans that are doing work. An operator that has +not started yet, such as one above a sort that is still reading its input, takes no share. The operators that cannot +spill take what Spark grants. So an operator that can spill sometimes spills even when there is memory to spare, in +order to leave it for the task's other operators. A limit that small can be less than one batch of wide rows, and a +sort cannot spill a batch it has not yet reserved, so it then fails with "Not enough memory to continue external +sort". Give the executors more off-heap memory, lower `spark.comet.batchSize`, or use `greedy_unified`, which leaves +every limit to Spark. The `greedy_unified` pool type implements a greedy first-come first-serve limit. This pool works well for queries that do not need to spill or have a single spillable operator. diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 93174ba10e9..317757c607e 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -45,12 +45,19 @@ use parking_lot::Mutex; /// ``` /// /// `running_tasks` counts the tasks running native plans, this one included, `unspillable` is the -/// bytes held by consumers that cannot spill and `num_spill` is the number of registered -/// consumers that can. A request from a consumer that can spill is refused when what the consumer -/// holds plus the request passes `limit`, or when the task's total plus the request passes -/// `task_share`. That keeps the memory that Spark grants the task for the consumers that cannot -/// spill, which take what Spark grants. A request from one of them is refused only when the task's -/// total would pass `pool_size`. +/// bytes held by consumers that cannot spill and `num_spill` is the number of active consumers that +/// can spill, counting the one asking. A consumer is active while it holds more than its low-water +/// mark (see [`ConsumerUsage::active`]). Unlike `FairSpillPool`, this leaves out the consumers that +/// have not started work. DataFusion registers each operator's consumer when it +/// builds the stream tree, before any batch flows, so the operators above a blocking one, such as a +/// sort, are registered while it reads its input, and would otherwise hold it to a fraction of the +/// task's share while their own shares stay unused (#7628). +/// +/// A request from a consumer that can spill is refused when what the consumer holds plus the +/// request passes `limit`, or when the task's total plus the request passes `task_share`. That +/// keeps the memory that Spark grants the task for the consumers that cannot spill, which take +/// what Spark grants. A request from one of them is refused only when the task's total would pass +/// `pool_size`. pub struct CometFairMemoryPool { spark: SparkMemory, /// The most that the task may reserve. @@ -66,23 +73,70 @@ struct CometFairPoolState { used: usize, /// Bytes held by consumers that cannot spill. unspillable: usize, - /// Registered consumers that can spill. + /// Consumers that can spill and are active; see [`ConsumerUsage::active`]. num_spill: usize, - /// Bytes held by each registered consumer, keyed by [`MemoryConsumer::id`]. The sibling + /// What each registered consumer holds, keyed by [`MemoryConsumer::id`]. The sibling /// reservations that `new_empty()`, `split()` and `take()` create belong to the same consumer, /// so they count against one share. The pool keeps these totals itself because /// `reservation.size()` covers only one reservation, and DataFusion updates it after calling /// `try_grow` but before calling `shrink`. - consumers: HashMap, + consumers: HashMap, +} + +/// What a registered consumer holds across all of its reservations. +#[derive(Default)] +struct ConsumerUsage { + used: usize, + /// The least the consumer has held since its first reservation, or `None` before it made one. + low_water: Option, +} + +impl ConsumerUsage { + /// Whether the consumer holds more than its low-water mark. DataFusion 56's aggregates reserve + /// their initial table when the plan starts, before any batch flows (apache/datafusion#25900), + /// so a first reservation does not show that an operator has started work. A consumer that has + /// emptied, such as an aggregate that emitted early, is active again once it holds memory, and + /// one that only gives memory back, such as a sort emitting what it buffered, is not. + fn active(&self) -> bool { + self.low_water + .is_some_and(|low_water| self.used > low_water) + } } impl CometFairPoolState { - /// The bytes held by `reservation`'s consumer across all of its reservations. - fn consumer_used(&mut self, reservation: &MemoryReservation) -> &mut usize { + /// What `reservation`'s consumer holds across all of its reservations. + fn consumer(&mut self, reservation: &MemoryReservation) -> &mut ConsumerUsage { self.consumers .get_mut(&reservation.consumer().id()) .expect("reservation's consumer is not registered with the pool") } + + /// Adds `additional` bytes to what `reservation`'s consumer holds, counting a consumer that + /// can spill in `num_spill` once it becomes active. + fn add_to_consumer(&mut self, reservation: &MemoryReservation, additional: usize) { + let consumer = self.consumer(reservation); + let was_active = consumer.active(); + consumer.used = consumer.used.saturating_add(additional); + consumer.low_water.get_or_insert(consumer.used); + if !was_active && consumer.active() && reservation.consumer().can_spill() { + self.num_spill += 1; + } + } + + /// Takes `subtractive` bytes off what `reservation`'s consumer holds, which must be at least + /// that much, and stops counting a consumer that can spill in `num_spill` once it is no + /// longer active. + fn subtract_from_consumer(&mut self, reservation: &MemoryReservation, subtractive: usize) { + let consumer = self.consumer(reservation); + let was_active = consumer.active(); + consumer.used -= subtractive; + consumer.low_water = consumer + .low_water + .map(|low_water| low_water.min(consumer.used)); + if was_active && !consumer.active() && reservation.consumer().can_spill() { + self.num_spill = self.num_spill.saturating_sub(1); + } + } } impl Debug for CometFairMemoryPool { @@ -167,11 +221,16 @@ impl Display for CometFairMemoryPool { /// pure `PartialMerge` that the planner runs as `Final` over input sorted on some of its keys /// (#6880). That one is not a `CometSortAggregateExec` and its input is only partially sorted, but /// it is checked only against the pool as well. The planner does not switch a `PartialMerge` over -/// fully sorted input, which would gain no spill path and only halve the limit of a sort below it. +/// fully sorted input, where the final aggregate would have no spill path. /// /// Only the check differs. The aggregate still counts as a consumer that can spill everywhere -/// else, so it is one of `num_spill` and its bytes are not `unspillable`. A sort below it then -/// stays limited to its fair share and leaves the aggregate the rest. +/// else, so it is one of `num_spill` once it is active, and its bytes are not `unspillable`. +/// A sort below it reads all of its input before the aggregate asks for any, so the sort can take +/// the task's whole share. It gives part of that back before the aggregate's first request, all of +/// it if it spilled. Otherwise `ExternalSorter`, which reserves about twice the bytes of each batch +/// it buffers, resizes each sorted run to its own size before it emits the first batch, and a +/// merge of the runs reserves only the batches it holds and their sort keys. +/// `a_sort_that_fills_the_share_leaves_room_for_the_final_aggregate_above_it` checks both. fn limited_to_its_share(consumer: &MemoryConsumer) -> bool { consumer.can_spill() && !consumer.name().starts_with(ORDERED_FINAL_AGGREGATE) } @@ -182,18 +241,21 @@ impl MemoryPool for CometFairMemoryPool { } fn register(&self, consumer: &MemoryConsumer) { - let mut state = self.state.lock(); - state.consumers.insert(consumer.id(), 0); - if consumer.can_spill() { - state.num_spill += 1; - } + self.state + .lock() + .consumers + .insert(consumer.id(), ConsumerUsage::default()); } fn unregister(&self, consumer: &MemoryConsumer) { // DataFusion unregisters a consumer after its last reservation has dropped and released // its bytes. If a release panicked, this runs while unwinding, so it must not panic too. let mut state = self.state.lock(); - if state.consumers.remove(&consumer.id()).is_some() && consumer.can_spill() { + let active = state + .consumers + .remove(&consumer.id()) + .is_some_and(|usage| usage.active()); + if active && consumer.can_spill() { state.num_spill = state.num_spill.saturating_sub(1); } } @@ -211,14 +273,13 @@ impl MemoryPool for CometFairMemoryPool { if !reservation.consumer().can_spill() { state.unspillable = state.unspillable.saturating_add(additional); } - let consumer_used = state.consumer_used(reservation); - *consumer_used = consumer_used.saturating_add(additional); + state.add_to_consumer(reservation, additional); } fn shrink(&self, reservation: &MemoryReservation, subtractive: usize) { if subtractive > 0 { let mut state = self.state.lock(); - let consumer_used = *state.consumer_used(reservation); + let consumer_used = state.consumer(reservation).used; if consumer_used < subtractive { panic!( "Failed to release {subtractive} bytes where only {consumer_used} bytes tracked for the consumer" @@ -231,7 +292,7 @@ impl MemoryPool for CometFairMemoryPool { if !reservation.consumer().can_spill() { state.unspillable = state.unspillable.saturating_sub(subtractive); } - *state.consumer_used(reservation) -= subtractive; + state.subtract_from_consumer(reservation, subtractive); } } @@ -246,19 +307,20 @@ impl MemoryPool for CometFairMemoryPool { let used = state.used; if limited_to_its_share(reservation.consumer()) { let (task_share, tasks) = self.task_share(); - // `num_spill` counts this consumer. - let limit = task_share.saturating_sub(state.unspillable) / state.num_spill.max(1); - let consumer_used = *state.consumer_used(reservation); + let consumer = state.consumer(reservation); + let (consumer_used, active) = (consumer.used, consumer.active()); + // `num_spill` counts this consumer only if it is already active. + let num_spill = (state.num_spill + usize::from(!active)).max(1); + let limit = task_share.saturating_sub(state.unspillable) / num_spill; if limit < consumer_used.saturating_add(additional) { return resources_err!( - "Failed to acquire {additional} bytes where this consumer already holds {consumer_used} bytes and the fair limit is {limit} bytes: the task's share of {task_share} bytes with {tasks} tasks running, less {} bytes held by consumers that cannot spill, divided among {} consumers that can ({} bytes overcommitted)", + "Failed to acquire {additional} bytes where this consumer already holds {consumer_used} bytes and the fair limit is {limit} bytes: the task's share of {task_share} bytes with {tasks} tasks running, less {} bytes held by consumers that cannot spill, divided among {num_spill} consumers that can and are doing work, this one included ({} bytes overcommitted)", state.unspillable, - state.num_spill, self.spark.overcommit() ); } // The shares alone do not bound the task's total, because a consumer keeps what - // it reserved before another consumer registered or another task started. + // it reserved before another consumer became active or another task started. if task_share < used.saturating_add(additional) { return resources_err!( "Failed to acquire {additional} bytes where {used} bytes already reserved ({} bytes overcommitted) and the task's share is {task_share} bytes with {tasks} tasks running", @@ -292,7 +354,7 @@ impl MemoryPool for CometFairMemoryPool { if !can_spill { state.unspillable += additional; } - *state.consumer_used(reservation) += additional; + state.add_to_consumer(reservation, additional); } Ok(()) } @@ -306,8 +368,23 @@ impl MemoryPool for CometFairMemoryPool { mod tests { use super::super::spark_memory::fake::FakeSpark; use super::super::task_shared::RunningTask; - use super::super::testing::spillable; + use super::super::testing::{hold, spillable, PeakMemoryPool}; use super::*; + use arrow::array::{ArrayRef, Int64Array}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; + use arrow::record_batch::RecordBatch; + use datafusion::common::Result as DFResult; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + use datafusion::execution::TaskContext; + use datafusion::functions_aggregate::count::count_udaf; + use datafusion::physical_expr::aggregate::AggregateExprBuilder; + use datafusion::physical_expr::expressions::col; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; + use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::{collect, ExecutionPlan, InputOrderMode}; + use datafusion::prelude::SessionConfig; use std::sync::Arc; /// A pool of `pool_size` bytes, all of the off-heap pool as with the default fraction, which @@ -343,14 +420,17 @@ mod tests { let _running = start(&tasks, 8); let pool = shared_pool(&fake, 1040, &tasks); let sort = spillable("ExternalSorter[0]", &pool); - let _other_sort = spillable("ExternalSorter[1]", &pool); + let other_sort = spillable("ExternalSorter[1]", &pool); let other_merge = MemoryConsumer::new("ExternalSorterMerge[1]").register(&pool); + hold(&other_sort, 10); - // The two sorts share the task's 130 bytes, so the first is refused, and spills, at 65. + // Both sorts are active, so they share the task's 130 bytes, and the first is refused, + // and spills, at 65. sort.try_grow(65).unwrap(); assert!(sort.try_grow(1).is_err()); - // That leaves the rest of what Spark grants the task for the other sort's merge, which - // cannot spill. + // The other sort spills before it merges. That leaves the rest of what Spark grants the + // task for its merge, which cannot spill. + other_sort.free(); other_merge.try_grow(65).unwrap(); assert_eq!(fake.held(), 130); } @@ -382,7 +462,8 @@ mod tests { let merge = MemoryConsumer::new("ExternalSorterMerge[0]").register(&pool); merge.try_grow(40).unwrap(); - // The two sorts share the 60 bytes that the merge leaves. + // Both sorts are active, so they share the 60 bytes that the merge leaves. + hold(&second, 10); first.try_grow(30).unwrap(); assert!(first.try_grow(1).is_err()); @@ -390,7 +471,7 @@ mod tests { merge.shrink(40); first.try_grow(20).unwrap(); assert!(first.try_grow(1).is_err()); - second.try_grow(50).unwrap(); + second.try_grow(40).unwrap(); assert!(second.try_grow(1).is_err()); assert_eq!(pool.reserved(), 100); } @@ -400,7 +481,8 @@ mod tests { let fake = FakeSpark::with(usize::MAX); let pool = pool(&fake, 100); let sort = spillable("ExternalSorter[0]", &pool); - let _other_sort = spillable("ExternalSorter[1]", &pool); + let other_sort = spillable("ExternalSorter[1]", &pool); + hold(&other_sort, 10); // Like a batch read back from a spill file, which already exists. let reader = MemoryConsumer::new("spill reader").register(&pool); reader.grow(40); @@ -443,7 +525,7 @@ mod tests { let _running = tasks.start(); let pool = shared_pool(&fake, 200, &tasks); let first = spillable("ExternalSorter[0]", &pool); - first.try_grow(100).unwrap(); + hold(&first, 100); // Another task starts, which halves the task's share to the 100 bytes that the sort holds. // A second sort is under its own limit of 50 bytes, but the task's total is at its share. @@ -490,28 +572,14 @@ mod tests { let pool = pool(&fake, 100); let sort = spillable("ExternalSorter[0]", &pool); let aggregate = spillable("OrderedFinalAggregateStream[0]", &pool); + hold(&sort, 10); - // Not held to half of the task's 100 bytes, as a second consumer that can spill would be. + // Not held to half of the task's 100 bytes, as a second consumer that can spill would be + // while the sort is active. aggregate.try_grow(60).unwrap(); // Its bytes come out of the sort's share instead. - assert!(sort.try_grow(41).is_err()); - sort.try_grow(40).unwrap(); - } - - #[test] - fn a_sort_below_an_ordered_final_aggregate_leaves_it_a_share() { - // In a `CometSortAggregateExec` over a native sort, the sort reads all of its input before - // the aggregate asks for anything, and keeps what it holds while it emits. - let fake = FakeSpark::with(usize::MAX); - let pool = pool(&fake, 100); - let sort = spillable("ExternalSorter[0]", &pool); - let aggregate = spillable("OrderedFinalAggregateStream[0]", &pool); - - // The aggregate counts as a consumer that can spill, so the sort gets half. - assert!(sort.try_grow(51).is_err()); - sort.try_grow(50).unwrap(); - // And the aggregate has the other half. - aggregate.try_grow(50).unwrap(); + assert!(sort.try_grow(31).is_err()); + sort.try_grow(30).unwrap(); } #[test] @@ -540,9 +608,9 @@ mod tests { let first = spillable("first", &pool); let second = spillable("second", &pool); - // Each consumer's share is 50 bytes, whatever the other one holds. - first.try_grow(40).unwrap(); - second.try_grow(20).unwrap(); + // Once both are active, each consumer's share is 50 bytes, whatever the other one holds. + hold(&first, 40); + hold(&second, 20); first.try_grow(10).unwrap(); assert!(first.try_grow(1).is_err()); second.try_grow(30).unwrap(); @@ -552,17 +620,18 @@ mod tests { } #[test] - fn a_consumer_registered_late_is_limited_by_the_tasks_share() { + fn a_consumer_that_starts_late_is_limited_by_the_tasks_share() { let fake = FakeSpark::with(usize::MAX); let pool = pool(&fake, 90); let first = spillable("first", &pool); - // Alone, the first consumer's share is the whole task's. - first.try_grow(60).unwrap(); - - // A second consumer halves both shares, but the first keeps the 60 bytes it holds. let second = spillable("second", &pool); + // While the second consumer is not active, the first consumer's share is the whole task's. + hold(&first, 60); + + // Once the second consumer is active, it halves both shares, but the first keeps the 60 + // bytes it holds. + hold(&second, 30); assert!(first.try_grow(1).is_err()); - second.try_grow(30).unwrap(); // The second consumer is 15 bytes under its share, but the task's share is full. assert!(second.try_grow(1).is_err()); assert_eq!(pool.reserved(), 90); @@ -572,13 +641,42 @@ mod tests { second.try_grow(15).unwrap(); assert!(second.try_grow(1).is_err()); - // Unregistering the second consumer gives the first the whole share again. - drop(second); + // Once the second consumer holds nothing, it is no longer active, and the first has the + // whole share again. + second.free(); first.try_grow(60).unwrap(); assert_eq!(pool.reserved(), 90); assert_eq!(fake.held(), 90); } + /// DataFusion 56's aggregates reserve their initial table when the plan starts, before any + /// batch flows (apache/datafusion#25900), so a consumer takes a share only while it holds more + /// than the least it has held since its first reservation. + #[test] + fn a_consumer_takes_a_share_while_it_holds_more_than_its_low_water_mark() { + let fake = FakeSpark::with(usize::MAX); + let pool = pool(&fake, 100); + let sort = spillable("ExternalSorter[0]", &pool); + let aggregate = spillable("SingleHashAggregateStream[0]", &pool); + aggregate.try_grow(10).unwrap(); + + // So the sort can use what the aggregate leaves of the task's share. + sort.try_grow(90).unwrap(); + + // Once the aggregate holds more than its initial table, it takes a share. + sort.shrink(40); + aggregate.try_grow(1).unwrap(); + assert!(sort.try_grow(1).is_err()); + + // An aggregate that emptied, as one that emitted early does, takes a share again as soon + // as it holds memory, even less than its initial table. + aggregate.free(); + sort.try_grow(1).unwrap(); + aggregate.try_grow(5).unwrap(); + assert!(sort.try_grow(1).is_err()); + assert_eq!(pool.reserved(), 56); + } + #[test] fn sibling_reservations_draw_on_one_share() { let fake = FakeSpark::with(usize::MAX); @@ -587,6 +685,7 @@ mod tests { // Like the reservation that a sort's streaming merge creates for each batch it reads. let sibling = first.new_empty(); let second = spillable("second", &pool); + hold(&second, 10); // Both of the first consumer's reservations draw on its one 50-byte share. first.try_grow(40).unwrap(); @@ -595,8 +694,8 @@ mod tests { assert!(first.try_grow(1).is_err()); assert!(sibling.try_grow(1).is_err()); - // So the second consumer can still reserve its whole share. - second.try_grow(50).unwrap(); + // So the second consumer can still reserve the rest of its share. + second.try_grow(40).unwrap(); assert_eq!(pool.reserved(), 100); // Dropping a reservation returns its bytes to the consumer's share. The consumer stays @@ -613,7 +712,8 @@ mod tests { let fake = FakeSpark::with(usize::MAX); let pool = pool(&fake, 100); let mut first = spillable("first", &pool); - let _second = spillable("second", &pool); + let second = spillable("second", &pool); + hold(&second, 10); first.try_grow(50).unwrap(); let split = first.split(20); @@ -628,7 +728,168 @@ mod tests { split.shrink(10); first.try_grow(10).unwrap(); assert!(taken.try_grow(1).is_err()); - assert_eq!(pool.reserved(), 50); - assert_eq!(fake.held(), 50); + assert_eq!(pool.reserved(), 60); + assert_eq!(fake.held(), 60); + } + + /// 16 batches of 1024 rows of `schema`'s two `Int64` columns, which `row` computes from a + /// distinct key in `0..16384`, in no particular order. + fn batches(schema: &SchemaRef, row: impl Fn(i64) -> (i64, i64)) -> Vec { + (0..16) + .map(|batch| { + let (first, second): (Vec, Vec) = (batch * 1024..(batch + 1) * 1024) + .map(|i| row(i * 7919 % 16384)) + .unzip(); + let columns: Vec = vec![ + Arc::new(Int64Array::from(first)), + Arc::new(Int64Array::from(second)), + ]; + RecordBatch::try_new(Arc::clone(schema), columns).unwrap() + }) + .collect() + } + + /// Batches of 1024 rows, without the sort's merge reservation, which cannot spill and so would + /// take its bytes out of the shares. + fn session_config() -> SessionConfig { + SessionConfig::new() + .with_batch_size(1024) + .with_sort_spill_reservation_bytes(0) + } + + async fn collect_with( + plan: Arc, + pool: Arc, + config: SessionConfig, + ) -> Vec { + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(pool) + .build_arc() + .unwrap(); + let context = TaskContext::default() + .with_session_config(config) + .with_runtime(runtime); + collect(plan, Arc::new(context)).await.unwrap() + } + + fn sort_on(column: &str, input: Arc) -> Arc { + let expr = col(column, &input.schema()).unwrap(); + let ordering = LexOrdering::new([PhysicalSortExpr::new_default(expr)]).unwrap(); + Arc::new(SortExec::new(ordering, input)) + } + + /// Sorts on `a` in a task whose share holds what the sort reserves for all of its input, under + /// the plan that `above` builds on the sort, and returns how many times the sort spilled. + async fn spills_of_a_sort_below( + above: impl FnOnce(Arc) -> DFResult>, + ) -> usize { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, false), + Field::new("b", DataType::Int64, false), + ])); + let batches = batches(&schema, |key| (key, key % 16)); + // `ExternalSorter` reserves about twice the bytes of each batch it buffers, so three times + // the input holds all of it, and half of that does not. + let input_bytes: usize = batches.iter().map(RecordBatch::get_array_memory_size).sum(); + let fake = FakeSpark::with(usize::MAX); + + let input = MemorySourceConfig::try_new_exec(&[batches], schema, None).unwrap(); + let sort = sort_on("a", input); + let plan = above(Arc::clone(&sort) as Arc).unwrap(); + let output = collect_with(plan, pool(&fake, 3 * input_bytes), session_config()).await; + assert!(output.iter().any(|batch| batch.num_rows() > 0)); + sort.metrics().unwrap().spill_count().unwrap() + } + + fn sorted_on_b(input: Arc) -> DFResult> { + Ok(sort_on("b", input)) + } + + fn counted_by_b(input: Arc) -> DFResult> { + let schema = input.schema(); + let count = AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?]) + .schema(Arc::clone(&schema)) + .alias("count(a)") + .build()?; + let group_by = PhysicalGroupBy::new_single(vec![(col("b", &schema)?, "b".to_string())]); + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::Single, + group_by, + vec![Arc::new(count)], + vec![None], + input, + schema, + )?)) + } + + /// DataFusion registers each operator's consumer when it builds the stream tree, before any + /// batch flows, so the consumers above a blocking operator are registered while it reads its + /// input, although they hold nothing until it emits (#7628). The pool asks Spark for memory in + /// `block_in_place`, which needs the multi-threaded runtime. + #[tokio::test(flavor = "multi_thread")] + async fn the_operators_above_a_sort_take_no_share_until_they_are_active() { + let spills = [ + spills_of_a_sort_below(Ok).await, + spills_of_a_sort_below(sorted_on_b).await, + spills_of_a_sort_below(counted_by_b).await, + ]; + // Alone, below a sort, and below an aggregate. + assert_eq!(spills, [0, 0, 0]); + } + + /// In a `CometSortAggregateExec` over a native sort, the final aggregate's input is sorted on + /// every grouping key, so DataFusion runs it as an `OrderedFinalAggregateStream` with no spill + /// path, and a refused request fails the query. The aggregate holds nothing while the sort + /// reads its input, so the sort can fill the task's share. It must give part of it back before + /// the aggregate's first request, which it does whether it emits from one sorted batch or + /// merges sorted runs. + #[tokio::test(flavor = "multi_thread")] + async fn a_sort_that_fills_the_share_leaves_room_for_the_final_aggregate_above_it() { + // The partial counts that a final aggregate reads: 4096 values of `b`, each in 4 rows. + let schema = Arc::new(Schema::new(vec![ + Field::new("b", DataType::Int64, false), + Field::new("count(a)[count]", DataType::Int64, false), + ])); + let batches = batches(&schema, |key| (key / 4, 1)); + let input: Arc = + MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap(); + // The schema that the partial aggregate read, which the final aggregate is built against. + let partial_input = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let a = col("a", &partial_input).unwrap(); + let count = AggregateExprBuilder::new(count_udaf(), vec![a]) + .schema(Arc::clone(&partial_input)) + .alias("count(a)") + .build() + .unwrap(); + let group_by = PhysicalGroupBy::new_single(vec![(col("b", &schema).unwrap(), "b".into())]); + + for in_place_bytes in [usize::MAX, 0] { + // Emits from one sorted batch, then merges sorted runs. + let config = session_config().with_sort_in_place_threshold_bytes(in_place_bytes); + // The most that the sort reserves on its own. + let alone = Arc::new(PeakMemoryPool::new(usize::MAX)); + let sort = sort_on("b", Arc::clone(&input)); + collect_with(sort, Arc::clone(&alone) as _, config.clone()).await; + + // A task share of exactly that. + let fake = FakeSpark::with(usize::MAX); + let sort = sort_on("b", Arc::clone(&input)); + let aggregate = AggregateExec::try_new( + AggregateMode::Final, + group_by.clone(), + vec![Arc::new(count.clone())], + vec![None], + Arc::clone(&sort) as _, + Arc::clone(&partial_input), + ) + .unwrap(); + // Sorted on every grouping key, so an `OrderedFinalAggregateStream` with no spill path. + assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Sorted); + let output = collect_with(Arc::new(aggregate), pool(&fake, alone.peak()), config).await; + let groups: usize = output.iter().map(RecordBatch::num_rows).sum(); + let path = format!("sort_in_place_threshold_bytes = {in_place_bytes}"); + assert_eq!(groups, 4096, "{path}"); + assert_eq!(sort.metrics().unwrap().spill_count(), Some(0), "{path}"); + } } } diff --git a/native/core/src/execution/memory_pools/spill_replay.rs b/native/core/src/execution/memory_pools/spill_replay.rs index c4805ce1a98..bcdddd91f98 100644 --- a/native/core/src/execution/memory_pools/spill_replay.rs +++ b/native/core/src/execution/memory_pools/spill_replay.rs @@ -196,7 +196,7 @@ fn is_final_aggregate(consumer: &MemoryConsumer) -> bool { #[cfg(test)] mod tests { use super::super::spark_memory::fake::FakeSpark; - use super::super::testing::spillable; + use super::super::testing::{hold, spillable}; use super::super::{create_pool, overcommit, MemoryPoolConfig, MemoryPoolType, RunningTasks}; use super::*; use datafusion::execution::memory_pool::UnboundedMemoryPool; @@ -281,7 +281,7 @@ mod tests { fn a_final_aggregate_reading_its_spill_files_back_may_pass_the_pool_limit() { let (pool, fake) = fair_pool(-3014, 100); let sort = spillable("ExternalSorter[0]", &pool); - sort.try_grow(70).unwrap(); + hold(&sort, 70); let merge = spillable("FinalHashAggregateStream[0]", &pool); let replay = merge.new_empty(); merge.try_grow(25).unwrap(); diff --git a/native/core/src/execution/memory_pools/testing.rs b/native/core/src/execution/memory_pools/testing.rs index 7284e2c1f18..e4bc560e4be 100644 --- a/native/core/src/execution/memory_pools/testing.rs +++ b/native/core/src/execution/memory_pools/testing.rs @@ -33,6 +33,14 @@ pub(crate) fn spillable(name: &str, pool: &Arc) -> MemoryReserva .register(pool) } +/// Reserves `bytes` for `reservation` in two requests, so that `CometFairMemoryPool` counts its +/// consumer as active, as an operator is once its input flows. A first reservation alone does not +/// make a consumer active, because DataFusion 56's aggregates make one when the plan starts. +pub(crate) fn hold(reservation: &MemoryReservation, bytes: usize) { + reservation.try_grow(1).unwrap(); + reservation.try_grow(bytes - 1).unwrap(); +} + /// A [`GreedyMemoryPool`] that also records the most it ever had reserved, so a test can check /// what an operator reserved while it ran, not only that it gave everything back. #[derive(Debug)] diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index c949b95cf66..a77c5a030cf 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -5321,7 +5321,7 @@ mod tests { /// refusal fails, are limited only by Spark. #[test] fn only_a_fanout_write_is_held_to_a_share_of_the_pool() { - use crate::execution::memory_pools::testing::spillable; + use crate::execution::memory_pools::testing::{hold, spillable}; use crate::execution::memory_pools::{ create_memory_pool_with_fake_spark, MemoryPoolConfig, MemoryPoolType, }; @@ -5339,10 +5339,11 @@ mod tests { let pool = create_memory_pool_with_fake_spark(&config, task_attempt_id, usize::MAX); let sort = spillable("ExternalSorter[0]", &pool); let write = write_consumer(0, writer_mode).register(&pool); + hold(&sort, 10); - // A write that can spill divides the task's 100 bytes with the sort, so it is - // refused at its half. One that cannot takes more, and what it holds comes out of - // the sort's share. + // A write that can spill divides the task's 100 bytes with the sort, which is + // active, so it is refused at its half. One that cannot takes more, and what it + // holds comes out of the sort's share. write.try_grow(50).unwrap(); assert_eq!( write.try_grow(1).is_err(), @@ -5350,7 +5351,7 @@ mod tests { "{writer_mode:?}" ); assert_eq!( - sort.try_grow(50).is_ok(), + sort.try_grow(40).is_ok(), held_to_a_share, "{writer_mode:?}" ); diff --git a/native/core/src/execution/planner/aggregate.rs b/native/core/src/execution/planner/aggregate.rs index 1c03dd0d7a6..2cf8bc6b980 100644 --- a/native/core/src/execution/planner/aggregate.rs +++ b/native/core/src/execution/planner/aggregate.rs @@ -253,10 +253,10 @@ impl OperatorBuilder for HashAggBuilder { // runs an OrderedFinalAggregateStream there, which spills, and which the fair pool exempts // from that limit (`limited_to_its_share`). MergeAsPartial evaluates to its state under // Final as under Single. - // Only partially sorted input switches: with fully sorted input Final has no spill path - // and would halve the limit of a sort below it. Final also reads states from the columns - // after the keys, so the input must be the keys followed by native states (Spark's binary - // collect buffers keep Single, and arrive unordered anyway). + // Only partially sorted input switches: with fully sorted input Final has no spill path. + // Final also reads states from the columns after the keys, so the input must be the keys + // followed by native states (Spark's binary collect buffers keep Single, and arrive + // unordered anyway). // Drop this on DataFusion 56, where the stream is ClusteredFinalAggregateStream // (apache/datafusion#24697), and update the name prefix in `spill_replay.rs` (#6410). let mode = if proto_mode == ProtoAggregateMode::PartialMerge diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index 58758ae4d93..f674a1a5e59 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -360,8 +360,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("PartialMerge over a sort-merge join on a prefix of its keys spills (issue #6880)") { - // The sorts under the join need more than a 2 or 8 MiB pool's fair share, so this uses 16 MiB. - withPartialMergeOverSortedPrefix(Seq(16)) { path => + withPartialMergeOverSortedPrefix(Seq(2, 8)) { path => // With the default rules, the join's hash partitioning on g satisfies the PartialMerge keyed // by (g, k), so the join, the partial aggregate and the PartialMerge run in one native plan, // and the PartialMerge sees its input sorted on g. The suite lets AQE broadcast up to 1g at From 44cda5112fef2487671862d61461e4a7014c5475 Mon Sep 17 00:00:00 2001 From: comphead Date: Sun, 15 Nov 2026 12:02:37 -0800 Subject: [PATCH 2/3] test: pin the count(DISTINCT) chain and the merge reservation for the fair pool Restore the count(DISTINCT) chain test, so a DataFusion bump that changes how much the lower aggregate frees fails a test instead of a query. Run the ordered final aggregate test with DataFusion's default merge reservation and with 1024 groups per batch, and check that the task's total keeps the bytes of a consumer that reserved only once. Say that the sort frees its merge reservation before emitting, and move the memory pool note to the 1.3.0 migration section, since #7096 is not in 1.2.0. --- .../contributor-guide/memory_management.md | 6 +- .../user-guide/latest/migration-guide.md | 54 ++--- .../src/execution/memory_pools/fair_pool.rs | 228 ++++++++++++++---- 3 files changed, 209 insertions(+), 79 deletions(-) diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index 432f1419b12..1d758aa42b9 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -415,9 +415,9 @@ over a native sort, a refusal fails the query, so the pool checks its requests o pool, as it does those of a consumer that cannot spill (`limited_to_its_share` in `fair_pool.rs`). It still counts as a consumer that can spill once it is active. A sort below it reads all of its input before the aggregate asks for any, so the sort can take the whole share, and it gives part of -that back before the aggregate's first request: all of it if it spilled, and otherwise what -`ExternalSorter` reserves beyond its sorted runs and, if it merges them, their sort keys, because it -reserves about twice the bytes of each batch it buffers. The partially sorted stream still spills +that back before the aggregate's first request: all of it if it spilled, and otherwise its merge +reservation and what `ExternalSorter` reserves beyond its sorted runs and, if it merges them, their +sort keys, because it reserves about twice the bytes of each batch it buffers. The partially sorted stream still spills when Spark refuses it. The exemption goes by the stream's name, so it also covers a `PartialMerge` that the planner runs as a final aggregate because its input is sorted on some of its grouping keys. That one is not a `CometSortAggregateExec` and its input is only partially sorted, but it is diff --git a/docs/source/user-guide/latest/migration-guide.md b/docs/source/user-guide/latest/migration-guide.md index 0bcf33292bb..685b8d9a4be 100644 --- a/docs/source/user-guide/latest/migration-guide.md +++ b/docs/source/user-guide/latest/migration-guide.md @@ -75,6 +75,33 @@ longer has any effect. Comet always converts Arrow batches to Spark rows on the already the default. The driver logs a warning when the setting is present in the Spark configuration. +### Memory Pool Limits + +The `fair_unified` memory pool, which is the default for `spark.comet.exec.memoryPool`, limited each +of a task's operators to a share of all of `spark.memory.offHeap.size`, while Spark limits each task +to an even share of it among the running tasks. Whenever more than one task was running, the share +was the looser limit, so one operator could take all of its task's memory, and another operator +that cannot spill, such as the final merge of a sort, then failed the task +([#7088](https://github.com/apache/datafusion-comet/issues/7088)). Spark retried such tasks, and the +retries usually succeeded. Comet 1.3.0 divides `spark.memory.offHeap.size` among the tasks running +native plans, as Spark does, and within a task limits only the operators that can spill, each to an +even share of what the operators that cannot spill leave, among those that are doing work. So sorts, +aggregations and the shuffle writer can spill sooner when several of them work at once in one task, +and the operators that cannot spill are limited only by Spark, unless the deprecated +`spark.comet.exec.memoryPool.fraction` is below `1.0`. The native Iceberg fanout writer is one of +the operators that can spill: it closes files rather than take memory from the others, so it can +write more files. + +An operator's limit can now be well below what Spark would grant its task, and a sort can fail +where it used to run. A sort fails with "Not enough memory to continue external sort" instead of +spilling when a single batch is larger than its share, which wide rows, many cores per executor, a +small `spark.memory.offHeap.size` and several operators that can spill in one task make likelier. +It can also fail when operators that cannot spill, such as the build side of a shuffled hash join, +hold the whole of the task's share. To avoid either, give the executors more off-heap memory, +lower `spark.comet.batchSize`, or leave every limit to Spark by setting +`spark.comet.exec.memoryPool=greedy_unified`. See +[Configuring Comet Memory](tuning/memory.md#configuring-comet-memory). + ## Upgrading to Comet 1.2.0 Comet `1.2.0` makes no behavior changes that need a `spark.comet.legacy.*` key. The changes below @@ -128,33 +155,6 @@ that still sets it to `true` keeps the split plan, with iceberg-java writing the on Spark 3.5+ it now also plans merge-on-read writes as the split plan. Unset it to plan Spark's own operator. -### Memory Pool Limits - -The `fair_unified` memory pool, which is the default for `spark.comet.exec.memoryPool`, limited each -of a task's operators to a share of all of `spark.memory.offHeap.size`, while Spark limits each task -to an even share of it among the running tasks. Whenever more than one task was running, the share -was the looser limit, so one operator could take all of its task's memory, and another operator -that cannot spill, such as the final merge of a sort, then failed the task -([#7088](https://github.com/apache/datafusion-comet/issues/7088)). Spark retried such tasks, and the -retries usually succeeded. Comet 1.2.0 divides `spark.memory.offHeap.size` among the tasks running -native plans, as Spark does, and within a task limits only the operators that can spill, each to an -even share of what the operators that cannot spill leave, among those that are doing work. So sorts, -aggregations and the shuffle writer can spill sooner when several of them work at once in one task, -and the operators that cannot spill are limited only by Spark, unless the deprecated -`spark.comet.exec.memoryPool.fraction` is below `1.0`. The native Iceberg fanout writer is one of -the operators that can spill: it closes files rather than take memory from the others, so it can -write more files. - -An operator's limit can now be well below what Spark would grant its task, and a sort can fail -where it used to run. A sort fails with "Not enough memory to continue external sort" instead of -spilling when a single batch is larger than its share, which wide rows, many cores per executor, a -small `spark.memory.offHeap.size` and several operators that can spill in one task make likelier. -It can also fail when operators that cannot spill, such as the build side of a shuffled hash join, -hold the whole of the task's share. To avoid either, give the executors more off-heap memory, -lower `spark.comet.batchSize`, or leave every limit to Spark by setting -`spark.comet.exec.memoryPool=greedy_unified`. See -[Configuring Comet Memory](tuning/memory.md#configuring-comet-memory). - ### Native Iceberg Scan File Concurrency `spark.comet.scan.icebergNative.dataFileConcurrencyLimit` now defaults to 2, where Comet `1.1.0` diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 317757c607e..2a25b230edf 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -227,9 +227,10 @@ impl Display for CometFairMemoryPool { /// else, so it is one of `num_spill` once it is active, and its bytes are not `unspillable`. /// A sort below it reads all of its input before the aggregate asks for any, so the sort can take /// the task's whole share. It gives part of that back before the aggregate's first request, all of -/// it if it spilled. Otherwise `ExternalSorter`, which reserves about twice the bytes of each batch -/// it buffers, resizes each sorted run to its own size before it emits the first batch, and a -/// merge of the runs reserves only the batches it holds and their sort keys. +/// it if it spilled. Otherwise `ExternalSorter` frees its merge reservation, and, since it reserves +/// about twice the bytes of each batch it buffers, resizes each sorted run to its own size before +/// it emits the first batch. A merge of the runs reserves only the batches it holds and their sort +/// keys. /// `a_sort_that_fills_the_share_leaves_room_for_the_final_aggregate_above_it` checks both. fn limited_to_its_share(consumer: &MemoryConsumer) -> bool { consumer.can_spill() && !consumer.name().starts_with(ORDERED_FINAL_AGGREGATE) @@ -369,8 +370,9 @@ mod tests { use super::super::spark_memory::fake::FakeSpark; use super::super::task_shared::RunningTask; use super::super::testing::{hold, spillable, PeakMemoryPool}; + use super::super::{create_pool, MemoryPoolConfig, MemoryPoolType}; use super::*; - use arrow::array::{ArrayRef, Int64Array}; + use arrow::array::{ArrayRef, Int64Array, StringArray}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion::common::Result as DFResult; @@ -660,8 +662,10 @@ mod tests { let aggregate = spillable("SingleHashAggregateStream[0]", &pool); aggregate.try_grow(10).unwrap(); - // So the sort can use what the aggregate leaves of the task's share. + // So the sort can use what the aggregate leaves of the task's share, and the task's total + // keeps the aggregate's bytes for it. sort.try_grow(90).unwrap(); + assert!(sort.try_grow(1).is_err()); // Once the aggregate holds more than its initial table, it takes a share. sort.shrink(40); @@ -732,23 +736,23 @@ mod tests { assert_eq!(fake.held(), 60); } - /// 16 batches of 1024 rows of `schema`'s two `Int64` columns, which `row` computes from a - /// distinct key in `0..16384`, in no particular order. - fn batches(schema: &SchemaRef, row: impl Fn(i64) -> (i64, i64)) -> Vec { + /// 16 batches of 1024 rows, whose columns `columns` builds from a distinct key in `0..16384` + /// for each row, in no particular order. + fn batches(schema: &SchemaRef, columns: impl Fn(&[i64]) -> Vec) -> Vec { (0..16) .map(|batch| { - let (first, second): (Vec, Vec) = (batch * 1024..(batch + 1) * 1024) - .map(|i| row(i * 7919 % 16384)) - .unzip(); - let columns: Vec = vec![ - Arc::new(Int64Array::from(first)), - Arc::new(Int64Array::from(second)), - ]; - RecordBatch::try_new(Arc::clone(schema), columns).unwrap() + let keys: Vec = (batch * 1024..(batch + 1) * 1024) + .map(|i| i * 7919 % 16384) + .collect(); + RecordBatch::try_new(Arc::clone(schema), columns(&keys)).unwrap() }) .collect() } + fn int64s(values: impl Iterator) -> ArrayRef { + Arc::new(Int64Array::from_iter_values(values)) + } + /// Batches of 1024 rows, without the sort's merge reservation, which cannot spill and so would /// take its bytes out of the shares. fn session_config() -> SessionConfig { @@ -787,7 +791,12 @@ mod tests { Field::new("a", DataType::Int64, false), Field::new("b", DataType::Int64, false), ])); - let batches = batches(&schema, |key| (key, key % 16)); + let batches = batches(&schema, |keys| { + vec![ + int64s(keys.iter().copied()), + int64s(keys.iter().map(|key| key % 16)), + ] + }); // `ExternalSorter` reserves about twice the bytes of each batch it buffers, so three times // the input holds all of it, and half of that does not. let input_bytes: usize = batches.iter().map(RecordBatch::get_array_memory_size).sum(); @@ -842,17 +851,15 @@ mod tests { /// path, and a refused request fails the query. The aggregate holds nothing while the sort /// reads its input, so the sort can fill the task's share. It must give part of it back before /// the aggregate's first request, which it does whether it emits from one sorted batch or - /// merges sorted runs. + /// merges sorted runs, with or without its merge reservation, and with 256 or 1024 groups in + /// each batch. #[tokio::test(flavor = "multi_thread")] async fn a_sort_that_fills_the_share_leaves_room_for_the_final_aggregate_above_it() { - // The partial counts that a final aggregate reads: 4096 values of `b`, each in 4 rows. + // The partial counts that a final aggregate reads. let schema = Arc::new(Schema::new(vec![ Field::new("b", DataType::Int64, false), Field::new("count(a)[count]", DataType::Int64, false), ])); - let batches = batches(&schema, |key| (key / 4, 1)); - let input: Arc = - MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap(); // The schema that the partial aggregate read, which the final aggregate is built against. let partial_input = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); let a = col("a", &partial_input).unwrap(); @@ -862,34 +869,157 @@ mod tests { .build() .unwrap(); let group_by = PhysicalGroupBy::new_single(vec![(col("b", &schema).unwrap(), "b".into())]); + let default_merge_reservation = SessionConfig::new() + .options() + .execution + .sort_spill_reservation_bytes; + + // 256 or 1024 values of `b` in each batch. + for rows_per_group in [4, 1] { + let batches = batches(&schema, |keys| { + vec![ + int64s(keys.iter().map(|key| key / rows_per_group)), + int64s(keys.iter().map(|_| 1)), + ] + }); + let input: Arc = + MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap(); + // The sort's merge reservation, which cannot spill, comes out of the share. Comet runs + // with DataFusion's default. + for merge_reservation in [0, default_merge_reservation] { + // Emits from one sorted batch, then merges sorted runs. + for in_place_bytes in [usize::MAX, 0] { + let config = SessionConfig::new() + .with_batch_size(1024) + .with_sort_spill_reservation_bytes(merge_reservation) + .with_sort_in_place_threshold_bytes(in_place_bytes); + // The most that the sort reserves on its own. + let alone = Arc::new(PeakMemoryPool::new(usize::MAX)); + let sort = sort_on("b", Arc::clone(&input)); + collect_with(sort, Arc::clone(&alone) as _, config.clone()).await; + + // A task share of exactly that. + let fake = FakeSpark::with(usize::MAX); + let sort = sort_on("b", Arc::clone(&input)); + let aggregate = AggregateExec::try_new( + AggregateMode::Final, + group_by.clone(), + vec![Arc::new(count.clone())], + vec![None], + Arc::clone(&sort) as _, + Arc::clone(&partial_input), + ) + .unwrap(); + // Sorted on every grouping key, so an `OrderedFinalAggregateStream` with no + // spill path. + assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Sorted); + let pool = pool(&fake, alone.peak()); + let output = collect_with(Arc::new(aggregate), pool, config).await; + let groups: usize = output.iter().map(RecordBatch::num_rows).sum(); + let case = format!( + "{rows_per_group} rows per group, merge reservation {merge_reservation}, \ + sort_in_place_threshold_bytes {in_place_bytes}" + ); + assert_eq!(groups, 16384 / rows_per_group as usize, "{case}"); + assert_eq!(sort.metrics().unwrap().spill_count(), Some(0), "{case}"); + } + } + } + } + + /// Spark's `count(DISTINCT d) GROUP BY k` as Comet runs it in one native plan when the input + /// is already partitioned on `k`: Spark's `PartialMerge` keyed by `(k, d)` as a `Single` + /// aggregate, then the `Partial` and `Final` aggregates keyed by `k`. + fn distinct_count(input: Arc) -> [Arc; 3] { + let schema = input.schema(); + let keys = vec![ + (col("k", &schema).unwrap(), "k".to_string()), + (col("d", &schema).unwrap(), "d".to_string()), + ]; + let group_by = PhysicalGroupBy::new_single(keys); + let distinct = AggregateExec::try_new( + AggregateMode::Single, + group_by, + vec![], + vec![], + input, + Arc::clone(&schema), + ); + let distinct = Arc::new(distinct.unwrap()); - for in_place_bytes in [usize::MAX, 0] { - // Emits from one sorted batch, then merges sorted runs. - let config = session_config().with_sort_in_place_threshold_bytes(in_place_bytes); - // The most that the sort reserves on its own. - let alone = Arc::new(PeakMemoryPool::new(usize::MAX)); - let sort = sort_on("b", Arc::clone(&input)); - collect_with(sort, Arc::clone(&alone) as _, config.clone()).await; - - // A task share of exactly that. - let fake = FakeSpark::with(usize::MAX); - let sort = sort_on("b", Arc::clone(&input)); - let aggregate = AggregateExec::try_new( - AggregateMode::Final, - group_by.clone(), - vec![Arc::new(count.clone())], - vec![None], - Arc::clone(&sort) as _, - Arc::clone(&partial_input), - ) + let schema = distinct.schema(); + let count = AggregateExprBuilder::new(count_udaf(), vec![col("d", &schema).unwrap()]) + .schema(Arc::clone(&schema)) + .alias("count(d)") + .build() .unwrap(); - // Sorted on every grouping key, so an `OrderedFinalAggregateStream` with no spill path. - assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Sorted); - let output = collect_with(Arc::new(aggregate), pool(&fake, alone.peak()), config).await; - let groups: usize = output.iter().map(RecordBatch::num_rows).sum(); - let path = format!("sort_in_place_threshold_bytes = {in_place_bytes}"); - assert_eq!(groups, 4096, "{path}"); - assert_eq!(sort.metrics().unwrap().spill_count(), Some(0), "{path}"); - } + let count = Arc::new(count); + let by_k = PhysicalGroupBy::new_single(vec![(col("k", &schema).unwrap(), "k".to_string())]); + let partial = AggregateExec::try_new( + AggregateMode::Partial, + by_k.clone(), + vec![Arc::clone(&count)], + vec![None], + Arc::clone(&distinct) as _, + Arc::clone(&schema), + ); + let partial = Arc::new(partial.unwrap()); + let final_agg = AggregateExec::try_new( + AggregateMode::Final, + by_k.as_final(), + vec![count], + vec![None], + Arc::clone(&partial) as _, + schema, + ); + [distinct, partial, Arc::new(final_agg.unwrap())] + } + + /// The aggregates above the first one in `distinct_count` hold nothing while it reads its + /// input, so it can fill the task's share, and it holds all of its output until it has emitted + /// it. With wide values that leaves the `Partial` aggregate too little room, so it emits early, + /// and the `Final` aggregate starts while the first one still holds nearly all of the share. + /// DataFusion 55.2's `FinalHashAggregateStream` then spills and completes rather than failing + /// the query. + #[tokio::test(flavor = "multi_thread")] + async fn a_final_aggregate_that_starts_while_the_one_below_fills_the_share_spills() { + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("d", DataType::Utf8, false), + ])); + // A distinct `k` and a 1 KiB `d` in each row. + let batches = batches(&schema, |keys| { + let d = keys.iter().map(|key| format!("{key:0>1024}")); + vec![ + int64s(keys.iter().copied()), + Arc::new(StringArray::from_iter_values(d)), + ] + }); + let input: Arc = + MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap(); + + // The most that the first aggregate reserves on its own. + let alone = Arc::new(PeakMemoryPool::new(usize::MAX)); + let [distinct, _, _] = distinct_count(Arc::clone(&input)); + collect_with(distinct, Arc::clone(&alone) as _, session_config()).await; + + // A task share of exactly that, in a pool built the way `createPlan` builds it, so that + // the final aggregate reads its spill files back as it would in a task. + let fake = FakeSpark::with(usize::MAX); + let config = MemoryPoolConfig::new(MemoryPoolType::FairUnified, alone.peak()); + let pool = create_pool(&config, -3020, &RunningTasks::default(), || fake.memory()); + let [distinct, _, final_agg] = distinct_count(input); + let output = collect_with(Arc::clone(&final_agg) as _, pool, session_config()).await; + let groups: usize = output.iter().map(RecordBatch::num_rows).sum(); + assert_eq!(groups, 16384); + + let spills = |agg: &Arc| agg.metrics().unwrap().spill_count().unwrap(); + assert_eq!(spills(&distinct), 0); + // Guards against a vacuous pass: the final aggregate had to spill, so it started short of + // memory. + assert!( + spills(&final_agg) > 0, + "the final aggregate had room to spare" + ); } } From 22aa4235e10fac26bbf8794b640db6bbe3bbdc10 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 09:50:07 -0800 Subject: [PATCH 3/3] fix: keep a share in fair_unified for sorts that have not started DataFusion 55.2's ExternalSorter fails with "Not enough memory to continue external sort" when its first request is refused, since it has nothing to spill yet. A hash aggregate below a sort keeps its output reserved until it has emitted all of it, and emits slices whose buffers the sort is charged for in full. With idle consumers taking no share, the aggregate could fill the task's share and fail the sort above it. A sort that has registered but not reserved now takes a share from every consumer that can spill except another sort, so the aggregate below it is held to half, as before #7628's fix. A sort below another sort is not, because a sort gives memory back before it emits and emits compact batches. A test runs a sort above a final and a single aggregate at 100% to 200% of the aggregate's own peak. CometTaskMetricsSuite's child-sort spill test now uses a smaller pool, since the idle shuffle writer above the sort no longer takes half of it. --- .ai/skills/review-comet-memory-pr/SKILL.md | 5 +- .../contributor-guide/memory_management.md | 24 ++- .../user-guide/latest/migration-guide.md | 4 +- .../source/user-guide/latest/tuning/memory.md | 4 +- .../src/execution/memory_pools/fair_pool.rs | 149 ++++++++++++++++-- .../sql/comet/CometTaskMetricsSuite.scala | 4 +- 6 files changed, 161 insertions(+), 29 deletions(-) diff --git a/.ai/skills/review-comet-memory-pr/SKILL.md b/.ai/skills/review-comet-memory-pr/SKILL.md index e0a04ab6132..e44cba32e8a 100644 --- a/.ai/skills/review-comet-memory-pr/SKILL.md +++ b/.ai/skills/review-comet-memory-pr/SKILL.md @@ -126,7 +126,10 @@ configuration, from the inside out: DataFusion 56 an aggregate also reserves its initial table then. A PR that divides by the registered consumers, or counts a consumer for its first reservation, again holds a blocking operator, such as a sort, to a fraction of the task's share while the operators above it have - not started (#7628). + not started (#7628). The exception is a sort that has not started, which counts for every + consumer but another sort: `ExternalSorter` fails when its first request is refused, and a hash + aggregate below it keeps its output reserved until it has emitted all of it. A PR that drops the + exception lets an aggregate fill the share and fails the sort above it. - [ ] **Task-shared pool lifetime.** `acquire_task_shared_pool` keeps a process-wide `HashMap>`. The returned `Arc` is the only lifetime handle, so there is no explicit release to forget and a `createPlan` that fails diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index 1d758aa42b9..55365a761e3 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -356,17 +356,18 @@ limit = (task_share - unspillable) / num_spill ``` Here `unspillable` is the bytes held by consumers that cannot spill, and `num_spill` is the number -of active consumers that can spill, counting the one asking. A consumer is active while it holds -more than its low-water mark, the least it has held since its first reservation. DataFusion -registers each operator's consumer when it builds the stream tree, before any batch flows, so the -operators above a blocking one, such as a sort, are registered while it reads its input. From -DataFusion 56 an aggregate also reserves its initial table then -([apache/datafusion#25900](https://github.com/apache/datafusion/pull/25900)). Neither shows that +of active consumers that can spill, counting the one asking and, unless it is a sort, the sorts that +have not started. A consumer is active while it holds more than its low-water mark, the least it has +held since its first reservation. DataFusion registers each operator's consumer when it builds the +stream tree, before any batch flows, so the operators above a blocking one, such as a sort, are +registered while it reads its input. From DataFusion 56 an aggregate also reserves its initial table +then ([apache/datafusion#25900](https://github.com/apache/datafusion/pull/25900)). Neither shows that the operator has started work, so it takes no share until it is active, which leaves its share to the operator doing the work ([issue #7628](https://github.com/apache/datafusion-comet/issues/7628)). An operator that emptied, such as an aggregate that emitted early, is active again once it holds memory, and one that only -gives memory back, such as a sort emitting what it buffered, takes no share. +gives memory back, such as a sort emitting what it buffered, takes no share. A sort that has not +started is the exception, for the reason given under the tuning details below. Like Spark's own divisor, the number of running tasks follows tasks as they start and finish, so a task that runs alone has the whole pool. It is still not Spark's number or Spark's pool, and the @@ -430,7 +431,14 @@ Three details matter for tuning: of every native plan in the task, not just the plan making the request. A consumer that can spill lowers the share of every other one once it is active, not when it registers, but does not take back memory they already hold, so a consumer that starts after another has filled the task's - share is refused until that one gives memory back. One that cannot spill changes the shares only + share is refused until that one gives memory back. Most consumers spill or emit early then, but + DataFusion 55.2's `ExternalSorter` fails with "Not enough memory to continue external sort" when + its first request is refused, since it has nothing to spill yet. So a sort that has registered + but not reserved keeps a share from every consumer but another sort. A hash aggregate below it, + which keeps its output reserved until it has emitted all of it and emits slices whose buffers the + sort is charged for in full, is held to its share and leaves the sort room. A sort below it gives + memory back before it emits and emits compact batches, so it needs to leave none. One that cannot + spill changes the shares only through the bytes it holds. - A consumer at its share is refused, and spills, even when the rest of the task's share is free. That keeps the memory for the task's other consumers, and is why `fair_unified` can spill earlier diff --git a/docs/source/user-guide/latest/migration-guide.md b/docs/source/user-guide/latest/migration-guide.md index 685b8d9a4be..609f39589f7 100644 --- a/docs/source/user-guide/latest/migration-guide.md +++ b/docs/source/user-guide/latest/migration-guide.md @@ -85,7 +85,9 @@ that cannot spill, such as the final merge of a sort, then failed the task ([#7088](https://github.com/apache/datafusion-comet/issues/7088)). Spark retried such tasks, and the retries usually succeeded. Comet 1.3.0 divides `spark.memory.offHeap.size` among the tasks running native plans, as Spark does, and within a task limits only the operators that can spill, each to an -even share of what the operators that cannot spill leave, among those that are doing work. So sorts, +even share of what the operators that cannot spill leave, among those that are doing work and the +sorts that have not started. A sort fails rather than spills when it cannot reserve its first batch, +so it keeps a share from the operators below it other than a sort, such as an aggregate. So sorts, aggregations and the shuffle writer can spill sooner when several of them work at once in one task, and the operators that cannot spill are limited only by Spark, unless the deprecated `spark.comet.exec.memoryPool.fraction` is below `1.0`. The native Iceberg fanout writer is one of diff --git a/docs/source/user-guide/latest/tuning/memory.md b/docs/source/user-guide/latest/tuning/memory.md index 91525b04fcd..f3247a0d588 100644 --- a/docs/source/user-guide/latest/tuning/memory.md +++ b/docs/source/user-guide/latest/tuning/memory.md @@ -84,7 +84,9 @@ build side of a hash join. Spark limits each task to an even share of `spark.mem using it, and the pool divides `spark.memory.offHeap.size` the same way among the tasks running native plans. Within a task, it limits each operator that can spill to an even share of what the operators that cannot spill leave of the task's share, counting the operators of all of the task's native plans that are doing work. An operator that has -not started yet, such as one above a sort that is still reading its input, takes no share. The operators that cannot +not started yet, such as one above a sort that is still reading its input, takes no share, except a sort: it fails +rather than spills when it cannot reserve its first batch, so a sort that has not started keeps a share from the +other operators, such as an aggregate below it. The operators that cannot spill take what Spark grants. So an operator that can spill sometimes spills even when there is memory to spare, in order to leave it for the task's other operators. A limit that small can be less than one batch of wide rows, and a sort cannot spill a batch it has not yet reserved, so it then fails with "Not enough memory to continue external diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 2a25b230edf..73e3b27edc8 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -46,12 +46,13 @@ use parking_lot::Mutex; /// /// `running_tasks` counts the tasks running native plans, this one included, `unspillable` is the /// bytes held by consumers that cannot spill and `num_spill` is the number of active consumers that -/// can spill, counting the one asking. A consumer is active while it holds more than its low-water -/// mark (see [`ConsumerUsage::active`]). Unlike `FairSpillPool`, this leaves out the consumers that -/// have not started work. DataFusion registers each operator's consumer when it -/// builds the stream tree, before any batch flows, so the operators above a blocking one, such as a -/// sort, are registered while it reads its input, and would otherwise hold it to a fraction of the -/// task's share while their own shares stay unused (#7628). +/// can spill, counting the one asking and, unless it is a sort, the sorts that have not started (see +/// [`is_sort`]). A consumer is active while it holds more than its low-water mark (see +/// [`ConsumerUsage::active`]). Unlike `FairSpillPool`, this leaves out the other consumers that have +/// not started work. DataFusion registers each operator's consumer when it builds the stream tree, +/// before any batch flows, so the operators above a blocking one are registered while it reads its +/// input, and would otherwise hold it to a fraction of the task's share while their own shares stay +/// unused (#7628). /// /// A request from a consumer that can spill is refused when what the consumer holds plus the /// request passes `limit`, or when the task's total plus the request passes `task_share`. That @@ -75,6 +76,8 @@ struct CometFairPoolState { unspillable: usize, /// Consumers that can spill and are active; see [`ConsumerUsage::active`]. num_spill: usize, + /// Registered sorts that have not reserved memory yet; see [`is_sort`]. + unstarted_sorts: usize, /// What each registered consumer holds, keyed by [`MemoryConsumer::id`]. The sibling /// reservations that `new_empty()`, `split()` and `take()` create belong to the same consumer, /// so they count against one share. The pool keeps these totals itself because @@ -116,11 +119,15 @@ impl CometFairPoolState { fn add_to_consumer(&mut self, reservation: &MemoryReservation, additional: usize) { let consumer = self.consumer(reservation); let was_active = consumer.active(); + let starts = consumer.low_water.is_none(); consumer.used = consumer.used.saturating_add(additional); consumer.low_water.get_or_insert(consumer.used); if !was_active && consumer.active() && reservation.consumer().can_spill() { self.num_spill += 1; } + if starts && is_sort(reservation.consumer()) { + self.unstarted_sorts = self.unstarted_sorts.saturating_sub(1); + } } /// Takes `subtractive` bytes off what `reservation`'s consumer holds, which must be at least @@ -171,6 +178,7 @@ impl CometFairMemoryPool { used: 0, unspillable: 0, num_spill: 0, + unstarted_sorts: 0, consumers: HashMap::new(), }), } @@ -236,29 +244,49 @@ fn limited_to_its_share(consumer: &MemoryConsumer) -> bool { consumer.can_spill() && !consumer.name().starts_with(ORDERED_FINAL_AGGREGATE) } +/// The name prefix of the consumer that DataFusion 55.2's `ExternalSorter` registers for what it +/// buffers. Its merge registers a separate consumer, `ExternalSorterMerge[`, that cannot spill. +const EXTERNAL_SORTER: &str = "ExternalSorter["; + +/// Whether `consumer` is a sort's. `ExternalSorter` fails when its first request is refused, +/// since it has nothing to spill yet, so a sort that has registered but not reserved takes a share +/// from every consumer that can spill except another sort. The others can keep what they hold +/// until they have emitted it: DataFusion 55.2's hash aggregates keep their output reserved until +/// they have emitted all of it, and emit slices of it, whose buffers a sort above is charged for in +/// full. A sort gives memory back before it emits (see [`limited_to_its_share`]) and emits compact +/// batches, so it needs to leave a sort above it no room. +fn is_sort(consumer: &MemoryConsumer) -> bool { + consumer.name().starts_with(EXTERNAL_SORTER) +} + impl MemoryPool for CometFairMemoryPool { fn name(&self) -> &str { "CometFairMemoryPool" } fn register(&self, consumer: &MemoryConsumer) { - self.state - .lock() + let mut state = self.state.lock(); + state .consumers .insert(consumer.id(), ConsumerUsage::default()); + if is_sort(consumer) { + state.unstarted_sorts += 1; + } } fn unregister(&self, consumer: &MemoryConsumer) { // DataFusion unregisters a consumer after its last reservation has dropped and released // its bytes. If a release panicked, this runs while unwinding, so it must not panic too. let mut state = self.state.lock(); - let active = state - .consumers - .remove(&consumer.id()) - .is_some_and(|usage| usage.active()); - if active && consumer.can_spill() { + let Some(usage) = state.consumers.remove(&consumer.id()) else { + return; + }; + if usage.active() && consumer.can_spill() { state.num_spill = state.num_spill.saturating_sub(1); } + if usage.low_water.is_none() && is_sort(consumer) { + state.unstarted_sorts = state.unstarted_sorts.saturating_sub(1); + } } /// Records memory that already exists, so it must not fail and ignores the fair limit and @@ -310,12 +338,18 @@ impl MemoryPool for CometFairMemoryPool { let (task_share, tasks) = self.task_share(); let consumer = state.consumer(reservation); let (consumer_used, active) = (consumer.used, consumer.active()); - // `num_spill` counts this consumer only if it is already active. - let num_spill = (state.num_spill + usize::from(!active)).max(1); + // `num_spill` counts this consumer only if it is already active. A sort that has not + // started takes a share from every consumer but another sort. + let unstarted_sorts = if is_sort(reservation.consumer()) { + 0 + } else { + state.unstarted_sorts + }; + let num_spill = (state.num_spill + usize::from(!active) + unstarted_sorts).max(1); let limit = task_share.saturating_sub(state.unspillable) / num_spill; if limit < consumer_used.saturating_add(additional) { return resources_err!( - "Failed to acquire {additional} bytes where this consumer already holds {consumer_used} bytes and the fair limit is {limit} bytes: the task's share of {task_share} bytes with {tasks} tasks running, less {} bytes held by consumers that cannot spill, divided among {num_spill} consumers that can and are doing work, this one included ({} bytes overcommitted)", + "Failed to acquire {additional} bytes where this consumer already holds {consumer_used} bytes and the fair limit is {limit} bytes: the task's share of {task_share} bytes with {tasks} tasks running, less {} bytes held by consumers that cannot spill, divided among {num_spill} consumers that can, this one included, counting those doing work and {unstarted_sorts} sorts that have not started ({} bytes overcommitted)", state.unspillable, self.spark.overcommit() ); @@ -766,6 +800,14 @@ mod tests { pool: Arc, config: SessionConfig, ) -> Vec { + try_collect_with(plan, pool, config).await.unwrap() + } + + async fn try_collect_with( + plan: Arc, + pool: Arc, + config: SessionConfig, + ) -> DFResult> { let runtime = RuntimeEnvBuilder::new() .with_memory_pool(pool) .build_arc() @@ -773,7 +815,7 @@ mod tests { let context = TaskContext::default() .with_session_config(config) .with_runtime(runtime); - collect(plan, Arc::new(context)).await.unwrap() + collect(plan, Arc::new(context)).await } fn sort_on(column: &str, input: Arc) -> Arc { @@ -1022,4 +1064,77 @@ mod tests { "the final aggregate had room to spare" ); } + + /// A sort above a hash aggregate, as for a window or a sort-merge join over an aggregate. + /// DataFusion 55.2's hash aggregates keep their output reserved until they have emitted all of + /// it, and `ExternalSorter` fails when its first request is refused, since it has nothing to + /// spill yet. + #[tokio::test(flavor = "multi_thread")] + async fn a_sort_above_a_hash_aggregate_gets_room_for_its_first_batch() { + // 16384 groups of a 64-byte `d` over 4096 values of `k`, one row each. A final aggregate + // reads `c` as partial counts, a single one counts its values. + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("d", DataType::Utf8, false), + Field::new("c", DataType::Int64, false), + ])); + let batches = batches(&schema, |keys| { + let d = keys.iter().map(|key| format!("{key:0>64}")); + vec![ + int64s(keys.iter().map(|key| key % 4096)), + Arc::new(StringArray::from_iter_values(d)), + int64s(keys.iter().map(|_| 1)), + ] + }); + let input: Arc = + MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap(); + // The schema that a partial aggregate would have read, which a final one is built against. + let partial_input = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, false)])); + let aggregate = |mode: AggregateMode| -> Arc { + let counted = if mode == AggregateMode::Final { + Arc::clone(&partial_input) + } else { + Arc::clone(&schema) + }; + let count = AggregateExprBuilder::new(count_udaf(), vec![col("c", &counted).unwrap()]) + .schema(Arc::clone(&counted)) + .alias("count(c)") + .build() + .unwrap(); + let keys = vec![ + (col("k", &schema).unwrap(), "k".to_string()), + (col("d", &schema).unwrap(), "d".to_string()), + ]; + Arc::new( + AggregateExec::try_new( + mode, + PhysicalGroupBy::new_single(keys), + vec![Arc::new(count)], + vec![None], + Arc::clone(&input), + counted, + ) + .unwrap(), + ) + }; + + for mode in [AggregateMode::Final, AggregateMode::Single] { + // The most that the aggregate reserves on its own. + let alone = Arc::new(PeakMemoryPool::new(usize::MAX)); + collect_with(aggregate(mode), Arc::clone(&alone) as _, session_config()).await; + for percent in [100, 110, 125, 150, 200] { + let fake = FakeSpark::with(usize::MAX); + let share = alone.peak() * percent / 100; + let plan = sort_on("k", aggregate(mode)); + let output = try_collect_with(plan, pool(&fake, share), session_config()).await; + let case = format!("{mode:?} aggregate, share {percent}% of its peak"); + let groups: usize = output + .unwrap_or_else(|e| panic!("{case}: {e}")) + .iter() + .map(RecordBatch::num_rows) + .sum(); + assert_eq!(groups, 16384, "{case}"); + } + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index d9452017130..936033313f2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -795,7 +795,9 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "zstd", CometConf.COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES.key -> "32k", CometConf.COMET_BATCH_SIZE.key -> "1024", - CometConf.COMET_OFFHEAP_MEMORY_POOL_FRACTION.key -> "0.002", + // About 1 MB per task, which a scan task's 4,000 rows overflow even though the idle + // shuffle writer above the sort takes no share. + CometConf.COMET_OFFHEAP_MEMORY_POOL_FRACTION.key -> "0.0005", CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", "spark.comet.datafusion.execution.spill_compression" -> "zstd", "spark.comet.datafusion.execution.sort_spill_reservation_bytes" -> "65536",