From 3e4d567422ddef228af6c786d081481ecb9a50de Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 8 Nov 2026 09:52:41 +0100 Subject: [PATCH 1/5] feat: make the dynamic task count sampling percentage configurable Adds the `dynamic_bytes_ready_sample_percentage` option to `DistributedConfig`, replacing the hardcoded `BYTES_READY_SAMPLE_PERCENTAGE` used while gathering runtime statistics for dynamic task count. The default stays at 0.2. Setting it to 1.0 makes the coordinator wait for every partition of a stage before estimating its statistics, which makes the estimations exact and deterministic. This is useful for tests that assert on plans decided from runtime statistics. # Conflicts: # datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs --- .../src/coordinator/prepare_dynamic_plan.rs | 21 +++++---- datafusion/distributed/src/distributed_ext.rs | 43 +++++++++++++++++++ .../distributed_planner/distributed_config.rs | 5 +++ .../advanced/08-adaptive-query-execution.md | 9 ++-- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index 033855e70..fb8de094e 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -146,8 +146,12 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { let (stats, consumer_tc) = if nb_type == TypeId::of::() { (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) } else { - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; + let (stats, new_metrics) = gather_runtime_statistics( + load_info_rxs, + &input_stage.plan, + nb_ctx.d_cfg.dynamic_bytes_ready_sample_percentage, + ) + .await?; metrics.extend(new_metrics); // returning Desired(0) here is our way to tell the planner that we don't care // about the task count assigned to the network boundary in the consumer stage, @@ -233,11 +237,10 @@ impl PlanReconstructor { async fn gather_runtime_statistics( per_task_load_info_stream: Vec + Unpin>, plan: &Arc, + // Fraction of partitions with non-zero sampled bytes that need to report before considering + // that there's enough samples collected to make a final decision. + bytes_ready_sample_percentage: f64, ) -> Result<(Statistics, MetricsSet)> { - /// How many LoadInfo samples with non-zero sampled bytes should be gathered before considering - /// that there's enough samples collected to make a final decision. The higher this value is, - /// the more execution is delayed because of sample collection, but better sampling precision. - const BYTES_READY_SAMPLE_PERCENTAGE: f32 = 0.2; /// If the leaf nodes in the provided `plan` do not provide any row count estimation, there's /// no way for us here to estimate the % of completion of the stage during sampling. This is /// the default % of completion for those cases. @@ -249,8 +252,8 @@ async fn gather_runtime_statistics( }; let n_cols = sampler.schema().fields.len(); - fn apply_pct(value: usize, pct: f32) -> usize { - (value as f32 * pct).round() as usize + fn apply_pct(value: usize, pct: f64) -> usize { + (value as f64 * pct).round() as usize } let partitions_per_task = sampler.partition_samplers.len(); @@ -283,7 +286,7 @@ async fn gather_runtime_statistics( // Short circuit if we collected enough bytes_ready measurements. if partitions_with_bytes_ready_done - >= apply_pct(total_partitions, BYTES_READY_SAMPLE_PERCENTAGE).max(1) + >= apply_pct(total_partitions, bytes_ready_sample_percentage).max(1) { break; } diff --git a/datafusion/distributed/src/distributed_ext.rs b/datafusion/distributed/src/distributed_ext.rs index 176bf62da..4565ee414 100644 --- a/datafusion/distributed/src/distributed_ext.rs +++ b/datafusion/distributed/src/distributed_ext.rs @@ -620,6 +620,21 @@ pub trait DistributedExt: Sized { dynamic_bytes_per_partition: usize, ) -> Result<(), DataFusionError>; + /// Fraction of a stage's partitions that need to report sampled bytes before the dynamic + /// task count allocator estimates the stage's runtime statistics. Higher values delay + /// execution longer while sampling, but produce more precise estimations. + fn with_distributed_dynamic_bytes_ready_sample_percentage( + self, + dynamic_bytes_ready_sample_percentage: f64, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_dynamic_bytes_ready_sample_percentage] but with + /// an in-place mutation. + fn set_distributed_dynamic_bytes_ready_sample_percentage( + &mut self, + dynamic_bytes_ready_sample_percentage: f64, + ) -> Result<(), DataFusionError>; + /// Target throughput in bytes per partition per second used by the dynamic task count /// allocator to decide how many tasks to assign to each stage based on runtime statistics. fn with_distributed_local_worker_context( @@ -1059,6 +1074,15 @@ impl DistributedExt for SessionConfig { Ok(()) } + fn set_distributed_dynamic_bytes_ready_sample_percentage( + &mut self, + dynamic_bytes_ready_sample_percentage: f64, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.dynamic_bytes_ready_sample_percentage = dynamic_bytes_ready_sample_percentage; + Ok(()) + } + fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext) { self.set_extension(Arc::new(local_worker_context)); } @@ -1191,6 +1215,10 @@ impl DistributedExt for SessionConfig { #[expr($?;Ok(self))] fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result; + #[call(set_distributed_dynamic_bytes_ready_sample_percentage)] + #[expr($?;Ok(self))] + fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result; + #[call(set_distributed_local_worker_context)] #[expr($;self)] fn with_distributed_local_worker_context(mut self, local_worker_context: LocalWorkerContext) -> Self; @@ -1362,6 +1390,11 @@ impl DistributedExt for SessionStateBuilder { #[expr($?;Ok(self))] fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result; + fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_dynamic_bytes_ready_sample_percentage)] + #[expr($?;Ok(self))] + fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result; + fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext); #[call(set_distributed_local_worker_context)] #[expr($;self)] @@ -1542,6 +1575,11 @@ impl DistributedExt for SessionState { #[expr($?;Ok(self))] fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result; + fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_dynamic_bytes_ready_sample_percentage)] + #[expr($?;Ok(self))] + fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result; + fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext); #[call(set_distributed_local_worker_context)] #[expr($;self)] @@ -1715,6 +1753,11 @@ impl DistributedExt for SessionContext { #[expr($?;Ok(self))] fn with_distributed_dynamic_bytes_per_partition(self, dynamic_bytes_per_partition: usize) -> Result; + fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_dynamic_bytes_ready_sample_percentage)] + #[expr($?;Ok(self))] + fn with_distributed_dynamic_bytes_ready_sample_percentage(self, dynamic_bytes_ready_sample_percentage: f64) -> Result; + fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext); #[call(set_distributed_local_worker_context)] #[expr($;self)] diff --git a/datafusion/distributed/src/distributed_planner/distributed_config.rs b/datafusion/distributed/src/distributed_planner/distributed_config.rs index deec18eb4..3d4e60cce 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_config.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_config.rs @@ -93,6 +93,11 @@ extensions_options! { /// If `dynamic_task_count` is enabled, this value is the amount of bytes each /// partition is expected to handle. Lower values will result in greater parallelism. pub dynamic_bytes_per_partition: usize, default = 16 * 1024 * 1024 + /// If `dynamic_task_count` is enabled, this is the fraction of a stage's partitions that + /// need to report sampled bytes before the stage's runtime statistics are estimated. Higher + /// values delay execution longer while sampling, but produce more precise estimations. With + /// `1.0`, every partition is sampled before estimating. + pub dynamic_bytes_ready_sample_percentage: f64, default = 0.2 } } diff --git a/docs/source/advanced/08-adaptive-query-execution.md b/docs/source/advanced/08-adaptive-query-execution.md index 4e5e4c80b..0ad186529 100644 --- a/docs/source/advanced/08-adaptive-query-execution.md +++ b/docs/source/advanced/08-adaptive-query-execution.md @@ -30,10 +30,11 @@ let state = SessionStateBuilder::new() The following settings affect AQE decisions: -| Setting | Default | Effect | -|-------------------------------------------|--------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `distributed.dynamic_task_count` | `false` | Enables runtime task-count selection. | -| `distributed.dynamic_bytes_per_partition` | 16 MiB | Sets the compute-cost budget per output partition. Lower values generally produce more tasks; higher values produce fewer. Configure it with `with_distributed_dynamic_bytes_per_partition`. | +| Setting | Default | Effect | +|-----------------------------------------------------|--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `distributed.dynamic_task_count` | `false` | Enables runtime task-count selection. | +| `distributed.dynamic_bytes_per_partition` | 16 MiB | Sets the compute-cost budget per output partition. Lower values generally produce more tasks; higher values produce fewer. Configure it with `with_distributed_dynamic_bytes_per_partition`. | +| `distributed.dynamic_bytes_ready_sample_percentage` | `0.2` | Sets the fraction of a stage's partitions that must report sampled bytes before its runtime statistics are estimated. Higher values delay execution longer but produce more precise estimates; `1.0` samples every partition. Configure it with `with_distributed_dynamic_bytes_ready_sample_percentage`. | ## Custom data sources From ffa335a786018c37108061a683f88d062a01a146 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 1 Nov 2026 12:27:42 +0100 Subject: [PATCH 2/5] Add a remainder replanner --- .../src/coordinator/prepare_dynamic_plan.rs | 3 + .../distributed_query_planner.rs | 4 +- .../inject_network_boundaries.rs | 348 ++++++++++++++++-- .../src/execution_plans/network_broadcast.rs | 10 +- .../src/execution_plans/network_coalesce.rs | 8 +- .../src/execution_plans/network_shuffle.rs | 10 +- 6 files changed, 340 insertions(+), 43 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index fb8de094e..5c8530534 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -41,6 +41,9 @@ pub(super) async fn prepare_dynamic_plan( let head_stage = inject_network_boundaries( Arc::clone(base_plan), + // TODO: Inject here a plan remainder replanner for reshaping the topology of the plan based on the runtime + // collected statistics. + Ok, DynamicPlanBuilder { query_coordinator, plans_for_viz: &plans_for_viz, diff --git a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs index 8ecbab25c..42128a8a4 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs @@ -157,8 +157,8 @@ pub fn create_distributed_plan( } // Compute per-node task counts and inject `Network*Exec` nodes at the stage boundaries. - plan = inject_network_boundaries(plan, CardinalityBasedNetworkBoundaryBuilder, session_cfg) - .await?; + let nb_builder = CardinalityBasedNetworkBoundaryBuilder; + plan = inject_network_boundaries(plan, Ok, nb_builder, session_cfg).await?; plan = prepare_network_boundaries(plan)?; if !plan.exists(|plan| Ok(plan.is_network_boundary()))? { diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index e796a7cca..4e0109f57 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -12,7 +12,8 @@ use crate::{ NetworkCoalesceExec, NetworkShuffleExec, Stage, }; use async_trait::async_trait; -use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::common::tree_node::TreeNodeRecursion::{Continue, Stop}; +use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::common::{HashMap, JoinType, Result, internal_err, plan_err}; use datafusion::physical_expr::Partitioning; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; @@ -144,7 +145,8 @@ use uuid::Uuid; /// the whole plan. This guarantees every node (including head-stage nodes that never sat /// directly above a boundary) has a task count recorded. pub(crate) async fn inject_network_boundaries( - plan: Arc, + mut plan: Arc, + remainder_replanner: impl Fn(Arc) -> Result>, nb_builder: impl NetworkBoundaryBuilder + Send + Sync, session_cfg: &SessionConfig, ) -> Result> { @@ -158,7 +160,15 @@ pub(crate) async fn inject_network_boundaries( stage_id: &AtomicUsize::new(1), }; - _inject_network_boundaries(plan, None, &ctx).await + let mut keep_transforming = true; + while keep_transforming { + let result = _inject_network_boundaries(plan, None, &ctx).await?; + plan = remainder_replanner(result.data)?; + // The _inject_network_boundaries pass needed to stop because it injected network + // boundaries, therefore, there might be more network boundaries to inject in a new pass. + keep_transforming = matches!(result.tnr, Stop); + } + Ok(plan) } type ConcurrentHashMap = Mutex>; @@ -205,6 +215,19 @@ impl<'a> InjectNetworkBoundaryContext<'a> { plan } + fn plan_with_inherited_task_count( + &self, + plan: Arc, + f: impl FnOnce(Arc) -> Result>, + ) -> Result> { + let task_count = self.task_count(&plan).ok(); + let mut plan = f(plan)?; + if let Some(task_count) = task_count { + plan = self.plan_with_task_count(plan, task_count); + } + Ok(plan) + } + pub(crate) fn task_count(&self, plan: &Arc) -> Result { let lock = self.task_counts.lock().expect("task counts mutex poisoned"); let Some((task_count, _)) = lock.get(&plan_ptr_key(plan)) else { @@ -235,7 +258,11 @@ async fn _inject_network_boundaries( plan: Arc, parent: Option<&Arc>, nb_ctx: &InjectNetworkBoundaryContext<'_>, -) -> Result> { +) -> Result>> { + if plan.is_network_boundary() { + return Ok(Transformed::no(plan)); + } + let broadcast_joins_enabled = nb_ctx.d_cfg.broadcast_joins; let mut futures = Vec::with_capacity(plan.children().len()); @@ -248,6 +275,20 @@ async fn _inject_network_boundaries( ))); } let processed_children = futures::future::try_join_all(futures).await?; + let plan = nb_ctx.plan_with_inherited_task_count(plan, |plan| { + let children = processed_children + .iter() + .map(|v| Arc::clone(&v.data)) + .collect(); + plan.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + })?; + + if processed_children.iter().any(|v| v.tnr == Stop) { + return Ok(Transformed::new(plan, true, Stop)); + } let ev = DesiredTaskCountEvent { plan: &plan, @@ -269,8 +310,8 @@ async fn _inject_network_boundaries( // the union's minimum task count. let mut load: f64 = 0.0; let mut min: Option = None; - for child in &processed_children { - let child_count = nb_ctx.task_count(child)?; + for processed_child in &processed_children { + let child_count = nb_ctx.task_count(&processed_child.data)?; load += match child_count.restriction { TaskCountRestriction::Exact(count) => child_count.soft.min(count.get() as f64), _ => child_count @@ -306,7 +347,7 @@ async fn _inject_network_boundaries( // sides have one output partition). let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? + task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? } task_count } else if let Some(node) = plan.downcast_ref::() @@ -323,7 +364,7 @@ async fn _inject_network_boundaries( // sides have one output partition). let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? + task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? } task_count } else if plan.is::() && !broadcast_joins_enabled { @@ -332,7 +373,7 @@ async fn _inject_network_boundaries( // broadcasts are unavailable. let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? + task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? } task_count } else { @@ -342,15 +383,11 @@ async fn _inject_network_boundaries( .map(|v| v.task_count) .unwrap_or(TaskCountAnnotation::soft(0.0)); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? + task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? } task_count }; - let plan = plan.replace_children( - processed_children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - )?; // Cap the reconciled task count by the configured max-per-stage budget. let max_tasks = nb_ctx.max_tasks()?; if let Some(min) = task_count.get_min() @@ -377,11 +414,12 @@ async fn _inject_network_boundaries( .nb_builder .build(input_stage, TypeId::of::(), nb_ctx) .await?; - let shuffle = Arc::new(NetworkShuffleExec::from_stage( + let nb = Arc::new(NetworkShuffleExec::from_stage( result.input_stage, result.input_properties, )); - Ok(nb_ctx.plan_with_task_count(shuffle, result.consumer_task_count)) + let nb = nb_ctx.plan_with_task_count(nb, result.consumer_task_count); + Ok(Transformed::new(nb, true, Stop)) } // Upon reaching a broadcast, we need to introduce a network broadcast right above it. else if let Some(_b_exec) = plan.downcast_ref::() { @@ -400,7 +438,8 @@ async fn _inject_network_boundaries( result.input_stage, result.input_properties, )); - Ok(nb_ctx.plan_with_task_count(nb, result.consumer_task_count)) + let nb = nb_ctx.plan_with_task_count(nb, result.consumer_task_count); + Ok(Transformed::new(nb, true, Stop)) } // If the parent of the current node is either a `CoalescePartitionsExec` or a // `SortPreservingMergeExec`, a network boundary below it is necessary. @@ -430,16 +469,19 @@ async fn _inject_network_boundaries( result.input_properties, 1, )?); - Ok(nb_ctx.plan_with_task_count(nb, result.consumer_task_count)) + let nb = nb_ctx.plan_with_task_count(nb, result.consumer_task_count); + Ok(Transformed::new(nb, true, Stop)) } else if parent.is_none() { // We've just finished walking the head stage's subplan. Run a final propagation so // every node in the head stage (which never crossed a stage boundary on the way up) // gets its task count recorded. - nb_ctx.propagate_task_count_until_network_boundaries(&plan, task_count) + let plan = nb_ctx.propagate_task_count_until_network_boundaries(&plan, task_count)?; + Ok(Transformed::yes(plan)) } else { // If this is not the root node, and it's also not a network boundary, then we don't need // to do anything else. - Ok(nb_ctx.plan_with_task_count(plan, task_count)) + let plan = nb_ctx.plan_with_task_count(plan, task_count); + Ok(Transformed::yes(plan)) } } @@ -497,7 +539,7 @@ impl InjectNetworkBoundaryContext<'_> { let scaled_up = response?.plan; scaled_up.apply(|plan| { self.set_task_count(plan, task_count); - Ok(TreeNodeRecursion::Continue) + Ok(Continue) })?; Ok(self.plan_with_task_count(scaled_up, task_count)) } @@ -798,6 +840,25 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + SortPreservingMergeExec: task_count=missing + ProjectionExec: task_count=missing + SortExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=2.67 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ + SortPreservingMergeExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=2.67, exact=1 + ProjectionExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 + SortExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 + AggregateExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ SortPreservingMergeExec: task_count=TaskCountAnnotation: soft=2.67, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=2.67, exact=1 ProjectionExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 @@ -823,6 +884,14 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=4.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=4.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -867,6 +936,48 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + RepartitionExec(Hash): task_count=missing + ProjectionExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.78 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + FilterExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + RepartitionExec(RoundRobinBatch): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + RepartitionExec(Hash): task_count=missing + ProjectionExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.78 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + FilterExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + RepartitionExec(RoundRobinBatch): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ + HashJoinExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.19 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=1.78, exact=2 + ProjectionExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + AggregateExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + FilterExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + RepartitionExec(RoundRobinBatch): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.19 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=1.78, exact=2 + ProjectionExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + AggregateExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.78, exact=2 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + FilterExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + RepartitionExec(RoundRobinBatch): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=1.19, exact=2 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.19, exact=2 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=1.78, exact=2 @@ -906,6 +1017,12 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -928,6 +1045,12 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=2.67 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ AggregateExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=2.67, exact=3 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1019,6 +1142,13 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + ProjectionExec: task_count=missing + BoundedWindowAggExec: task_count=missing + SortExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=4.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + ------------------------------------ ProjectionExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 BoundedWindowAggExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 SortExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1097,6 +1227,19 @@ mod tests { .desired_task_count_handler(fractional_leaf_desired_task_count_handler); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=0.27 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=0.40, exact=1 + AggregateExec: task_count=TaskCountAnnotation: soft=0.40, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=0.40, exact=1 + ProjectionExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=0.27 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=0.40, exact=1 + AggregateExec: task_count=TaskCountAnnotation: soft=0.40, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=0.40, exact=1 + ------------------------------------ ChildrenIsolatorUnionExec: task_count=TaskCountAnnotation: soft=0.53, exact=1 AggregateExec: task_count=TaskCountAnnotation: soft=0.53, exact=1 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=0.53, exact=1 @@ -1126,6 +1269,12 @@ mod tests { .desired_task_count_handler(repartition_max_one_desired_task_count_handler); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=1 + AggregateExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + ------------------------------------ AggregateExec: task_count=TaskCountAnnotation: soft=1.00, exact=1 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=1.00, exact=1 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -1203,6 +1352,17 @@ mod tests { "#; let annotated = annotate_test_plan(test_plan_builder, nested_query).await; assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=missing + CrossJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=0.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=0.00, exact=1 + DataSourceExec: task_count=TaskCountAnnotation: soft=0.00, exact=1 + ChildrenIsolatorUnionExec: task_count=TaskCountAnnotation: soft=0.00, min=2 + DataSourceExec: task_count=TaskCountAnnotation: soft=0.00, exact=2 + DataSourceExec: task_count=TaskCountAnnotation: soft=0.00, exact=2 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ ChildrenIsolatorUnionExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 CrossJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=2 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=2 @@ -1264,6 +1424,13 @@ mod tests { .broadcast_joins(true); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1305,6 +1472,13 @@ mod tests { let annotated = annotate_test_plan(test_plan_builder, query).await; assert!(annotated.contains("Broadcast")); assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1330,6 +1504,13 @@ mod tests { .desired_task_count_handler(build_side_one_desired_task_count_handler); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=0.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=0.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=0.00, exact=1 + DataSourceExec: task_count=TaskCountAnnotation: soft=3.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 @@ -1355,12 +1536,19 @@ mod tests { .desired_task_count_handler(broadcast_build_coalesce_max_desired_task_count_handler); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" - HashJoinExec: task_count=TaskCountAnnotation: soft=3.00, exact=1 - CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=3.00, exact=1 - NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=3.00, exact=1 + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=3.00 BroadcastExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 DistributedLeafExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 - DistributedLeafExec: task_count=TaskCountAnnotation: soft=3.00, exact=1 + DataSourceExec: task_count=TaskCountAnnotation: soft=3.00 + ------------------------------------ + HashJoinExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 + CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 + BroadcastExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=3.00, exact=3 "); } @@ -1381,6 +1569,12 @@ mod tests { .broadcast_joins(true); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -1406,6 +1600,13 @@ mod tests { .broadcast_joins(true); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + NestedLoopJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ NestedLoopJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1433,6 +1634,12 @@ mod tests { .broadcast_joins(true); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + NestedLoopJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ NestedLoopJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -1457,6 +1664,12 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + CrossJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ CrossJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -1482,6 +1695,12 @@ mod tests { // With broadcast disabled, no broadcast annotation should appear assert!(!annotated.contains("Broadcast")); assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 NetworkCoalesceExec: task_count=TaskCountAnnotation: soft=4.00, exact=1 @@ -1506,6 +1725,29 @@ mod tests { .broadcast_joins(true); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + BroadcastExec: task_count=missing + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 @@ -1546,6 +1788,26 @@ mod tests { // With ChildrenIsolatorUnionExec, each broadcast task_count should be limited to their // context. assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=missing + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + HashJoinExec: task_count=missing + CoalescePartitionsExec: task_count=missing + NetworkBroadcastExec: task_count=TaskCountAnnotation: soft=4.00 + BroadcastExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 + DataSourceExec: task_count=TaskCountAnnotation: soft=4.00 + ------------------------------------ ChildrenIsolatorUnionExec: task_count=TaskCountAnnotation: soft=4.00, exact=4 HashJoinExec: task_count=TaskCountAnnotation: soft=4.00, exact=2 CoalescePartitionsExec: task_count=TaskCountAnnotation: soft=4.00, exact=2 @@ -1580,6 +1842,13 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + ProjectionExec: task_count=missing + AggregateExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=20.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=20.00, exact=20 + AggregateExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 + ------------------------------------ ProjectionExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 AggregateExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=20.00, exact=20 @@ -1602,6 +1871,14 @@ mod tests { .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; assert_snapshot!(annotated, @r" + HashJoinExec: task_count=missing + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=20.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=20.00, exact=20 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 + NetworkShuffleExec: task_count=TaskCountAnnotation: soft=20.00 + RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=20.00, exact=20 + DistributedLeafExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 + ------------------------------------ HashJoinExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 RepartitionExec(Hash): task_count=TaskCountAnnotation: soft=20.00, exact=20 NetworkShuffleExec: task_count=TaskCountAnnotation: soft=20.00, exact=20 @@ -1658,7 +1935,7 @@ mod tests { .expect("failed to normalize collect joins"); let plan = insert_broadcast_execs(plan, session_config.options()) .expect("failed to insert broadcasts"); - let plan = insert_children_isolator_unions(plan, session_config.options()) + let mut plan = insert_children_isolator_unions(plan, session_config.options()) .expect("failed to insert children isolator unions"); let network_boundaries_ctx = InjectNetworkBoundaryContext { cfg: &session_config, @@ -1670,10 +1947,21 @@ mod tests { nb_builder: &CardinalityBasedNetworkBoundaryBuilder, }; - let annotated = _inject_network_boundaries(plan, None, &network_boundaries_ctx) - .await - .expect("failed to annotate plan"); - debug_annotated(&annotated, 0, &network_boundaries_ctx) + let mut annotated = String::new(); + + let mut keep_transforming = true; + while keep_transforming { + let result = _inject_network_boundaries(plan, None, &network_boundaries_ctx) + .await + .expect("failed to annotate plan"); + plan = result.data; + keep_transforming = matches!(result.tnr, Stop); + annotated += &debug_annotated(&plan, 0, &network_boundaries_ctx); + if keep_transforming { + annotated += "------------------------------------\n"; + } + } + annotated } fn debug_annotated( diff --git a/datafusion/distributed/src/execution_plans/network_broadcast.rs b/datafusion/distributed/src/execution_plans/network_broadcast.rs index 10581506a..cae8401de 100644 --- a/datafusion/distributed/src/execution_plans/network_broadcast.rs +++ b/datafusion/distributed/src/execution_plans/network_broadcast.rs @@ -227,19 +227,21 @@ impl ExecutionPlan for NetworkBroadcastExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + ) -> Result> { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_coalesce.rs b/datafusion/distributed/src/execution_plans/network_coalesce.rs index db9b964a8..18340d386 100644 --- a/datafusion/distributed/src/execution_plans/network_coalesce.rs +++ b/datafusion/distributed/src/execution_plans/network_coalesce.rs @@ -251,18 +251,20 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_shuffle.rs b/datafusion/distributed/src/execution_plans/network_shuffle.rs index 16e73ad73..786ca55c9 100644 --- a/datafusion/distributed/src/execution_plans/network_shuffle.rs +++ b/datafusion/distributed/src/execution_plans/network_shuffle.rs @@ -299,19 +299,21 @@ impl ExecutionPlan for NetworkShuffleExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + ) -> Result> { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( From 2b35ef1ebaec3d82f18dee3d9039191a15d8be2a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 6 Nov 2026 12:37:44 +0100 Subject: [PATCH 3/5] Optimize planning traversals, avoiding recomputations --- .../src/coordinator/prepare_dynamic_plan.rs | 5 +- .../inject_network_boundaries.rs | 118 +++++++++++------- .../src/execution_plans/network_broadcast.rs | 10 +- .../src/execution_plans/network_coalesce.rs | 8 +- .../src/execution_plans/network_shuffle.rs | 10 +- 5 files changed, 85 insertions(+), 66 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index 5c8530534..b74f263ce 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -111,8 +111,9 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { // Propagate the final task_count inferred based on runtime statistics and compute cost. // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the // plan is finally left ready for distribution. - input_stage.plan = - nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; + input_stage.plan = nb_ctx + .propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)? + .data; input_stage.tasks = task_count.as_usize(); // In order to infer the compute the cost of the stage above this one, here a sampler // is injected to gather runtime statistics. diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index 4e0109f57..be76ec4e2 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -220,12 +220,17 @@ impl<'a> InjectNetworkBoundaryContext<'a> { plan: Arc, f: impl FnOnce(Arc) -> Result>, ) -> Result> { - let task_count = self.task_count(&plan).ok(); - let mut plan = f(plan)?; - if let Some(task_count) = task_count { - plan = self.plan_with_task_count(plan, task_count); - } - Ok(plan) + let task_count_opt = { + let key = plan_ptr_key(&plan); + let lock = self.task_counts.lock().expect("task counts mutex poisoned"); + lock.get(&key).map(|(v, _)| *v) + }; + let plan = f(plan)?; + + Ok(match task_count_opt { + Some(task_count) => self.plan_with_task_count(plan, task_count), + None => plan, + }) } pub(crate) fn task_count(&self, plan: &Arc) -> Result { @@ -255,7 +260,7 @@ fn plan_ptr_key(plan: &Arc) -> usize { /// (or [InjectNetworkBoundaryContext::set_task_count] on the way through) so the returned node has /// a recorded task count. Callers downstream depend on this invariant. async fn _inject_network_boundaries( - plan: Arc, + mut plan: Arc, parent: Option<&Arc>, nb_ctx: &InjectNetworkBoundaryContext<'_>, ) -> Result>> { @@ -275,18 +280,20 @@ async fn _inject_network_boundaries( ))); } let processed_children = futures::future::try_join_all(futures).await?; - let plan = nb_ctx.plan_with_inherited_task_count(plan, |plan| { - let children = processed_children - .iter() - .map(|v| Arc::clone(&v.data)) - .collect(); - plan.replace_children( - children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - ) - })?; + let any_children_transformed = processed_children.iter().any(|v| v.transformed); + let any_children_stopped = processed_children.iter().any(|v| v.tnr == Stop); + let processed_children: Vec<_> = processed_children.into_iter().map(|v| v.data).collect(); + + if any_children_transformed { + plan = nb_ctx.plan_with_inherited_task_count(plan, |plan| { + plan.replace_children( + processed_children.clone(), + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + })?; + } - if processed_children.iter().any(|v| v.tnr == Stop) { + if any_children_stopped { return Ok(Transformed::new(plan, true, Stop)); } @@ -311,7 +318,7 @@ async fn _inject_network_boundaries( let mut load: f64 = 0.0; let mut min: Option = None; for processed_child in &processed_children { - let child_count = nb_ctx.task_count(&processed_child.data)?; + let child_count = nb_ctx.task_count(processed_child)?; load += match child_count.restriction { TaskCountRestriction::Exact(count) => child_count.soft.min(count.get() as f64), _ => child_count @@ -347,7 +354,7 @@ async fn _inject_network_boundaries( // sides have one output partition). let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? + task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? } task_count } else if let Some(node) = plan.downcast_ref::() @@ -364,7 +371,7 @@ async fn _inject_network_boundaries( // sides have one output partition). let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? + task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? } task_count } else if plan.is::() && !broadcast_joins_enabled { @@ -373,7 +380,7 @@ async fn _inject_network_boundaries( // broadcasts are unavailable. let mut task_count = TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? + task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? } task_count } else { @@ -383,7 +390,7 @@ async fn _inject_network_boundaries( .map(|v| v.task_count) .unwrap_or(TaskCountAnnotation::soft(0.0)); for processed_child in &processed_children { - task_count = task_count.merge(nb_ctx.task_count(&processed_child.data)?)? + task_count = task_count.merge(nb_ctx.task_count(processed_child)?)? } task_count }; @@ -475,13 +482,12 @@ async fn _inject_network_boundaries( // We've just finished walking the head stage's subplan. Run a final propagation so // every node in the head stage (which never crossed a stage boundary on the way up) // gets its task count recorded. - let plan = nb_ctx.propagate_task_count_until_network_boundaries(&plan, task_count)?; - Ok(Transformed::yes(plan)) + nb_ctx.propagate_task_count_until_network_boundaries(&plan, task_count) } else { // If this is not the root node, and it's also not a network boundary, then we don't need // to do anything else. let plan = nb_ctx.plan_with_task_count(plan, task_count); - Ok(Transformed::yes(plan)) + Ok(Transformed::new(plan, any_children_transformed, Continue)) } } @@ -523,7 +529,7 @@ impl InjectNetworkBoundaryContext<'_> { &self, plan: &Arc, task_count: TaskCountAnnotation, - ) -> Result> { + ) -> Result>> { let task_count = task_count.into_sealed(); // Remote network boundaries have no children, but still need shuffle preparation. if plan.children().is_empty() && !plan.is_network_boundary() { @@ -533,7 +539,10 @@ impl InjectNetworkBoundaryContext<'_> { session_config: self.cfg, }; match ScaleUpLeafNodeHandlers::handle(ev) { - None => Ok(self.plan_with_task_count(Arc::clone(plan), task_count)), + None => { + let plan = self.plan_with_task_count(Arc::clone(plan), task_count); + Ok(Transformed::no(plan)) + } Some(response) => { // The scaled up subtree may contain more than 1 node. let scaled_up = response?.plan; @@ -541,7 +550,8 @@ impl InjectNetworkBoundaryContext<'_> { self.set_task_count(plan, task_count); Ok(Continue) })?; - Ok(self.plan_with_task_count(scaled_up, task_count)) + let plan = self.plan_with_task_count(scaled_up, task_count); + Ok(Transformed::yes(plan)) } } @@ -571,11 +581,13 @@ impl InjectNetworkBoundaryContext<'_> { // NetworkShuffleExec below. .with_preserve_order(), ) as Arc; - return Ok(self.plan_with_task_count(consumer_repartition, task_count)); + let plan = self.plan_with_task_count(consumer_repartition, task_count); + return Ok(Transformed::yes(plan)); } } // Just annotate the network boundary and stop recursion here. - Ok(self.plan_with_task_count(Arc::clone(plan), task_count)) + let plan = self.plan_with_task_count(Arc::clone(plan), task_count); + Ok(Transformed::no(plan)) // Finalize a pre-inserted ChildrenIsolatorUnionExec. } else if plan.is::() { @@ -613,24 +625,35 @@ impl InjectNetworkBoundaryContext<'_> { new_children .push(self.propagate_task_count_until_network_boundaries(child, task_count)?); } - let c_i_union = Arc::new(c_i_union).replace_children( - new_children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - )?; - Ok(self.plan_with_task_count(c_i_union, task_count)) + let mut c_i_union = Arc::new(c_i_union) as Arc; + if new_children.iter().any(|v| v.transformed) { + c_i_union = c_i_union.replace_children( + new_children.into_iter().map(|v| v.data).collect(), + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; + } + let plan = self.plan_with_task_count(c_i_union, task_count); + Ok(Transformed::yes(plan)) // Handle middle nodes. } else { - let mut new_children = Vec::with_capacity(plan.children().len()); - for child in plan.children() { - new_children - .push(self.propagate_task_count_until_network_boundaries(child, task_count)?); + let new_children = plan + .children() + .into_iter() + .map(|child| self.propagate_task_count_until_network_boundaries(child, task_count)) + .collect::>>()?; + + if new_children.iter().any(|v| v.transformed) { + let plan = Arc::clone(plan).replace_children( + new_children.into_iter().map(|v| v.data).collect(), + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; + let plan = self.plan_with_task_count(plan, task_count); + Ok(Transformed::yes(plan)) + } else { + let plan = self.plan_with_task_count(Arc::clone(plan), task_count); + Ok(Transformed::no(plan)) } - let plan = Arc::clone(plan).replace_children( - new_children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - )?; - Ok(self.plan_with_task_count(plan, task_count)) } } } @@ -726,8 +749,9 @@ impl NetworkBoundaryBuilder for CardinalityBasedNetworkBoundaryBuilder { nb_ctx: &'a InjectNetworkBoundaryContext<'a>, ) -> Result { let tc = nb_ctx.task_count(&input_stage.plan)?; - input_stage.plan = - nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, tc)?; + input_stage.plan = nb_ctx + .propagate_task_count_until_network_boundaries(&input_stage.plan, tc)? + .data; let input_properties = Arc::clone(input_stage.plan.properties()); if nb_type == TypeId::of::() { diff --git a/datafusion/distributed/src/execution_plans/network_broadcast.rs b/datafusion/distributed/src/execution_plans/network_broadcast.rs index cae8401de..10581506a 100644 --- a/datafusion/distributed/src/execution_plans/network_broadcast.rs +++ b/datafusion/distributed/src/execution_plans/network_broadcast.rs @@ -227,21 +227,19 @@ impl ExecutionPlan for NetworkBroadcastExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result> { - match &self.input_stage { + ) -> Result, DataFusionError> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { Stage::Local(local) => { - let mut local = local.clone(); local.plan = require_one_child(children)?; - self.with_input_stage(Stage::Local(local)) - .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } - Ok(self) } } + Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_coalesce.rs b/datafusion/distributed/src/execution_plans/network_coalesce.rs index 18340d386..db9b964a8 100644 --- a/datafusion/distributed/src/execution_plans/network_coalesce.rs +++ b/datafusion/distributed/src/execution_plans/network_coalesce.rs @@ -251,20 +251,18 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result> { - match &self.input_stage { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { Stage::Local(local) => { - let mut local = local.clone(); local.plan = require_one_child(children)?; - self.with_input_stage(Stage::Local(local)) - .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } - Ok(self) } } + Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_shuffle.rs b/datafusion/distributed/src/execution_plans/network_shuffle.rs index 786ca55c9..16e73ad73 100644 --- a/datafusion/distributed/src/execution_plans/network_shuffle.rs +++ b/datafusion/distributed/src/execution_plans/network_shuffle.rs @@ -299,21 +299,19 @@ impl ExecutionPlan for NetworkShuffleExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result> { - match &self.input_stage { + ) -> Result, DataFusionError> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { Stage::Local(local) => { - let mut local = local.clone(); local.plan = require_one_child(children)?; - self.with_input_stage(Stage::Local(local)) - .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } - Ok(self) } } + Ok(Arc::new(self_clone)) } fn execute( From 308ecebbc160b2a1ae46cedca1c5ca41634dfada Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 8 Nov 2026 10:43:05 +0100 Subject: [PATCH 4/5] Keep remote network boundaries unchanged in with_new_children A network boundary whose stage is already remote has no children, but DataFusion rules like EnsureRequirements still call `with_new_children` on it with an empty list. Returning a fresh clone gave the node a new identity, which dropped the task count annotations that the network boundary injection keys by node pointer, failing the next planning pass. Remote boundaries now return themselves, and local ones are rebuilt through `with_input_stage`. --- .../src/execution_plans/network_broadcast.rs | 10 ++++++---- .../src/execution_plans/network_coalesce.rs | 8 +++++--- .../distributed/src/execution_plans/network_shuffle.rs | 10 ++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/datafusion/distributed/src/execution_plans/network_broadcast.rs b/datafusion/distributed/src/execution_plans/network_broadcast.rs index 10581506a..cae8401de 100644 --- a/datafusion/distributed/src/execution_plans/network_broadcast.rs +++ b/datafusion/distributed/src/execution_plans/network_broadcast.rs @@ -227,19 +227,21 @@ impl ExecutionPlan for NetworkBroadcastExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + ) -> Result> { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_coalesce.rs b/datafusion/distributed/src/execution_plans/network_coalesce.rs index db9b964a8..18340d386 100644 --- a/datafusion/distributed/src/execution_plans/network_coalesce.rs +++ b/datafusion/distributed/src/execution_plans/network_coalesce.rs @@ -251,18 +251,20 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/execution_plans/network_shuffle.rs b/datafusion/distributed/src/execution_plans/network_shuffle.rs index 16e73ad73..786ca55c9 100644 --- a/datafusion/distributed/src/execution_plans/network_shuffle.rs +++ b/datafusion/distributed/src/execution_plans/network_shuffle.rs @@ -299,19 +299,21 @@ impl ExecutionPlan for NetworkShuffleExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + ) -> Result> { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( From 878ea353bb482fa6ef8d3e0bd2845472c579f1cf Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 8 Nov 2026 10:48:10 +0100 Subject: [PATCH 5/5] Pass the network boundary injection context to the remainder replanner The remainder replanner runs between network boundary injection passes, and replanning decisions need the same context those passes use: the session and distributed configuration, and the task counts annotated so far. --- .../distributed/src/coordinator/prepare_dynamic_plan.rs | 2 +- .../src/distributed_planner/distributed_query_planner.rs | 9 +++++++-- .../src/distributed_planner/inject_network_boundaries.rs | 7 +++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index b74f263ce..6d893a4a1 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -43,7 +43,7 @@ pub(super) async fn prepare_dynamic_plan( Arc::clone(base_plan), // TODO: Inject here a plan remainder replanner for reshaping the topology of the plan based on the runtime // collected statistics. - Ok, + |plan, _nb_ctx| Ok(plan), DynamicPlanBuilder { query_coordinator, plans_for_viz: &plans_for_viz, diff --git a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs index 42128a8a4..c09ed37fa 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs @@ -157,8 +157,13 @@ pub fn create_distributed_plan( } // Compute per-node task counts and inject `Network*Exec` nodes at the stage boundaries. - let nb_builder = CardinalityBasedNetworkBoundaryBuilder; - plan = inject_network_boundaries(plan, Ok, nb_builder, session_cfg).await?; + plan = inject_network_boundaries( + plan, + |plan, _nb_ctx| Ok(plan), + CardinalityBasedNetworkBoundaryBuilder, + session_cfg, + ) + .await?; plan = prepare_network_boundaries(plan)?; if !plan.exists(|plan| Ok(plan.is_network_boundary()))? { diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index be76ec4e2..c25a6c003 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -146,7 +146,10 @@ use uuid::Uuid; /// directly above a boundary) has a task count recorded. pub(crate) async fn inject_network_boundaries( mut plan: Arc, - remainder_replanner: impl Fn(Arc) -> Result>, + remainder_replanner: impl Fn( + Arc, + &InjectNetworkBoundaryContext, + ) -> Result>, nb_builder: impl NetworkBoundaryBuilder + Send + Sync, session_cfg: &SessionConfig, ) -> Result> { @@ -163,7 +166,7 @@ pub(crate) async fn inject_network_boundaries( let mut keep_transforming = true; while keep_transforming { let result = _inject_network_boundaries(plan, None, &ctx).await?; - plan = remainder_replanner(result.data)?; + plan = remainder_replanner(result.data, &ctx)?; // The _inject_network_boundaries pass needed to stop because it injected network // boundaries, therefore, there might be more network boundaries to inject in a new pass. keep_transforming = matches!(result.tnr, Stop);