From 1c74880d046ef35e0aaafeca1180d3363066de27 Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Thu, 5 Nov 2026 22:05:17 +0000 Subject: [PATCH 01/10] perf: overlap worker execution with task setup Start coordinator inputs after their own connection-aware setup and start AQE sampling as each task becomes ready. Keep concrete child destinations in worker plans and preserve task reporting and dynamic-filter stage sealing. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- src/coordinator/prepare_dynamic_plan.rs | 240 +++++------ src/coordinator/prepare_static_plan.rs | 173 +++++--- src/coordinator/query_coordinator.rs | 41 +- .../inject_network_boundaries.rs | 15 + src/distributed_planner/mod.rs | 3 +- src/distributed_planner/network_boundary.rs | 4 +- src/execution_plans/mod.rs | 2 + src/execution_plans/network_broadcast.rs | 54 ++- src/execution_plans/network_coalesce.rs | 106 ++--- src/execution_plans/network_shuffle.rs | 108 ++--- .../pending_network_boundary.rs | 137 ++++++ src/worker/mod.rs | 2 +- src/worker/worker_connection_pool.rs | 170 ++++++-- tests/startup_overlap.rs | 395 ++++++++++++++++++ 14 files changed, 1116 insertions(+), 334 deletions(-) create mode 100644 src/execution_plans/pending_network_boundary.rs create mode 100644 tests/startup_overlap.rs diff --git a/src/coordinator/prepare_dynamic_plan.rs b/src/coordinator/prepare_dynamic_plan.rs index e8453d4d9..ea2149890 100644 --- a/src/coordinator/prepare_dynamic_plan.rs +++ b/src/coordinator/prepare_dynamic_plan.rs @@ -1,9 +1,10 @@ use crate::common::{TreeNodeExt, element_wise_sum, vec_avg_reduce, vec_div, vec_mul}; use crate::coordinator::distributed::PreparedPlan; +use crate::coordinator::prepare_static_plan::prepare_stages; use crate::coordinator::query_coordinator::QueryCoordinator; use crate::distributed_planner::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, ProducerHead, calculate_cost, - inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + ProducerHead, calculate_cost, inject_network_boundaries, }; use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, @@ -14,10 +15,11 @@ use crate::{ BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, TaskCountAnnotation, }; +use async_trait::async_trait; use dashmap::DashMap; use datafusion::common::stats::Precision; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{Result, exec_err, plan_err}; +use datafusion::common::{DataFusionError, Result, exec_err, plan_err}; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Statistics, StatisticsArgs, @@ -33,128 +35,17 @@ pub(super) async fn prepare_dynamic_plan( query_coordinator: &Arc, base_plan: &Arc, ) -> Result { - let plans_for_viz = Arc::new(PlanReconstructor::default()); - + let plans_for_viz = PlanReconstructor::default(); let head_stage = inject_network_boundaries( Arc::clone(base_plan), - |mut input_stage: LocalStage, nb_type: TypeId, nb_ctx: &InjectNetworkBoundaryContext| { - let mut metrics = MetricsSet::new(); - - // At this point, input_stage.plan has two kind of leaf nodes: - // - The ones that naturally do not read from any children, like DataSourceExec - // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration - // of this same function. - // Both types of leaf nodes contain very valuable and accurate statistics that are used - // here for computing an estimation of the compute cost (measured in bytes): - // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their - // data source, like parquet files. - // - Network boundaries contain statistics collected from runtime information, gathered - // by the SamplerExec injected by this same function. - let cost = calculate_cost(&input_stage.plan)?; - metrics.push(BytesCounterMetric::new_metric( - "cpu_cost", - *cost.cpu.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "memory_cost", - *cost.memory.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "network_cost", - *cost.network.get_value().unwrap_or(&0), - )); - let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 - / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 - / input_stage - .plan - .output_partitioning() - .partition_count() - .max(1) as f64; - let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); - let task_count = nb_ctx - .task_count(&input_stage.plan)? - .merge(TaskCountAnnotation::soft(compute_based_task_count))?; - - // Propagate the final task_count inferred based on runtime statistics and compute cost. - // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the - // plan is finally left ready for distribution. - input_stage.plan = nb_ctx - .propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; - input_stage.tasks = task_count.as_usize(); - // In order to infer the compute the cost of the stage above this one, here a sampler - // is injected to gather runtime statistics. - input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&input_stage.plan)? - } else { - vec![] - }; - - let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); - - let query_coordinator = Arc::clone(query_coordinator); - let plans_for_viz = Arc::clone(&plans_for_viz); - - Ok(async move { - let mut stage_coordinator = query_coordinator.stage_coordinator(&input_stage); - - let mut futures = Vec::with_capacity(input_stage.tasks); - for task_i in 0..input_stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(input_stage.tasks); - for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() - { - workers.push(url); - load_info_rxs.push({ - let rx = - stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); - UnboundedReceiverStream::new(rx) - }); - let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; - } - stage_coordinator.seal_dynamic_filter_stage()?; - - let (stats, consumer_tc) = if nb_type == TypeId::of::() { - (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) - } else { - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; - metrics.extend(new_metrics); - // returning Desired(0) here is our way to tell the planner that we don't care - // about the task count assigned to the network boundary in the consumer stage, - // and we don't want it to affect other task count decisions. A 0.0 hint does - // not mask smaller fractional values via max-merge, and does not inflate - // UNION sums when this boundary sits under a child of a - // ChildrenIsolatorUnionExec. - (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) - }; - - // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan - // before it's moved: the returned stage is remote and carries no plan to read it - // back from. - let input_properties = Arc::clone(input_stage.plan.properties()); - plans_for_viz.insert(input_stage.num, input_stage.plan, metrics); - Ok(NetworkBoundaryBuilderResult { - consumer_task_count: consumer_tc, - input_stage: Stage::Remote(RemoteStage { - query_id: input_stage.query_id, - num: input_stage.num, - workers, - runtime_stats: stats, - dynamic_filter_anchors, - }), - input_properties, - }) - }) + DynamicPlanBuilder { + query_coordinator, + plans_for_viz: &plans_for_viz, }, query_coordinator.session_config(), ) .await?; + let head_stage = prepare_stages(query_coordinator, &head_stage, true).await?; Ok(PreparedPlan { plan_for_viz: plans_for_viz.reconstruct(&head_stage)?, @@ -162,6 +53,117 @@ pub(super) async fn prepare_dynamic_plan( }) } +struct DynamicPlanBuilder<'a> { + query_coordinator: &'a Arc, + plans_for_viz: &'a PlanReconstructor, +} + +#[async_trait] +impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { + async fn build<'a>( + &'a self, + mut input_stage: LocalStage, + nb_type: TypeId, + nb_ctx: &'a InjectNetworkBoundaryContext<'a>, + ) -> Result { + // A deferred coalesce becomes a concrete destination before another worker stage + // consumes it. Preserve the planner's pointer-keyed annotations when replacing nodes. + let prepared = prepare_stages(self.query_coordinator, &input_stage.plan, false).await?; + nb_ctx.copy_task_counts(&input_stage.plan, &prepared)?; + input_stage.plan = prepared; + let mut metrics = MetricsSet::new(); + + // Leaf statistics and runtime statistics from sampled network boundaries estimate + // the compute cost of the next stage. + let cost = calculate_cost(&input_stage.plan)?; + metrics.push(BytesCounterMetric::new_metric( + "cpu_cost", + *cost.cpu.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "memory_cost", + *cost.memory.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "network_cost", + *cost.network.get_value().unwrap_or(&0), + )); + let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 + / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 + / input_stage + .plan + .output_partitioning() + .partition_count() + .max(1) as f64; + let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); + let task_count = nb_ctx + .task_count(&input_stage.plan)? + .merge(TaskCountAnnotation::soft(compute_based_task_count))?; + + input_stage.plan = + nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; + input_stage.tasks = task_count.as_usize(); + + if nb_type == TypeId::of::() { + let input_properties = Arc::clone(input_stage.plan.properties()); + self.plans_for_viz + .insert(input_stage.num, Arc::clone(&input_stage.plan), metrics); + // Coalesces require no sampling. Leave their setup to the consuming stage; + // the coordinator can then execute each task as its own setup completes. + return Ok(NetworkBoundaryBuilderResult { + consumer_task_count: TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0), + input_stage: Stage::Local(input_stage), + input_properties, + }); + } + + input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; + let dynamic_filter_anchors = + if is_remote_dynamic_filtering_enabled(self.query_coordinator.session_config())? { + orphan_dynamic_filter_consumers(&input_stage.plan)? + } else { + vec![] + }; + let futures = (0..input_stage.tasks).map(|task_i| { + let input_stage = &input_stage; + async move { + let coordinator = self.query_coordinator.stage_coordinator(input_stage); + let (url, tx, stream) = coordinator.init_bidirectional_stream(task_i).await?; + let load_info = coordinator.worker_to_coordinator_task(task_i, stream); + coordinator.coordinator_to_worker_task(task_i, tx.clone())?; + let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling); + Ok::<_, DataFusionError>((url, UnboundedReceiverStream::new(load_info))) + } + }); + let (workers, load_info_rxs): (Vec<_>, Vec<_>) = futures::future::try_join_all(futures) + .await? + .into_iter() + .unzip(); + self.query_coordinator + .stage_coordinator(&input_stage) + .seal_dynamic_filter_stage()?; + + let (stats, new_metrics) = + gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; + metrics.extend(new_metrics); + let input_properties = Arc::clone(input_stage.plan.properties()); + self.plans_for_viz + .insert(input_stage.num, input_stage.plan, metrics); + Ok(NetworkBoundaryBuilderResult { + // A zero soft hint leaves the consumer's task count to its other inputs. + consumer_task_count: TaskCountAnnotation::soft(0.0), + input_stage: Stage::Remote(RemoteStage { + query_id: input_stage.query_id, + num: input_stage.num, + workers, + runtime_stats: Some(Arc::new(stats)), + dynamic_filter_anchors, + }), + input_properties, + }) + } +} + /// Reconstructs the plan dynamically as stages get transitioned to Remote and get sent to the /// respective workers. /// diff --git a/src/coordinator/prepare_static_plan.rs b/src/coordinator/prepare_static_plan.rs index c0a5e193f..0e5b9883e 100644 --- a/src/coordinator/prepare_static_plan.rs +++ b/src/coordinator/prepare_static_plan.rs @@ -4,71 +4,142 @@ use crate::coordinator::query_coordinator::QueryCoordinator; use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, }; -use crate::stage::RemoteStage; +use crate::execution_plans::PendingNetworkBoundaryExec; +use crate::stage::{LocalStage, RemoteStage}; +use crate::worker::PendingTaskUrl; use crate::{NetworkBoundaryExt, Stage}; -use datafusion::common::tree_node::Transformed; -use datafusion::common::{Result, exec_err}; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; +use datafusion::common::{DataFusionError, HashSet, Result, exec_datafusion_err}; use datafusion::physical_plan::ExecutionPlan; +use futures::FutureExt; +use futures::future::try_join_all; use std::sync::Arc; +use tokio::sync::oneshot; -/// Prepares the distributed plan for execution, which implies: -/// 1. Perform some worker URL assignation, choosing either: -/// - The workers selected by the user's [crate::RouteTaskHandler]. -/// - Randomly otherwise -/// 2. Sending the sliced subplans to the assigned URLs. For each URL assigned to a task, a -/// network call feeding the subplan is necessary. -/// 3. In each network boundary, set the input plan to `None`. That way, network boundaries -/// become nodes without children and traversing them will not go further down in. -/// 4. Spawn a background task per worker that waits for the worker to finish and collects -/// its metrics into [DistributedExec::task_metrics] via the coordinator channel. pub(super) async fn prepare_static_plan( - query_coordinator: &QueryCoordinator, + query_coordinator: &Arc, base_plan: &Arc, ) -> Result { - let prepared = Arc::clone(base_plan).transform_up_async(async |plan| { - // The following logic is just applied on network boundaries. - let Some(plan) = plan.as_network_boundary() else { + Ok(PreparedPlan { + head_stage: prepare_stages(query_coordinator, base_plan, true).await?, + plan_for_viz: Arc::clone(base_plan), + }) +} + +/// Worker plans need concrete child destinations. Only inputs consumed directly by the +/// coordinator can remain pending, letting each task execute after its own setup succeeds. +pub(super) async fn prepare_stages( + query_coordinator: &Arc, + plan: &Arc, + defer_head: bool, +) -> Result> { + let mut head_stages = HashSet::new(); + if defer_head { + plan.apply(|plan| { + if let Some(boundary) = plan.as_network_boundary() { + head_stages.insert(boundary.input_stage().num()); + Ok(TreeNodeRecursion::Jump) + } else { + Ok(TreeNodeRecursion::Continue) + } + })?; + } + let prepared = Arc::clone(plan).transform_up_async(async |plan| { + let Some(boundary) = plan.as_network_boundary() else { return Ok(Transformed::no(plan)); }; - - let Stage::Local(stage) = plan.input_stage() else { - return exec_err!("Input stage from network boundary was not in Local state"); + let Stage::Local(stage) = boundary.input_stage() else { + return Ok(Transformed::no(plan)); }; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&stage.plan)? - } else { - vec![] - }; - - let mut stage_coordinator = query_coordinator.stage_coordinator(stage); - let mut futures = Vec::with_capacity(stage.tasks); - for task_i in 0..stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(stage.tasks); - for (task_i, (url, worker_tx, worker_stream)) in results.into_iter().enumerate() { - workers.push(url); - stage_coordinator.worker_to_coordinator_task(task_i, worker_stream); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; + let anchors = if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { + orphan_dynamic_filter_consumers(&stage.plan)? + } else { + vec![] + }; + if head_stages.contains(&stage.num) { + let workers = start_pending_stage(query_coordinator, stage); + return Ok(Transformed::yes(Arc::new(PendingNetworkBoundaryExec::new( + boundary.with_input_stage(Stage::Local(stage.clone()))?, + workers, + anchors, + query_coordinator.spawner.query_finished(), + )) as Arc)); } - stage_coordinator.seal_dynamic_filter_stage()?; - Ok(Transformed::yes(plan.with_input_stage(Stage::Remote( - RemoteStage { + let coordinator = query_coordinator.stage_coordinator(stage); + let coordinator = &coordinator; + let workers = try_join_all((0..stage.tasks).map(|task| async move { + let (url, tx, stream) = coordinator.init_bidirectional_stream(task).await?; + coordinator.worker_to_coordinator_task(task, stream); + coordinator.coordinator_to_worker_task(task, tx)?; + Ok::<_, DataFusionError>(url) + })) + .await?; + coordinator.seal_dynamic_filter_stage()?; + Ok(Transformed::yes(boundary.with_input_stage( + Stage::Remote(RemoteStage { query_id: stage.query_id, num: stage.num, workers, runtime_stats: None, - dynamic_filter_anchors, - }, - ))?)) + dynamic_filter_anchors: anchors, + }), + )?)) }); - Ok(PreparedPlan { - head_stage: prepared.await?.data, - // If the plan was statically planned, the base plan is the same one that will be used for - // visualization. - plan_for_viz: Arc::clone(base_plan), - }) + Ok(prepared.await?.data) +} + +fn start_pending_stage( + query_coordinator: &Arc, + stage: &LocalStage, +) -> Vec { + let mut workers = Vec::with_capacity(stage.tasks); + let mut setups = Vec::with_capacity(stage.tasks); + for task in 0..stage.tasks { + let (tx, rx) = oneshot::channel(); + workers.push( + async move { + rx.await + .map_err(|_| Arc::new(exec_datafusion_err!("Task setup was cancelled")))? + } + .boxed() + .shared(), + ); + // Construct cleanup before spawning: even an unpolled setup must complete report waits. + let reports = query_coordinator + .stage_coordinator(stage) + .pending_task_reports(task); + setups.push((task, tx, reports)); + } + let coordinator = Arc::clone(query_coordinator); + let stage = stage.clone(); + let finished = query_coordinator.spawner.query_finished(); + query_coordinator.spawner.spawn(async move { + let stage_coordinator = coordinator.stage_coordinator(&stage); + let setup = async { + try_join_all(setups.into_iter().map(|(task, tx, mut reports)| { + let stage_coordinator = &stage_coordinator; + async move { + let result = async { + let (url, worker_tx, stream) = + stage_coordinator.init_bidirectional_stream(task).await?; + stage_coordinator.worker_to_coordinator_task(task, stream); + reports.reader_started(); + stage_coordinator.coordinator_to_worker_task(task, worker_tx)?; + Ok::<_, DataFusionError>(url) + } + .await + .map_err(Arc::new); + let _ = tx.send(result.clone()); + result.map_err(DataFusionError::Shared) + } + })) + .await?; + stage_coordinator.seal_dynamic_filter_stage() + }; + tokio::select! { + result = setup => result, + _ = finished.cancelled() => Ok(()), + } + }); + workers } diff --git a/src/coordinator/query_coordinator.rs b/src/coordinator/query_coordinator.rs index c1e6080dd..80e049819 100644 --- a/src/coordinator/query_coordinator.rs +++ b/src/coordinator/query_coordinator.rs @@ -135,6 +135,18 @@ pub(super) struct StageCoordinator<'a> { } impl<'a> StageCoordinator<'a> { + pub(super) fn pending_task_reports(&self, task_i: usize) -> PendingTaskReports { + PendingTaskReports { + task_key: TaskKey { + query_id: self.query_id, + stage_id: self.stage_id, + task_number: task_i, + }, + metrics_store: self.metrics_store.clone(), + completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), + } + } + /// Sends a plan to a specific worker and sets up the bidirectional stream. /// /// Its returns are: @@ -280,7 +292,7 @@ impl<'a> StageCoordinator<'a> { /// Spawns background tasks in charge of collecting messages sent by a worker. pub(super) fn worker_to_coordinator_task( - &mut self, + &self, task_i: usize, mut worker_to_coordinator_stream: BoxStream<'static, Result>, ) -> UnboundedReceiver { @@ -342,7 +354,7 @@ impl<'a> StageCoordinator<'a> { /// to workers here are: /// - WorkUnits collected from [WorkUnitFeeds] present in the plan. pub(super) fn coordinator_to_worker_task( - &mut self, + &self, task_i: usize, tx: UnboundedSender, ) -> Result<()> { @@ -498,6 +510,31 @@ impl<'a> StageCoordinator<'a> { } } +/// Completes report waits if setup ends before a worker report reader owns them. +pub(super) struct PendingTaskReports { + task_key: TaskKey, + metrics_store: Option>>, + completed_dynamic_filter_store: Option>>, +} + +impl PendingTaskReports { + pub(super) fn reader_started(&mut self) { + self.metrics_store = None; + self.completed_dynamic_filter_store = None; + } +} + +impl Drop for PendingTaskReports { + fn drop(&mut self) { + if let Some(store) = self.metrics_store.take() { + store.insert(self.task_key, TaskMetrics::default()); + } + if let Some(store) = self.completed_dynamic_filter_store.take() { + store.insert(self.task_key, TaskCompletedDynamicFilters::default()); + } + } +} + struct TaskSpecializedPlan { plan: Arc, work_unit_feed_declarations: Vec, diff --git a/src/distributed_planner/inject_network_boundaries.rs b/src/distributed_planner/inject_network_boundaries.rs index e796a7cca..d75af4450 100644 --- a/src/distributed_planner/inject_network_boundaries.rs +++ b/src/distributed_planner/inject_network_boundaries.rs @@ -216,6 +216,21 @@ impl<'a> InjectNetworkBoundaryContext<'a> { Ok(*task_count) } + /// Retains annotations when stage preparation replaces nodes without changing topology. + pub(crate) fn copy_task_counts( + &self, + original: &Arc, + prepared: &Arc, + ) -> Result<()> { + self.set_task_count(prepared, self.task_count(original)?); + if !original.is_network_boundary() { + for (original, prepared) in original.children().into_iter().zip(prepared.children()) { + self.copy_task_counts(original, prepared)?; + } + } + Ok(()) + } + fn fetch_add_stage_id(&self) -> usize { self.stage_id.fetch_add(1, Ordering::Acquire) } diff --git a/src/distributed_planner/mod.rs b/src/distributed_planner/mod.rs index b49645214..756ab456f 100644 --- a/src/distributed_planner/mod.rs +++ b/src/distributed_planner/mod.rs @@ -14,7 +14,8 @@ mod statistics; pub use distributed_config::DistributedConfig; pub use distributed_query_planner::create_distributed_plan; pub(crate) use inject_network_boundaries::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + inject_network_boundaries, }; pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt, ProducerHead}; pub use session_state_builder_ext::SessionStateBuilderExt; diff --git a/src/distributed_planner/network_boundary.rs b/src/distributed_planner/network_boundary.rs index 259c802b2..0eca5a8f9 100644 --- a/src/distributed_planner/network_boundary.rs +++ b/src/distributed_planner/network_boundary.rs @@ -1,4 +1,4 @@ -use crate::execution_plans::SamplerExec; +use crate::execution_plans::{PendingNetworkBoundaryExec, SamplerExec}; use crate::{ BroadcastExec, MaybeEncoded, NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage, @@ -67,6 +67,8 @@ impl NetworkBoundaryExt for dyn ExecutionPlan { Some(node) } else if let Some(node) = self.downcast_ref::() { Some(node) + } else if let Some(node) = self.downcast_ref::() { + Some(node) } else { None } diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index e21747dd4..1fbb15382 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -6,6 +6,7 @@ mod metrics; mod network_broadcast; mod network_coalesce; mod network_shuffle; +mod pending_network_boundary; mod sampler; #[cfg(any(test, feature = "integration"))] @@ -18,4 +19,5 @@ pub(crate) use metrics::MetricsWrapperExec; pub use network_broadcast::NetworkBroadcastExec; pub use network_coalesce::NetworkCoalesceExec; pub use network_shuffle::{NetworkShuffleExec, PRODUCER_SALT_DEFAULT, ShuffleMode}; +pub(crate) use pending_network_boundary::PendingNetworkBoundaryExec; pub use sampler::SamplerExec; diff --git a/src/execution_plans/network_broadcast.rs b/src/execution_plans/network_broadcast.rs index 10581506a..ef4ed6201 100644 --- a/src/execution_plans/network_broadcast.rs +++ b/src/execution_plans/network_broadcast.rs @@ -1,7 +1,7 @@ use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::stage::{LocalStage, Stage}; -use crate::worker::WorkerConnectionPool; +use crate::worker::{RemoteStageRef, WorkerConnectionPool}; use crate::{BroadcastExec, DistributedTaskContext}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Result, not_impl_err, plan_err}; @@ -201,6 +201,36 @@ impl DisplayAs for NetworkBroadcastExec { } } +impl NetworkBroadcastExec { + pub(crate) fn execute_remote( + &self, + partition: usize, + context: Arc, + remote_stage: RemoteStageRef<'_>, + ) -> Result { + let task_context = DistributedTaskContext::from_ctx(&context); + let out_partitions = self.properties.partitioning.partition_count(); + let off = out_partitions * task_context.task_index; + let mut streams = Vec::with_capacity(self.input_stage.task_count()); + + for input_task_index in 0..self.input_stage.task_count() { + streams.push(self.worker_connections.execute( + remote_stage, + off..(off + self.properties.partitioning.partition_count()), + input_task_index, + off + partition, + self.producer_head(task_context.task_count)?, + &context, + )?); + } + + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + futures::stream::select_all(streams), + ))) + } +} + impl ExecutionPlan for NetworkBroadcastExec { fn name(&self) -> &str { "NetworkBroadcastExec" @@ -251,27 +281,7 @@ impl ExecutionPlan for NetworkBroadcastExec { Stage::Local(local) => return local.execute(partition, context), Stage::Remote(remote_stage) => remote_stage, }; - - let task_context = DistributedTaskContext::from_ctx(&context); - let out_partitions = self.properties.partitioning.partition_count(); - let off = out_partitions * task_context.task_index; - let mut streams = Vec::with_capacity(self.input_stage.task_count()); - - for input_task_index in 0..self.input_stage.task_count() { - streams.push(self.worker_connections.execute( - remote_stage, - off..(off + self.properties.partitioning.partition_count()), - input_task_index, - off + partition, - self.producer_head(task_context.task_count)?, - &context, - )?); - } - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - futures::stream::select_all(streams), - ))) + self.execute_remote(partition, context, remote_stage.into()) } fn metrics(&self) -> Option { diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index db9b964a8..f5b370017 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -3,7 +3,7 @@ use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::execution_plans::common::scale_partitioning_props; use crate::stage::{LocalStage, Stage}; -use crate::worker::WorkerConnectionPool; +use crate::worker::{RemoteStageRef, WorkerConnectionPool}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{exec_err, not_impl_err, plan_err}; use datafusion::error::Result; @@ -224,57 +224,13 @@ impl DisplayAs for NetworkCoalesceExec { } } -impl ExecutionPlan for NetworkCoalesceExec { - fn name(&self) -> &str { - "NetworkCoalesceExec" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - match &self.input_stage.local_plan() { - Some(v) => vec![v], - None => vec![], - } - } - - fn apply_expressions( - &self, - f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { - Stage::Local(local) => { - local.plan = require_one_child(children)?; - } - Stage::Remote(_) => { - if !children.is_empty() { - not_impl_err!("NetworkBoundary cannot accept children")? - } - } - } - Ok(Arc::new(self_clone)) - } - - fn execute( +impl NetworkCoalesceExec { + pub(crate) fn execute_remote( &self, partition: usize, context: Arc, + remote_stage: RemoteStageRef<'_>, ) -> Result { - let remote_stage = match &self.input_stage { - Stage::Local(local) => return local.execute(partition, context), - Stage::Remote(remote_stage) => remote_stage, - }; - let task_context = DistributedTaskContext::from_ctx(&context); if task_context.task_index >= task_context.task_count { return exec_err!( @@ -341,6 +297,60 @@ impl ExecutionPlan for NetworkCoalesceExec { stream, ))) } +} + +impl ExecutionPlan for NetworkCoalesceExec { + fn name(&self) -> &str { + "NetworkCoalesceExec" + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + match &self.input_stage.local_plan() { + Some(v) => vec![v], + None => vec![], + } + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { + Stage::Local(local) => { + local.plan = require_one_child(children)?; + } + Stage::Remote(_) => { + if !children.is_empty() { + not_impl_err!("NetworkBoundary cannot accept children")? + } + } + } + Ok(Arc::new(self_clone)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let remote_stage = match &self.input_stage { + Stage::Local(local) => return local.execute(partition, context), + Stage::Remote(remote_stage) => remote_stage, + }; + self.execute_remote(partition, context, remote_stage.into()) + } fn metrics(&self) -> Option { Some(self.worker_connections.metrics.clone_inner()) diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 16e73ad73..e82297972 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -2,7 +2,7 @@ use crate::common::require_one_child; use crate::distributed_planner::ProducerHead; use crate::execution_plans::common::{salted_partitioning, scale_partitioning}; use crate::stage::{LocalStage, Stage}; -use crate::worker::WorkerConnectionPool; +use crate::worker::{RemoteStageRef, WorkerConnectionPool}; use crate::{DistributedTaskContext, MaybeEncoded, NetworkBoundary}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Result, not_impl_err, plan_err}; @@ -273,60 +273,16 @@ impl DisplayAs for NetworkShuffleExec { } } -impl ExecutionPlan for NetworkShuffleExec { - fn name(&self) -> &str { - "NetworkShuffleExec" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - match &self.input_stage.local_plan() { - Some(v) => vec![v], - None => vec![], - } - } - - fn apply_expressions( - &self, - f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { - Stage::Local(local) => { - local.plan = require_one_child(children)?; - } - Stage::Remote(_) => { - if !children.is_empty() { - not_impl_err!("NetworkBoundary cannot accept children")? - } - } - } - Ok(Arc::new(self_clone)) - } - - fn execute( +impl NetworkShuffleExec { + pub(crate) fn execute_remote( &self, partition: usize, context: Arc, + remote_stage: RemoteStageRef<'_>, ) -> Result { - let remote_stage = match &self.input_stage { - Stage::Local(local) => return local.execute(partition, context), - Stage::Remote(remote_stage) => remote_stage, - }; - let task_context = DistributedTaskContext::from_ctx(&context); let task_index = task_context.task_index; - let producer_task_count = remote_stage.workers.len(); + let producer_task_count = remote_stage.task_count(); let schema = self.schema(); let mut streams: Vec = @@ -401,6 +357,60 @@ impl ExecutionPlan for NetworkShuffleExec { ))) } } +} + +impl ExecutionPlan for NetworkShuffleExec { + fn name(&self) -> &str { + "NetworkShuffleExec" + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + match &self.input_stage.local_plan() { + Some(v) => vec![v], + None => vec![], + } + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result, DataFusionError> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { + Stage::Local(local) => { + local.plan = require_one_child(children)?; + } + Stage::Remote(_) => { + if !children.is_empty() { + not_impl_err!("NetworkBoundary cannot accept children")? + } + } + } + Ok(Arc::new(self_clone)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let remote_stage = match &self.input_stage { + Stage::Local(local) => return local.execute(partition, context), + Stage::Remote(remote_stage) => remote_stage, + }; + self.execute_remote(partition, context, remote_stage.into()) + } fn metrics(&self) -> Option { Some(self.worker_connections.metrics.clone_inner()) diff --git a/src/execution_plans/pending_network_boundary.rs b/src/execution_plans/pending_network_boundary.rs new file mode 100644 index 000000000..cd493ee03 --- /dev/null +++ b/src/execution_plans/pending_network_boundary.rs @@ -0,0 +1,137 @@ +use crate::distributed_planner::{NetworkBoundary, ProducerHead}; +use crate::worker::{PendingTaskUrl, RemoteStageRef}; +use crate::{NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{Result, internal_err}; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr_common::metrics::MetricsSet; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, Statistics, StatisticsArgs, + apply_expression_roots, +}; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; +use tokio_util::sync::CancellationToken; + +/// A coordinator-only boundary whose tasks become executable independently of their peers. +/// Boundaries inside worker plans keep concrete destinations and never use this wrapper. +pub(crate) struct PendingNetworkBoundaryExec { + boundary: Arc, + workers: Vec, + dynamic_filter_anchors: Vec>, + query_finished: CancellationToken, +} + +impl PendingNetworkBoundaryExec { + pub(crate) fn new( + boundary: Arc, + workers: Vec, + dynamic_filter_anchors: Vec>, + query_finished: CancellationToken, + ) -> Self { + Self { + boundary, + workers, + dynamic_filter_anchors, + query_finished, + } + } +} + +impl Debug for PendingNetworkBoundaryExec { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingNetworkBoundaryExec") + .field("boundary", &self.boundary) + .field("tasks", &self.workers.len()) + .finish() + } +} + +impl DisplayAs for PendingNetworkBoundaryExec { + fn fmt_as(&self, display: DisplayFormatType, f: &mut Formatter<'_>) -> std::fmt::Result { + self.boundary.fmt_as(display, f) + } +} + +impl NetworkBoundary for PendingNetworkBoundaryExec { + fn input_stage(&self) -> &Stage { + self.boundary.input_stage() + } + + fn with_input_stage(&self, stage: Stage) -> Result> { + self.boundary.with_input_stage(stage) + } + + fn producer_head(&self, consumer_tasks: usize) -> Result { + self.boundary.producer_head(consumer_tasks) + } +} + +impl ExecutionPlan for PendingNetworkBoundaryExec { + fn name(&self) -> &str { + self.boundary.name() + } + + fn properties(&self) -> &Arc { + self.boundary.properties() + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + if !children.is_empty() { + return internal_err!("A pending network boundary cannot accept children"); + } + Ok(self) + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.dynamic_filter_anchors.iter(), f) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let stage = self.input_stage(); + let pending = RemoteStageRef::Pending { + query_id: stage.query_id(), + num: stage.num(), + workers: &self.workers, + query_finished: &self.query_finished, + }; + let boundary: &dyn ExecutionPlan = self.boundary.as_ref(); + if let Some(boundary) = boundary.downcast_ref::() { + return boundary.execute_remote(partition, context, pending); + } + if let Some(boundary) = boundary.downcast_ref::() { + return boundary.execute_remote(partition, context, pending); + } + if let Some(boundary) = boundary.downcast_ref::() { + return boundary.execute_remote(partition, context, pending); + } + internal_err!("Unsupported pending network boundary") + } + + fn metrics(&self) -> Option { + self.boundary.metrics() + } + + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> Result> { + Ok(Arc::new(Statistics::new_unknown(&self.schema()))) + } +} diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 7893097ec..f831d9bce 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -9,7 +9,7 @@ mod worker_connection_pool; mod worker_service; pub(crate) use single_write_multi_read::SingleWriteMultiRead; -pub(crate) use worker_connection_pool::WorkerConnectionPool; +pub(crate) use worker_connection_pool::{PendingTaskUrl, RemoteStageRef, WorkerConnectionPool}; pub use impl_coordinator_channel::CoordinatorChannelResult; pub use session_builder::{ diff --git a/src/worker/worker_connection_pool.rs b/src/worker/worker_connection_pool.rs index 05ea11693..9d31fa911 100644 --- a/src/worker/worker_connection_pool.rs +++ b/src/worker/worker_connection_pool.rs @@ -6,7 +6,9 @@ use crate::{ExecuteTaskRequest, LocalWorkerContext, TaskKey, get_distributed_cha use dashmap::DashMap; use datafusion::arrow::array::RecordBatch; use datafusion::common::runtime::SpawnedTask; -use datafusion::common::{DataFusionError, Result, internal_datafusion_err, internal_err}; +use datafusion::common::{ + DataFusionError, Result, exec_err, internal_datafusion_err, internal_err, +}; use datafusion::execution::TaskContext; use datafusion::physical_expr_common::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::metrics::MetricBuilder; @@ -16,6 +18,10 @@ use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use std::fmt::{Debug, Formatter}; use std::ops::Range; use std::sync::{Arc, Mutex}; +use tokio::select; +use tokio_util::sync::CancellationToken; +use url::Url; +use uuid::Uuid; /// Manages connections to remote workers. /// - Handles a range of partitions at a time in other to give the chance to [crate::WorkerChannel] @@ -46,6 +52,83 @@ struct StreamSlot { /// Just some boilerplate for a shared future. type SharedBoxFuture = Shared>>>; +pub(crate) type PendingTaskUrl = SharedBoxFuture; + +/// Coordinator-local destinations may still be routing; wire plans always contain resolved URLs. +#[derive(Clone, Copy)] +pub(crate) enum RemoteStageRef<'a> { + Ready(&'a RemoteStage), + Pending { + query_id: Uuid, + num: usize, + workers: &'a [PendingTaskUrl], + query_finished: &'a CancellationToken, + }, +} + +impl<'a> From<&'a RemoteStage> for RemoteStageRef<'a> { + fn from(stage: &'a RemoteStage) -> Self { + Self::Ready(stage) + } +} + +impl RemoteStageRef<'_> { + pub(crate) fn task_count(self) -> usize { + match self { + Self::Ready(stage) => stage.workers.len(), + Self::Pending { workers, .. } => workers.len(), + } + } + + fn task_key(self, task_number: usize) -> TaskKey { + let (query_id, stage_id) = match self { + Self::Ready(stage) => (stage.query_id, stage.num), + Self::Pending { query_id, num, .. } => (query_id, num), + }; + TaskKey { + query_id, + stage_id, + task_number, + } + } + + fn destination(self, task: usize) -> Result { + let destination = match self { + Self::Ready(stage) => stage.workers.get(task).cloned().map(TaskDestination::Ready), + Self::Pending { + workers, + query_finished, + .. + } => workers + .get(task) + .cloned() + .map(|url| TaskDestination::Pending(url, query_finished.clone())), + }; + destination.ok_or_else(|| internal_datafusion_err!("input stage task {task} out of range")) + } +} + +enum TaskDestination { + Ready(Url), + Pending(PendingTaskUrl, CancellationToken), +} + +impl TaskDestination { + fn query_finished(&self) -> Option { + match self { + Self::Ready(_) => None, + Self::Pending(_, query_finished) => Some(query_finished.clone()), + } + } + + async fn resolve(self) -> Result { + match self { + Self::Ready(url) => Ok(url), + Self::Pending(url, _) => url.await.map_err(DataFusionError::Shared), + } + } +} + impl WorkerConnectionPool { /// Builds a new [WorkerConnectionPool] with as many empty slots for worker stream entries as /// the provided `input_tasks`. @@ -56,23 +139,18 @@ impl WorkerConnectionPool { } } - pub(crate) fn execute( + pub(crate) fn execute<'a>( &self, - input_stage: &RemoteStage, + input_stage: impl Into>, target_partitions: Range, target_task: usize, target_partition: usize, producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let Some(target_url) = input_stage.workers.get(target_task).cloned() else { - internal_err!("input_stage.workers[{target_task}] out of range.")? - }; - let task_key = TaskKey { - query_id: input_stage.query_id, - stage_id: input_stage.num, - task_number: target_task, - }; + let input_stage = input_stage.into(); + let destination = input_stage.destination(target_task)?; + let task_key = input_stage.task_key(target_task); let bdr = || MetricBuilder::new(&self.metrics); let output_bytes = bdr().output_bytes(target_partition); let output_rows = bdr().output_rows(target_partition); @@ -106,38 +184,50 @@ impl WorkerConnectionPool { // // Note that this does not start polling the returned streams, it just instantiates them. let streams_task = SpawnedTask::spawn(async move { - let request = ExecuteTaskRequest { - task_key, - target_partition_start: target_partitions.start, - target_partition_end: target_partitions.end, - producer_head, - }; - let mut client = match LocalWorkerContext::from_ctx(&ctx) { - Some(lw) if lw.self_url == target_url => { - local_connections_used.add(1); - Ok(lw.to_worker_channel()) - } - _ => { - let ch_resolver = get_distributed_channel_resolver(ctx.as_ref()); - ch_resolver.get_worker_client_for_url(&target_url).await + let query_finished = destination.query_finished(); + let setup = async move { + let target_url = destination.resolve().await?; + let request = ExecuteTaskRequest { + task_key, + target_partition_start: target_partitions.start, + target_partition_end: target_partitions.end, + producer_head, + }; + let mut client = match LocalWorkerContext::from_ctx(&ctx) { + Some(lw) if lw.self_url == target_url => { + local_connections_used.add(1); + Ok(lw.to_worker_channel()) + } + _ => { + let ch_resolver = get_distributed_channel_resolver(ctx.as_ref()); + ch_resolver.get_worker_client_for_url(&target_url).await + } + }?; + let headers = get_passthrough_headers(ctx.session_config()); + let streams = client.execute_task(headers, request, metrics, &ctx).await?; + if streams.len() != task_slots.len() { + return internal_err!( + "Worker returned {} streams for {} partitions", + streams.len(), + task_slots.len() + ); } - }?; - let headers = get_passthrough_headers(ctx.session_config()); - let streams = client.execute_task(headers, request, metrics, &ctx).await?; - if streams.len() != task_slots.len() { - return internal_err!( - "Worker returned {} streams for {} partitions", - streams.len(), - task_slots.len() - ); - } - for (slot, stream) in task_slots.iter().zip(streams) { - let mut slot = slot.lock().unwrap(); - if !slot.closed { - slot.stream = Some(stream); + for (slot, stream) in task_slots.iter().zip(streams) { + let mut slot = slot.lock().unwrap(); + if !slot.closed { + slot.stream = Some(stream); + } } + Ok(()) + }; + match query_finished { + Some(query_finished) => select! { + biased; + _ = query_finished.cancelled() => exec_err!("Query finished during task setup"), + result = setup => result, + }, + None => setup.await, } - Ok(()) }); let ready = async move { diff --git a/tests/startup_overlap.rs b/tests/startup_overlap.rs new file mode 100644 index 000000000..be0b6f1f6 --- /dev/null +++ b/tests/startup_overlap.rs @@ -0,0 +1,395 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use std::hint::black_box; + use std::slice::from_ref; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, Instant}; + + use arrow::array::{Int64Array, RecordBatch}; + use arrow::datatypes::DataType; + use async_trait::async_trait; + use datafusion::common::runtime::SpawnedTask; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::common::{Result, assert_contains, exec_err}; + use datafusion::execution::SessionState; + use datafusion::logical_expr::{ScalarUDF, Volatility, create_udf}; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, + }; + use datafusion_distributed::{ + DistributedExec, DistributedExt, MappedWorkerSessionBuilderExt, NetworkBoundaryExt, + NetworkCoalesceExec, RouteTaskEvent, RouteTaskEventResponse, RouteTaskHandler, Stage, + WorkerQueryContext, WorkerSessionBuilder, WorkerToCoordinatorMsg, + }; + use futures::{StreamExt, TryStreamExt}; + use tokio::time::{sleep, timeout}; + use tokio_util::sync::CancellationToken; + + const WAIT: Duration = Duration::from_secs(5); + const SCAN: &str = "SELECT task_index FROM url_emitter(2, 2, 'scan')"; + + #[tokio::test] + async fn static_scan_returns_output_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(gate.clone()); + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(from_ref(&first)), vec![0]); + assert!(!gate.release.is_cancelled()); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn adaptive_scan_returns_output_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(from_ref(&first)), vec![0]); + assert!(!gate.release.is_cancelled()); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn adaptive_shuffle_samples_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + let query = ctx + .sql("SELECT task_index FROM url_emitter(2, 2, 'scan') GROUP BY task_index") + .await?; + let execution = SpawnedTask::spawn(async move { query.collect().await }); + + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + // A positive LoadInfo report proves the remote sampler has produced rows. + timeout(WAIT, gate.sampled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let batches = timeout(WAIT, execution.join()).await.unwrap().unwrap()?; + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn late_setup_failure_reaches_output_and_finishes_metrics() -> Result<()> { + let gate = SetupGate { + fail: true, + ..Default::default() + }; + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(&[first]), vec![0]); + gate.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "injected setup failure"); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn dropping_output_cancels_pending_setup_and_finishes_metrics() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(true)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(&[first]), vec![0]); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + drop(output); + timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn limit_completion_cancels_pending_setup_and_finishes_metrics() -> Result<()> { + let gate = SetupGate { + wait_for_sibling: true, + ..Default::default() + }; + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + let plan = ctx + .sql(&format!("{SCAN} LIMIT 1")) + .await? + .create_physical_plan() + .await?; + let output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + let batches = timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?; + + assert_eq!(task_indices(&batches), vec![0]); + timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn adaptive_nested_coalesce_has_concrete_worker_destinations() -> Result<()> { + let gate = SetupGate::default(); + gate.release.cancel(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + let query = ctx + .sql( + "SELECT task_index FROM url_emitter(2, 2, 'scan') WHERE task_index = \ + (SELECT MAX(task_index) FROM url_emitter(2, 2, 'inner'))", + ) + .await?; + + let batches = timeout(WAIT, query.collect()).await.unwrap()?; + assert_eq!(task_indices(&batches), vec![1]); + assert!(gate.nested_coalesces.load(Ordering::Relaxed) > 0); + Ok(()) + } + + // Iteration zero is warmup. Compare the same test on both revisions. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[ignore = "controlled startup benchmark; run with --release --ignored --nocapture"] + async fn startup_overlap_benchmark() -> Result<()> { + for adaptive in [false, true] { + for delayed_task in [None, Some(0), Some(1)] { + for iteration in 0..6 { + let gate = SetupGate { + delayed_task, + ..Default::default() + }; + let activity = Arc::new(AtomicUsize::new(0)); + let observation = Some((Arc::clone(&activity), gate.release.clone())); + let udf = Arc::new(cpu_work(observation)); + let worker = build_state.map(move |builder| { + Ok(builder.with_scalar_functions(vec![udf.clone()]).build()) + }); + let ctx = context(worker) + .await + .with_distributed_dynamic_task_count(adaptive)? + .with_distributed_route_task_handler(gate.clone()); + ctx.register_udf(cpu_work(None)); + let plan = ctx + .sql("SELECT cpu_work(task_index) FROM url_emitter(2, 2, 'scan')") + .await? + .create_physical_plan() + .await?; + if delayed_task.is_none() { + gate.release.cancel(); + } + let release = gate.clone(); + let timer = SpawnedTask::spawn(async move { + if delayed_task.is_some() { + release.entered.cancelled().await; + sleep(Duration::from_millis(250)).await; + release.release.cancel(); + } + }); + let start = Instant::now(); + let mut output = execute_stream(plan, ctx.task_ctx())?; + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + let first_ms = start.elapsed().as_secs_f64() * 1_000.0; + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + let total_ms = start.elapsed().as_secs_f64() * 1_000.0; + timer.join().await.unwrap(); + assert_eq!(task_indices(&batches), vec![0, 1]); + let observed = activity.load(Ordering::Relaxed); + assert!(observed > 0, "heavy task must execute remotely"); + println!( + "adaptive={adaptive},delayed_task={delayed_task:?},iteration={iteration},first_ms={first_ms:.3},total_ms={total_ms:.3},heavy_started_before_release={}", + observed == 2 + ); + } + } + } + Ok(()) + } + + fn cpu_work(observation: Option<(Arc, CancellationToken)>) -> ScalarUDF { + create_udf( + "cpu_work", + vec![DataType::Int64], + DataType::Int64, + Volatility::Volatile, + Arc::new(move |args| { + let Some((activity, release)) = &observation else { + return exec_err!("CPU work must execute on a remote worker"); + }; + let input = args[0].clone().into_array(1)?; + let input = input.as_any().downcast_ref::().unwrap(); + for task in input.values() { + if *task == 0 { + activity.store(1 + usize::from(!release.is_cancelled()), Ordering::Relaxed); + let mut value = 1u64; + for _ in 0..100_000_000 { + value = black_box(value.wrapping_mul(17).wrapping_add(1)); + } + } + } + Ok(args[0].clone()) + }), + ) + } + + async fn context(builder: impl WorkerSessionBuilder + Send + Sync + 'static) -> SessionContext { + let mut ctx = start_in_memory_context(2, builder).await; + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + ctx + } + + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) + } + + fn task_indices(batches: &[RecordBatch]) -> Vec { + let mut values = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + values.sort_unstable(); + values + } + + async fn assert_terminal_metrics(plan: &Arc) { + let distributed = plan.downcast_ref::().unwrap(); + let reports = timeout(WAIT, distributed.wait_for_metrics()) + .await + .unwrap() + .unwrap(); + assert_eq!(reports.len(), 2); + } + + #[derive(Clone, Default)] + struct SetupGate { + entered: CancellationToken, + release: CancellationToken, + cancelled: CancellationToken, + sampled: CancellationToken, + nested_coalesces: Arc, + fail: bool, + delayed_task: Option, + wait_for_sibling: bool, + } + + #[async_trait] + impl RouteTaskHandler for SetupGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + Some(self.route(event).await) + } + } + + impl SetupGate { + async fn route(&self, event: RouteTaskEvent<'_>) -> Result { + event.task_specialized_plan.apply(|node| { + if let Some(boundary) = node.as_network_boundary() { + assert!(matches!(boundary.input_stage(), Stage::Remote(_))); + } + if node.is::() { + self.nested_coalesces.fetch_add(1, Ordering::Relaxed); + } + Ok(TreeNodeRecursion::Continue) + })?; + if event.task_key.task_number == 0 && self.wait_for_sibling { + self.entered.cancelled().await; + } + if event.task_key.task_number == self.delayed_task.unwrap_or(1) { + let cancelled = self.cancelled.clone().drop_guard(); + self.entered.cancel(); + self.release.cancelled().await; + cancelled.disarm(); + if self.fail { + return exec_err!("injected setup failure"); + } + } + let mut response = UrlEmitterRouteTaskHandler.handle(event).await.unwrap()?; + let sampled = self.sampled.clone(); + response.worker_to_coordinator_stream = response.worker_to_coordinator_stream + .inspect_ok(move |message| { + if matches!(message, WorkerToCoordinatorMsg::LoadInfo(info) if info.rows_ready > 0) { + sampled.cancel(); + } + }).boxed(); + Ok(response) + } + } +} From 6e4da2d9a61d970a3054f019556051e97eed05c4 Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Fri, 6 Nov 2026 17:16:27 +0000 Subject: [PATCH 02/10] refactor: unify task startup and report cleanup Share task startup across adaptive and static planning. Create report cleanup before startup can be spawned, then transfer ownership into the worker reader so missing reports complete on setup cancellation and response errors. Cover a worker response error after output begins, including metrics and dynamic filter report completion. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- src/coordinator/prepare_dynamic_plan.rs | 7 +- src/coordinator/prepare_static_plan.rs | 45 ++++--------- src/coordinator/query_coordinator.rs | 90 ++++++++++++------------- tests/startup_overlap.rs | 74 +++++++++++++++++++- 4 files changed, 132 insertions(+), 84 deletions(-) diff --git a/src/coordinator/prepare_dynamic_plan.rs b/src/coordinator/prepare_dynamic_plan.rs index ea2149890..6b5875260 100644 --- a/src/coordinator/prepare_dynamic_plan.rs +++ b/src/coordinator/prepare_dynamic_plan.rs @@ -125,12 +125,9 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { vec![] }; let futures = (0..input_stage.tasks).map(|task_i| { - let input_stage = &input_stage; + let setup = self.query_coordinator.start_task(&input_stage, task_i); async move { - let coordinator = self.query_coordinator.stage_coordinator(input_stage); - let (url, tx, stream) = coordinator.init_bidirectional_stream(task_i).await?; - let load_info = coordinator.worker_to_coordinator_task(task_i, stream); - coordinator.coordinator_to_worker_task(task_i, tx.clone())?; + let (url, tx, load_info) = setup.await?; let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling); Ok::<_, DataFusionError>((url, UnboundedReceiverStream::new(load_info))) } diff --git a/src/coordinator/prepare_static_plan.rs b/src/coordinator/prepare_static_plan.rs index 0e5b9883e..84972d2c7 100644 --- a/src/coordinator/prepare_static_plan.rs +++ b/src/coordinator/prepare_static_plan.rs @@ -65,16 +65,15 @@ pub(super) async fn prepare_stages( query_coordinator.spawner.query_finished(), )) as Arc)); } - let coordinator = query_coordinator.stage_coordinator(stage); - let coordinator = &coordinator; - let workers = try_join_all((0..stage.tasks).map(|task| async move { - let (url, tx, stream) = coordinator.init_bidirectional_stream(task).await?; - coordinator.worker_to_coordinator_task(task, stream); - coordinator.coordinator_to_worker_task(task, tx)?; - Ok::<_, DataFusionError>(url) - })) - .await?; - coordinator.seal_dynamic_filter_stage()?; + let workers = + try_join_all((0..stage.tasks).map(|task| query_coordinator.start_task(stage, task))) + .await? + .into_iter() + .map(|(url, ..)| url) + .collect(); + query_coordinator + .stage_coordinator(stage) + .seal_dynamic_filter_stage()?; Ok(Transformed::yes(boundary.with_input_stage( Stage::Remote(RemoteStage { query_id: stage.query_id, @@ -104,11 +103,7 @@ fn start_pending_stage( .boxed() .shared(), ); - // Construct cleanup before spawning: even an unpolled setup must complete report waits. - let reports = query_coordinator - .stage_coordinator(stage) - .pending_task_reports(task); - setups.push((task, tx, reports)); + setups.push((tx, query_coordinator.start_task(stage, task))); } let coordinator = Arc::clone(query_coordinator); let stage = stage.clone(); @@ -116,22 +111,10 @@ fn start_pending_stage( query_coordinator.spawner.spawn(async move { let stage_coordinator = coordinator.stage_coordinator(&stage); let setup = async { - try_join_all(setups.into_iter().map(|(task, tx, mut reports)| { - let stage_coordinator = &stage_coordinator; - async move { - let result = async { - let (url, worker_tx, stream) = - stage_coordinator.init_bidirectional_stream(task).await?; - stage_coordinator.worker_to_coordinator_task(task, stream); - reports.reader_started(); - stage_coordinator.coordinator_to_worker_task(task, worker_tx)?; - Ok::<_, DataFusionError>(url) - } - .await - .map_err(Arc::new); - let _ = tx.send(result.clone()); - result.map_err(DataFusionError::Shared) - } + try_join_all(setups.into_iter().map(|(tx, setup)| async move { + let result = setup.await.map(|(url, ..)| url).map_err(Arc::new); + let _ = tx.send(result.clone()); + result.map_err(DataFusionError::Shared) })) .await?; stage_coordinator.seal_dynamic_filter_stage() diff --git a/src/coordinator/query_coordinator.rs b/src/coordinator/query_coordinator.rs index 80e049819..3ed700c73 100644 --- a/src/coordinator/query_coordinator.rs +++ b/src/coordinator/query_coordinator.rs @@ -99,8 +99,6 @@ impl QueryCoordinator { task_ctx: &self.task_ctx, metrics_set: &self.metrics, metrics: &self.coordinator_to_worker_metrics, - metrics_store: &self.metrics_store, - completed_dynamic_filter_store: &self.completed_dynamic_filter_store, dynamic_filter_registry: &self.dynamic_filter_registry, spawner: &self.spawner, } @@ -110,6 +108,39 @@ impl QueryCoordinator { pub(super) fn session_config(&self) -> &SessionConfig { self.task_ctx.session_config() } + + /// Builds task startup with report cleanup owned even before its first poll. + pub(super) fn start_task( + self: &Arc, + stage: &LocalStage, + task_i: usize, + ) -> impl Future< + Output = Result<( + Url, + UnboundedSender, + UnboundedReceiver, + )>, + > + Send + + 'static { + let reports = TaskReports { + task_key: TaskKey { + query_id: stage.query_id, + stage_id: stage.num, + task_number: task_i, + }, + metrics_store: self.metrics_store.clone(), + completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), + }; + let coordinator = Arc::clone(self); + let stage = stage.clone(); + async move { + let stage_coordinator = coordinator.stage_coordinator(&stage); + let (url, tx, stream) = stage_coordinator.init_bidirectional_stream(task_i).await?; + let load_info = stage_coordinator.worker_to_coordinator_task(stream, reports); + stage_coordinator.coordinator_to_worker_task(task_i, tx.clone())?; + Ok((url, tx, load_info)) + } + } } /// Manages all the coordinator->worker and worker->coordinator comms that happen during the @@ -128,32 +159,18 @@ pub(super) struct StageCoordinator<'a> { task_ctx: &'a Arc, metrics_set: &'a ExecutionPlanMetricsSet, metrics: &'a CoordinatorToWorkerMetrics, - metrics_store: &'a Option>>, - completed_dynamic_filter_store: &'a Option>>, dynamic_filter_registry: &'a Arc, spawner: &'a Spawner, } impl<'a> StageCoordinator<'a> { - pub(super) fn pending_task_reports(&self, task_i: usize) -> PendingTaskReports { - PendingTaskReports { - task_key: TaskKey { - query_id: self.query_id, - stage_id: self.stage_id, - task_number: task_i, - }, - metrics_store: self.metrics_store.clone(), - completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), - } - } - /// Sends a plan to a specific worker and sets up the bidirectional stream. /// /// Its returns are: /// - The worker URL in which the task got allocated /// - The coordinator-to-worker stream for producing messages that reach the worker. /// - The worker-to-coordinator stream for receiving messages from the worker. - pub(super) async fn init_bidirectional_stream( + async fn init_bidirectional_stream( &self, task_i: usize, ) -> Result<( @@ -291,18 +308,12 @@ impl<'a> StageCoordinator<'a> { } /// Spawns background tasks in charge of collecting messages sent by a worker. - pub(super) fn worker_to_coordinator_task( + fn worker_to_coordinator_task( &self, - task_i: usize, mut worker_to_coordinator_stream: BoxStream<'static, Result>, + mut reports: TaskReports, ) -> UnboundedReceiver { - let task_key = TaskKey { - query_id: self.query_id, - stage_id: self.stage_id, - task_number: task_i, - }; - let mut task_metrics = self.metrics_store.clone(); - let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone(); + let task_key = reports.task_key; let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); let task_ctx = Arc::clone(self.task_ctx); let (load_info_tx, load_info_rx) = unbounded_channel(); @@ -313,12 +324,12 @@ impl<'a> StageCoordinator<'a> { while let Some(msg) = worker_to_coordinator_stream.try_next().await? { match msg { WorkerToCoordinatorMsg::TaskMetrics(metrics) => { - if let Some(task_metrics) = task_metrics.take() { + if let Some(task_metrics) = reports.metrics_store.take() { task_metrics.insert(task_key, metrics); } } WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { - if let Some(store) = completed_dynamic_filter_store.take() { + if let Some(store) = reports.completed_dynamic_filter_store.take() { store.insert(task_key, filters); } } @@ -336,14 +347,6 @@ impl<'a> StageCoordinator<'a> { } }; } - // An unexecuted task sends no final reports; still complete its waits. - if let Some(store) = task_metrics { - store.insert(task_key, TaskMetrics::default()); - } - if let Some(store) = completed_dynamic_filter_store { - store.insert(task_key, TaskCompletedDynamicFilters::default()); - } - let _ = load_info_tx_opt.take(); Ok::<_, DataFusionError>(()) }); @@ -353,7 +356,7 @@ impl<'a> StageCoordinator<'a> { /// Spawns a background task in charge of sending messages to workers. Some things that are sent /// to workers here are: /// - WorkUnits collected from [WorkUnitFeeds] present in the plan. - pub(super) fn coordinator_to_worker_task( + fn coordinator_to_worker_task( &self, task_i: usize, tx: UnboundedSender, @@ -510,21 +513,14 @@ impl<'a> StageCoordinator<'a> { } } -/// Completes report waits if setup ends before a worker report reader owns them. -pub(super) struct PendingTaskReports { +/// Completes missing report waits when setup or its worker response reader ends. +struct TaskReports { task_key: TaskKey, metrics_store: Option>>, completed_dynamic_filter_store: Option>>, } -impl PendingTaskReports { - pub(super) fn reader_started(&mut self) { - self.metrics_store = None; - self.completed_dynamic_filter_store = None; - } -} - -impl Drop for PendingTaskReports { +impl Drop for TaskReports { fn drop(&mut self) { if let Some(store) = self.metrics_store.take() { store.insert(self.task_key, TaskMetrics::default()); diff --git a/tests/startup_overlap.rs b/tests/startup_overlap.rs index be0b6f1f6..3885f98e5 100644 --- a/tests/startup_overlap.rs +++ b/tests/startup_overlap.rs @@ -25,8 +25,9 @@ mod tests { DistributedExec, DistributedExt, MappedWorkerSessionBuilderExt, NetworkBoundaryExt, NetworkCoalesceExec, RouteTaskEvent, RouteTaskEventResponse, RouteTaskHandler, Stage, WorkerQueryContext, WorkerSessionBuilder, WorkerToCoordinatorMsg, + rewrite_distributed_plan_with_dynamic_filters, }; - use futures::{StreamExt, TryStreamExt}; + use futures::{StreamExt, TryStreamExt, stream}; use tokio::time::{sleep, timeout}; use tokio_util::sync::CancellationToken; @@ -129,6 +130,48 @@ mod tests { Ok(()) } + #[tokio::test] + async fn worker_response_error_after_output_finishes_reports() -> Result<()> { + let response_error = ResponseErrorGate::default(); + let ctx = context(build_state) + .await + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_dynamic_filter_collection(true)? + .with_distributed_route_task_handler(response_error.clone()); + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert!(matches!(task_indices(&[first]).as_slice(), [0] | [1])); + response_error.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "injected worker response error"); + + let distributed = plan.downcast_ref::().unwrap(); + let reports = timeout(WAIT, distributed.wait_for_metrics()) + .await + .unwrap() + .unwrap(); + assert_eq!(reports.len(), 2); + assert!( + reports + .values() + .all(|report| report.pre_order_plan_metrics.is_empty()) + ); + let task_ctx = ctx.task_ctx(); + timeout( + WAIT, + rewrite_distributed_plan_with_dynamic_filters(plan, &task_ctx), + ) + .await + .unwrap()?; + Ok(()) + } + #[tokio::test] async fn dropping_output_cancels_pending_setup_and_finishes_metrics() -> Result<()> { let gate = SetupGate::default(); @@ -336,6 +379,35 @@ mod tests { assert_eq!(reports.len(), 2); } + #[derive(Clone, Default)] + struct ResponseErrorGate { + release: CancellationToken, + } + + #[async_trait] + impl RouteTaskHandler for ResponseErrorGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + Some( + UrlEmitterRouteTaskHandler + .handle(event) + .await? + .map(|mut response| { + let release = self.release.clone(); + let error = stream::once(async move { + release.cancelled().await; + exec_err!("injected worker response error") + }); + response.worker_to_coordinator_stream = + error.chain(response.worker_to_coordinator_stream).boxed(); + response + }), + ) + } + } + #[derive(Clone, Default)] struct SetupGate { entered: CancellationToken, From bd7a7c5c81dcd38acd2bbcdcaefa337aa0091ae9 Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Fri, 6 Nov 2026 18:43:26 +0000 Subject: [PATCH 03/10] test: make startup overlap configuration explicit Show worker and partition counts in each startup regression and benchmark setup. Keep the shared fixture limited to codec and routing registration. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/startup_overlap.rs | 93 ++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/tests/startup_overlap.rs b/tests/startup_overlap.rs index 3885f98e5..ec09a218d 100644 --- a/tests/startup_overlap.rs +++ b/tests/startup_overlap.rs @@ -24,8 +24,7 @@ mod tests { use datafusion_distributed::{ DistributedExec, DistributedExt, MappedWorkerSessionBuilderExt, NetworkBoundaryExt, NetworkCoalesceExec, RouteTaskEvent, RouteTaskEventResponse, RouteTaskHandler, Stage, - WorkerQueryContext, WorkerSessionBuilder, WorkerToCoordinatorMsg, - rewrite_distributed_plan_with_dynamic_filters, + WorkerQueryContext, WorkerToCoordinatorMsg, rewrite_distributed_plan_with_dynamic_filters, }; use futures::{StreamExt, TryStreamExt, stream}; use tokio::time::{sleep, timeout}; @@ -37,10 +36,15 @@ mod tests { #[tokio::test] async fn static_scan_returns_output_before_sibling_setup() -> Result<()> { let gate = SetupGate::default(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(false)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let mut output = ctx.sql(SCAN).await?.execute_stream().await?; let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); @@ -61,10 +65,15 @@ mod tests { #[tokio::test] async fn adaptive_scan_returns_output_before_sibling_setup() -> Result<()> { let gate = SetupGate::default(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let mut output = ctx.sql(SCAN).await?.execute_stream().await?; let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); @@ -85,10 +94,15 @@ mod tests { #[tokio::test] async fn adaptive_shuffle_samples_before_sibling_setup() -> Result<()> { let gate = SetupGate::default(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let query = ctx .sql("SELECT task_index FROM url_emitter(2, 2, 'scan') GROUP BY task_index") .await?; @@ -110,11 +124,16 @@ mod tests { fail: true, ..Default::default() }; - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(false)? .with_distributed_metrics_collection(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; @@ -133,12 +152,17 @@ mod tests { #[tokio::test] async fn worker_response_error_after_output_finishes_reports() -> Result<()> { let response_error = ResponseErrorGate::default(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(false)? .with_distributed_metrics_collection(true)? .with_distributed_dynamic_filter_collection(true)? .with_distributed_route_task_handler(response_error.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; @@ -175,11 +199,16 @@ mod tests { #[tokio::test] async fn dropping_output_cancels_pending_setup_and_finishes_metrics() -> Result<()> { let gate = SetupGate::default(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(true)? .with_distributed_metrics_collection(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; @@ -199,11 +228,16 @@ mod tests { wait_for_sibling: true, ..Default::default() }; - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(false)? .with_distributed_metrics_collection(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let plan = ctx .sql(&format!("{SCAN} LIMIT 1")) .await? @@ -225,10 +259,15 @@ mod tests { async fn adaptive_nested_coalesce_has_concrete_worker_destinations() -> Result<()> { let gate = SetupGate::default(); gate.release.cancel(); - let ctx = context(build_state) - .await + let ctx = context(start_in_memory_context(2, build_state).await) .with_distributed_dynamic_task_count(true)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; let query = ctx .sql( "SELECT task_index FROM url_emitter(2, 2, 'scan') WHERE task_index = \ @@ -259,10 +298,15 @@ mod tests { let worker = build_state.map(move |builder| { Ok(builder.with_scalar_functions(vec![udf.clone()]).build()) }); - let ctx = context(worker) - .await + let ctx = context(start_in_memory_context(2, worker).await) .with_distributed_dynamic_task_count(adaptive)? .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; ctx.register_udf(cpu_work(None)); let plan = ctx .sql("SELECT cpu_work(task_index) FROM url_emitter(2, 2, 'scan')") @@ -331,18 +375,11 @@ mod tests { ) } - async fn context(builder: impl WorkerSessionBuilder + Send + Sync + 'static) -> SessionContext { - let mut ctx = start_in_memory_context(2, builder).await; + fn context(mut ctx: SessionContext) -> SessionContext { ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); ctx.set_distributed_user_codec(URLEmitterExtensionCodec); ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; ctx } From 673ab7944ab7acdee2d848c14bcfefb3d40a1c0e Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Fri, 6 Nov 2026 19:17:03 +0000 Subject: [PATCH 04/10] refactor: keep pending task routing in the connection pool Configure the existing connection pool with shared pending destinations and query cancellation when building a coordinator-only boundary. Delegate to ordinary boundary execution and remove the remote-stage and task-destination forwarding types. Preserve stream sharing, pool reset behavior, dynamic-filter anchors, and concrete destinations in worker plans. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- src/coordinator/prepare_static_plan.rs | 14 +- src/execution_plans/network_broadcast.rs | 62 +++----- src/execution_plans/network_coalesce.rs | 108 ++++++------- src/execution_plans/network_shuffle.rs | 112 ++++++------- .../pending_network_boundary.rs | 63 ++++---- src/worker/mod.rs | 2 +- src/worker/worker_connection_pool.rs | 150 +++++++----------- 7 files changed, 231 insertions(+), 280 deletions(-) diff --git a/src/coordinator/prepare_static_plan.rs b/src/coordinator/prepare_static_plan.rs index 84972d2c7..16d6986b9 100644 --- a/src/coordinator/prepare_static_plan.rs +++ b/src/coordinator/prepare_static_plan.rs @@ -58,12 +58,14 @@ pub(super) async fn prepare_stages( }; if head_stages.contains(&stage.num) { let workers = start_pending_stage(query_coordinator, stage); - return Ok(Transformed::yes(Arc::new(PendingNetworkBoundaryExec::new( - boundary.with_input_stage(Stage::Local(stage.clone()))?, - workers, - anchors, - query_coordinator.spawner.query_finished(), - )) as Arc)); + return Ok(Transformed::yes( + Arc::new(PendingNetworkBoundaryExec::try_new( + boundary.with_input_stage(Stage::Local(stage.clone()))?, + workers, + anchors, + query_coordinator.spawner.query_finished(), + )?) as Arc, + )); } let workers = try_join_all((0..stage.tasks).map(|task| query_coordinator.start_task(stage, task))) diff --git a/src/execution_plans/network_broadcast.rs b/src/execution_plans/network_broadcast.rs index ef4ed6201..343ef39c0 100644 --- a/src/execution_plans/network_broadcast.rs +++ b/src/execution_plans/network_broadcast.rs @@ -1,7 +1,7 @@ use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::stage::{LocalStage, Stage}; -use crate::worker::{RemoteStageRef, WorkerConnectionPool}; +use crate::worker::WorkerConnectionPool; use crate::{BroadcastExec, DistributedTaskContext}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Result, not_impl_err, plan_err}; @@ -201,36 +201,6 @@ impl DisplayAs for NetworkBroadcastExec { } } -impl NetworkBroadcastExec { - pub(crate) fn execute_remote( - &self, - partition: usize, - context: Arc, - remote_stage: RemoteStageRef<'_>, - ) -> Result { - let task_context = DistributedTaskContext::from_ctx(&context); - let out_partitions = self.properties.partitioning.partition_count(); - let off = out_partitions * task_context.task_index; - let mut streams = Vec::with_capacity(self.input_stage.task_count()); - - for input_task_index in 0..self.input_stage.task_count() { - streams.push(self.worker_connections.execute( - remote_stage, - off..(off + self.properties.partitioning.partition_count()), - input_task_index, - off + partition, - self.producer_head(task_context.task_count)?, - &context, - )?); - } - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - futures::stream::select_all(streams), - ))) - } -} - impl ExecutionPlan for NetworkBroadcastExec { fn name(&self) -> &str { "NetworkBroadcastExec" @@ -277,11 +247,31 @@ impl ExecutionPlan for NetworkBroadcastExec { partition: usize, context: Arc, ) -> Result { - let remote_stage = match &self.input_stage { - Stage::Local(local) => return local.execute(partition, context), - Stage::Remote(remote_stage) => remote_stage, - }; - self.execute_remote(partition, context, remote_stage.into()) + if let Stage::Local(local) = &self.input_stage + && !self.worker_connections.has_pending_workers() + { + return local.execute(partition, context); + } + let task_context = DistributedTaskContext::from_ctx(&context); + let out_partitions = self.properties.partitioning.partition_count(); + let off = out_partitions * task_context.task_index; + let mut streams = Vec::with_capacity(self.input_stage.task_count()); + + for input_task_index in 0..self.input_stage.task_count() { + streams.push(self.worker_connections.execute( + &self.input_stage, + off..(off + self.properties.partitioning.partition_count()), + input_task_index, + off + partition, + self.producer_head(task_context.task_count)?, + &context, + )?); + } + + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + futures::stream::select_all(streams), + ))) } fn metrics(&self) -> Option { diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index f5b370017..dc43bd402 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -3,7 +3,7 @@ use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::execution_plans::common::scale_partitioning_props; use crate::stage::{LocalStage, Stage}; -use crate::worker::{RemoteStageRef, WorkerConnectionPool}; +use crate::worker::WorkerConnectionPool; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{exec_err, not_impl_err, plan_err}; use datafusion::error::Result; @@ -224,13 +224,57 @@ impl DisplayAs for NetworkCoalesceExec { } } -impl NetworkCoalesceExec { - pub(crate) fn execute_remote( +impl ExecutionPlan for NetworkCoalesceExec { + fn name(&self) -> &str { + "NetworkCoalesceExec" + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + match &self.input_stage.local_plan() { + Some(v) => vec![v], + None => vec![], + } + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { + Stage::Local(local) => { + local.plan = require_one_child(children)?; + } + Stage::Remote(_) => { + if !children.is_empty() { + not_impl_err!("NetworkBoundary cannot accept children")? + } + } + } + Ok(Arc::new(self_clone)) + } + + fn execute( &self, partition: usize, context: Arc, - remote_stage: RemoteStageRef<'_>, ) -> Result { + if let Stage::Local(local) = &self.input_stage + && !self.worker_connections.has_pending_workers() + { + return local.execute(partition, context); + } let task_context = DistributedTaskContext::from_ctx(&context); if task_context.task_index >= task_context.task_count { return exec_err!( @@ -284,7 +328,7 @@ impl NetworkCoalesceExec { let target_task = group.start_task + input_task_offset; let stream = self.worker_connections.execute( - remote_stage, + &self.input_stage, 0..partitions_per_task, target_task, target_partition, @@ -297,60 +341,6 @@ impl NetworkCoalesceExec { stream, ))) } -} - -impl ExecutionPlan for NetworkCoalesceExec { - fn name(&self) -> &str { - "NetworkCoalesceExec" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - match &self.input_stage.local_plan() { - Some(v) => vec![v], - None => vec![], - } - } - - fn apply_expressions( - &self, - f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { - Stage::Local(local) => { - local.plan = require_one_child(children)?; - } - Stage::Remote(_) => { - if !children.is_empty() { - not_impl_err!("NetworkBoundary cannot accept children")? - } - } - } - Ok(Arc::new(self_clone)) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> Result { - let remote_stage = match &self.input_stage { - Stage::Local(local) => return local.execute(partition, context), - Stage::Remote(remote_stage) => remote_stage, - }; - self.execute_remote(partition, context, remote_stage.into()) - } fn metrics(&self) -> Option { Some(self.worker_connections.metrics.clone_inner()) diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index e82297972..8ba6cc3a3 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -2,7 +2,7 @@ use crate::common::require_one_child; use crate::distributed_planner::ProducerHead; use crate::execution_plans::common::{salted_partitioning, scale_partitioning}; use crate::stage::{LocalStage, Stage}; -use crate::worker::{RemoteStageRef, WorkerConnectionPool}; +use crate::worker::WorkerConnectionPool; use crate::{DistributedTaskContext, MaybeEncoded, NetworkBoundary}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Result, not_impl_err, plan_err}; @@ -273,16 +273,60 @@ impl DisplayAs for NetworkShuffleExec { } } -impl NetworkShuffleExec { - pub(crate) fn execute_remote( +impl ExecutionPlan for NetworkShuffleExec { + fn name(&self) -> &str { + "NetworkShuffleExec" + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + match &self.input_stage.local_plan() { + Some(v) => vec![v], + None => vec![], + } + } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result, DataFusionError> { + let mut self_clone = self.as_ref().clone(); + match &mut self_clone.input_stage { + Stage::Local(local) => { + local.plan = require_one_child(children)?; + } + Stage::Remote(_) => { + if !children.is_empty() { + not_impl_err!("NetworkBoundary cannot accept children")? + } + } + } + Ok(Arc::new(self_clone)) + } + + fn execute( &self, partition: usize, context: Arc, - remote_stage: RemoteStageRef<'_>, ) -> Result { + if let Stage::Local(local) = &self.input_stage + && !self.worker_connections.has_pending_workers() + { + return local.execute(partition, context); + } let task_context = DistributedTaskContext::from_ctx(&context); let task_index = task_context.task_index; - let producer_task_count = remote_stage.task_count(); + let producer_task_count = self.input_stage.task_count(); let schema = self.schema(); let mut streams: Vec = @@ -296,7 +340,7 @@ impl NetworkShuffleExec { let mut streams = Vec::with_capacity(producer_task_count); for input_task_index in 0..producer_task_count { let stream = self.worker_connections.execute( - remote_stage, + &self.input_stage, off..(off + partition_count), input_task_index, global_partition, @@ -312,7 +356,7 @@ impl NetworkShuffleExec { } else { // TwoPhase: each consumer task reads from exactly one producer task. let stream = self.worker_connections.execute( - remote_stage, + &self.input_stage, task_index..task_index + 1, partition, task_index, @@ -357,60 +401,6 @@ impl NetworkShuffleExec { ))) } } -} - -impl ExecutionPlan for NetworkShuffleExec { - fn name(&self) -> &str { - "NetworkShuffleExec" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - match &self.input_stage.local_plan() { - Some(v) => vec![v], - None => vec![], - } - } - - fn apply_expressions( - &self, - f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - apply_expression_roots(self.input_stage.dynamic_filter_anchors().iter(), f) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result, DataFusionError> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { - Stage::Local(local) => { - local.plan = require_one_child(children)?; - } - Stage::Remote(_) => { - if !children.is_empty() { - not_impl_err!("NetworkBoundary cannot accept children")? - } - } - } - Ok(Arc::new(self_clone)) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> Result { - let remote_stage = match &self.input_stage { - Stage::Local(local) => return local.execute(partition, context), - Stage::Remote(remote_stage) => remote_stage, - }; - self.execute_remote(partition, context, remote_stage.into()) - } fn metrics(&self) -> Option { Some(self.worker_connections.metrics.clone_inner()) diff --git a/src/execution_plans/pending_network_boundary.rs b/src/execution_plans/pending_network_boundary.rs index cd493ee03..9a802f739 100644 --- a/src/execution_plans/pending_network_boundary.rs +++ b/src/execution_plans/pending_network_boundary.rs @@ -1,5 +1,5 @@ use crate::distributed_planner::{NetworkBoundary, ProducerHead}; -use crate::worker::{PendingTaskUrl, RemoteStageRef}; +use crate::worker::{PendingTaskUrl, WorkerConnectionPool}; use crate::{NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Result, internal_err}; @@ -18,24 +18,48 @@ use tokio_util::sync::CancellationToken; /// Boundaries inside worker plans keep concrete destinations and never use this wrapper. pub(crate) struct PendingNetworkBoundaryExec { boundary: Arc, - workers: Vec, dynamic_filter_anchors: Vec>, - query_finished: CancellationToken, } impl PendingNetworkBoundaryExec { - pub(crate) fn new( + pub(crate) fn try_new( boundary: Arc, workers: Vec, dynamic_filter_anchors: Vec>, query_finished: CancellationToken, - ) -> Self { - Self { + ) -> Result { + let Stage::Local(stage) = boundary.input_stage() else { + return internal_err!("A pending network boundary requires a local stage"); + }; + if stage.tasks != workers.len() { + return internal_err!( + "Pending network boundary has {} workers for {} tasks", + workers.len(), + stage.tasks + ); + } + let worker_connections = WorkerConnectionPool::new_pending(workers, query_finished); + let plan: &dyn ExecutionPlan = boundary.as_ref(); + let boundary: Arc = + if let Some(plan) = plan.downcast_ref::() { + let mut plan = plan.clone(); + plan.worker_connections = worker_connections; + Arc::new(plan) + } else if let Some(plan) = plan.downcast_ref::() { + let mut plan = plan.clone(); + plan.worker_connections = worker_connections; + Arc::new(plan) + } else if let Some(plan) = plan.downcast_ref::() { + let mut plan = plan.clone(); + plan.worker_connections = worker_connections; + Arc::new(plan) + } else { + return internal_err!("Unsupported pending network boundary"); + }; + Ok(Self { boundary, - workers, dynamic_filter_anchors, - query_finished, - } + }) } } @@ -43,7 +67,7 @@ impl Debug for PendingNetworkBoundaryExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("PendingNetworkBoundaryExec") .field("boundary", &self.boundary) - .field("tasks", &self.workers.len()) + .field("tasks", &self.boundary.input_stage().task_count()) .finish() } } @@ -103,24 +127,7 @@ impl ExecutionPlan for PendingNetworkBoundaryExec { partition: usize, context: Arc, ) -> Result { - let stage = self.input_stage(); - let pending = RemoteStageRef::Pending { - query_id: stage.query_id(), - num: stage.num(), - workers: &self.workers, - query_finished: &self.query_finished, - }; - let boundary: &dyn ExecutionPlan = self.boundary.as_ref(); - if let Some(boundary) = boundary.downcast_ref::() { - return boundary.execute_remote(partition, context, pending); - } - if let Some(boundary) = boundary.downcast_ref::() { - return boundary.execute_remote(partition, context, pending); - } - if let Some(boundary) = boundary.downcast_ref::() { - return boundary.execute_remote(partition, context, pending); - } - internal_err!("Unsupported pending network boundary") + self.boundary.execute(partition, context) } fn metrics(&self) -> Option { diff --git a/src/worker/mod.rs b/src/worker/mod.rs index f831d9bce..3df566a83 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -9,7 +9,7 @@ mod worker_connection_pool; mod worker_service; pub(crate) use single_write_multi_read::SingleWriteMultiRead; -pub(crate) use worker_connection_pool::{PendingTaskUrl, RemoteStageRef, WorkerConnectionPool}; +pub(crate) use worker_connection_pool::{PendingTaskUrl, WorkerConnectionPool}; pub use impl_coordinator_channel::CoordinatorChannelResult; pub use session_builder::{ diff --git a/src/worker/worker_connection_pool.rs b/src/worker/worker_connection_pool.rs index 9d31fa911..6ad1a98a3 100644 --- a/src/worker/worker_connection_pool.rs +++ b/src/worker/worker_connection_pool.rs @@ -1,7 +1,7 @@ use crate::common::on_drop_stream; use crate::distributed_planner::ProducerHead; use crate::passthrough_headers::get_passthrough_headers; -use crate::stage::RemoteStage; +use crate::stage::Stage; use crate::{ExecuteTaskRequest, LocalWorkerContext, TaskKey, get_distributed_channel_resolver}; use dashmap::DashMap; use datafusion::arrow::array::RecordBatch; @@ -12,7 +12,7 @@ use datafusion::common::{ use datafusion::execution::TaskContext; use datafusion::physical_expr_common::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::metrics::MetricBuilder; -use futures::future::{BoxFuture, Shared}; +use futures::future::{BoxFuture, Shared, ready}; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use std::fmt::{Debug, Formatter}; @@ -21,7 +21,6 @@ use std::sync::{Arc, Mutex}; use tokio::select; use tokio_util::sync::CancellationToken; use url::Url; -use uuid::Uuid; /// Manages connections to remote workers. /// - Handles a range of partitions at a time in other to give the chance to [crate::WorkerChannel] @@ -32,9 +31,16 @@ use uuid::Uuid; /// - Lazily initializes connections per remote worker and requested partition range. pub(crate) struct WorkerConnectionPool { lazy_stream_groups: Vec, StreamGroup>>, + pending_workers: Option>, pub(crate) metrics: ExecutionPlanMetricsSet, } +/// Coordinator-local destinations may still be routing; wire plans always contain resolved URLs. +struct PendingWorkers { + workers: Vec, + query_finished: CancellationToken, +} + /// Slots exist before asynchronous connection setup, so dropping a consumer can cancel /// its receiver both before and after the worker returns the partition streams. struct StreamGroup { @@ -54,103 +60,66 @@ type SharedBoxFuture = Shared; -/// Coordinator-local destinations may still be routing; wire plans always contain resolved URLs. -#[derive(Clone, Copy)] -pub(crate) enum RemoteStageRef<'a> { - Ready(&'a RemoteStage), - Pending { - query_id: Uuid, - num: usize, - workers: &'a [PendingTaskUrl], - query_finished: &'a CancellationToken, - }, -} - -impl<'a> From<&'a RemoteStage> for RemoteStageRef<'a> { - fn from(stage: &'a RemoteStage) -> Self { - Self::Ready(stage) - } -} - -impl RemoteStageRef<'_> { - pub(crate) fn task_count(self) -> usize { - match self { - Self::Ready(stage) => stage.workers.len(), - Self::Pending { workers, .. } => workers.len(), - } - } - - fn task_key(self, task_number: usize) -> TaskKey { - let (query_id, stage_id) = match self { - Self::Ready(stage) => (stage.query_id, stage.num), - Self::Pending { query_id, num, .. } => (query_id, num), - }; - TaskKey { - query_id, - stage_id, - task_number, - } - } - - fn destination(self, task: usize) -> Result { - let destination = match self { - Self::Ready(stage) => stage.workers.get(task).cloned().map(TaskDestination::Ready), - Self::Pending { - workers, - query_finished, - .. - } => workers - .get(task) - .cloned() - .map(|url| TaskDestination::Pending(url, query_finished.clone())), - }; - destination.ok_or_else(|| internal_datafusion_err!("input stage task {task} out of range")) - } -} - -enum TaskDestination { - Ready(Url), - Pending(PendingTaskUrl, CancellationToken), -} - -impl TaskDestination { - fn query_finished(&self) -> Option { - match self { - Self::Ready(_) => None, - Self::Pending(_, query_finished) => Some(query_finished.clone()), - } - } - - async fn resolve(self) -> Result { - match self { - Self::Ready(url) => Ok(url), - Self::Pending(url, _) => url.await.map_err(DataFusionError::Shared), - } - } -} - impl WorkerConnectionPool { /// Builds a new [WorkerConnectionPool] with as many empty slots for worker stream entries as /// the provided `input_tasks`. pub(crate) fn new(input_tasks: usize) -> Self { Self { lazy_stream_groups: (0..input_tasks).map(|_| Default::default()).collect(), + pending_workers: None, metrics: ExecutionPlanMetricsSet::default(), } } - pub(crate) fn execute<'a>( + pub(crate) fn new_pending( + workers: Vec, + query_finished: CancellationToken, + ) -> Self { + let input_tasks = workers.len(); + Self { + pending_workers: Some(Arc::new(PendingWorkers { + workers, + query_finished, + })), + ..Self::new(input_tasks) + } + } + + pub(crate) fn has_pending_workers(&self) -> bool { + self.pending_workers.is_some() + } + + pub(crate) fn execute( &self, - input_stage: impl Into>, + input_stage: &Stage, target_partitions: Range, target_task: usize, target_partition: usize, producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let input_stage = input_stage.into(); - let destination = input_stage.destination(target_task)?; - let task_key = input_stage.task_key(target_task); + let (destination, query_finished) = if let Some(pending) = &self.pending_workers { + let destination = pending.workers.get(target_task).cloned().ok_or_else(|| { + internal_datafusion_err!("input stage task {target_task} out of range") + })?; + ( + destination.map_err(DataFusionError::Shared).left_future(), + Some(pending.query_finished.clone()), + ) + } else { + let Stage::Remote(stage) = input_stage else { + return internal_err!("A local stage requires pending worker destinations"); + }; + let destination = stage.workers.get(target_task).cloned().ok_or_else(|| { + internal_datafusion_err!("input stage task {target_task} out of range") + })?; + (ready(Ok(destination)).right_future(), None) + }; + let task_key = TaskKey { + query_id: input_stage.query_id(), + stage_id: input_stage.num(), + task_number: target_task, + }; let bdr = || MetricBuilder::new(&self.metrics); let output_bytes = bdr().output_bytes(target_partition); let output_rows = bdr().output_rows(target_partition); @@ -184,9 +153,8 @@ impl WorkerConnectionPool { // // Note that this does not start polling the returned streams, it just instantiates them. let streams_task = SpawnedTask::spawn(async move { - let query_finished = destination.query_finished(); let setup = async move { - let target_url = destination.resolve().await?; + let target_url = destination.await?; let request = ExecuteTaskRequest { task_key, target_partition_start: target_partitions.start, @@ -309,7 +277,10 @@ impl Debug for WorkerConnectionPool { impl Clone for WorkerConnectionPool { fn clone(&self) -> Self { - Self::new(self.lazy_stream_groups.len()) + Self { + pending_workers: self.pending_workers.clone(), + ..Self::new(self.lazy_stream_groups.len()) + } } } @@ -317,6 +288,7 @@ impl Clone for WorkerConnectionPool { mod tests { use super::*; use crate::grpc::DefaultChannelResolver; + use crate::stage::RemoteStage; use crate::worker::test_utils::worker_handles::TcpWorkerHandle; use crate::{DistributedConfig, DistributedExt}; use datafusion::arrow::array::Int32Array; @@ -404,7 +376,7 @@ mod tests { struct Fixture { pool: WorkerConnectionPool, ctx: Arc, - stage: RemoteStage, + stage: Stage, _worker: TcpWorkerHandle, } @@ -428,13 +400,13 @@ mod tests { .with_distributed_channel_resolver(DefaultChannelResolver::default()) .with_distributed_worker_connection_buffer_budget_bytes(128)? .task_ctx(); - let stage = RemoteStage { + let stage = Stage::Remote(RemoteStage { query_id, num: 0, workers: vec![worker.url.clone()], runtime_stats: None, dynamic_filter_anchors: vec![], - }; + }); Ok(Self { pool: WorkerConnectionPool::new(1), ctx, From 94f286a2d0e87f0b7f7f83976900314aa1eeb978 Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Fri, 6 Nov 2026 19:59:29 +0000 Subject: [PATCH 05/10] refactor: represent pending workers in remote stages Store ready and pending worker locations in RemoteStage so ordinary network boundaries can consume tasks independently after routing succeeds. Remove the pending boundary wrapper and separate connection-pool routing state. Keep the dialer selection contract and URL wire format. Document migration from Vec and reject pending locations when encoding worker plans. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- docs/upgrade/5.0.0.md | 21 +++ src/codec/distributed_codec.rs | 45 +++++- src/coordinator/prepare_dynamic_plan.rs | 4 +- src/coordinator/prepare_static_plan.rs | 48 +++--- src/distributed_planner/network_boundary.rs | 4 +- src/events/defaults/routing.rs | 3 +- .../benchmarks/shuffle_bench.rs | 7 +- .../benchmarks/transport_bench.rs | 7 +- src/execution_plans/mod.rs | 2 - src/execution_plans/network_broadcast.rs | 12 +- src/execution_plans/network_coalesce.rs | 12 +- src/execution_plans/network_shuffle.rs | 16 +- .../pending_network_boundary.rs | 144 ------------------ src/lib.rs | 3 +- src/stage.rs | 50 +++++- src/worker/mod.rs | 2 +- src/worker/worker_connection_pool.rs | 80 +++------- tests/startup_overlap.rs | 10 +- 18 files changed, 200 insertions(+), 270 deletions(-) delete mode 100644 src/execution_plans/pending_network_boundary.rs diff --git a/docs/upgrade/5.0.0.md b/docs/upgrade/5.0.0.md index b26d50681..938059d63 100644 --- a/docs/upgrade/5.0.0.md +++ b/docs/upgrade/5.0.0.md @@ -154,3 +154,24 @@ A coordinator and a worker on different versions cannot decode each other's sketches, and with metrics collection enabled (the default) the decode error fails the query. Upgrade coordinators and workers together, or disable metrics collection with `with_distributed_metrics_collection(false)` during the rollout. + +## 7. Remote stage destinations can still be routing + +`RemoteStage::workers` now contains `WorkerLocation` values instead of `Url`s. +When constructing a stage with resolved worker URLs, convert them with `into()` +or `WorkerLocation::from`: + +```rust +use datafusion_distributed::WorkerLocation; + +let workers = urls.into_iter().map(WorkerLocation::from).collect(); +``` + +When inspecting a destination, use `worker.ready_url()?` to borrow its resolved +URL. This returns an error for `WorkerLocation::Pending`; coordinator input +stages may contain these destinations while task routing finishes. Best-effort +colocation handlers should skip pending destinations instead of treating them +as resolved URLs. + +Worker plans require `WorkerLocation::Ready` destinations. The wire format still +contains URL strings, and encoding a pending destination returns an error. diff --git a/src/codec/distributed_codec.rs b/src/codec/distributed_codec.rs index 85cd9b623..251c9371e 100644 --- a/src/codec/distributed_codec.rs +++ b/src/codec/distributed_codec.rs @@ -99,7 +99,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let Ok(url) = Url::parse(&url_str) else { return Err(proto_error("Invalid URL in task")); }; - worker_urls.push(url); + worker_urls.push(url.into()); } Ok(Stage::Remote(RemoteStage { query_id: deserialize_uuid(proto.query_id.as_ref())?, @@ -347,7 +347,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let mut tasks = Vec::with_capacity(remote.workers.len()); for worker in &remote.workers { tasks.push(ExecutionTaskProto { - url_str: Some(worker.to_string()), + url_str: Some(worker.ready_url()?.to_string()), }) } StageProto { @@ -806,6 +806,7 @@ mod tests { }; use super::*; + use crate::WorkerLocation; use crate::execution_plans::PRODUCER_SALT_DEFAULT; const DEFAULT_MODE: ShuffleMode = ShuffleMode::Direct; @@ -824,6 +825,8 @@ mod tests { sorts::sort::SortExec, union::UnionExec, }, }; + use futures::FutureExt; + use tokio_util::sync::CancellationToken; fn empty_exec() -> Arc { Arc::new(EmptyExec::new(SchemaRef::new(Schema::empty()))) @@ -865,13 +868,20 @@ mod tests { fn test_roundtrip_single_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; let ctx = create_context(); + let worker_url = Url::parse("http://worker:50051").unwrap(); let schema = schema_i32("a"); let part = Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 4); let plan: Arc = Arc::new(new_network_hash_shuffle_exec( part, EquivalenceProperties::new(schema), - dummy_stage(), + Stage::Remote(RemoteStage { + query_id: Default::default(), + num: 0, + workers: vec![worker_url.clone().into()], + runtime_stats: None, + dynamic_filter_anchors: vec![], + }), DEFAULT_MODE, )); @@ -880,10 +890,39 @@ mod tests { let decoded = codec.try_decode(&buf, &[], &ctx, &default_proto_converter())?; assert_eq!(repr(&plan), repr(&decoded)); + let boundary = decoded.downcast_ref::().unwrap(); + let Stage::Remote(stage) = boundary.input_stage() else { + panic!("decoded input should be remote"); + }; + assert_eq!(stage.workers[0].ready_url()?, &worker_url); Ok(()) } + #[test] + fn pending_worker_location_cannot_be_encoded() { + let stage = Stage::Remote(RemoteStage { + query_id: Default::default(), + num: 0, + workers: vec![WorkerLocation::Pending { + url: futures::future::pending().boxed().shared(), + query_finished: CancellationToken::new(), + }], + runtime_stats: None, + dynamic_filter_anchors: vec![], + }); + let plan: Arc = Arc::new(new_network_hash_shuffle_exec( + Partitioning::RoundRobinBatch(1), + EquivalenceProperties::new(schema_i32("a")), + stage, + DEFAULT_MODE, + )); + let error = DistributedCodec + .try_encode(plan, &mut vec![], &default_proto_converter()) + .unwrap_err(); + crate::assert_snapshot!(error, @"Execution error: Worker location is pending"); + } + #[test] fn test_roundtrip_network_dynamic_filter_anchor() -> datafusion::common::Result<()> { let ctx = create_context(); diff --git a/src/coordinator/prepare_dynamic_plan.rs b/src/coordinator/prepare_dynamic_plan.rs index 6b5875260..3ee83e9ac 100644 --- a/src/coordinator/prepare_dynamic_plan.rs +++ b/src/coordinator/prepare_dynamic_plan.rs @@ -10,7 +10,7 @@ use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, }; use crate::execution_plans::SamplerExec; -use crate::stage::{LocalStage, RemoteStage}; +use crate::stage::{LocalStage, RemoteStage, WorkerLocation}; use crate::{ BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, TaskCountAnnotation, @@ -152,7 +152,7 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { input_stage: Stage::Remote(RemoteStage { query_id: input_stage.query_id, num: input_stage.num, - workers, + workers: workers.into_iter().map(WorkerLocation::from).collect(), runtime_stats: Some(Arc::new(stats)), dynamic_filter_anchors, }), diff --git a/src/coordinator/prepare_static_plan.rs b/src/coordinator/prepare_static_plan.rs index 16d6986b9..7e2f01454 100644 --- a/src/coordinator/prepare_static_plan.rs +++ b/src/coordinator/prepare_static_plan.rs @@ -4,10 +4,8 @@ use crate::coordinator::query_coordinator::QueryCoordinator; use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, }; -use crate::execution_plans::PendingNetworkBoundaryExec; use crate::stage::{LocalStage, RemoteStage}; -use crate::worker::PendingTaskUrl; -use crate::{NetworkBoundaryExt, Stage}; +use crate::{NetworkBoundaryExt, Stage, WorkerLocation}; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::common::{DataFusionError, HashSet, Result, exec_datafusion_err}; use datafusion::physical_plan::ExecutionPlan; @@ -56,26 +54,21 @@ pub(super) async fn prepare_stages( } else { vec![] }; - if head_stages.contains(&stage.num) { - let workers = start_pending_stage(query_coordinator, stage); - return Ok(Transformed::yes( - Arc::new(PendingNetworkBoundaryExec::try_new( - boundary.with_input_stage(Stage::Local(stage.clone()))?, - workers, - anchors, - query_coordinator.spawner.query_finished(), - )?) as Arc, - )); - } - let workers = - try_join_all((0..stage.tasks).map(|task| query_coordinator.start_task(stage, task))) - .await? - .into_iter() - .map(|(url, ..)| url) - .collect(); - query_coordinator - .stage_coordinator(stage) - .seal_dynamic_filter_stage()?; + let workers = if head_stages.contains(&stage.num) { + start_pending_stage(query_coordinator, stage) + } else { + let workers = try_join_all( + (0..stage.tasks).map(|task| query_coordinator.start_task(stage, task)), + ) + .await? + .into_iter() + .map(|(url, ..)| url.into()) + .collect(); + query_coordinator + .stage_coordinator(stage) + .seal_dynamic_filter_stage()?; + workers + }; Ok(Transformed::yes(boundary.with_input_stage( Stage::Remote(RemoteStage { query_id: stage.query_id, @@ -92,19 +85,20 @@ pub(super) async fn prepare_stages( fn start_pending_stage( query_coordinator: &Arc, stage: &LocalStage, -) -> Vec { +) -> Vec { let mut workers = Vec::with_capacity(stage.tasks); let mut setups = Vec::with_capacity(stage.tasks); for task in 0..stage.tasks { let (tx, rx) = oneshot::channel(); - workers.push( - async move { + workers.push(WorkerLocation::Pending { + url: async move { rx.await .map_err(|_| Arc::new(exec_datafusion_err!("Task setup was cancelled")))? } .boxed() .shared(), - ); + query_finished: query_coordinator.spawner.query_finished(), + }); setups.push((tx, query_coordinator.start_task(stage, task))); } let coordinator = Arc::clone(query_coordinator); diff --git a/src/distributed_planner/network_boundary.rs b/src/distributed_planner/network_boundary.rs index 0eca5a8f9..259c802b2 100644 --- a/src/distributed_planner/network_boundary.rs +++ b/src/distributed_planner/network_boundary.rs @@ -1,4 +1,4 @@ -use crate::execution_plans::{PendingNetworkBoundaryExec, SamplerExec}; +use crate::execution_plans::SamplerExec; use crate::{ BroadcastExec, MaybeEncoded, NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage, @@ -67,8 +67,6 @@ impl NetworkBoundaryExt for dyn ExecutionPlan { Some(node) } else if let Some(node) = self.downcast_ref::() { Some(node) - } else if let Some(node) = self.downcast_ref::() { - Some(node) } else { None } diff --git a/src/events/defaults/routing.rs b/src/events/defaults/routing.rs index 795e2ee84..35473f4e2 100644 --- a/src/events/defaults/routing.rs +++ b/src/events/defaults/routing.rs @@ -172,8 +172,9 @@ impl RouteTaskHandler for SingleTaskChildUrlRouteTaskHandler { if let Stage::Remote(remote) = nb.input_stage() && remote.workers.len() == 1 + && let Ok(url) = remote.workers[0].ready_url() { - single_stage_url = Some(remote.workers[0].clone()); + single_stage_url = Some(url.clone()); return Ok(TreeNodeRecursion::Stop); } diff --git a/src/execution_plans/benchmarks/shuffle_bench.rs b/src/execution_plans/benchmarks/shuffle_bench.rs index 1a9f8b299..ee8e4b5fd 100644 --- a/src/execution_plans/benchmarks/shuffle_bench.rs +++ b/src/execution_plans/benchmarks/shuffle_bench.rs @@ -216,7 +216,12 @@ impl ShuffleFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_workers.clone(), + workers: self + .input_stage_workers + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/src/execution_plans/benchmarks/transport_bench.rs b/src/execution_plans/benchmarks/transport_bench.rs index 5c624473b..0e8047df6 100644 --- a/src/execution_plans/benchmarks/transport_bench.rs +++ b/src/execution_plans/benchmarks/transport_bench.rs @@ -265,7 +265,12 @@ impl TransportFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_tasks.clone(), + workers: self + .input_stage_tasks + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index 1fbb15382..e21747dd4 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -6,7 +6,6 @@ mod metrics; mod network_broadcast; mod network_coalesce; mod network_shuffle; -mod pending_network_boundary; mod sampler; #[cfg(any(test, feature = "integration"))] @@ -19,5 +18,4 @@ pub(crate) use metrics::MetricsWrapperExec; pub use network_broadcast::NetworkBroadcastExec; pub use network_coalesce::NetworkCoalesceExec; pub use network_shuffle::{NetworkShuffleExec, PRODUCER_SALT_DEFAULT, ShuffleMode}; -pub(crate) use pending_network_boundary::PendingNetworkBoundaryExec; pub use sampler::SamplerExec; diff --git a/src/execution_plans/network_broadcast.rs b/src/execution_plans/network_broadcast.rs index 343ef39c0..10581506a 100644 --- a/src/execution_plans/network_broadcast.rs +++ b/src/execution_plans/network_broadcast.rs @@ -247,11 +247,11 @@ impl ExecutionPlan for NetworkBroadcastExec { partition: usize, context: Arc, ) -> Result { - if let Stage::Local(local) = &self.input_stage - && !self.worker_connections.has_pending_workers() - { - return local.execute(partition, context); - } + let remote_stage = match &self.input_stage { + Stage::Local(local) => return local.execute(partition, context), + Stage::Remote(remote_stage) => remote_stage, + }; + let task_context = DistributedTaskContext::from_ctx(&context); let out_partitions = self.properties.partitioning.partition_count(); let off = out_partitions * task_context.task_index; @@ -259,7 +259,7 @@ impl ExecutionPlan for NetworkBroadcastExec { for input_task_index in 0..self.input_stage.task_count() { streams.push(self.worker_connections.execute( - &self.input_stage, + remote_stage, off..(off + self.properties.partitioning.partition_count()), input_task_index, off + partition, diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index dc43bd402..db9b964a8 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -270,11 +270,11 @@ impl ExecutionPlan for NetworkCoalesceExec { partition: usize, context: Arc, ) -> Result { - if let Stage::Local(local) = &self.input_stage - && !self.worker_connections.has_pending_workers() - { - return local.execute(partition, context); - } + let remote_stage = match &self.input_stage { + Stage::Local(local) => return local.execute(partition, context), + Stage::Remote(remote_stage) => remote_stage, + }; + let task_context = DistributedTaskContext::from_ctx(&context); if task_context.task_index >= task_context.task_count { return exec_err!( @@ -328,7 +328,7 @@ impl ExecutionPlan for NetworkCoalesceExec { let target_task = group.start_task + input_task_offset; let stream = self.worker_connections.execute( - &self.input_stage, + remote_stage, 0..partitions_per_task, target_task, target_partition, diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 8ba6cc3a3..16e73ad73 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -319,14 +319,14 @@ impl ExecutionPlan for NetworkShuffleExec { partition: usize, context: Arc, ) -> Result { - if let Stage::Local(local) = &self.input_stage - && !self.worker_connections.has_pending_workers() - { - return local.execute(partition, context); - } + let remote_stage = match &self.input_stage { + Stage::Local(local) => return local.execute(partition, context), + Stage::Remote(remote_stage) => remote_stage, + }; + let task_context = DistributedTaskContext::from_ctx(&context); let task_index = task_context.task_index; - let producer_task_count = self.input_stage.task_count(); + let producer_task_count = remote_stage.workers.len(); let schema = self.schema(); let mut streams: Vec = @@ -340,7 +340,7 @@ impl ExecutionPlan for NetworkShuffleExec { let mut streams = Vec::with_capacity(producer_task_count); for input_task_index in 0..producer_task_count { let stream = self.worker_connections.execute( - &self.input_stage, + remote_stage, off..(off + partition_count), input_task_index, global_partition, @@ -356,7 +356,7 @@ impl ExecutionPlan for NetworkShuffleExec { } else { // TwoPhase: each consumer task reads from exactly one producer task. let stream = self.worker_connections.execute( - &self.input_stage, + remote_stage, task_index..task_index + 1, partition, task_index, diff --git a/src/execution_plans/pending_network_boundary.rs b/src/execution_plans/pending_network_boundary.rs deleted file mode 100644 index 9a802f739..000000000 --- a/src/execution_plans/pending_network_boundary.rs +++ /dev/null @@ -1,144 +0,0 @@ -use crate::distributed_planner::{NetworkBoundary, ProducerHead}; -use crate::worker::{PendingTaskUrl, WorkerConnectionPool}; -use crate::{NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage}; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::common::{Result, internal_err}; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_expr_common::metrics::MetricsSet; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, Statistics, StatisticsArgs, - apply_expression_roots, -}; -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; -use tokio_util::sync::CancellationToken; - -/// A coordinator-only boundary whose tasks become executable independently of their peers. -/// Boundaries inside worker plans keep concrete destinations and never use this wrapper. -pub(crate) struct PendingNetworkBoundaryExec { - boundary: Arc, - dynamic_filter_anchors: Vec>, -} - -impl PendingNetworkBoundaryExec { - pub(crate) fn try_new( - boundary: Arc, - workers: Vec, - dynamic_filter_anchors: Vec>, - query_finished: CancellationToken, - ) -> Result { - let Stage::Local(stage) = boundary.input_stage() else { - return internal_err!("A pending network boundary requires a local stage"); - }; - if stage.tasks != workers.len() { - return internal_err!( - "Pending network boundary has {} workers for {} tasks", - workers.len(), - stage.tasks - ); - } - let worker_connections = WorkerConnectionPool::new_pending(workers, query_finished); - let plan: &dyn ExecutionPlan = boundary.as_ref(); - let boundary: Arc = - if let Some(plan) = plan.downcast_ref::() { - let mut plan = plan.clone(); - plan.worker_connections = worker_connections; - Arc::new(plan) - } else if let Some(plan) = plan.downcast_ref::() { - let mut plan = plan.clone(); - plan.worker_connections = worker_connections; - Arc::new(plan) - } else if let Some(plan) = plan.downcast_ref::() { - let mut plan = plan.clone(); - plan.worker_connections = worker_connections; - Arc::new(plan) - } else { - return internal_err!("Unsupported pending network boundary"); - }; - Ok(Self { - boundary, - dynamic_filter_anchors, - }) - } -} - -impl Debug for PendingNetworkBoundaryExec { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PendingNetworkBoundaryExec") - .field("boundary", &self.boundary) - .field("tasks", &self.boundary.input_stage().task_count()) - .finish() - } -} - -impl DisplayAs for PendingNetworkBoundaryExec { - fn fmt_as(&self, display: DisplayFormatType, f: &mut Formatter<'_>) -> std::fmt::Result { - self.boundary.fmt_as(display, f) - } -} - -impl NetworkBoundary for PendingNetworkBoundaryExec { - fn input_stage(&self) -> &Stage { - self.boundary.input_stage() - } - - fn with_input_stage(&self, stage: Stage) -> Result> { - self.boundary.with_input_stage(stage) - } - - fn producer_head(&self, consumer_tasks: usize) -> Result { - self.boundary.producer_head(consumer_tasks) - } -} - -impl ExecutionPlan for PendingNetworkBoundaryExec { - fn name(&self) -> &str { - self.boundary.name() - } - - fn properties(&self) -> &Arc { - self.boundary.properties() - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - if !children.is_empty() { - return internal_err!("A pending network boundary cannot accept children"); - } - Ok(self) - } - - fn apply_expressions( - &self, - f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - apply_expression_roots(self.dynamic_filter_anchors.iter(), f) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> Result { - self.boundary.execute(partition, context) - } - - fn metrics(&self) -> Option { - self.boundary.metrics() - } - - fn statistics_from_inputs( - &self, - _input_stats: &[Arc], - _args: &StatisticsArgs, - ) -> Result> { - Ok(Arc::new(Statistics::new_unknown(&self.schema()))) - } -} diff --git a/src/lib.rs b/src/lib.rs index 88397fb89..b43e81f27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -64,7 +64,8 @@ pub use protocol::{ get_distributed_channel_resolver, }; pub use stage::{ - DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, + DistributedTaskContext, Stage, WorkerLocation, display_plan_ascii, display_plan_graphviz, + explain_analyze, }; pub use work_unit_feed::{ DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, WorkUnitFeedProvider, diff --git a/src/stage.rs b/src/stage.rs index 17132c1dc..c880e41a2 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -1,7 +1,7 @@ use crate::coordinator::DistributedExec; use crate::execution_plans::{DistributedLeafExec, NetworkCoalesceExec}; use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; -use datafusion::common::{HashMap, Statistics, config_err}; +use datafusion::common::{DataFusionError, HashMap, Statistics, config_err}; use datafusion::common::{exec_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; @@ -12,9 +12,12 @@ use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext, displayable, }; +use futures::future::{BoxFuture, Shared}; use itertools::Either; use std::collections::VecDeque; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use tokio_util::sync::CancellationToken; use url::Url; use uuid::Uuid; @@ -111,14 +114,54 @@ pub struct RemoteStage { pub query_id: Uuid, /// Our stage number pub num: usize, - /// The worker URLs to which queries should be issued. - pub workers: Vec, + /// The worker locations to which queries should be issued. + pub workers: Vec, /// Statistics collected at runtime, if any. pub runtime_stats: Option>, /// Dynamic-filter consumers retained after the stage's plan is moved to its workers. pub dynamic_filter_anchors: Vec>, } +/// A task's selected worker URL or its in-progress routing result. +/// Pending locations stay in the coordinator; worker plans require ready locations. +#[derive(Clone)] +pub enum WorkerLocation { + /// A successfully connected worker selected by the task-routing handler. + Ready(Url), + /// A coordinator-consumed task whose routing is still in progress. + Pending { + /// The shared result of task setup, including connection-aware routing. + url: Shared>>>, + /// Cancels location resolution and result-stream setup when the query ends. + query_finished: CancellationToken, + }, +} + +impl WorkerLocation { + /// Returns a concrete URL, or an error for a pending location. + pub fn ready_url(&self) -> Result<&Url> { + match self { + Self::Ready(url) => Ok(url), + Self::Pending { .. } => exec_err!("Worker location is pending"), + } + } +} + +impl From for WorkerLocation { + fn from(url: Url) -> Self { + Self::Ready(url) + } +} + +impl Debug for WorkerLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Ready(url) => f.debug_tuple("Ready").field(url).finish(), + Self::Pending { .. } => f.write_str("Pending"), + } + } +} + impl Stage { pub fn query_id(&self) -> Uuid { match &self { @@ -236,7 +279,6 @@ impl DistributedTaskContext { use crate::{DistributedMetricsFormat, NetworkShuffleExec, rewrite_distributed_plan_with_metrics}; use crate::{NetworkBoundary, NetworkBoundaryExt}; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::DataFusionError; use datafusion::common::stats::Precision; use datafusion::physical_expr::Partitioning; /// Be able to display a nice tree for stages. diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 3df566a83..7893097ec 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -9,7 +9,7 @@ mod worker_connection_pool; mod worker_service; pub(crate) use single_write_multi_read::SingleWriteMultiRead; -pub(crate) use worker_connection_pool::{PendingTaskUrl, WorkerConnectionPool}; +pub(crate) use worker_connection_pool::WorkerConnectionPool; pub use impl_coordinator_channel::CoordinatorChannelResult; pub use session_builder::{ diff --git a/src/worker/worker_connection_pool.rs b/src/worker/worker_connection_pool.rs index 6ad1a98a3..87db51f4a 100644 --- a/src/worker/worker_connection_pool.rs +++ b/src/worker/worker_connection_pool.rs @@ -1,7 +1,7 @@ use crate::common::on_drop_stream; use crate::distributed_planner::ProducerHead; use crate::passthrough_headers::get_passthrough_headers; -use crate::stage::Stage; +use crate::stage::{RemoteStage, WorkerLocation}; use crate::{ExecuteTaskRequest, LocalWorkerContext, TaskKey, get_distributed_channel_resolver}; use dashmap::DashMap; use datafusion::arrow::array::RecordBatch; @@ -19,8 +19,6 @@ use std::fmt::{Debug, Formatter}; use std::ops::Range; use std::sync::{Arc, Mutex}; use tokio::select; -use tokio_util::sync::CancellationToken; -use url::Url; /// Manages connections to remote workers. /// - Handles a range of partitions at a time in other to give the chance to [crate::WorkerChannel] @@ -31,16 +29,9 @@ use url::Url; /// - Lazily initializes connections per remote worker and requested partition range. pub(crate) struct WorkerConnectionPool { lazy_stream_groups: Vec, StreamGroup>>, - pending_workers: Option>, pub(crate) metrics: ExecutionPlanMetricsSet, } -/// Coordinator-local destinations may still be routing; wire plans always contain resolved URLs. -struct PendingWorkers { - workers: Vec, - query_finished: CancellationToken, -} - /// Slots exist before asynchronous connection setup, so dropping a consumer can cancel /// its receiver both before and after the worker returns the partition streams. struct StreamGroup { @@ -58,66 +49,41 @@ struct StreamSlot { /// Just some boilerplate for a shared future. type SharedBoxFuture = Shared>>>; -pub(crate) type PendingTaskUrl = SharedBoxFuture; - impl WorkerConnectionPool { /// Builds a new [WorkerConnectionPool] with as many empty slots for worker stream entries as /// the provided `input_tasks`. pub(crate) fn new(input_tasks: usize) -> Self { Self { lazy_stream_groups: (0..input_tasks).map(|_| Default::default()).collect(), - pending_workers: None, metrics: ExecutionPlanMetricsSet::default(), } } - pub(crate) fn new_pending( - workers: Vec, - query_finished: CancellationToken, - ) -> Self { - let input_tasks = workers.len(); - Self { - pending_workers: Some(Arc::new(PendingWorkers { - workers, - query_finished, - })), - ..Self::new(input_tasks) - } - } - - pub(crate) fn has_pending_workers(&self) -> bool { - self.pending_workers.is_some() - } - pub(crate) fn execute( &self, - input_stage: &Stage, + input_stage: &RemoteStage, target_partitions: Range, target_task: usize, target_partition: usize, producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let (destination, query_finished) = if let Some(pending) = &self.pending_workers { - let destination = pending.workers.get(target_task).cloned().ok_or_else(|| { - internal_datafusion_err!("input stage task {target_task} out of range") - })?; - ( - destination.map_err(DataFusionError::Shared).left_future(), - Some(pending.query_finished.clone()), - ) - } else { - let Stage::Remote(stage) = input_stage else { - return internal_err!("A local stage requires pending worker destinations"); - }; - let destination = stage.workers.get(target_task).cloned().ok_or_else(|| { - internal_datafusion_err!("input stage task {target_task} out of range") - })?; - (ready(Ok(destination)).right_future(), None) + let worker = input_stage.workers.get(target_task).ok_or_else(|| { + internal_datafusion_err!("input stage task {target_task} out of range") + })?; + let (destination, query_finished) = match worker { + WorkerLocation::Ready(url) => (ready(Ok(url.clone())).left_future(), None), + WorkerLocation::Pending { + url, + query_finished, + } => ( + url.clone().map_err(DataFusionError::Shared).right_future(), + Some(query_finished.clone()), + ), }; let task_key = TaskKey { - query_id: input_stage.query_id(), - stage_id: input_stage.num(), + query_id: input_stage.query_id, + stage_id: input_stage.num, task_number: target_task, }; let bdr = || MetricBuilder::new(&self.metrics); @@ -277,10 +243,7 @@ impl Debug for WorkerConnectionPool { impl Clone for WorkerConnectionPool { fn clone(&self) -> Self { - Self { - pending_workers: self.pending_workers.clone(), - ..Self::new(self.lazy_stream_groups.len()) - } + Self::new(self.lazy_stream_groups.len()) } } @@ -288,7 +251,6 @@ impl Clone for WorkerConnectionPool { mod tests { use super::*; use crate::grpc::DefaultChannelResolver; - use crate::stage::RemoteStage; use crate::worker::test_utils::worker_handles::TcpWorkerHandle; use crate::{DistributedConfig, DistributedExt}; use datafusion::arrow::array::Int32Array; @@ -376,7 +338,7 @@ mod tests { struct Fixture { pool: WorkerConnectionPool, ctx: Arc, - stage: Stage, + stage: RemoteStage, _worker: TcpWorkerHandle, } @@ -400,13 +362,13 @@ mod tests { .with_distributed_channel_resolver(DefaultChannelResolver::default()) .with_distributed_worker_connection_buffer_budget_bytes(128)? .task_ctx(); - let stage = Stage::Remote(RemoteStage { + let stage = RemoteStage { query_id, num: 0, - workers: vec![worker.url.clone()], + workers: vec![worker.url.clone().into()], runtime_stats: None, dynamic_filter_anchors: vec![], - }); + }; Ok(Self { pool: WorkerConnectionPool::new(1), ctx, diff --git a/tests/startup_overlap.rs b/tests/startup_overlap.rs index ec09a218d..1beba134b 100644 --- a/tests/startup_overlap.rs +++ b/tests/startup_overlap.rs @@ -471,7 +471,15 @@ mod tests { async fn route(&self, event: RouteTaskEvent<'_>) -> Result { event.task_specialized_plan.apply(|node| { if let Some(boundary) = node.as_network_boundary() { - assert!(matches!(boundary.input_stage(), Stage::Remote(_))); + let Stage::Remote(stage) = boundary.input_stage() else { + panic!("worker plans require a remote input stage"); + }; + assert!( + stage + .workers + .iter() + .all(|worker| worker.ready_url().is_ok()) + ); } if node.is::() { self.nested_coalesces.fetch_add(1, Ordering::Relaxed); From e96e0a329bd76f89e1cc41e4ef23764be135fcbd Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Fri, 6 Nov 2026 20:03:22 +0000 Subject: [PATCH 06/10] docs: group worker location migration with routing updates Keep the worker-location API migration beside the routing API changes to avoid an append conflict with newer upgrade-guide entries on main. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- docs/upgrade/5.0.0.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/upgrade/5.0.0.md b/docs/upgrade/5.0.0.md index 938059d63..03a00ed53 100644 --- a/docs/upgrade/5.0.0.md +++ b/docs/upgrade/5.0.0.md @@ -79,6 +79,27 @@ let state = SessionStateBuilder::new() coordinator-to-worker connection is currently established. Implementations can choose to retry on other worker if the current one is not available. +### Remote stage destinations can still be routing + +`RemoteStage::workers` now contains `WorkerLocation` values instead of `Url`s. +When constructing a stage with resolved worker URLs, convert them with `into()` +or `WorkerLocation::from`: + +```rust +use datafusion_distributed::WorkerLocation; + +let workers = urls.into_iter().map(WorkerLocation::from).collect(); +``` + +When inspecting a destination, use `worker.ready_url()?` to borrow its resolved +URL. This returns an error for `WorkerLocation::Pending`; coordinator input +stages may contain these destinations while task routing finishes. Best-effort +colocation handlers should skip pending destinations instead of treating them +as resolved URLs. + +Worker plans require `WorkerLocation::Ready` destinations. The wire format still +contains URL strings, and encoding a pending destination returns an error. + ## 2. `WorkerPlanRewriteHandler::rewrite_worker_plan` renamed to `handle` and is now async `WorkerPlanRewriteHandler::rewrite_worker_plan` has been renamed to `handle` to @@ -154,24 +175,3 @@ A coordinator and a worker on different versions cannot decode each other's sketches, and with metrics collection enabled (the default) the decode error fails the query. Upgrade coordinators and workers together, or disable metrics collection with `with_distributed_metrics_collection(false)` during the rollout. - -## 7. Remote stage destinations can still be routing - -`RemoteStage::workers` now contains `WorkerLocation` values instead of `Url`s. -When constructing a stage with resolved worker URLs, convert them with `into()` -or `WorkerLocation::from`: - -```rust -use datafusion_distributed::WorkerLocation; - -let workers = urls.into_iter().map(WorkerLocation::from).collect(); -``` - -When inspecting a destination, use `worker.ready_url()?` to borrow its resolved -URL. This returns an error for `WorkerLocation::Pending`; coordinator input -stages may contain these destinations while task routing finishes. Best-effort -colocation handlers should skip pending destinations instead of treating them -as resolved URLs. - -Worker plans require `WorkerLocation::Ready` destinations. The wire format still -contains URL strings, and encoding a pending destination returns an error. From 0291e08809168c4e772a0e4f8a16e886b8e5e7ec Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Tue, 10 Nov 2026 15:40:32 +0000 Subject: [PATCH 07/10] perf: benchmark task startup overlap Measure completed queries with a delayed light task, a delayed heavy task, and no setup delay in static and adaptive execution. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- datafusion/distributed-benchmarks/Cargo.toml | 4 + datafusion/distributed-benchmarks/README.md | 29 +++ .../benches/startup_overlap.rs | 200 ++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100644 datafusion/distributed-benchmarks/benches/startup_overlap.rs diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index 08e548e02..7508b104e 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -66,3 +66,7 @@ harness = false [[bench]] name = "metrics_collection" harness = false + +[[bench]] +name = "startup_overlap" +harness = false diff --git a/datafusion/distributed-benchmarks/README.md b/datafusion/distributed-benchmarks/README.md index d5c01fb6f..3e1c6f1a3 100644 --- a/datafusion/distributed-benchmarks/README.md +++ b/datafusion/distributed-benchmarks/README.md @@ -1,5 +1,34 @@ # Local DataFusion benchmarks +### Task startup overlap + +```shell +cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --save-baseline main +# Run the same benchmark on the candidate revision: +cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --baseline main +``` + +This controlled benchmark runs two tasks over localhost gRPC in static and AQE +mode. Task 0 performs 100 million dependent integer operations; task 1 returns +one row. The existing localhost fixture starts three endpoints so both active +tasks can route away from the coordinator's local endpoint. Each query must +return task indices `[0, 1]`, and the CPU function must execute remotely once. + +`delay_light_task` adds 250 ms to task 1's routing setup, allowing task 0's CPU +work to overlap that wait. `delay_heavy_task` delays task 0 instead, keeping the +wait on the critical path; `no_delay` provides the other control. The delay is +only in routing setup, not in the CPU workload. Criterion measures execution +through complete output collection; worker startup and query planning are +excluded. An untimed preflight also reports first-batch time and whether remote +CPU work started and finished before the delayed task returned a ready +destination. The witnesses are reported without requiring overlap, so the +benchmark also runs before the optimization. + +Run both revisions on a quiet host and alternate comparisons. Verify a positive +overlap witness in the candidate's `delay_light_task` preflight and its absence +in the baseline before interpreting timings. This isolates scheduling value +under controlled setup skew; it does not replace representative suite results. + ### Generating Benchmarking data The `dfbench prepare-*` commands generate datasets under `--root` (by default diff --git a/datafusion/distributed-benchmarks/benches/startup_overlap.rs b/datafusion/distributed-benchmarks/benches/startup_overlap.rs new file mode 100644 index 000000000..54b1de817 --- /dev/null +++ b/datafusion/distributed-benchmarks/benches/startup_overlap.rs @@ -0,0 +1,200 @@ +use arrow::array::Int64Array; +use arrow::datatypes::DataType; +use async_trait::async_trait; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion::common::{Result, exec_err}; +use datafusion::execution::SessionState; +use datafusion::logical_expr::{ScalarUDF, Volatility, create_udf}; +use datafusion::physical_plan::execute_stream; +use datafusion::prelude::SessionContext; +use datafusion_distributed::test_utils::localhost::start_localhost_context; +use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, +}; +use datafusion_distributed::{ + DistributedExt, MappedWorkerSessionBuilderExt, RouteTaskEvent, RouteTaskEventResponse, + RouteTaskHandler, WorkerQueryContext, +}; +use futures::TryStreamExt; +use std::hint::black_box; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::time::timeout; + +const QUERY: &str = "SELECT cpu_work(task_index) FROM url_emitter(2, 2, 'startup')"; +const SETUP_DELAY: Duration = Duration::from_millis(250); +const WAIT: Duration = Duration::from_secs(10); + +fn startup_overlap(c: &mut Criterion) { + let rt = RuntimeBuilder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("startup_overlap"); + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(5)); + + for (mode, adaptive) in [("static", false), ("aqe", true)] { + for (scenario, delayed_task) in [ + ("no_delay", None), + ("delay_light_task", Some(1)), + ("delay_heavy_task", Some(0)), + ] { + let probe = Arc::new(Probe::default()); + let udf = Arc::new(cpu_work(Some(Arc::clone(&probe)))); + let worker = build_state.map(move |builder| { + Ok(builder + .with_scalar_functions(vec![Arc::clone(&udf)]) + .build()) + }); + // Routing uses URLs 2 and 1, avoiding URL 0's local-worker shortcut. + let (mut ctx, _workers, _) = rt.block_on(start_localhost_context(3, worker)); + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx.register_udf(cpu_work(None)); + let ctx = ctx + .with_distributed_dynamic_task_count(adaptive) + .unwrap() + .with_distributed_route_task_handler(SetupDelay { + delayed_task, + probe: Arc::clone(&probe), + }); + rt.block_on(ctx.sql("SET datafusion.execution.target_partitions = 2")) + .unwrap(); + + let (first, total) = rt.block_on(run_query(&ctx, &probe)).unwrap(); + println!( + "preflight {mode}/{scenario}: first={first:?} total={total:?} \ + heavy_started_before_ready={} heavy_finished_before_ready={}", + probe.started_before_ready.load(Ordering::SeqCst), + probe.finished_before_ready.load(Ordering::SeqCst), + ); + group.bench_function(BenchmarkId::new(mode, scenario), |b| { + b.iter_custom(|iterations| { + rt.block_on(async { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + elapsed += run_query(&ctx, &probe).await.unwrap().1; + } + elapsed + }) + }) + }); + } + } + group.finish(); +} + +async fn run_query(ctx: &SessionContext, probe: &Probe) -> Result<(Duration, Duration)> { + let plan = ctx.sql(QUERY).await?.create_physical_plan().await?; + probe.delayed_ready.store(false, Ordering::SeqCst); + probe.started_before_ready.store(false, Ordering::SeqCst); + probe.finished_before_ready.store(false, Ordering::SeqCst); + probe.heavy_executions.store(0, Ordering::SeqCst); + let start = Instant::now(); + let mut output = execute_stream(plan, ctx.task_ctx())?; + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + let first_elapsed = start.elapsed(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + let total_elapsed = start.elapsed(); + let mut values: Vec<_> = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + values.sort_unstable(); + assert_eq!(values, vec![0, 1]); + assert_eq!(probe.heavy_executions.load(Ordering::SeqCst), 1); + Ok((first_elapsed, total_elapsed)) +} + +async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) +} + +#[derive(Default)] +struct Probe { + delayed_ready: AtomicBool, + started_before_ready: AtomicBool, + finished_before_ready: AtomicBool, + heavy_executions: AtomicUsize, +} + +struct SetupDelay { + delayed_task: Option, + probe: Arc, +} + +#[async_trait] +impl RouteTaskHandler for SetupDelay { + async fn handle(&self, event: RouteTaskEvent<'_>) -> Option> { + let delayed = self.delayed_task == Some(event.task_key.task_number); + if delayed { + tokio::time::sleep(SETUP_DELAY).await; + } + let response = UrlEmitterRouteTaskHandler.handle(event).await?; + if delayed || self.delayed_task.is_none() { + self.probe.delayed_ready.store(true, Ordering::SeqCst); + } + Some(response) + } +} + +fn cpu_work(probe: Option>) -> ScalarUDF { + create_udf( + "cpu_work", + vec![DataType::Int64], + DataType::Int64, + Volatility::Volatile, + Arc::new(move |args| { + let Some(probe) = &probe else { + return exec_err!("cpu_work must execute on a remote worker"); + }; + let input = args[0].clone().into_array(1)?; + let input = input.as_any().downcast_ref::().unwrap(); + for task in input.values() { + if *task == 0 { + probe.heavy_executions.fetch_add(1, Ordering::SeqCst); + probe.started_before_ready.store( + !probe.delayed_ready.load(Ordering::SeqCst), + Ordering::SeqCst, + ); + let mut value = 1u64; + for _ in 0..100_000_000 { + value = black_box(value.wrapping_mul(17).wrapping_add(1)); + } + probe.finished_before_ready.store( + !probe.delayed_ready.load(Ordering::SeqCst), + Ordering::SeqCst, + ); + } + } + Ok(args[0].clone()) + }), + ) +} + +criterion_group!(benches, startup_overlap); +criterion_main!(benches); From 18b47adac79d8e6ccd11f6454d41349f48b364f9 Mon Sep 17 00:00:00 2001 From: "joy.bestourous" Date: Tue, 10 Nov 2026 15:45:26 +0000 Subject: [PATCH 08/10] refactor: name the adaptive network boundary builder Move the existing AQE boundary callback into an implementation of NetworkBoundaryBuilder without changing setup or sampling order. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../src/coordinator/prepare_dynamic_plan.rs | 247 +++++++++--------- .../src/distributed_planner/mod.rs | 3 +- 2 files changed, 132 insertions(+), 118 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index e8453d4d9..f29fa8d01 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -2,8 +2,8 @@ use crate::common::{TreeNodeExt, element_wise_sum, vec_avg_reduce, vec_div, vec_ use crate::coordinator::distributed::PreparedPlan; use crate::coordinator::query_coordinator::QueryCoordinator; use crate::distributed_planner::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, ProducerHead, calculate_cost, - inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + ProducerHead, calculate_cost, inject_network_boundaries, }; use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, @@ -14,6 +14,7 @@ use crate::{ BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, TaskCountAnnotation, }; +use async_trait::async_trait; use dashmap::DashMap; use datafusion::common::stats::Precision; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; @@ -33,124 +34,13 @@ pub(super) async fn prepare_dynamic_plan( query_coordinator: &Arc, base_plan: &Arc, ) -> Result { - let plans_for_viz = Arc::new(PlanReconstructor::default()); + let plans_for_viz = PlanReconstructor::default(); let head_stage = inject_network_boundaries( Arc::clone(base_plan), - |mut input_stage: LocalStage, nb_type: TypeId, nb_ctx: &InjectNetworkBoundaryContext| { - let mut metrics = MetricsSet::new(); - - // At this point, input_stage.plan has two kind of leaf nodes: - // - The ones that naturally do not read from any children, like DataSourceExec - // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration - // of this same function. - // Both types of leaf nodes contain very valuable and accurate statistics that are used - // here for computing an estimation of the compute cost (measured in bytes): - // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their - // data source, like parquet files. - // - Network boundaries contain statistics collected from runtime information, gathered - // by the SamplerExec injected by this same function. - let cost = calculate_cost(&input_stage.plan)?; - metrics.push(BytesCounterMetric::new_metric( - "cpu_cost", - *cost.cpu.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "memory_cost", - *cost.memory.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "network_cost", - *cost.network.get_value().unwrap_or(&0), - )); - let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 - / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 - / input_stage - .plan - .output_partitioning() - .partition_count() - .max(1) as f64; - let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); - let task_count = nb_ctx - .task_count(&input_stage.plan)? - .merge(TaskCountAnnotation::soft(compute_based_task_count))?; - - // Propagate the final task_count inferred based on runtime statistics and compute cost. - // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the - // plan is finally left ready for distribution. - input_stage.plan = nb_ctx - .propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; - input_stage.tasks = task_count.as_usize(); - // In order to infer the compute the cost of the stage above this one, here a sampler - // is injected to gather runtime statistics. - input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&input_stage.plan)? - } else { - vec![] - }; - - let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); - - let query_coordinator = Arc::clone(query_coordinator); - let plans_for_viz = Arc::clone(&plans_for_viz); - - Ok(async move { - let mut stage_coordinator = query_coordinator.stage_coordinator(&input_stage); - - let mut futures = Vec::with_capacity(input_stage.tasks); - for task_i in 0..input_stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(input_stage.tasks); - for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() - { - workers.push(url); - load_info_rxs.push({ - let rx = - stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); - UnboundedReceiverStream::new(rx) - }); - let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; - } - stage_coordinator.seal_dynamic_filter_stage()?; - - let (stats, consumer_tc) = if nb_type == TypeId::of::() { - (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) - } else { - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; - metrics.extend(new_metrics); - // returning Desired(0) here is our way to tell the planner that we don't care - // about the task count assigned to the network boundary in the consumer stage, - // and we don't want it to affect other task count decisions. A 0.0 hint does - // not mask smaller fractional values via max-merge, and does not inflate - // UNION sums when this boundary sits under a child of a - // ChildrenIsolatorUnionExec. - (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) - }; - - // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan - // before it's moved: the returned stage is remote and carries no plan to read it - // back from. - let input_properties = Arc::clone(input_stage.plan.properties()); - plans_for_viz.insert(input_stage.num, input_stage.plan, metrics); - Ok(NetworkBoundaryBuilderResult { - consumer_task_count: consumer_tc, - input_stage: Stage::Remote(RemoteStage { - query_id: input_stage.query_id, - num: input_stage.num, - workers, - runtime_stats: stats, - dynamic_filter_anchors, - }), - input_properties, - }) - }) + DynamicPlanBuilder { + query_coordinator, + plans_for_viz: &plans_for_viz, }, query_coordinator.session_config(), ) @@ -162,6 +52,129 @@ pub(super) async fn prepare_dynamic_plan( }) } +struct DynamicPlanBuilder<'a> { + query_coordinator: &'a Arc, + plans_for_viz: &'a PlanReconstructor, +} + +#[async_trait] +impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { + async fn build<'a>( + &'a self, + mut input_stage: LocalStage, + nb_type: TypeId, + nb_ctx: &'a InjectNetworkBoundaryContext<'a>, + ) -> Result { + let mut metrics = MetricsSet::new(); + + // At this point, input_stage.plan has two kind of leaf nodes: + // - The ones that naturally do not read from any children, like DataSourceExec + // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration + // of this same function. + // Both types of leaf nodes contain very valuable and accurate statistics that are used + // here for computing an estimation of the compute cost (measured in bytes): + // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their + // data source, like parquet files. + // - Network boundaries contain statistics collected from runtime information, gathered + // by the SamplerExec injected by this same function. + let cost = calculate_cost(&input_stage.plan)?; + metrics.push(BytesCounterMetric::new_metric( + "cpu_cost", + *cost.cpu.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "memory_cost", + *cost.memory.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "network_cost", + *cost.network.get_value().unwrap_or(&0), + )); + let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 + / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 + / input_stage + .plan + .output_partitioning() + .partition_count() + .max(1) as f64; + let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); + let task_count = nb_ctx + .task_count(&input_stage.plan)? + .merge(TaskCountAnnotation::soft(compute_based_task_count))?; + + // Propagate the final task_count inferred based on runtime statistics and compute cost. + // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the + // plan is finally left ready for distribution. + input_stage.plan = + nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; + input_stage.tasks = task_count.as_usize(); + // In order to infer the compute the cost of the stage above this one, here a sampler + // is injected to gather runtime statistics. + input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; + let dynamic_filter_anchors = + if is_remote_dynamic_filtering_enabled(self.query_coordinator.session_config())? { + orphan_dynamic_filter_consumers(&input_stage.plan)? + } else { + vec![] + }; + + let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); + + let mut stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage); + + let mut futures = Vec::with_capacity(input_stage.tasks); + for task_i in 0..input_stage.tasks { + futures.push(stage_coordinator.init_bidirectional_stream(task_i)); + } + let results = futures::future::try_join_all(futures).await?; + + let mut workers = Vec::with_capacity(input_stage.tasks); + for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() { + workers.push(url); + load_info_rxs.push({ + let rx = stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); + UnboundedReceiverStream::new(rx) + }); + let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); + stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; + } + stage_coordinator.seal_dynamic_filter_stage()?; + + let (stats, consumer_tc) = if nb_type == TypeId::of::() { + (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) + } else { + let (stats, new_metrics) = + gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; + metrics.extend(new_metrics); + // returning Desired(0) here is our way to tell the planner that we don't care + // about the task count assigned to the network boundary in the consumer stage, + // and we don't want it to affect other task count decisions. A 0.0 hint does + // not mask smaller fractional values via max-merge, and does not inflate + // UNION sums when this boundary sits under a child of a + // ChildrenIsolatorUnionExec. + (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) + }; + + // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan + // before it's moved: the returned stage is remote and carries no plan to read it + // back from. + let input_properties = Arc::clone(input_stage.plan.properties()); + self.plans_for_viz + .insert(input_stage.num, input_stage.plan, metrics); + Ok(NetworkBoundaryBuilderResult { + consumer_task_count: consumer_tc, + input_stage: Stage::Remote(RemoteStage { + query_id: input_stage.query_id, + num: input_stage.num, + workers, + runtime_stats: stats, + dynamic_filter_anchors, + }), + input_properties, + }) + } +} + /// Reconstructs the plan dynamically as stages get transitioned to Remote and get sent to the /// respective workers. /// diff --git a/datafusion/distributed/src/distributed_planner/mod.rs b/datafusion/distributed/src/distributed_planner/mod.rs index b49645214..756ab456f 100644 --- a/datafusion/distributed/src/distributed_planner/mod.rs +++ b/datafusion/distributed/src/distributed_planner/mod.rs @@ -14,7 +14,8 @@ mod statistics; pub use distributed_config::DistributedConfig; pub use distributed_query_planner::create_distributed_plan; pub(crate) use inject_network_boundaries::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + inject_network_boundaries, }; pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt, ProducerHead}; pub use session_state_builder_ext::SessionStateBuilderExt; From 9d0d0b545e36fdd12aac706941686861986fb172 Mon Sep 17 00:00:00 2001 From: Joy Bestourous <32601978+joybestourous@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:55:33 -0500 Subject: [PATCH 09/10] perf: simplify task startup with existing worker readers Drive setup in each worker response reader and share its eventual address with consumers. Preserve routing retries, query cancellation and report collection while allowing ready tasks to execute independently. Keep input resolution in the existing specialization pass and preserve source comments. Cover early execution, AQE sampling, cancellation and late setup errors with focused regressions. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../src/codec/distributed_codec.rs | 46 +- .../src/coordinator/prepare_dynamic_plan.rs | 161 +++---- .../src/coordinator/prepare_static_plan.rs | 76 ++-- .../src/coordinator/query_coordinator.rs | 186 ++++---- .../inject_network_boundaries.rs | 15 - .../src/events/defaults/routing.rs | 3 +- datafusion/distributed/src/stage.rs | 101 +++-- .../src/worker/worker_connection_pool.rs | 93 ++-- datafusion/distributed/tests/main.rs | 2 +- .../distributed/tests/startup_overlap.rs | 414 ------------------ datafusion/distributed/tests/task_startup.rs | 259 +++++++++++ docs/upgrade/5.0.0.md | 23 +- 12 files changed, 541 insertions(+), 838 deletions(-) delete mode 100644 datafusion/distributed/tests/startup_overlap.rs create mode 100644 datafusion/distributed/tests/task_startup.rs diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 251c9371e..37272dc13 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -346,8 +346,11 @@ impl PhysicalExtensionCodec for DistributedCodec { Stage::Remote(remote) => { let mut tasks = Vec::with_capacity(remote.workers.len()); for worker in &remote.workers { + let url = worker.ready_url().ok_or_else(|| { + proto_error("Worker location must be resolved before encoding") + })?; tasks.push(ExecutionTaskProto { - url_str: Some(worker.ready_url()?.to_string()), + url_str: Some(url.to_string()), }) } StageProto { @@ -806,7 +809,6 @@ mod tests { }; use super::*; - use crate::WorkerLocation; use crate::execution_plans::PRODUCER_SALT_DEFAULT; const DEFAULT_MODE: ShuffleMode = ShuffleMode::Direct; @@ -825,8 +827,6 @@ mod tests { sorts::sort::SortExec, union::UnionExec, }, }; - use futures::FutureExt; - use tokio_util::sync::CancellationToken; fn empty_exec() -> Arc { Arc::new(EmptyExec::new(SchemaRef::new(Schema::empty()))) @@ -868,20 +868,13 @@ mod tests { fn test_roundtrip_single_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; let ctx = create_context(); - let worker_url = Url::parse("http://worker:50051").unwrap(); let schema = schema_i32("a"); let part = Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 4); let plan: Arc = Arc::new(new_network_hash_shuffle_exec( part, EquivalenceProperties::new(schema), - Stage::Remote(RemoteStage { - query_id: Default::default(), - num: 0, - workers: vec![worker_url.clone().into()], - runtime_stats: None, - dynamic_filter_anchors: vec![], - }), + dummy_stage(), DEFAULT_MODE, )); @@ -890,39 +883,10 @@ mod tests { let decoded = codec.try_decode(&buf, &[], &ctx, &default_proto_converter())?; assert_eq!(repr(&plan), repr(&decoded)); - let boundary = decoded.downcast_ref::().unwrap(); - let Stage::Remote(stage) = boundary.input_stage() else { - panic!("decoded input should be remote"); - }; - assert_eq!(stage.workers[0].ready_url()?, &worker_url); Ok(()) } - #[test] - fn pending_worker_location_cannot_be_encoded() { - let stage = Stage::Remote(RemoteStage { - query_id: Default::default(), - num: 0, - workers: vec![WorkerLocation::Pending { - url: futures::future::pending().boxed().shared(), - query_finished: CancellationToken::new(), - }], - runtime_stats: None, - dynamic_filter_anchors: vec![], - }); - let plan: Arc = Arc::new(new_network_hash_shuffle_exec( - Partitioning::RoundRobinBatch(1), - EquivalenceProperties::new(schema_i32("a")), - stage, - DEFAULT_MODE, - )); - let error = DistributedCodec - .try_encode(plan, &mut vec![], &default_proto_converter()) - .unwrap_err(); - crate::assert_snapshot!(error, @"Execution error: Worker location is pending"); - } - #[test] fn test_roundtrip_network_dynamic_filter_anchor() -> datafusion::common::Result<()> { let ctx = create_context(); diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index 04ab0343b..a0ff76a52 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -9,22 +9,21 @@ use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, }; use crate::execution_plans::SamplerExec; -use crate::stage::{LocalStage, RemoteStage, WorkerLocation}; +use crate::stage::{LocalStage, RemoteStage}; use crate::{ - BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, - NetworkCoalesceExec, Stage, TaskCountAnnotation, + BytesCounterMetric, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, + TaskCountAnnotation, }; use async_trait::async_trait; use dashmap::DashMap; use datafusion::common::stats::Precision; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{DataFusionError, Result, exec_err, plan_err}; +use datafusion::common::{Result, exec_err, plan_err}; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Statistics, StatisticsArgs, StatisticsContext, }; -use futures::future::try_join_all; use futures::{Stream, StreamExt}; use std::any::TypeId; use std::num::NonZeroUsize; @@ -36,6 +35,7 @@ pub(super) async fn prepare_dynamic_plan( base_plan: &Arc, ) -> Result { let plans_for_viz = PlanReconstructor::default(); + let head_stage = inject_network_boundaries( Arc::clone(base_plan), DynamicPlanBuilder { @@ -45,31 +45,6 @@ pub(super) async fn prepare_dynamic_plan( query_coordinator.session_config(), ) .await?; - let head_stage = head_stage - .transform_down(|plan| { - let Some(boundary) = plan.as_network_boundary() else { - return Ok(Transformed::no(plan)); - }; - let Stage::Local(stage) = boundary.input_stage() else { - return Ok(Transformed::no(plan)); - }; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&stage.plan)? - } else { - vec![] - }; - Ok(Transformed::yes(boundary.with_input_stage( - Stage::Remote(RemoteStage { - query_id: stage.query_id, - num: stage.num, - workers: query_coordinator.start_pending_stage(stage), - runtime_stats: None, - dynamic_filter_anchors, - }), - )?)) - })? - .data; Ok(PreparedPlan { plan_for_viz: plans_for_viz.reconstruct(&head_stage)?, @@ -90,15 +65,18 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { nb_type: TypeId, nb_ctx: &'a InjectNetworkBoundaryContext<'a>, ) -> Result { - // A deferred coalesce becomes a concrete destination before another worker stage - // consumes it. Preserve the planner's pointer-keyed annotations when replacing nodes. - let prepared = prepare_worker_inputs(self.query_coordinator, &input_stage.plan).await?; - nb_ctx.copy_task_counts(&input_stage.plan, &prepared)?; - input_stage.plan = prepared; let mut metrics = MetricsSet::new(); - // Leaf statistics and runtime statistics from sampled network boundaries estimate - // the compute cost of the next stage. + // At this point, input_stage.plan has two kind of leaf nodes: + // - The ones that naturally do not read from any children, like DataSourceExec + // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration + // of this same function. + // Both types of leaf nodes contain very valuable and accurate statistics that are used + // here for computing an estimation of the compute cost (measured in bytes): + // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their + // data source, like parquet files. + // - Network boundaries contain statistics collected from runtime information, gathered + // by the SamplerExec injected by this same function. let cost = calculate_cost(&input_stage.plan)?; metrics.push(BytesCounterMetric::new_metric( "cpu_cost", @@ -124,23 +102,14 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { .task_count(&input_stage.plan)? .merge(TaskCountAnnotation::soft(compute_based_task_count))?; + // Propagate the final task_count inferred based on runtime statistics and compute cost. + // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the + // plan is finally left ready for distribution. input_stage.plan = nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; input_stage.tasks = task_count.as_usize(); - - if nb_type == TypeId::of::() { - let input_properties = Arc::clone(input_stage.plan.properties()); - self.plans_for_viz - .insert(input_stage.num, Arc::clone(&input_stage.plan), metrics); - // Coalesces require no sampling. Leave their setup to the consuming stage; - // the coordinator can then execute each task as its own setup completes. - return Ok(NetworkBoundaryBuilderResult { - consumer_task_count: TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0), - input_stage: Stage::Local(input_stage), - input_properties, - }); - } - + // In order to infer the compute the cost of the stage above this one, here a sampler + // is injected to gather runtime statistics. input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; let dynamic_filter_anchors = if is_remote_dynamic_filtering_enabled(self.query_coordinator.session_config())? { @@ -148,34 +117,43 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { } else { vec![] }; - let futures = (0..input_stage.tasks).map(|task_i| { - let setup = self.query_coordinator.start_task(&input_stage, task_i); - async move { - let (url, tx, load_info) = setup.await?; - let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling); - Ok::<_, DataFusionError>((url, UnboundedReceiverStream::new(load_info))) - } - }); - let (workers, load_info_rxs): (Vec<_>, Vec<_>) = - try_join_all(futures).await?.into_iter().unzip(); - self.query_coordinator - .stage_coordinator(&input_stage) - .seal_dynamic_filter_stage()?; - - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; - metrics.extend(new_metrics); + + let (workers, load_info_rxs) = self.query_coordinator.start_stage(&input_stage); + + let (stats, consumer_tc) = if nb_type == TypeId::of::() { + (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) + } else { + let (stats, new_metrics) = gather_runtime_statistics( + load_info_rxs + .into_iter() + .map(UnboundedReceiverStream::new) + .collect(), + &input_stage.plan, + ) + .await?; + metrics.extend(new_metrics); + // returning Desired(0) here is our way to tell the planner that we don't care + // about the task count assigned to the network boundary in the consumer stage, + // and we don't want it to affect other task count decisions. A 0.0 hint does + // not mask smaller fractional values via max-merge, and does not inflate + // UNION sums when this boundary sits under a child of a + // ChildrenIsolatorUnionExec. + (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) + }; + + // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan + // before it's moved: the returned stage is remote and carries no plan to read it + // back from. let input_properties = Arc::clone(input_stage.plan.properties()); self.plans_for_viz .insert(input_stage.num, input_stage.plan, metrics); Ok(NetworkBoundaryBuilderResult { - // A zero soft hint leaves the consumer's task count to its other inputs. - consumer_task_count: TaskCountAnnotation::soft(0.0), + consumer_task_count: consumer_tc, input_stage: Stage::Remote(RemoteStage { query_id: input_stage.query_id, num: input_stage.num, - workers: workers.into_iter().map(WorkerLocation::from).collect(), - runtime_stats: Some(Arc::new(stats)), + workers, + runtime_stats: stats, dynamic_filter_anchors, }), input_properties, @@ -183,45 +161,6 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { } } -async fn prepare_worker_inputs( - query_coordinator: &Arc, - plan: &Arc, -) -> Result> { - let prepared = Arc::clone(plan).transform_up_async(async |plan| { - let Some(boundary) = plan.as_network_boundary() else { - return Ok(Transformed::no(plan)); - }; - let Stage::Local(stage) = boundary.input_stage() else { - return Ok(Transformed::no(plan)); - }; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&stage.plan)? - } else { - vec![] - }; - let workers = - try_join_all((0..stage.tasks).map(|task| query_coordinator.start_task(stage, task))) - .await? - .into_iter() - .map(|(url, ..)| WorkerLocation::from(url)) - .collect(); - query_coordinator - .stage_coordinator(stage) - .seal_dynamic_filter_stage()?; - Ok(Transformed::yes(boundary.with_input_stage( - Stage::Remote(RemoteStage { - query_id: stage.query_id, - num: stage.num, - workers, - runtime_stats: None, - dynamic_filter_anchors, - }), - )?)) - }); - Ok(prepared.await?.data) -} - /// Reconstructs the plan dynamically as stages get transitioned to Remote and get sent to the /// respective workers. /// diff --git a/datafusion/distributed/src/coordinator/prepare_static_plan.rs b/datafusion/distributed/src/coordinator/prepare_static_plan.rs index 3cef2c808..6f69ab829 100644 --- a/datafusion/distributed/src/coordinator/prepare_static_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_static_plan.rs @@ -6,64 +6,60 @@ use crate::dynamic_filtering::{ }; use crate::stage::RemoteStage; use crate::{NetworkBoundaryExt, Stage}; -use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{HashSet, Result, exec_err}; +use datafusion::common::tree_node::Transformed; +use datafusion::common::{Result, exec_err}; use datafusion::physical_plan::ExecutionPlan; -use futures::future::try_join_all; use std::sync::Arc; +/// Prepares the distributed plan for execution, which implies: +/// 1. Perform some worker URL assignation, choosing either: +/// - The workers selected by the user's [crate::RouteTaskHandler]. +/// - Randomly otherwise +/// 2. Sending the sliced subplans to the assigned URLs. For each URL assigned to a task, a +/// network call feeding the subplan is necessary. +/// 3. In each network boundary, set the input plan to `None`. That way, network boundaries +/// become nodes without children and traversing them will not go further down in. +/// 4. Spawn a background task per worker that waits for the worker to finish and collects +/// its metrics into [DistributedExec::task_metrics] via the coordinator channel. +/// +/// Setup is scheduled independently for each task. The returned remote stages contain pending +/// locations; a consumer waits only for the task it reads. A plan sent to another worker must +/// first resolve the input addresses it includes. pub(super) async fn prepare_static_plan( query_coordinator: &Arc, base_plan: &Arc, ) -> Result { - let mut coordinator_input_stage_ids = HashSet::new(); - base_plan.apply(|plan| { - if let Some(boundary) = plan.as_network_boundary() { - coordinator_input_stage_ids.insert(boundary.input_stage().num()); - Ok(TreeNodeRecursion::Jump) - } else { - Ok(TreeNodeRecursion::Continue) - } - })?; let prepared = Arc::clone(base_plan).transform_up_async(async |plan| { - let Some(boundary) = plan.as_network_boundary() else { + // The following logic is just applied on network boundaries. + let Some(plan) = plan.as_network_boundary() else { return Ok(Transformed::no(plan)); }; - let Stage::Local(stage) = boundary.input_stage() else { - return exec_err!("Expected a local stage during static plan preparation"); - }; - let anchors = if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&stage.plan)? - } else { - vec![] - }; - let workers = if coordinator_input_stage_ids.contains(&stage.num) { - query_coordinator.start_pending_stage(stage) - } else { - let workers = try_join_all( - (0..stage.tasks).map(|task| query_coordinator.start_task(stage, task)), - ) - .await? - .into_iter() - .map(|(url, ..)| url.into()) - .collect(); - query_coordinator - .stage_coordinator(stage) - .seal_dynamic_filter_stage()?; - workers + + let Stage::Local(stage) = plan.input_stage() else { + return exec_err!("Input stage from network boundary was not in Local state"); }; - Ok(Transformed::yes(boundary.with_input_stage( - Stage::Remote(RemoteStage { + let dynamic_filter_anchors = + if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { + orphan_dynamic_filter_consumers(&stage.plan)? + } else { + vec![] + }; + + let (workers, _) = query_coordinator.start_stage(stage); + Ok(Transformed::yes(plan.with_input_stage(Stage::Remote( + RemoteStage { query_id: stage.query_id, num: stage.num, workers, runtime_stats: None, - dynamic_filter_anchors: anchors, - }), - )?)) + dynamic_filter_anchors, + }, + ))?)) }); Ok(PreparedPlan { head_stage: prepared.await?.data, + // If the plan was statically planned, the base plan is the same one that will be used for + // visualization. plan_for_viz: Arc::clone(base_plan), }) } diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index 4a4a733ef..d0f8d1527 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -21,14 +21,14 @@ use crate::work_unit_feed::{build_work_unit_batch_msg, set_work_unit_send_time}; use crate::{ CoordinatorToWorkerMsg, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedGetterExt, DistributedTaskContext, DistributedWorkUnitFeedContext, LoadInfo, LocalWorkerContext, - MaybeEncoded, SetPlanRequest, TaskCompletedDynamicFilters, TaskKey, TaskMetrics, - WorkUnitFeedDeclaration, WorkerLocation, WorkerToCoordinatorMsg, + MaybeEncoded, NetworkBoundaryExt, SetPlanRequest, Stage, TaskCompletedDynamicFilters, TaskKey, + TaskMetrics, WorkUnitFeedDeclaration, WorkerLocation, WorkerToCoordinatorMsg, get_distributed_channel_resolver, }; use datafusion::common::Result; use datafusion::common::instant::Instant; use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; -use datafusion::common::{DataFusionError, exec_datafusion_err, internal_err}; +use datafusion::common::{DataFusionError, internal_err}; use datafusion::execution::TaskContext; use datafusion::physical_expr_common::metrics::{ExecutionPlanMetricsSet, Label, MetricBuilder}; use datafusion::physical_plan::metrics::Count; @@ -37,7 +37,7 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStreamBuilder; use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; use datafusion::prelude::SessionConfig; use futures::future::try_join_all; -use futures::stream::BoxStream; +use futures::stream::{BoxStream, empty}; use futures::{FutureExt, StreamExt, TryStreamExt}; use std::sync::{Arc, Mutex}; use tokio::select; @@ -101,6 +101,8 @@ impl QueryCoordinator { task_ctx: &self.task_ctx, metrics_set: &self.metrics, metrics: &self.coordinator_to_worker_metrics, + metrics_store: &self.metrics_store, + completed_dynamic_filter_store: &self.completed_dynamic_filter_store, dynamic_filter_registry: &self.dynamic_filter_registry, spawner: &self.spawner, } @@ -111,76 +113,71 @@ impl QueryCoordinator { self.task_ctx.session_config() } - /// Builds task startup with report cleanup owned even before its first poll. - pub(super) fn start_task( + /// Starts every task independently, using the same lifecycle for all stages. + /// + /// Each reader drives its task's setup before collecting worker messages. Cancellation + /// stops pending setup, while connected streams drain their final reports as usual. + /// Locations retain only receivers, so cached plans cannot keep setup running. + /// Successful tasks can be consumed immediately, while stage sealing waits for all tasks. + /// Worker plans still wait for the input addresses they must serialize before routing. + pub(super) fn start_stage( self: &Arc, stage: &LocalStage, - task_i: usize, - ) -> impl Future< - Output = Result<( - Url, - UnboundedSender, - UnboundedReceiver, - )>, - > + Send - + 'static { - let reports = TaskReports { - task_key: TaskKey { - query_id: stage.query_id, - stage_id: stage.num, - task_number: task_i, - }, - metrics_store: self.metrics_store.clone(), - completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), - }; - let coordinator = Arc::clone(self); - let stage = stage.clone(); - async move { - let stage_coordinator = coordinator.stage_coordinator(&stage); - let (url, tx, stream) = stage_coordinator.init_bidirectional_stream(task_i).await?; - let load_info = stage_coordinator.worker_to_coordinator_task(stream, reports); - stage_coordinator.coordinator_to_worker_task(task_i, tx.clone())?; - Ok((url, tx, load_info)) - } - } - - /// Starts coordinator inputs independently while supervising all setup failures. - pub(super) fn start_pending_stage(self: &Arc, stage: &LocalStage) -> Vec { + ) -> (Vec, Vec>) { let mut workers = Vec::with_capacity(stage.tasks); - let mut setups = Vec::with_capacity(stage.tasks); + let mut load_info = Vec::with_capacity(stage.tasks); + let mut stage_coordinator = self.stage_coordinator(stage); for task in 0..stage.tasks { - let (tx, rx) = oneshot::channel(); - workers.push(WorkerLocation::Pending { - url: async move { - rx.await - .map_err(|_| Arc::new(exec_datafusion_err!("Task setup was cancelled")))? + let (location_tx, location_rx) = oneshot::channel(); + workers.push(WorkerLocation::pending(location_rx)); + let coordinator = Arc::clone(self); + let stage = stage.clone(); + let finished = self.spawner.query_finished(); + let stream = async move { + let mut stage_coordinator = coordinator.stage_coordinator(&stage); + let result = select! { + biased; + _ = finished.cancelled() => return empty().boxed(), + result = stage_coordinator.init_bidirectional_stream(task) => result, + }; + match result { + Ok((url, tx, stream)) => { + let location = stage_coordinator + .coordinator_to_worker_task(task, tx.clone()) + .map(|()| { + let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling); + url + }); + let _ = location_tx.send(location.map_err(Arc::new)); + stream + } + Err(error) => { + let _ = location_tx.send(Err(Arc::new(error))); + empty().boxed() + } } - .boxed() - .shared(), - query_finished: self.spawner.query_finished(), - }); - setups.push((tx, self.start_task(stage, task))); + } + .flatten_stream() + .boxed(); + load_info.push(stage_coordinator.worker_to_coordinator_task(task, stream)); } + let coordinator = Arc::clone(self); let stage = stage.clone(); + let locations = workers.clone(); let finished = self.spawner.query_finished(); self.spawner.spawn(async move { - let stage_coordinator = coordinator.stage_coordinator(&stage); - let setup = async { - try_join_all(setups.into_iter().map(|(tx, setup)| async move { - let result = setup.await.map(|(url, ..)| url).map_err(Arc::new); - let _ = tx.send(result.clone()); - result.map_err(DataFusionError::Shared) - })) - .await?; - stage_coordinator.seal_dynamic_filter_stage() - }; - tokio::select! { - result = setup => result, - _ = finished.cancelled() => Ok(()), + select! { + biased; + _ = finished.cancelled() => {} + result = try_join_all(locations.iter().map(WorkerLocation::resolve)) => { + result?; + coordinator.stage_coordinator(&stage).seal_dynamic_filter_stage()?; + } } + Ok(()) }); - workers + (workers, load_info) } } @@ -200,6 +197,8 @@ pub(super) struct StageCoordinator<'a> { task_ctx: &'a Arc, metrics_set: &'a ExecutionPlanMetricsSet, metrics: &'a CoordinatorToWorkerMetrics, + metrics_store: &'a Option>>, + completed_dynamic_filter_store: &'a Option>>, dynamic_filter_registry: &'a Arc, spawner: &'a Spawner, } @@ -211,7 +210,7 @@ impl<'a> StageCoordinator<'a> { /// - The worker URL in which the task got allocated /// - The coordinator-to-worker stream for producing messages that reach the worker. /// - The worker-to-coordinator stream for receiving messages from the worker. - async fn init_bidirectional_stream( + pub(super) async fn init_bidirectional_stream( &self, task_i: usize, ) -> Result<( @@ -225,7 +224,7 @@ impl<'a> StageCoordinator<'a> { plan, work_unit_feed_declarations, dynamic_filter_remote_producer_ids, - } = self.task_specialized_plan(task_i)?; + } = self.task_specialized_plan(task_i).await?; let task_key = TaskKey { query_id: self.query_id, @@ -350,12 +349,18 @@ impl<'a> StageCoordinator<'a> { } /// Spawns background tasks in charge of collecting messages sent by a worker. - fn worker_to_coordinator_task( - &self, + pub(super) fn worker_to_coordinator_task( + &mut self, + task_i: usize, mut worker_to_coordinator_stream: BoxStream<'static, Result>, - mut reports: TaskReports, ) -> UnboundedReceiver { - let task_key = reports.task_key; + let task_key = TaskKey { + query_id: self.query_id, + stage_id: self.stage_id, + task_number: task_i, + }; + let mut task_metrics = self.metrics_store.clone(); + let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone(); let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); let task_ctx = Arc::clone(self.task_ctx); let (load_info_tx, load_info_rx) = unbounded_channel(); @@ -366,12 +371,12 @@ impl<'a> StageCoordinator<'a> { while let Some(msg) = worker_to_coordinator_stream.try_next().await? { match msg { WorkerToCoordinatorMsg::TaskMetrics(metrics) => { - if let Some(task_metrics) = reports.metrics_store.take() { + if let Some(task_metrics) = task_metrics.take() { task_metrics.insert(task_key, metrics); } } WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { - if let Some(store) = reports.completed_dynamic_filter_store.take() { + if let Some(store) = completed_dynamic_filter_store.take() { store.insert(task_key, filters); } } @@ -389,6 +394,14 @@ impl<'a> StageCoordinator<'a> { } }; } + // An unexecuted task sends no final reports; still complete its waits. + if let Some(store) = task_metrics { + store.insert(task_key, TaskMetrics::default()); + } + if let Some(store) = completed_dynamic_filter_store { + store.insert(task_key, TaskCompletedDynamicFilters::default()); + } + let _ = load_info_tx_opt.take(); Ok::<_, DataFusionError>(()) }); @@ -398,8 +411,8 @@ impl<'a> StageCoordinator<'a> { /// Spawns a background task in charge of sending messages to workers. Some things that are sent /// to workers here are: /// - WorkUnits collected from [WorkUnitFeeds] present in the plan. - fn coordinator_to_worker_task( - &self, + pub(super) fn coordinator_to_worker_task( + &mut self, task_i: usize, tx: UnboundedSender, ) -> Result<()> { @@ -477,7 +490,7 @@ impl<'a> StageCoordinator<'a> { /// trimming down any unnecessary information that the specific `task_i` task is not going to /// need, like unexecuted branches in [ChildrenIsolatorUnionExec], or unexecuted variants of /// [DistributedLeafExec]. - fn task_specialized_plan(&self, task_i: usize) -> Result { + async fn task_specialized_plan(&self, task_i: usize) -> Result { let session_config = self.task_ctx.session_config(); let wuf_registry = session_config .get_extension::() @@ -492,7 +505,13 @@ impl<'a> StageCoordinator<'a> { }; let plan = Arc::clone(self.plan); + let mut input_locations = vec![]; let transformed = plan.transform_down_with_dt_ctx(d_ctx, |plan, d_ctx| { + if let Some(boundary) = plan.as_network_boundary() + && let Stage::Remote(stage) = boundary.input_stage() + { + input_locations.extend(stage.workers.iter().cloned()); + } if let Some(wuf) = wuf_registry.get_work_unit_feed(&plan) { work_unit_feed_declarations.push(WorkUnitFeedDeclaration { id: wuf.id(), @@ -533,6 +552,9 @@ impl<'a> StageCoordinator<'a> { // we are explicitly not retransforming the entire plan. if other operators cause shared state errors they will error out. Ok(Transformed::no(plan)) })?; + // Worker plans carry concrete input addresses. Resolve them during the existing + // specialization pass, after unused task branches have been removed. + try_join_all(input_locations.iter().map(WorkerLocation::resolve)).await?; let plan = if dynamic_filtering_enabled { maybe_roundtrip_plan_to_sever_in_memory_dynamic_filter_relationships( Arc::clone(&transformed.data), @@ -555,24 +577,6 @@ impl<'a> StageCoordinator<'a> { } } -/// Completes missing report waits when setup or its worker response reader ends. -struct TaskReports { - task_key: TaskKey, - metrics_store: Option>>, - completed_dynamic_filter_store: Option>>, -} - -impl Drop for TaskReports { - fn drop(&mut self) { - if let Some(store) = self.metrics_store.take() { - store.insert(self.task_key, TaskMetrics::default()); - } - if let Some(store) = self.completed_dynamic_filter_store.take() { - store.insert(self.task_key, TaskCompletedDynamicFilters::default()); - } - } -} - struct TaskSpecializedPlan { plan: Arc, work_unit_feed_declarations: Vec, diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index d75af4450..e796a7cca 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -216,21 +216,6 @@ impl<'a> InjectNetworkBoundaryContext<'a> { Ok(*task_count) } - /// Retains annotations when stage preparation replaces nodes without changing topology. - pub(crate) fn copy_task_counts( - &self, - original: &Arc, - prepared: &Arc, - ) -> Result<()> { - self.set_task_count(prepared, self.task_count(original)?); - if !original.is_network_boundary() { - for (original, prepared) in original.children().into_iter().zip(prepared.children()) { - self.copy_task_counts(original, prepared)?; - } - } - Ok(()) - } - fn fetch_add_stage_id(&self) -> usize { self.stage_id.fetch_add(1, Ordering::Acquire) } diff --git a/datafusion/distributed/src/events/defaults/routing.rs b/datafusion/distributed/src/events/defaults/routing.rs index 35473f4e2..44846fab0 100644 --- a/datafusion/distributed/src/events/defaults/routing.rs +++ b/datafusion/distributed/src/events/defaults/routing.rs @@ -172,9 +172,8 @@ impl RouteTaskHandler for SingleTaskChildUrlRouteTaskHandler { if let Stage::Remote(remote) = nb.input_stage() && remote.workers.len() == 1 - && let Ok(url) = remote.workers[0].ready_url() { - single_stage_url = Some(url.clone()); + single_stage_url = remote.workers[0].ready_url().cloned(); return Ok(TreeNodeRecursion::Stop); } diff --git a/datafusion/distributed/src/stage.rs b/datafusion/distributed/src/stage.rs index c880e41a2..058b3e4e7 100644 --- a/datafusion/distributed/src/stage.rs +++ b/datafusion/distributed/src/stage.rs @@ -1,8 +1,8 @@ use crate::coordinator::DistributedExec; use crate::execution_plans::{DistributedLeafExec, NetworkCoalesceExec}; use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; -use datafusion::common::{DataFusionError, HashMap, Statistics, config_err}; -use datafusion::common::{exec_err, plan_err}; +use datafusion::common::{HashMap, Statistics, config_err}; +use datafusion::common::{exec_datafusion_err, exec_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; @@ -12,15 +12,62 @@ use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext, displayable, }; -use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; +use futures::future::Shared; use itertools::Either; use std::collections::VecDeque; -use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use tokio_util::sync::CancellationToken; +use tokio::sync::oneshot; use url::Url; use uuid::Uuid; +/// The eventual committed address of a worker whose task may still be setting up. +/// +/// Routing may try several workers before committing to one. Consumers await only the +/// location they need; plans sent to another worker must first resolve all their inputs. +#[derive(Debug, Clone)] +pub enum WorkerLocation { + /// An address obtained from a decoded plan or supplied by the caller. + Ready(Url), + /// The result of query-owned routing and plan delivery. + Pending(Shared>>>), +} + +impl WorkerLocation { + pub(crate) fn pending(rx: oneshot::Receiver>>) -> Self { + Self::Pending(rx.shared()) + } + + /// Waits for successful routing, retaining the original setup error on failure. + pub async fn resolve(&self) -> Result { + match self { + Self::Ready(url) => Ok(url.clone()), + Self::Pending(url) => url + .clone() + .await + .map_err(|_| exec_datafusion_err!("Task setup was cancelled"))? + .map_err(DataFusionError::Shared), + } + } + + /// Returns a location that has already been resolved, without waiting for setup. + pub fn ready_url(&self) -> Option<&Url> { + match self { + Self::Ready(url) => Some(url), + Self::Pending(url) => match url.peek() { + Some(Ok(Ok(url))) => Some(url), + _ => None, + }, + } + } +} + +impl From for WorkerLocation { + fn from(url: Url) -> Self { + Self::Ready(url) + } +} + /// A unit of isolation for a portion of a physical execution plan /// that can be executed independently and across a network boundary. /// It implements [`ExecutionPlan`] and can be executed to produce a @@ -114,7 +161,8 @@ pub struct RemoteStage { pub query_id: Uuid, /// Our stage number pub num: usize, - /// The worker locations to which queries should be issued. + /// The worker URLs to which queries should be issued. + /// Locations are in task-number order and may still be awaiting setup. pub workers: Vec, /// Statistics collected at runtime, if any. pub runtime_stats: Option>, @@ -122,46 +170,6 @@ pub struct RemoteStage { pub dynamic_filter_anchors: Vec>, } -/// A task's selected worker URL or its in-progress routing result. -/// Pending locations stay in the coordinator; worker plans require ready locations. -#[derive(Clone)] -pub enum WorkerLocation { - /// A successfully connected worker selected by the task-routing handler. - Ready(Url), - /// A coordinator-consumed task whose routing is still in progress. - Pending { - /// The shared result of task setup, including connection-aware routing. - url: Shared>>>, - /// Cancels location resolution and result-stream setup when the query ends. - query_finished: CancellationToken, - }, -} - -impl WorkerLocation { - /// Returns a concrete URL, or an error for a pending location. - pub fn ready_url(&self) -> Result<&Url> { - match self { - Self::Ready(url) => Ok(url), - Self::Pending { .. } => exec_err!("Worker location is pending"), - } - } -} - -impl From for WorkerLocation { - fn from(url: Url) -> Self { - Self::Ready(url) - } -} - -impl Debug for WorkerLocation { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Self::Ready(url) => f.debug_tuple("Ready").field(url).finish(), - Self::Pending { .. } => f.write_str("Pending"), - } - } -} - impl Stage { pub fn query_id(&self) -> Uuid { match &self { @@ -279,6 +287,7 @@ impl DistributedTaskContext { use crate::{DistributedMetricsFormat, NetworkShuffleExec, rewrite_distributed_plan_with_metrics}; use crate::{NetworkBoundary, NetworkBoundaryExt}; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::DataFusionError; use datafusion::common::stats::Precision; use datafusion::physical_expr::Partitioning; /// Be able to display a nice tree for stages. diff --git a/datafusion/distributed/src/worker/worker_connection_pool.rs b/datafusion/distributed/src/worker/worker_connection_pool.rs index 87db51f4a..9239c9184 100644 --- a/datafusion/distributed/src/worker/worker_connection_pool.rs +++ b/datafusion/distributed/src/worker/worker_connection_pool.rs @@ -1,24 +1,21 @@ use crate::common::on_drop_stream; use crate::distributed_planner::ProducerHead; use crate::passthrough_headers::get_passthrough_headers; -use crate::stage::{RemoteStage, WorkerLocation}; +use crate::stage::RemoteStage; use crate::{ExecuteTaskRequest, LocalWorkerContext, TaskKey, get_distributed_channel_resolver}; use dashmap::DashMap; use datafusion::arrow::array::RecordBatch; use datafusion::common::runtime::SpawnedTask; -use datafusion::common::{ - DataFusionError, Result, exec_err, internal_datafusion_err, internal_err, -}; +use datafusion::common::{DataFusionError, Result, internal_datafusion_err, internal_err}; use datafusion::execution::TaskContext; use datafusion::physical_expr_common::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::metrics::MetricBuilder; -use futures::future::{BoxFuture, Shared, ready}; +use futures::future::{BoxFuture, Shared}; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use std::fmt::{Debug, Formatter}; use std::ops::Range; use std::sync::{Arc, Mutex}; -use tokio::select; /// Manages connections to remote workers. /// - Handles a range of partitions at a time in other to give the chance to [crate::WorkerChannel] @@ -68,18 +65,8 @@ impl WorkerConnectionPool { producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let worker = input_stage.workers.get(target_task).ok_or_else(|| { - internal_datafusion_err!("input stage task {target_task} out of range") - })?; - let (destination, query_finished) = match worker { - WorkerLocation::Ready(url) => (ready(Ok(url.clone())).left_future(), None), - WorkerLocation::Pending { - url, - query_finished, - } => ( - url.clone().map_err(DataFusionError::Shared).right_future(), - Some(query_finished.clone()), - ), + let Some(worker_location) = input_stage.workers.get(target_task).cloned() else { + internal_err!("input_stage.workers[{target_task}] out of range.")? }; let task_key = TaskKey { query_id: input_stage.query_id, @@ -119,49 +106,39 @@ impl WorkerConnectionPool { // // Note that this does not start polling the returned streams, it just instantiates them. let streams_task = SpawnedTask::spawn(async move { - let setup = async move { - let target_url = destination.await?; - let request = ExecuteTaskRequest { - task_key, - target_partition_start: target_partitions.start, - target_partition_end: target_partitions.end, - producer_head, - }; - let mut client = match LocalWorkerContext::from_ctx(&ctx) { - Some(lw) if lw.self_url == target_url => { - local_connections_used.add(1); - Ok(lw.to_worker_channel()) - } - _ => { - let ch_resolver = get_distributed_channel_resolver(ctx.as_ref()); - ch_resolver.get_worker_client_for_url(&target_url).await - } - }?; - let headers = get_passthrough_headers(ctx.session_config()); - let streams = client.execute_task(headers, request, metrics, &ctx).await?; - if streams.len() != task_slots.len() { - return internal_err!( - "Worker returned {} streams for {} partitions", - streams.len(), - task_slots.len() - ); + let target_url = worker_location.resolve().await?; + let request = ExecuteTaskRequest { + task_key, + target_partition_start: target_partitions.start, + target_partition_end: target_partitions.end, + producer_head, + }; + let mut client = match LocalWorkerContext::from_ctx(&ctx) { + Some(lw) if lw.self_url == target_url => { + local_connections_used.add(1); + Ok(lw.to_worker_channel()) } - for (slot, stream) in task_slots.iter().zip(streams) { - let mut slot = slot.lock().unwrap(); - if !slot.closed { - slot.stream = Some(stream); - } + _ => { + let ch_resolver = get_distributed_channel_resolver(ctx.as_ref()); + ch_resolver.get_worker_client_for_url(&target_url).await + } + }?; + let headers = get_passthrough_headers(ctx.session_config()); + let streams = client.execute_task(headers, request, metrics, &ctx).await?; + if streams.len() != task_slots.len() { + return internal_err!( + "Worker returned {} streams for {} partitions", + streams.len(), + task_slots.len() + ); + } + for (slot, stream) in task_slots.iter().zip(streams) { + let mut slot = slot.lock().unwrap(); + if !slot.closed { + slot.stream = Some(stream); } - Ok(()) - }; - match query_finished { - Some(query_finished) => select! { - biased; - _ = query_finished.cancelled() => exec_err!("Query finished during task setup"), - result = setup => result, - }, - None => setup.await, } + Ok(()) }); let ready = async move { diff --git a/datafusion/distributed/tests/main.rs b/datafusion/distributed/tests/main.rs index c7aea036f..33a6f86bd 100644 --- a/datafusion/distributed/tests/main.rs +++ b/datafusion/distributed/tests/main.rs @@ -21,11 +21,11 @@ mod multi_task_collect_join_repros; mod passthrough_headers; mod shuffle; mod sorted_data_source; -mod startup_overlap; mod stateful_data_cleanup; mod stateful_execution_plan; mod task_estimator_test; mod task_finished_handler; +mod task_startup; mod tpcds_correctness_test; mod tpcds_plans_test; mod tpch_correctness_test; diff --git a/datafusion/distributed/tests/startup_overlap.rs b/datafusion/distributed/tests/startup_overlap.rs deleted file mode 100644 index b6a60ffe4..000000000 --- a/datafusion/distributed/tests/startup_overlap.rs +++ /dev/null @@ -1,414 +0,0 @@ -#[cfg(all(feature = "integration", test))] -mod tests { - use std::slice::from_ref; - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::Duration; - - use arrow::array::{Int64Array, RecordBatch}; - use async_trait::async_trait; - use datafusion::common::runtime::SpawnedTask; - use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; - use datafusion::common::{Result, assert_contains, exec_err}; - use datafusion::execution::SessionState; - use datafusion::physical_plan::{ExecutionPlan, execute_stream}; - use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; - use datafusion_distributed::test_utils::routing::{ - URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, - url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, - }; - use datafusion_distributed::{ - DistributedExec, DistributedExt, NetworkBoundaryExt, NetworkCoalesceExec, RouteTaskEvent, - RouteTaskEventResponse, RouteTaskHandler, Stage, WorkerQueryContext, - WorkerToCoordinatorMsg, rewrite_distributed_plan_with_dynamic_filters, - }; - use futures::{StreamExt, TryStreamExt, stream}; - use tokio::time::timeout; - use tokio_util::sync::CancellationToken; - - const WAIT: Duration = Duration::from_secs(5); - const SCAN: &str = "SELECT task_index FROM url_emitter(2, 2, 'scan')"; - - #[tokio::test] - async fn static_scan_returns_output_before_sibling_setup() -> Result<()> { - let gate = SetupGate::default(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(false)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let mut output = ctx.sql(SCAN).await?.execute_stream().await?; - - let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); - assert_eq!(task_indices(from_ref(&first)), vec![0]); - assert!(!gate.release.is_cancelled()); - timeout(WAIT, gate.entered.cancelled()).await.unwrap(); - gate.release.cancel(); - let mut batches = vec![first]; - batches.extend( - timeout(WAIT, output.try_collect::>()) - .await - .unwrap()?, - ); - assert_eq!(task_indices(&batches), vec![0, 1]); - Ok(()) - } - - #[tokio::test] - async fn adaptive_scan_returns_output_before_sibling_setup() -> Result<()> { - let gate = SetupGate::default(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let mut output = ctx.sql(SCAN).await?.execute_stream().await?; - - let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); - assert_eq!(task_indices(from_ref(&first)), vec![0]); - assert!(!gate.release.is_cancelled()); - timeout(WAIT, gate.entered.cancelled()).await.unwrap(); - gate.release.cancel(); - let mut batches = vec![first]; - batches.extend( - timeout(WAIT, output.try_collect::>()) - .await - .unwrap()?, - ); - assert_eq!(task_indices(&batches), vec![0, 1]); - Ok(()) - } - - #[tokio::test] - async fn adaptive_shuffle_samples_before_sibling_setup() -> Result<()> { - let gate = SetupGate::default(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let query = ctx - .sql("SELECT task_index FROM url_emitter(2, 2, 'scan') GROUP BY task_index") - .await?; - let execution = SpawnedTask::spawn(async move { query.collect().await }); - - timeout(WAIT, gate.entered.cancelled()).await.unwrap(); - // A positive LoadInfo report proves the remote sampler has produced rows. - timeout(WAIT, gate.sampled.cancelled()).await.unwrap(); - assert!(!gate.release.is_cancelled()); - gate.release.cancel(); - let batches = timeout(WAIT, execution.join()).await.unwrap().unwrap()?; - assert_eq!(task_indices(&batches), vec![0, 1]); - Ok(()) - } - - #[tokio::test] - async fn late_setup_failure_reaches_output_and_finishes_metrics() -> Result<()> { - let gate = SetupGate { - fail: true, - ..Default::default() - }; - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(false)? - .with_distributed_metrics_collection(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; - let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; - - let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); - assert_eq!(task_indices(&[first]), vec![0]); - gate.release.cancel(); - let error = timeout(WAIT, output.try_collect::>()) - .await - .unwrap() - .unwrap_err(); - assert_contains!(error.to_string(), "injected setup failure"); - assert_terminal_metrics(&plan).await; - Ok(()) - } - - #[tokio::test] - async fn worker_response_error_after_output_finishes_reports() -> Result<()> { - let response_error = ResponseErrorGate::default(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(false)? - .with_distributed_metrics_collection(true)? - .with_distributed_dynamic_filter_collection(true)? - .with_distributed_route_task_handler(response_error.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; - let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; - - let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); - assert!(matches!(task_indices(&[first]).as_slice(), [0] | [1])); - response_error.release.cancel(); - let error = timeout(WAIT, output.try_collect::>()) - .await - .unwrap() - .unwrap_err(); - assert_contains!(error.to_string(), "injected worker response error"); - - let distributed = plan.downcast_ref::().unwrap(); - let reports = timeout(WAIT, distributed.wait_for_metrics()) - .await - .unwrap() - .unwrap(); - assert_eq!(reports.len(), 2); - assert!( - reports - .values() - .all(|report| report.pre_order_plan_metrics.is_empty()) - ); - let task_ctx = ctx.task_ctx(); - timeout( - WAIT, - rewrite_distributed_plan_with_dynamic_filters(plan, &task_ctx), - ) - .await - .unwrap()?; - Ok(()) - } - - #[tokio::test] - async fn dropping_output_cancels_pending_setup_and_finishes_metrics() -> Result<()> { - let gate = SetupGate::default(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(true)? - .with_distributed_metrics_collection(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; - let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; - - let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); - assert_eq!(task_indices(&[first]), vec![0]); - timeout(WAIT, gate.entered.cancelled()).await.unwrap(); - drop(output); - timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); - assert!(!gate.release.is_cancelled()); - assert_terminal_metrics(&plan).await; - Ok(()) - } - - #[tokio::test] - async fn limit_completion_cancels_pending_setup_and_finishes_metrics() -> Result<()> { - let gate = SetupGate { - wait_for_sibling: true, - ..Default::default() - }; - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(false)? - .with_distributed_metrics_collection(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let plan = ctx - .sql(&format!("{SCAN} LIMIT 1")) - .await? - .create_physical_plan() - .await?; - let output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; - let batches = timeout(WAIT, output.try_collect::>()) - .await - .unwrap()?; - - assert_eq!(task_indices(&batches), vec![0]); - timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); - assert!(!gate.release.is_cancelled()); - assert_terminal_metrics(&plan).await; - Ok(()) - } - - #[tokio::test] - async fn adaptive_nested_coalesce_has_concrete_worker_destinations() -> Result<()> { - let gate = SetupGate::default(); - gate.release.cancel(); - let ctx = context(start_in_memory_context(2, build_state).await) - .with_distributed_dynamic_task_count(true)? - .with_distributed_route_task_handler(gate.clone()); - ctx.state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = 2; - let query = ctx - .sql( - "SELECT task_index FROM url_emitter(2, 2, 'scan') WHERE task_index = \ - (SELECT MAX(task_index) FROM url_emitter(2, 2, 'inner'))", - ) - .await?; - - let batches = timeout(WAIT, query.collect()).await.unwrap()?; - assert_eq!(task_indices(&batches), vec![1]); - assert!(gate.nested_coalesces.load(Ordering::Relaxed) > 0); - Ok(()) - } - - fn context(mut ctx: SessionContext) -> SessionContext { - ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); - ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); - ctx.set_distributed_user_codec(URLEmitterExtensionCodec); - ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); - ctx - } - - async fn build_state(ctx: WorkerQueryContext) -> Result { - Ok(ctx - .builder - .with_distributed_user_codec(URLEmitterExtensionCodec) - .build()) - } - - fn task_indices(batches: &[RecordBatch]) -> Vec { - let mut values = batches - .iter() - .flat_map(|batch| { - batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .values() - .to_vec() - }) - .collect::>(); - values.sort_unstable(); - values - } - - async fn assert_terminal_metrics(plan: &Arc) { - let distributed = plan.downcast_ref::().unwrap(); - let reports = timeout(WAIT, distributed.wait_for_metrics()) - .await - .unwrap() - .unwrap(); - assert_eq!(reports.len(), 2); - } - - #[derive(Clone, Default)] - struct ResponseErrorGate { - release: CancellationToken, - } - - #[async_trait] - impl RouteTaskHandler for ResponseErrorGate { - async fn handle( - &self, - event: RouteTaskEvent<'_>, - ) -> Option> { - Some( - UrlEmitterRouteTaskHandler - .handle(event) - .await? - .map(|mut response| { - let release = self.release.clone(); - let error = stream::once(async move { - release.cancelled().await; - exec_err!("injected worker response error") - }); - response.worker_to_coordinator_stream = - error.chain(response.worker_to_coordinator_stream).boxed(); - response - }), - ) - } - } - - #[derive(Clone, Default)] - struct SetupGate { - entered: CancellationToken, - release: CancellationToken, - cancelled: CancellationToken, - sampled: CancellationToken, - nested_coalesces: Arc, - fail: bool, - wait_for_sibling: bool, - } - - #[async_trait] - impl RouteTaskHandler for SetupGate { - async fn handle( - &self, - event: RouteTaskEvent<'_>, - ) -> Option> { - Some(self.route(event).await) - } - } - - impl SetupGate { - async fn route(&self, event: RouteTaskEvent<'_>) -> Result { - event.task_specialized_plan.apply(|node| { - if let Some(boundary) = node.as_network_boundary() { - let Stage::Remote(stage) = boundary.input_stage() else { - panic!("worker plans require a remote input stage"); - }; - assert!( - stage - .workers - .iter() - .all(|worker| worker.ready_url().is_ok()) - ); - } - if node.is::() { - self.nested_coalesces.fetch_add(1, Ordering::Relaxed); - } - Ok(TreeNodeRecursion::Continue) - })?; - if event.task_key.task_number == 0 && self.wait_for_sibling { - self.entered.cancelled().await; - } - if event.task_key.task_number == 1 { - let cancelled = self.cancelled.clone().drop_guard(); - self.entered.cancel(); - self.release.cancelled().await; - cancelled.disarm(); - if self.fail { - return exec_err!("injected setup failure"); - } - } - let mut response = UrlEmitterRouteTaskHandler.handle(event).await.unwrap()?; - let sampled = self.sampled.clone(); - response.worker_to_coordinator_stream = response.worker_to_coordinator_stream - .inspect_ok(move |message| { - if matches!(message, WorkerToCoordinatorMsg::LoadInfo(info) if info.rows_ready > 0) { - sampled.cancel(); - } - }).boxed(); - Ok(response) - } - } -} diff --git a/datafusion/distributed/tests/task_startup.rs b/datafusion/distributed/tests/task_startup.rs new file mode 100644 index 000000000..0b227b139 --- /dev/null +++ b/datafusion/distributed/tests/task_startup.rs @@ -0,0 +1,259 @@ +#[cfg(test)] +mod tests { + use arrow::array::Int64Array; + use async_trait::async_trait; + use datafusion::common::runtime::SpawnedTask; + use datafusion::common::{Result, assert_contains, exec_err}; + use datafusion::execution::SessionState; + use datafusion::physical_plan::execute_stream; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, + }; + use datafusion_distributed::{ + DistributedExt, DistributedMetricsFormat, RouteTaskEvent, RouteTaskEventResponse, + RouteTaskHandler, WorkerQueryContext, WorkerToCoordinatorMsg, + rewrite_distributed_plan_with_metrics, + }; + use futures::{StreamExt, TryStreamExt}; + use std::sync::Arc; + use std::time::Duration; + use tokio::time::timeout; + use tokio_util::sync::CancellationToken; + + const WAIT: Duration = Duration::from_secs(5); + const SCAN: &str = "SELECT task_index FROM url_emitter(2, 2, 'startup')"; + + #[tokio::test] + async fn static_task_produces_output_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let rest = output.try_collect::>().await?; + assert_eq!(rest.iter().map(|batch| batch.num_rows()).sum::(), 1); + Ok(()) + } + + #[tokio::test] + async fn adaptive_task_produces_output_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let rest = output.try_collect::>().await?; + assert_eq!(rest.iter().map(|batch| batch.num_rows()).sum::(), 1); + Ok(()) + } + + #[tokio::test] + async fn adaptive_task_samples_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let query = ctx + .sql("SELECT task_index FROM url_emitter(2, 2, 'startup') GROUP BY task_index") + .await?; + let execution = SpawnedTask::spawn(async move { query.collect().await }); + + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + timeout(WAIT, gate.sampled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let batches = timeout(WAIT, execution).await.unwrap().unwrap()?; + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn limit_cancels_unstarted_tasks_and_completes_reports() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let plan = ctx + .sql(&format!("{SCAN} LIMIT 1")) + .await? + .create_physical_plan() + .await?; + let output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let batches = timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?; + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + timeout(WAIT, gate.exited.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + timeout( + WAIT, + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated), + ) + .await + .unwrap()?; + Ok(()) + } + + #[tokio::test] + async fn late_setup_error_reaches_the_query() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(FailSecondTask(gate.clone())); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + gate.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "second task setup failed"); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn setup_error_survives_parent_stage_cancellation() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(FailSecondTask(gate.clone())); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let execution = SpawnedTask::spawn(async move { + ctx.sql("SELECT count(*) FROM url_emitter(2, 2, 'startup') GROUP BY task_index") + .await? + .collect() + .await + }); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let error = timeout(WAIT, execution) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "second task setup failed"); + Ok(()) + } + + fn with_emitter(mut ctx: SessionContext) -> SessionContext { + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx + } + + async fn build_worker(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) + } + + #[derive(Clone, Default)] + struct TaskGate { + entered: CancellationToken, + release: CancellationToken, + exited: CancellationToken, + sampled: CancellationToken, + } + + #[async_trait] + impl RouteTaskHandler for TaskGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + // Make the stalled routing attempt observable before the ready task produces data. + if event.task_key.task_number == 0 { + self.entered.cancelled().await; + } + if event.task_key.task_number == 1 { + let _exited = self.exited.clone().drop_guard(); + self.entered.cancel(); + self.release.cancelled().await; + } + let sampled = self.sampled.clone(); + let response = UrlEmitterRouteTaskHandler.handle(event).await?; + Some(response.map(|mut response| { + response.worker_to_coordinator_stream = response.worker_to_coordinator_stream + .inspect_ok(move |message| { + if matches!(message, WorkerToCoordinatorMsg::LoadInfo(info) if info.rows_ready > 0) { + sampled.cancel(); + } + }) + .boxed(); + response + })) + } + } + + struct FailSecondTask(TaskGate); + + #[async_trait] + impl RouteTaskHandler for FailSecondTask { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + if event.task_key.task_number == 1 { + self.0.entered.cancel(); + self.0.release.cancelled().await; + return Some(exec_err!("second task setup failed")); + } + UrlEmitterRouteTaskHandler.handle(event).await + } + } +} diff --git a/docs/upgrade/5.0.0.md b/docs/upgrade/5.0.0.md index 18f45cd86..bb940e46e 100644 --- a/docs/upgrade/5.0.0.md +++ b/docs/upgrade/5.0.0.md @@ -79,26 +79,11 @@ let state = SessionStateBuilder::new() coordinator-to-worker connection is currently established. Implementations can choose to retry on other worker if the current one is not available. -### Remote stage destinations can still be routing +### `RemoteStage::workers` now holds `WorkerLocation` values -`RemoteStage::workers` now contains `WorkerLocation` values instead of `Url`s. -When constructing a stage with resolved worker URLs, convert them with `into()` -or `WorkerLocation::from`: - -```rust -use datafusion_distributed::WorkerLocation; - -let workers = urls.into_iter().map(WorkerLocation::from).collect(); -``` - -When inspecting a destination, use `worker.ready_url()?` to borrow its resolved -URL. This returns an error for `WorkerLocation::Pending`; coordinator input -stages may contain these destinations while task routing finishes. Best-effort -colocation handlers should skip pending destinations instead of treating them -as resolved URLs. - -Worker plans require `WorkerLocation::Ready` destinations. The wire format still -contains URL strings, and encoding a pending destination returns an error. +Convert an existing `Vec` with `urls.into_iter().map(Into::into).collect()`. +Use `worker_location.resolve().await?` to obtain its committed URL, or +`worker_location.ready_url()` to inspect a location without waiting. ## 2. `WorkerPlanRewriteHandler::rewrite_worker_plan` renamed to `handle` and is now async From c637f16503969b4e5986177521e01a6f1ebc921c Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 19 Nov 2026 14:27:39 -0500 Subject: [PATCH 10/10] test: restore cancellation coverage for task startup --- .../src/coordinator/query_coordinator.rs | 2 ++ datafusion/distributed/tests/task_startup.rs | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index d0f8d1527..2f6681975 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -152,6 +152,8 @@ impl QueryCoordinator { stream } Err(error) => { + // The stage supervisor propagates the location error. End the reader + // normally so it completes any missing final reports. let _ = location_tx.send(Err(Arc::new(error))); empty().boxed() } diff --git a/datafusion/distributed/tests/task_startup.rs b/datafusion/distributed/tests/task_startup.rs index 0b227b139..107775700 100644 --- a/datafusion/distributed/tests/task_startup.rs +++ b/datafusion/distributed/tests/task_startup.rs @@ -107,6 +107,33 @@ mod tests { Ok(()) } + #[tokio::test] + async fn dropping_output_cancels_unstarted_tasks_and_completes_reports() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + drop(output); + + timeout(WAIT, gate.exited.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + timeout( + WAIT, + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated), + ) + .await + .unwrap()?; + Ok(()) + } + #[tokio::test] async fn limit_cancels_unstarted_tasks_and_completes_reports() -> Result<()> { let gate = TaskGate::default();