From ce672e830eb705d41bb6e6f08b6cf001bdfaaf29 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 12 Nov 2026 14:03:34 +0100 Subject: [PATCH] perf: overlap ready task execution with setup Task routing no longer blocks stage preparation. init_bidirectional_stream now spawns each task's routing and returns a WorkerLocation right away, which is pending until the task is routed. RemoteStage::workers becomes a Vec. - The coordinator reads a task's output as soon as that task is routed, instead of waiting for all its siblings. - Each task gets KickOffSampling as soon as it is routed, so AQE sampling starts per task. - A task waits for its child stages' locations (ensure_all_resolved) before routing, since worker plans embed child stage URLs. - Routing and work unit feeds stop when the query finishes. --- .../src/codec/distributed_codec.rs | 4 +- .../src/coordinator/prepare_dynamic_plan.rs | 24 +- .../src/coordinator/prepare_static_plan.rs | 16 +- .../src/coordinator/query_coordinator.rs | 244 ++++++----- .../distributed/src/coordinator/spawner.rs | 12 +- .../src/events/defaults/routing.rs | 4 +- .../benchmarks/shuffle_bench.rs | 7 +- .../benchmarks/transport_bench.rs | 7 +- datafusion/distributed/src/lib.rs | 3 +- datafusion/distributed/src/stage.rs | 78 +++- .../src/worker/worker_connection_pool.rs | 5 +- datafusion/distributed/tests/main.rs | 1 + .../distributed/tests/startup_overlap.rs | 414 ++++++++++++++++++ docs/upgrade/5.0.0.md | 16 + 14 files changed, 682 insertions(+), 153 deletions(-) create mode 100644 datafusion/distributed/tests/startup_overlap.rs diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 114273638..fe795dbab 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -101,7 +101,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let Ok(url) = Url::parse(&url_str) else { return Err(proto_error("Invalid URL in task")); }; - worker_urls.push(url); + worker_urls.push(url.into()); } Ok(Stage::Remote(RemoteStage { query_id: deserialize_uuid(proto.query_id.as_ref())?, @@ -349,7 +349,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let mut tasks = Vec::with_capacity(remote.workers.len()); for worker in &remote.workers { tasks.push(ExecutionTaskProto { - url_str: Some(worker.to_string()), + url_str: Some(worker.ready_url()?.to_string()), }) } StageProto { diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index 033855e70..cda6cae8d 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -11,8 +11,8 @@ use crate::dynamic_filtering::{ use crate::execution_plans::SamplerExec; use crate::stage::{LocalStage, RemoteStage}; use crate::{ - BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, - NetworkCoalesceExec, Stage, TaskCountAnnotation, + BytesCounterMetric, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, + TaskCountAnnotation, }; use async_trait::async_trait; use dashmap::DashMap; @@ -123,23 +123,13 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); - let mut stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage); - - let mut futures = Vec::with_capacity(input_stage.tasks); - for task_i in 0..input_stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; + let stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage); let mut workers = Vec::with_capacity(input_stage.tasks); - for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() { - workers.push(url); - load_info_rxs.push({ - let rx = stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); - UnboundedReceiverStream::new(rx) - }); - let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; + for task_i in 0..input_stage.tasks { + let (worker_loc, load_info_rx) = stage_coordinator.init_bidirectional_stream(task_i)?; + workers.push(worker_loc); + load_info_rxs.push(UnboundedReceiverStream::new(load_info_rx)); } stage_coordinator.seal_dynamic_filter_stage()?; diff --git a/datafusion/distributed/src/coordinator/prepare_static_plan.rs b/datafusion/distributed/src/coordinator/prepare_static_plan.rs index c0a5e193f..a9c0abbd2 100644 --- a/datafusion/distributed/src/coordinator/prepare_static_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_static_plan.rs @@ -41,20 +41,14 @@ pub(super) async fn prepare_static_plan( vec![] }; - let mut stage_coordinator = query_coordinator.stage_coordinator(stage); - let mut futures = Vec::with_capacity(stage.tasks); - for task_i in 0..stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - + let stage_coordinator = query_coordinator.stage_coordinator(stage); let mut workers = Vec::with_capacity(stage.tasks); - for (task_i, (url, worker_tx, worker_stream)) in results.into_iter().enumerate() { - workers.push(url); - stage_coordinator.worker_to_coordinator_task(task_i, worker_stream); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; + for task_i in 0..stage.tasks { + let (worker_loc, _) = stage_coordinator.init_bidirectional_stream(task_i)?; + workers.push(worker_loc); } stage_coordinator.seal_dynamic_filter_stage()?; + Ok(Transformed::yes(plan.with_input_stage(Stage::Remote( RemoteStage { query_id: stage.query_id, diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index f205a5567..5511bfc00 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -20,7 +20,8 @@ use crate::{ CoordinatorToWorkerDialer, CoordinatorToWorkerMsg, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedGetterExt, DistributedTaskContext, DistributedWorkUnitFeedContext, LoadInfo, LocalWorkerContext, MaybeEncoded, SetPlanRequest, TaskCompletedDynamicFilters, TaskKey, - TaskMetrics, WorkUnitFeedDeclaration, WorkerToCoordinatorMsg, get_distributed_channel_resolver, + TaskMetrics, WorkUnitFeedDeclaration, WorkerLocation, WorkerToCoordinatorMsg, + get_distributed_channel_resolver, }; use async_trait::async_trait; use datafusion::common::instant::Instant; @@ -35,11 +36,9 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStreamBuilder; use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; use datafusion::prelude::SessionConfig; use futures::future::try_join_all; -use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use http::HeaderMap; use std::sync::{Arc, LazyLock, Mutex}; -use tokio::select; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tokio_stream::wrappers::UnboundedReceiverStream; use tokio_util::sync::CancellationToken; @@ -139,21 +138,17 @@ impl<'a> StageCoordinator<'a> { /// Sends a plan to a specific worker and sets up the bidirectional stream. /// /// Its returns are: - /// - The worker URL in which the task got allocated + /// - The worker location in which the task got allocated. /// - The coordinator-to-worker stream for producing messages that reach the worker. /// - The worker-to-coordinator stream for receiving messages from the worker. - pub(super) async fn init_bidirectional_stream( + pub(super) fn init_bidirectional_stream( &self, task_i: usize, - ) -> Result<( - Url, - UnboundedSender, - BoxStream<'static, Result>, - )> { + ) -> Result<(WorkerLocation, UnboundedReceiver)> { let session_config = self.task_ctx.session_config(); let TaskSpecializedPlan { - plan, + plan: task_specialized_plan, work_unit_feed_declarations, dynamic_filter_remote_producer_ids, } = self.task_specialized_plan(task_i)?; @@ -166,7 +161,7 @@ impl<'a> StageCoordinator<'a> { if is_remote_dynamic_filtering_enabled(session_config)? { self.dynamic_filter_registry - .register_task(&plan, task_key)?; + .register_task(&task_specialized_plan, task_key)?; } let mut headers = get_config_extension_propagation_headers(session_config)?; @@ -181,125 +176,156 @@ impl<'a> StageCoordinator<'a> { set_plan_request_template: SetPlanRequest { task_key, task_count: self.task_count, - plan: MaybeEncoded::Decoded(Arc::clone(&plan)), - dynamic_filter_remote_producer_ids, - work_unit_feed_declarations, + plan: MaybeEncoded::Decoded(Arc::clone(&task_specialized_plan)), + dynamic_filter_remote_producer_ids: dynamic_filter_remote_producer_ids.clone(), + work_unit_feed_declarations: work_unit_feed_declarations.clone(), target_worker_url: URL_PLACEHOLDER.clone(), query_start_time_ns: self.metrics.instantiation_time, }, coordinator_to_worker_tx: Mutex::new(None), }; + // The dialer moved the copies above, so the spawned task gets its own. let worker_resolver = session_config.get_distributed_worker_resolver()?; + let task_ctx = Arc::clone(self.task_ctx); + let task_count = self.task_count; + let metrics = self.metrics.clone(); + let metrics_set = self.metrics_set.clone(); + let stage_plan = Arc::clone(self.plan); + let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); + let cancel = self.spawner.query_finished(); - let ev = RouteTaskEvent { - task_ctx: self.task_ctx, - metrics: self.metrics_set, - worker_resolver: worker_resolver.as_ref(), - stage_plan: self.plan, - task_specialized_plan: &plan, - task_key, - task_count: self.task_count, - dialer: &dialer, - }; - - let start = Instant::now(); - let Some(response) = RouteTaskHandlers::handle(ev).await.transpose()? else { - return internal_err!("No RouteTaskHandler returned a response"); - }; - self.metrics.plan_send_latency.record(&start); - - let coordinator_to_worker_tx = dialer.coordinator_to_worker_tx()?; - self.dynamic_filter_registry - .register_sender(task_key, coordinator_to_worker_tx.clone())?; - - Ok(( - response.url, - coordinator_to_worker_tx, - response.worker_to_coordinator_stream, - )) - } - - pub(super) fn seal_dynamic_filter_stage(&self) -> Result<()> { - self.dynamic_filter_registry.seal_stage(self.stage_id) - } - - /// Spawns background tasks in charge of collecting messages sent by a worker. - pub(super) fn worker_to_coordinator_task( - &mut self, - task_i: usize, - mut worker_to_coordinator_stream: BoxStream<'static, Result>, - ) -> UnboundedReceiver { - let task_key = TaskKey { - query_id: self.query_id, - stage_id: self.stage_id, - task_number: task_i, - }; + // Results to be threaded around. + let (worker_location, worker_location_tx) = WorkerLocation::new_pending(); + let (load_info_tx, load_info_rx) = unbounded_channel(); // Completes this task's report waits however the task below ends, even if it's never polled. let mut reports = TaskReports { task_key, metrics_store: self.metrics_store.clone(), completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), }; - let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); - let task_ctx = Arc::clone(self.task_ctx); - let (load_info_tx, load_info_rx) = unbounded_channel(); - let mut load_info_tx_opt = Some(load_info_tx); - - // Read from the worker -> coordinator stream and fail the query on error. - self.spawner.spawn_unbounded(async move { - while let Some(msg) = worker_to_coordinator_stream.try_next().await? { - match msg { - WorkerToCoordinatorMsg::TaskMetrics(metrics) => { - if let Some(store) = reports.metrics_store.take() { - store.insert(task_key, metrics); + + // Spawns the task that establishes the connection to the remote worker and sends message + // to it. + self.spawner.spawn(|spawner| async move { + // The only WorkerLocations that we need resolved at this point are those reachable + // through the current task-specialized plan. We don't care if there's other parts of + // the stage plan that have not resolved yet. + let fut = WorkerLocation::ensure_all_resolved(&task_specialized_plan); + + let Some(_) = cancel.run_until_cancelled(fut).await.transpose()? else { + // Query was canceled, nothing else to do. + return Ok(()); + }; + + let ev = RouteTaskEvent { + task_ctx: &task_ctx, + metrics: &metrics_set, + worker_resolver: worker_resolver.as_ref(), + stage_plan: &stage_plan, + task_specialized_plan: &task_specialized_plan, + task_key, + task_count, + dialer: &dialer, + }; + + let start = Instant::now(); + + let fut = RouteTaskHandlers::handle(ev); + let Some(routing_response) = cancel.run_until_cancelled(fut).await else { + // Query was canceled, nothing else to do. + return Ok(()); + }; + let Some(mut response) = routing_response.transpose()? else { + return internal_err!("No RouteTaskHandler returned a response"); + }; + metrics.plan_send_latency.record(&start); + let _ = worker_location_tx.send(response.url); + + let mut load_info_tx_opt = Some(load_info_tx); + let dynamic_filter_registry_clone = Arc::clone(&dynamic_filter_registry); + let task_ctx_clone = Arc::clone(&task_ctx); + + spawner.spawn_unbounded(async move { + // Read from the worker -> coordinator stream and fail the query on error. + while let Some(msg) = response.worker_to_coordinator_stream.try_next().await? { + match msg { + WorkerToCoordinatorMsg::TaskMetrics(metrics) => { + if let Some(store) = reports.metrics_store.take() { + store.insert(task_key, metrics); + } } - } - WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { - if let Some(store) = reports.completed_dynamic_filter_store.take() { - store.insert(task_key, filters); + WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { + if let Some(store) = reports.completed_dynamic_filter_store.take() { + store.insert(task_key, filters); + } } - } - WorkerToCoordinatorMsg::ProducedDynamicFilter(filter) => { - dynamic_filter_registry - .record_dynamic_filter_update(task_key, *filter, &task_ctx)?; - } - WorkerToCoordinatorMsg::LoadInfo(info) => { - if let Some(tx) = &load_info_tx_opt { - let _ = tx.send(info); + WorkerToCoordinatorMsg::ProducedDynamicFilter(filter) => { + dynamic_filter_registry_clone.record_dynamic_filter_update( + task_key, + *filter, + &task_ctx_clone, + )?; } - } - WorkerToCoordinatorMsg::LoadInfoEos => { - let _ = load_info_tx_opt.take(); - } - }; - } - let _ = load_info_tx_opt.take(); - Ok::<_, DataFusionError>(()) + WorkerToCoordinatorMsg::LoadInfo(info) => { + if let Some(tx) = &load_info_tx_opt { + let _ = tx.send(info); + } + } + WorkerToCoordinatorMsg::LoadInfoEos => { + let _ = load_info_tx_opt.take(); + } + }; + } + Ok::<_, DataFusionError>(()) + }); + + let coordinator_to_worker_tx = dialer.coordinator_to_worker_tx()?; + + dynamic_filter_registry.register_sender(task_key, coordinator_to_worker_tx.clone())?; + + let _ = coordinator_to_worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); + + Self::send_work_unit_feeds( + &task_ctx, + task_i, + task_count, + &stage_plan, + cancel, + coordinator_to_worker_tx, + ) + .await }); - load_info_rx + Ok((worker_location, load_info_rx)) } - /// Spawns a background task in charge of sending messages to workers. Some things that are sent - /// to workers here are: - /// - WorkUnits collected from [WorkUnitFeeds] present in the plan. - pub(super) fn coordinator_to_worker_task( - &mut self, + pub(super) fn seal_dynamic_filter_stage(&self) -> Result<()> { + self.dynamic_filter_registry.seal_stage(self.stage_id) + } + + /// Recurses the plan looking for all the nodes containing WorkUnitFeeds and streams all the + /// WorkUnits for those plans into `tx` until they are exhausted. This function will block up + /// until all WorkUnits have been fully generated and produced into `tx`. + async fn send_work_unit_feeds( + task_ctx: &Arc, task_i: usize, + task_count: usize, + stage_plan: &Arc, + query_finished: CancellationToken, tx: UnboundedSender, ) -> Result<()> { - let session_config = self.task_ctx.session_config(); + let session_config = task_ctx.session_config(); let wuf_registry = session_config .get_extension::() .unwrap_or_default(); let d_ctx = DistributedTaskContext { task_index: task_i, - task_count: self.task_count, + task_count, }; let mut futures = vec![]; - self.plan.apply_with_dt_ctx(d_ctx, |plan, d_ctx| { + stage_plan.apply_with_dt_ctx(d_ctx, |plan, d_ctx| { let Some(wuf) = wuf_registry.get_work_unit_feed(plan) else { return Ok(TreeNodeRecursion::Continue); }; @@ -311,7 +337,7 @@ impl<'a> StageCoordinator<'a> { let dist_feed_ctx = DistributedWorkUnitFeedContext { fan_out_tasks: d_ctx.task_count, }; - let t_ctx = Arc::new(task_ctx_with_extension(self.task_ctx, dist_feed_ctx)); + let t_ctx = Arc::new(task_ctx_with_extension(task_ctx, dist_feed_ctx)); let mut feeds = Vec::with_capacity(end_partition - start_partition); for (partition, feed_idx) in (start_partition..end_partition).enumerate() { @@ -347,16 +373,12 @@ impl<'a> StageCoordinator<'a> { } // Cancel any feeds if the query finishes or aborts. - let query_finished = self.spawner.query_finished(); - self.spawner.spawn(async move { - let _guard = WorkUnitEosOnDrop(tx); - select! { - result = try_join_all(futures) => { result?; } - _ = query_finished.cancelled() => {} - } - Ok(()) - }); - Ok(()) + let _guard = WorkUnitEosOnDrop(tx); + let fut = try_join_all(futures); + let Some(result) = query_finished.run_until_cancelled(fut).await else { + return Ok(()); + }; + result.map(|_| ()) } /// Specializes the [Arc] for this stage to provided task index. This implies @@ -442,7 +464,7 @@ impl<'a> StageCoordinator<'a> { } /// Completes missing report waits when the worker-to-coordinator task ends, including when it -/// fails or when the task is dropped before its first poll. +/// fails, when routing never finishes, or when the task is dropped before its first poll. struct TaskReports { task_key: TaskKey, metrics_store: Option>>, diff --git a/datafusion/distributed/src/coordinator/spawner.rs b/datafusion/distributed/src/coordinator/spawner.rs index de7d31b71..6fc412529 100644 --- a/datafusion/distributed/src/coordinator/spawner.rs +++ b/datafusion/distributed/src/coordinator/spawner.rs @@ -77,9 +77,17 @@ impl Spawner { /// [`Spawner::end_query_guard`] or [`Spawner::query_finished`]. /// /// Otherwise, they may choose to terminate gracefully, blocking the response stream. - pub(super) fn spawn(&self, task: impl Future> + Send + 'static) { + pub(super) fn spawn(&self, task_f: impl FnOnce(Spawner) -> F) + where + F: Future> + Send + 'static, + { + let self_clone = Spawner { + query_finished: self.query_finished.clone(), + task_tx: self.task_tx.clone(), + error_tx: self.error_tx.clone(), + }; // Once the supervisor stops, sending fails and drops the future without running it. - let _ = self.task_tx.send(task.boxed()); + let _ = self.task_tx.send(task_f(self_clone).boxed()); } /// Spawns a task that keeps running if the query fails or is cancelled. diff --git a/datafusion/distributed/src/events/defaults/routing.rs b/datafusion/distributed/src/events/defaults/routing.rs index 795e2ee84..7e8fd27bb 100644 --- a/datafusion/distributed/src/events/defaults/routing.rs +++ b/datafusion/distributed/src/events/defaults/routing.rs @@ -172,8 +172,10 @@ impl RouteTaskHandler for SingleTaskChildUrlRouteTaskHandler { if let Stage::Remote(remote) = nb.input_stage() && remote.workers.len() == 1 + // TODO: we should resolve the URL here. + && let Ok(url) = remote.workers[0].ready_url() { - single_stage_url = Some(remote.workers[0].clone()); + single_stage_url = Some(url.clone()); return Ok(TreeNodeRecursion::Stop); } diff --git a/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs b/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs index 1a9f8b299..ee8e4b5fd 100644 --- a/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs +++ b/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs @@ -216,7 +216,12 @@ impl ShuffleFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_workers.clone(), + workers: self + .input_stage_workers + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs b/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs index 5c624473b..0e8047df6 100644 --- a/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs +++ b/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs @@ -265,7 +265,12 @@ impl TransportFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_tasks.clone(), + workers: self + .input_stage_tasks + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/datafusion/distributed/src/lib.rs b/datafusion/distributed/src/lib.rs index d4d7a401a..a2fce525e 100644 --- a/datafusion/distributed/src/lib.rs +++ b/datafusion/distributed/src/lib.rs @@ -65,7 +65,8 @@ pub use protocol::{ get_distributed_channel_resolver, }; pub use stage::{ - DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, + DistributedTaskContext, Stage, WorkerLocation, display_plan_ascii, display_plan_graphviz, + explain_analyze, }; pub use work_unit_feed::{ DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, WorkUnitFeedProvider, diff --git a/datafusion/distributed/src/stage.rs b/datafusion/distributed/src/stage.rs index 2b4e132a2..a24272a9b 100644 --- a/datafusion/distributed/src/stage.rs +++ b/datafusion/distributed/src/stage.rs @@ -1,7 +1,7 @@ use crate::coordinator::DistributedExec; use crate::execution_plans::{DistributedLeafExec, NetworkCoalesceExec}; use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; -use datafusion::common::{HashMap, Statistics, config_err}; +use datafusion::common::{DataFusionError, HashMap, Statistics, config_err, exec_datafusion_err}; use datafusion::common::{exec_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; @@ -12,8 +12,10 @@ use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext, displayable, }; +use futures::future::{BoxFuture, Shared, try_join_all}; use itertools::Either; use std::collections::VecDeque; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; use url::Url; use uuid::Uuid; @@ -111,14 +113,80 @@ pub struct RemoteStage { pub query_id: Uuid, /// Our stage number pub num: usize, - /// The worker URLs to which queries should be issued. - pub workers: Vec, + /// The worker locations to which queries should be issued. + pub workers: Vec, /// Statistics collected at runtime, if any. pub runtime_stats: Option>, /// Dynamic-filter consumers retained after the stage's plan is moved to its workers. pub dynamic_filter_anchors: Vec>, } +/// A task's selected worker URL or its in-progress routing result. +/// Pending locations stay in the coordinator; worker plans require ready locations. +#[derive(Clone)] +pub struct WorkerLocation(Shared>>>); + +impl WorkerLocation { + pub(crate) fn new_pending() -> (Self, oneshot::Sender) { + let (tx, rx) = oneshot::channel::(); + let fut = async move { + rx.await + .map_err(|err| Arc::new(exec_datafusion_err!("{err}"))) + }; + + (Self(fut.boxed().shared()), tx) + } + + /// Returns a concrete URL, or an error for a location whose routing has not completed yet. + /// A pending location whose routing already finished yields its result without polling. + pub fn ready_url(&self) -> Result<&Url> { + match self.0.peek() { + Some(Ok(url)) => Ok(url), + Some(Err(err)) => Err(DataFusionError::Shared(Arc::clone(err))), + None => exec_err!("Worker location is pending"), + } + } + + pub(crate) async fn ensure_all_resolved(plan: &Arc) -> Result<()> { + let mut remote_locations = vec![]; + let _ = plan.apply(|plan| { + if let Some(nb) = plan.as_network_boundary() + && let Stage::Remote(remote) = nb.input_stage() + { + remote_locations.extend(remote.workers.clone()); + } + Ok(TreeNodeRecursion::Continue) + }); + let futures = remote_locations.into_iter().map(|loc| loc.0.clone()); + try_join_all(futures) + .await + .map_err(DataFusionError::Shared)?; + Ok(()) + } + + pub(crate) async fn resolve(&self) -> Result { + self.0.clone().await.map_err(DataFusionError::Shared) + } +} + +impl From for WorkerLocation { + fn from(url: Url) -> Self { + let fut = futures::future::ready(Ok(url)).boxed().shared(); + // Do a polling cycle to the future so that it shows up as "Ready" + let _ = fut.clone().now_or_never(); + Self(fut) + } +} + +impl Debug for WorkerLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.ready_url() { + Ok(url) => f.debug_tuple("Ready").field(url).finish(), + Err(_) => f.write_str("Pending"), + } + } +} + impl Stage { pub fn query_id(&self) -> Uuid { match &self { @@ -236,9 +304,10 @@ impl DistributedTaskContext { use crate::{DistributedMetricsFormat, NetworkShuffleExec, rewrite_distributed_plan_with_metrics}; use crate::{NetworkBoundary, NetworkBoundaryExt}; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::DataFusionError; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion::physical_expr::Partitioning; +use futures::FutureExt; /// Be able to display a nice tree for stages. /// /// The challenge to doing this at the moment is that `TreeRenderVisitor` @@ -252,6 +321,7 @@ use datafusion::physical_expr::Partitioning; /// In the meantime, we can make a dummy ExecutionPlan that will let us render /// the Stage tree. use std::fmt::Write; +use tokio::sync::oneshot; /// explain_analyze renders an [ExecutionPlan] with metrics. pub async fn explain_analyze( diff --git a/datafusion/distributed/src/worker/worker_connection_pool.rs b/datafusion/distributed/src/worker/worker_connection_pool.rs index 05ea11693..9239c9184 100644 --- a/datafusion/distributed/src/worker/worker_connection_pool.rs +++ b/datafusion/distributed/src/worker/worker_connection_pool.rs @@ -65,7 +65,7 @@ impl WorkerConnectionPool { producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let Some(target_url) = input_stage.workers.get(target_task).cloned() else { + let Some(worker_location) = input_stage.workers.get(target_task).cloned() else { internal_err!("input_stage.workers[{target_task}] out of range.")? }; let task_key = TaskKey { @@ -106,6 +106,7 @@ impl WorkerConnectionPool { // // Note that this does not start polling the returned streams, it just instantiates them. let streams_task = SpawnedTask::spawn(async move { + let target_url = worker_location.resolve().await?; let request = ExecuteTaskRequest { task_key, target_partition_start: target_partitions.start, @@ -341,7 +342,7 @@ mod tests { let stage = RemoteStage { query_id, num: 0, - workers: vec![worker.url.clone()], + workers: vec![worker.url.clone().into()], runtime_stats: None, dynamic_filter_anchors: vec![], }; diff --git a/datafusion/distributed/tests/main.rs b/datafusion/distributed/tests/main.rs index d3e5e896b..030d9bab4 100644 --- a/datafusion/distributed/tests/main.rs +++ b/datafusion/distributed/tests/main.rs @@ -21,6 +21,7 @@ mod multi_task_collect_join_repros; mod passthrough_headers; mod shuffle; mod sorted_data_source; +mod startup_overlap; mod stateful_data_cleanup; mod stateful_execution_plan; mod task_estimator_test; diff --git a/datafusion/distributed/tests/startup_overlap.rs b/datafusion/distributed/tests/startup_overlap.rs new file mode 100644 index 000000000..b6a60ffe4 --- /dev/null +++ b/datafusion/distributed/tests/startup_overlap.rs @@ -0,0 +1,414 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use std::slice::from_ref; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use arrow::array::{Int64Array, RecordBatch}; + use async_trait::async_trait; + use datafusion::common::runtime::SpawnedTask; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::common::{Result, assert_contains, exec_err}; + use datafusion::execution::SessionState; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, + }; + use datafusion_distributed::{ + DistributedExec, DistributedExt, NetworkBoundaryExt, NetworkCoalesceExec, RouteTaskEvent, + RouteTaskEventResponse, RouteTaskHandler, Stage, WorkerQueryContext, + WorkerToCoordinatorMsg, rewrite_distributed_plan_with_dynamic_filters, + }; + use futures::{StreamExt, TryStreamExt, stream}; + use tokio::time::timeout; + use tokio_util::sync::CancellationToken; + + const WAIT: Duration = Duration::from_secs(5); + const SCAN: &str = "SELECT task_index FROM url_emitter(2, 2, 'scan')"; + + #[tokio::test] + async fn static_scan_returns_output_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(from_ref(&first)), vec![0]); + assert!(!gate.release.is_cancelled()); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn adaptive_scan_returns_output_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(from_ref(&first)), vec![0]); + assert!(!gate.release.is_cancelled()); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn adaptive_shuffle_samples_before_sibling_setup() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let query = ctx + .sql("SELECT task_index FROM url_emitter(2, 2, 'scan') GROUP BY task_index") + .await?; + let execution = SpawnedTask::spawn(async move { query.collect().await }); + + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + // A positive LoadInfo report proves the remote sampler has produced rows. + timeout(WAIT, gate.sampled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let batches = timeout(WAIT, execution.join()).await.unwrap().unwrap()?; + assert_eq!(task_indices(&batches), vec![0, 1]); + Ok(()) + } + + #[tokio::test] + async fn late_setup_failure_reaches_output_and_finishes_metrics() -> Result<()> { + let gate = SetupGate { + fail: true, + ..Default::default() + }; + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(&[first]), vec![0]); + gate.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "injected setup failure"); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn worker_response_error_after_output_finishes_reports() -> Result<()> { + let response_error = ResponseErrorGate::default(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_dynamic_filter_collection(true)? + .with_distributed_route_task_handler(response_error.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert!(matches!(task_indices(&[first]).as_slice(), [0] | [1])); + response_error.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "injected worker response error"); + + let distributed = plan.downcast_ref::().unwrap(); + let reports = timeout(WAIT, distributed.wait_for_metrics()) + .await + .unwrap() + .unwrap(); + assert_eq!(reports.len(), 2); + assert!( + reports + .values() + .all(|report| report.pre_order_plan_metrics.is_empty()) + ); + let task_ctx = ctx.task_ctx(); + timeout( + WAIT, + rewrite_distributed_plan_with_dynamic_filters(plan, &task_ctx), + ) + .await + .unwrap()?; + Ok(()) + } + + #[tokio::test] + async fn dropping_output_cancels_pending_setup_and_finishes_metrics() -> Result<()> { + let gate = SetupGate::default(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(task_indices(&[first]), vec![0]); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + drop(output); + timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn limit_completion_cancels_pending_setup_and_finishes_metrics() -> Result<()> { + let gate = SetupGate { + wait_for_sibling: true, + ..Default::default() + }; + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let plan = ctx + .sql(&format!("{SCAN} LIMIT 1")) + .await? + .create_physical_plan() + .await?; + let output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + let batches = timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?; + + assert_eq!(task_indices(&batches), vec![0]); + timeout(WAIT, gate.cancelled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + assert_terminal_metrics(&plan).await; + Ok(()) + } + + #[tokio::test] + async fn adaptive_nested_coalesce_has_concrete_worker_destinations() -> Result<()> { + let gate = SetupGate::default(); + gate.release.cancel(); + let ctx = context(start_in_memory_context(2, build_state).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 2; + let query = ctx + .sql( + "SELECT task_index FROM url_emitter(2, 2, 'scan') WHERE task_index = \ + (SELECT MAX(task_index) FROM url_emitter(2, 2, 'inner'))", + ) + .await?; + + let batches = timeout(WAIT, query.collect()).await.unwrap()?; + assert_eq!(task_indices(&batches), vec![1]); + assert!(gate.nested_coalesces.load(Ordering::Relaxed) > 0); + Ok(()) + } + + fn context(mut ctx: SessionContext) -> SessionContext { + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx + } + + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) + } + + fn task_indices(batches: &[RecordBatch]) -> Vec { + let mut values = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + values.sort_unstable(); + values + } + + async fn assert_terminal_metrics(plan: &Arc) { + let distributed = plan.downcast_ref::().unwrap(); + let reports = timeout(WAIT, distributed.wait_for_metrics()) + .await + .unwrap() + .unwrap(); + assert_eq!(reports.len(), 2); + } + + #[derive(Clone, Default)] + struct ResponseErrorGate { + release: CancellationToken, + } + + #[async_trait] + impl RouteTaskHandler for ResponseErrorGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + Some( + UrlEmitterRouteTaskHandler + .handle(event) + .await? + .map(|mut response| { + let release = self.release.clone(); + let error = stream::once(async move { + release.cancelled().await; + exec_err!("injected worker response error") + }); + response.worker_to_coordinator_stream = + error.chain(response.worker_to_coordinator_stream).boxed(); + response + }), + ) + } + } + + #[derive(Clone, Default)] + struct SetupGate { + entered: CancellationToken, + release: CancellationToken, + cancelled: CancellationToken, + sampled: CancellationToken, + nested_coalesces: Arc, + fail: bool, + wait_for_sibling: bool, + } + + #[async_trait] + impl RouteTaskHandler for SetupGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + Some(self.route(event).await) + } + } + + impl SetupGate { + async fn route(&self, event: RouteTaskEvent<'_>) -> Result { + event.task_specialized_plan.apply(|node| { + if let Some(boundary) = node.as_network_boundary() { + let Stage::Remote(stage) = boundary.input_stage() else { + panic!("worker plans require a remote input stage"); + }; + assert!( + stage + .workers + .iter() + .all(|worker| worker.ready_url().is_ok()) + ); + } + if node.is::() { + self.nested_coalesces.fetch_add(1, Ordering::Relaxed); + } + Ok(TreeNodeRecursion::Continue) + })?; + if event.task_key.task_number == 0 && self.wait_for_sibling { + self.entered.cancelled().await; + } + if event.task_key.task_number == 1 { + let cancelled = self.cancelled.clone().drop_guard(); + self.entered.cancel(); + self.release.cancelled().await; + cancelled.disarm(); + if self.fail { + return exec_err!("injected setup failure"); + } + } + let mut response = UrlEmitterRouteTaskHandler.handle(event).await.unwrap()?; + let sampled = self.sampled.clone(); + response.worker_to_coordinator_stream = response.worker_to_coordinator_stream + .inspect_ok(move |message| { + if matches!(message, WorkerToCoordinatorMsg::LoadInfo(info) if info.rows_ready > 0) { + sampled.cancel(); + } + }).boxed(); + Ok(response) + } + } +} diff --git a/docs/upgrade/5.0.0.md b/docs/upgrade/5.0.0.md index 71ef6d15c..98bc09ae2 100644 --- a/docs/upgrade/5.0.0.md +++ b/docs/upgrade/5.0.0.md @@ -163,3 +163,19 @@ the metrics of nodes that run on workers have no `partition`, and only the `task_id` label for `DistributedMetricsFormat::PerTask`. To keep the previous per-partition metrics, disable it with `with_distributed_aggregate_metrics_on_workers_before_collection(false)`. + +## 8. Remote stage workers are `WorkerLocation`s + +`RemoteStage::workers` is now a `Vec` instead of a `Vec`, +since a task's worker can still be routing on the coordinator while sibling +tasks already run. Use `WorkerLocation::ready_url()` to read the URL; it +returns an error while the task is still being routed. Worker plans, and the +plans seen by a `RouteTaskHandler`, only contain ready locations. To build a +location from a known URL, use `WorkerLocation::from(url)`. + +```rust +let Stage::Remote(stage) = boundary.input_stage() else { return }; +for worker in &stage.workers { + let url: &Url = worker.ready_url()?; +} +```