diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 85cd9b623..37272dc13 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -99,7 +99,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let Ok(url) = Url::parse(&url_str) else { return Err(proto_error("Invalid URL in task")); }; - worker_urls.push(url); + worker_urls.push(url.into()); } Ok(Stage::Remote(RemoteStage { query_id: deserialize_uuid(proto.query_id.as_ref())?, @@ -346,8 +346,11 @@ impl PhysicalExtensionCodec for DistributedCodec { Stage::Remote(remote) => { let mut tasks = Vec::with_capacity(remote.workers.len()); for worker in &remote.workers { + let url = worker.ready_url().ok_or_else(|| { + proto_error("Worker location must be resolved before encoding") + })?; tasks.push(ExecutionTaskProto { - url_str: Some(worker.to_string()), + url_str: Some(url.to_string()), }) } StageProto { diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index f29fa8d01..a0ff76a52 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -11,8 +11,8 @@ use crate::dynamic_filtering::{ use crate::execution_plans::SamplerExec; use crate::stage::{LocalStage, RemoteStage}; use crate::{ - BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, - NetworkCoalesceExec, Stage, TaskCountAnnotation, + BytesCounterMetric, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, + TaskCountAnnotation, }; use async_trait::async_trait; use dashmap::DashMap; @@ -118,33 +118,19 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { vec![] }; - let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); - - let mut stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage); - - let mut futures = Vec::with_capacity(input_stage.tasks); - for task_i in 0..input_stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(input_stage.tasks); - for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() { - workers.push(url); - load_info_rxs.push({ - let rx = stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); - UnboundedReceiverStream::new(rx) - }); - let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; - } - stage_coordinator.seal_dynamic_filter_stage()?; + let (workers, load_info_rxs) = self.query_coordinator.start_stage(&input_stage); let (stats, consumer_tc) = if nb_type == TypeId::of::() { (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) } else { - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; + let (stats, new_metrics) = gather_runtime_statistics( + load_info_rxs + .into_iter() + .map(UnboundedReceiverStream::new) + .collect(), + &input_stage.plan, + ) + .await?; metrics.extend(new_metrics); // returning Desired(0) here is our way to tell the planner that we don't care // about the task count assigned to the network boundary in the consumer stage, diff --git a/datafusion/distributed/src/coordinator/prepare_static_plan.rs b/datafusion/distributed/src/coordinator/prepare_static_plan.rs index c0a5e193f..6f69ab829 100644 --- a/datafusion/distributed/src/coordinator/prepare_static_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_static_plan.rs @@ -21,8 +21,12 @@ use std::sync::Arc; /// become nodes without children and traversing them will not go further down in. /// 4. Spawn a background task per worker that waits for the worker to finish and collects /// its metrics into [DistributedExec::task_metrics] via the coordinator channel. +/// +/// Setup is scheduled independently for each task. The returned remote stages contain pending +/// locations; a consumer waits only for the task it reads. A plan sent to another worker must +/// first resolve the input addresses it includes. pub(super) async fn prepare_static_plan( - query_coordinator: &QueryCoordinator, + query_coordinator: &Arc, base_plan: &Arc, ) -> Result { let prepared = Arc::clone(base_plan).transform_up_async(async |plan| { @@ -41,20 +45,7 @@ pub(super) async fn prepare_static_plan( vec![] }; - let mut stage_coordinator = query_coordinator.stage_coordinator(stage); - let mut futures = Vec::with_capacity(stage.tasks); - for task_i in 0..stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(stage.tasks); - for (task_i, (url, worker_tx, worker_stream)) in results.into_iter().enumerate() { - workers.push(url); - stage_coordinator.worker_to_coordinator_task(task_i, worker_stream); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; - } - stage_coordinator.seal_dynamic_filter_stage()?; + let (workers, _) = query_coordinator.start_stage(stage); Ok(Transformed::yes(plan.with_input_stage(Stage::Remote( RemoteStage { query_id: stage.query_id, diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index 8274259ec..2f6681975 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -21,8 +21,9 @@ use crate::work_unit_feed::{build_work_unit_batch_msg, set_work_unit_send_time}; use crate::{ CoordinatorToWorkerMsg, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedGetterExt, DistributedTaskContext, DistributedWorkUnitFeedContext, LoadInfo, LocalWorkerContext, - MaybeEncoded, SetPlanRequest, TaskCompletedDynamicFilters, TaskKey, TaskMetrics, - WorkUnitFeedDeclaration, WorkerToCoordinatorMsg, get_distributed_channel_resolver, + MaybeEncoded, NetworkBoundaryExt, SetPlanRequest, Stage, TaskCompletedDynamicFilters, TaskKey, + TaskMetrics, WorkUnitFeedDeclaration, WorkerLocation, WorkerToCoordinatorMsg, + get_distributed_channel_resolver, }; use datafusion::common::Result; use datafusion::common::instant::Instant; @@ -36,11 +37,12 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStreamBuilder; use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; use datafusion::prelude::SessionConfig; use futures::future::try_join_all; -use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; +use futures::stream::{BoxStream, empty}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use std::sync::{Arc, Mutex}; use tokio::select; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio::sync::oneshot; use tokio_stream::wrappers::UnboundedReceiverStream; use url::Url; use uuid::Uuid; @@ -110,6 +112,75 @@ impl QueryCoordinator { pub(super) fn session_config(&self) -> &SessionConfig { self.task_ctx.session_config() } + + /// Starts every task independently, using the same lifecycle for all stages. + /// + /// Each reader drives its task's setup before collecting worker messages. Cancellation + /// stops pending setup, while connected streams drain their final reports as usual. + /// Locations retain only receivers, so cached plans cannot keep setup running. + /// Successful tasks can be consumed immediately, while stage sealing waits for all tasks. + /// Worker plans still wait for the input addresses they must serialize before routing. + pub(super) fn start_stage( + self: &Arc, + stage: &LocalStage, + ) -> (Vec, Vec>) { + let mut workers = Vec::with_capacity(stage.tasks); + let mut load_info = Vec::with_capacity(stage.tasks); + let mut stage_coordinator = self.stage_coordinator(stage); + for task in 0..stage.tasks { + let (location_tx, location_rx) = oneshot::channel(); + workers.push(WorkerLocation::pending(location_rx)); + let coordinator = Arc::clone(self); + let stage = stage.clone(); + let finished = self.spawner.query_finished(); + let stream = async move { + let mut stage_coordinator = coordinator.stage_coordinator(&stage); + let result = select! { + biased; + _ = finished.cancelled() => return empty().boxed(), + result = stage_coordinator.init_bidirectional_stream(task) => result, + }; + match result { + Ok((url, tx, stream)) => { + let location = stage_coordinator + .coordinator_to_worker_task(task, tx.clone()) + .map(|()| { + let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling); + url + }); + let _ = location_tx.send(location.map_err(Arc::new)); + stream + } + Err(error) => { + // The stage supervisor propagates the location error. End the reader + // normally so it completes any missing final reports. + let _ = location_tx.send(Err(Arc::new(error))); + empty().boxed() + } + } + } + .flatten_stream() + .boxed(); + load_info.push(stage_coordinator.worker_to_coordinator_task(task, stream)); + } + + let coordinator = Arc::clone(self); + let stage = stage.clone(); + let locations = workers.clone(); + let finished = self.spawner.query_finished(); + self.spawner.spawn(async move { + select! { + biased; + _ = finished.cancelled() => {} + result = try_join_all(locations.iter().map(WorkerLocation::resolve)) => { + result?; + coordinator.stage_coordinator(&stage).seal_dynamic_filter_stage()?; + } + } + Ok(()) + }); + (workers, load_info) + } } /// Manages all the coordinator->worker and worker->coordinator comms that happen during the @@ -155,7 +226,7 @@ impl<'a> StageCoordinator<'a> { plan, work_unit_feed_declarations, dynamic_filter_remote_producer_ids, - } = self.task_specialized_plan(task_i)?; + } = self.task_specialized_plan(task_i).await?; let task_key = TaskKey { query_id: self.query_id, @@ -421,7 +492,7 @@ impl<'a> StageCoordinator<'a> { /// trimming down any unnecessary information that the specific `task_i` task is not going to /// need, like unexecuted branches in [ChildrenIsolatorUnionExec], or unexecuted variants of /// [DistributedLeafExec]. - fn task_specialized_plan(&self, task_i: usize) -> Result { + async fn task_specialized_plan(&self, task_i: usize) -> Result { let session_config = self.task_ctx.session_config(); let wuf_registry = session_config .get_extension::() @@ -436,7 +507,13 @@ impl<'a> StageCoordinator<'a> { }; let plan = Arc::clone(self.plan); + let mut input_locations = vec![]; let transformed = plan.transform_down_with_dt_ctx(d_ctx, |plan, d_ctx| { + if let Some(boundary) = plan.as_network_boundary() + && let Stage::Remote(stage) = boundary.input_stage() + { + input_locations.extend(stage.workers.iter().cloned()); + } if let Some(wuf) = wuf_registry.get_work_unit_feed(&plan) { work_unit_feed_declarations.push(WorkUnitFeedDeclaration { id: wuf.id(), @@ -477,6 +554,9 @@ impl<'a> StageCoordinator<'a> { // we are explicitly not retransforming the entire plan. if other operators cause shared state errors they will error out. Ok(Transformed::no(plan)) })?; + // Worker plans carry concrete input addresses. Resolve them during the existing + // specialization pass, after unused task branches have been removed. + try_join_all(input_locations.iter().map(WorkerLocation::resolve)).await?; let plan = if dynamic_filtering_enabled { maybe_roundtrip_plan_to_sever_in_memory_dynamic_filter_relationships( Arc::clone(&transformed.data), diff --git a/datafusion/distributed/src/events/defaults/routing.rs b/datafusion/distributed/src/events/defaults/routing.rs index 795e2ee84..44846fab0 100644 --- a/datafusion/distributed/src/events/defaults/routing.rs +++ b/datafusion/distributed/src/events/defaults/routing.rs @@ -173,7 +173,7 @@ impl RouteTaskHandler for SingleTaskChildUrlRouteTaskHandler { if let Stage::Remote(remote) = nb.input_stage() && remote.workers.len() == 1 { - single_stage_url = Some(remote.workers[0].clone()); + single_stage_url = remote.workers[0].ready_url().cloned(); return Ok(TreeNodeRecursion::Stop); } diff --git a/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs b/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs index 1a9f8b299..ee8e4b5fd 100644 --- a/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs +++ b/datafusion/distributed/src/execution_plans/benchmarks/shuffle_bench.rs @@ -216,7 +216,12 @@ impl ShuffleFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_workers.clone(), + workers: self + .input_stage_workers + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs b/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs index 5c624473b..0e8047df6 100644 --- a/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs +++ b/datafusion/distributed/src/execution_plans/benchmarks/transport_bench.rs @@ -265,7 +265,12 @@ impl TransportFixture { let input_stage = Stage::Remote(RemoteStage { query_id, num: 0, - workers: self.input_stage_tasks.clone(), + workers: self + .input_stage_tasks + .iter() + .cloned() + .map(Into::into) + .collect(), runtime_stats: None, dynamic_filter_anchors: vec![], }); diff --git a/datafusion/distributed/src/lib.rs b/datafusion/distributed/src/lib.rs index dbdc91ea6..eea4cf88e 100644 --- a/datafusion/distributed/src/lib.rs +++ b/datafusion/distributed/src/lib.rs @@ -64,7 +64,8 @@ pub use protocol::{ get_distributed_channel_resolver, }; pub use stage::{ - DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, + DistributedTaskContext, Stage, WorkerLocation, display_plan_ascii, display_plan_graphviz, + explain_analyze, }; pub use work_unit_feed::{ DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, WorkUnitFeedProvider, diff --git a/datafusion/distributed/src/stage.rs b/datafusion/distributed/src/stage.rs index 17132c1dc..058b3e4e7 100644 --- a/datafusion/distributed/src/stage.rs +++ b/datafusion/distributed/src/stage.rs @@ -2,7 +2,7 @@ use crate::coordinator::DistributedExec; use crate::execution_plans::{DistributedLeafExec, NetworkCoalesceExec}; use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; use datafusion::common::{HashMap, Statistics, config_err}; -use datafusion::common::{exec_err, plan_err}; +use datafusion::common::{exec_datafusion_err, exec_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; @@ -12,12 +12,62 @@ use datafusion::physical_plan::{ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext, displayable, }; +use futures::FutureExt; +use futures::future::Shared; use itertools::Either; use std::collections::VecDeque; use std::sync::Arc; +use tokio::sync::oneshot; use url::Url; use uuid::Uuid; +/// The eventual committed address of a worker whose task may still be setting up. +/// +/// Routing may try several workers before committing to one. Consumers await only the +/// location they need; plans sent to another worker must first resolve all their inputs. +#[derive(Debug, Clone)] +pub enum WorkerLocation { + /// An address obtained from a decoded plan or supplied by the caller. + Ready(Url), + /// The result of query-owned routing and plan delivery. + Pending(Shared>>>), +} + +impl WorkerLocation { + pub(crate) fn pending(rx: oneshot::Receiver>>) -> Self { + Self::Pending(rx.shared()) + } + + /// Waits for successful routing, retaining the original setup error on failure. + pub async fn resolve(&self) -> Result { + match self { + Self::Ready(url) => Ok(url.clone()), + Self::Pending(url) => url + .clone() + .await + .map_err(|_| exec_datafusion_err!("Task setup was cancelled"))? + .map_err(DataFusionError::Shared), + } + } + + /// Returns a location that has already been resolved, without waiting for setup. + pub fn ready_url(&self) -> Option<&Url> { + match self { + Self::Ready(url) => Some(url), + Self::Pending(url) => match url.peek() { + Some(Ok(Ok(url))) => Some(url), + _ => None, + }, + } + } +} + +impl From for WorkerLocation { + fn from(url: Url) -> Self { + Self::Ready(url) + } +} + /// A unit of isolation for a portion of a physical execution plan /// that can be executed independently and across a network boundary. /// It implements [`ExecutionPlan`] and can be executed to produce a @@ -112,7 +162,8 @@ pub struct RemoteStage { /// Our stage number pub num: usize, /// The worker URLs to which queries should be issued. - pub workers: Vec, + /// Locations are in task-number order and may still be awaiting setup. + pub workers: Vec, /// Statistics collected at runtime, if any. pub runtime_stats: Option>, /// Dynamic-filter consumers retained after the stage's plan is moved to its workers. diff --git a/datafusion/distributed/src/worker/worker_connection_pool.rs b/datafusion/distributed/src/worker/worker_connection_pool.rs index 05ea11693..9239c9184 100644 --- a/datafusion/distributed/src/worker/worker_connection_pool.rs +++ b/datafusion/distributed/src/worker/worker_connection_pool.rs @@ -65,7 +65,7 @@ impl WorkerConnectionPool { producer_head: ProducerHead, ctx: &Arc, ) -> Result>> { - let Some(target_url) = input_stage.workers.get(target_task).cloned() else { + let Some(worker_location) = input_stage.workers.get(target_task).cloned() else { internal_err!("input_stage.workers[{target_task}] out of range.")? }; let task_key = TaskKey { @@ -106,6 +106,7 @@ impl WorkerConnectionPool { // // Note that this does not start polling the returned streams, it just instantiates them. let streams_task = SpawnedTask::spawn(async move { + let target_url = worker_location.resolve().await?; let request = ExecuteTaskRequest { task_key, target_partition_start: target_partitions.start, @@ -341,7 +342,7 @@ mod tests { let stage = RemoteStage { query_id, num: 0, - workers: vec![worker.url.clone()], + workers: vec![worker.url.clone().into()], runtime_stats: None, dynamic_filter_anchors: vec![], }; diff --git a/datafusion/distributed/tests/main.rs b/datafusion/distributed/tests/main.rs index bfe99619a..33a6f86bd 100644 --- a/datafusion/distributed/tests/main.rs +++ b/datafusion/distributed/tests/main.rs @@ -25,6 +25,7 @@ mod stateful_data_cleanup; mod stateful_execution_plan; mod task_estimator_test; mod task_finished_handler; +mod task_startup; mod tpcds_correctness_test; mod tpcds_plans_test; mod tpch_correctness_test; diff --git a/datafusion/distributed/tests/task_startup.rs b/datafusion/distributed/tests/task_startup.rs new file mode 100644 index 000000000..107775700 --- /dev/null +++ b/datafusion/distributed/tests/task_startup.rs @@ -0,0 +1,286 @@ +#[cfg(test)] +mod tests { + use arrow::array::Int64Array; + use async_trait::async_trait; + use datafusion::common::runtime::SpawnedTask; + use datafusion::common::{Result, assert_contains, exec_err}; + use datafusion::execution::SessionState; + use datafusion::physical_plan::execute_stream; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, + }; + use datafusion_distributed::{ + DistributedExt, DistributedMetricsFormat, RouteTaskEvent, RouteTaskEventResponse, + RouteTaskHandler, WorkerQueryContext, WorkerToCoordinatorMsg, + rewrite_distributed_plan_with_metrics, + }; + use futures::{StreamExt, TryStreamExt}; + use std::sync::Arc; + use std::time::Duration; + use tokio::time::timeout; + use tokio_util::sync::CancellationToken; + + const WAIT: Duration = Duration::from_secs(5); + const SCAN: &str = "SELECT task_index FROM url_emitter(2, 2, 'startup')"; + + #[tokio::test] + async fn static_task_produces_output_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let rest = output.try_collect::>().await?; + assert_eq!(rest.iter().map(|batch| batch.num_rows()).sum::(), 1); + Ok(()) + } + + #[tokio::test] + async fn adaptive_task_produces_output_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 0 + ); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let rest = output.try_collect::>().await?; + assert_eq!(rest.iter().map(|batch| batch.num_rows()).sum::(), 1); + Ok(()) + } + + #[tokio::test] + async fn adaptive_task_samples_before_sibling_setup() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let query = ctx + .sql("SELECT task_index FROM url_emitter(2, 2, 'startup') GROUP BY task_index") + .await?; + let execution = SpawnedTask::spawn(async move { query.collect().await }); + + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + timeout(WAIT, gate.sampled.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + gate.release.cancel(); + let batches = timeout(WAIT, execution).await.unwrap().unwrap()?; + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn dropping_output_cancels_unstarted_tasks_and_completes_reports() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(true)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let plan = ctx.sql(SCAN).await?.create_physical_plan().await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + drop(output); + + timeout(WAIT, gate.exited.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + timeout( + WAIT, + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated), + ) + .await + .unwrap()?; + Ok(()) + } + + #[tokio::test] + async fn limit_cancels_unstarted_tasks_and_completes_reports() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_metrics_collection(true)? + .with_distributed_route_task_handler(gate.clone()); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let plan = ctx + .sql(&format!("{SCAN} LIMIT 1")) + .await? + .create_physical_plan() + .await?; + let output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + let batches = timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?; + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + timeout(WAIT, gate.exited.cancelled()).await.unwrap(); + assert!(!gate.release.is_cancelled()); + timeout( + WAIT, + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated), + ) + .await + .unwrap()?; + Ok(()) + } + + #[tokio::test] + async fn late_setup_error_reaches_the_query() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(FailSecondTask(gate.clone())); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let mut output = ctx.sql(SCAN).await?.execute_stream().await?; + + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + assert_eq!(first.num_rows(), 1); + gate.release.cancel(); + let error = timeout(WAIT, output.try_collect::>()) + .await + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "second task setup failed"); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn setup_error_survives_parent_stage_cancellation() -> Result<()> { + let gate = TaskGate::default(); + let ctx = with_emitter(start_in_memory_context(2, build_worker).await) + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(FailSecondTask(gate.clone())); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await?; + let execution = SpawnedTask::spawn(async move { + ctx.sql("SELECT count(*) FROM url_emitter(2, 2, 'startup') GROUP BY task_index") + .await? + .collect() + .await + }); + timeout(WAIT, gate.entered.cancelled()).await.unwrap(); + gate.release.cancel(); + let error = timeout(WAIT, execution) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_contains!(error.to_string(), "second task setup failed"); + Ok(()) + } + + fn with_emitter(mut ctx: SessionContext) -> SessionContext { + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx + } + + async fn build_worker(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) + } + + #[derive(Clone, Default)] + struct TaskGate { + entered: CancellationToken, + release: CancellationToken, + exited: CancellationToken, + sampled: CancellationToken, + } + + #[async_trait] + impl RouteTaskHandler for TaskGate { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + // Make the stalled routing attempt observable before the ready task produces data. + if event.task_key.task_number == 0 { + self.entered.cancelled().await; + } + if event.task_key.task_number == 1 { + let _exited = self.exited.clone().drop_guard(); + self.entered.cancel(); + self.release.cancelled().await; + } + let sampled = self.sampled.clone(); + let response = UrlEmitterRouteTaskHandler.handle(event).await?; + Some(response.map(|mut response| { + response.worker_to_coordinator_stream = response.worker_to_coordinator_stream + .inspect_ok(move |message| { + if matches!(message, WorkerToCoordinatorMsg::LoadInfo(info) if info.rows_ready > 0) { + sampled.cancel(); + } + }) + .boxed(); + response + })) + } + } + + struct FailSecondTask(TaskGate); + + #[async_trait] + impl RouteTaskHandler for FailSecondTask { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + if event.task_key.task_number == 1 { + self.0.entered.cancel(); + self.0.release.cancelled().await; + return Some(exec_err!("second task setup failed")); + } + UrlEmitterRouteTaskHandler.handle(event).await + } + } +} diff --git a/docs/upgrade/5.0.0.md b/docs/upgrade/5.0.0.md index 71ef6d15c..bb940e46e 100644 --- a/docs/upgrade/5.0.0.md +++ b/docs/upgrade/5.0.0.md @@ -79,6 +79,12 @@ let state = SessionStateBuilder::new() coordinator-to-worker connection is currently established. Implementations can choose to retry on other worker if the current one is not available. +### `RemoteStage::workers` now holds `WorkerLocation` values + +Convert an existing `Vec` with `urls.into_iter().map(Into::into).collect()`. +Use `worker_location.resolve().await?` to obtain its committed URL, or +`worker_location.ready_url()` to inspect a location without waiting. + ## 2. `WorkerPlanRewriteHandler::rewrite_worker_plan` renamed to `handle` and is now async `WorkerPlanRewriteHandler::rewrite_worker_plan` has been renamed to `handle` to