Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions datafusion/distributed/src/codec/distributed_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ impl PhysicalExtensionCodec for DistributedCodec {
let Ok(url) = Url::parse(&url_str) else {
return Err(proto_error("Invalid URL in task"));
};
worker_urls.push(url);
worker_urls.push(url.into());
}
Ok(Stage::Remote(RemoteStage {
query_id: deserialize_uuid(proto.query_id.as_ref())?,
Expand Down Expand Up @@ -346,8 +346,11 @@ impl PhysicalExtensionCodec for DistributedCodec {
Stage::Remote(remote) => {
let mut tasks = Vec::with_capacity(remote.workers.len());
for worker in &remote.workers {
let url = worker.ready_url().ok_or_else(|| {
proto_error("Worker location must be resolved before encoding")
})?;
tasks.push(ExecutionTaskProto {
url_str: Some(worker.to_string()),
url_str: Some(url.to_string()),
})
}
StageProto {
Expand Down
36 changes: 11 additions & 25 deletions datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ use crate::dynamic_filtering::{
use crate::execution_plans::SamplerExec;
use crate::stage::{LocalStage, RemoteStage};
use crate::{
BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt,
NetworkCoalesceExec, Stage, TaskCountAnnotation,
BytesCounterMetric, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage,
TaskCountAnnotation,
};
use async_trait::async_trait;
use dashmap::DashMap;
Expand Down Expand Up @@ -118,33 +118,19 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> {
vec![]
};

let mut load_info_rxs = Vec::with_capacity(input_stage.tasks);

let mut stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage);

let mut futures = Vec::with_capacity(input_stage.tasks);
for task_i in 0..input_stage.tasks {
futures.push(stage_coordinator.init_bidirectional_stream(task_i));
}
let results = futures::future::try_join_all(futures).await?;

let mut workers = Vec::with_capacity(input_stage.tasks);
for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() {
workers.push(url);
load_info_rxs.push({
let rx = stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream);
UnboundedReceiverStream::new(rx)
});
let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling);
stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?;
}
stage_coordinator.seal_dynamic_filter_stage()?;
let (workers, load_info_rxs) = self.query_coordinator.start_stage(&input_stage);

let (stats, consumer_tc) = if nb_type == TypeId::of::<NetworkCoalesceExec>() {
(None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0))
} else {
let (stats, new_metrics) =
gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?;
let (stats, new_metrics) = gather_runtime_statistics(
load_info_rxs
.into_iter()
.map(UnboundedReceiverStream::new)
.collect(),
&input_stage.plan,
)
.await?;
metrics.extend(new_metrics);
// returning Desired(0) here is our way to tell the planner that we don't care
// about the task count assigned to the network boundary in the consumer stage,
Expand Down
21 changes: 6 additions & 15 deletions datafusion/distributed/src/coordinator/prepare_static_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,12 @@ use std::sync::Arc;
/// become nodes without children and traversing them will not go further down in.
/// 4. Spawn a background task per worker that waits for the worker to finish and collects
/// its metrics into [DistributedExec::task_metrics] via the coordinator channel.
///
/// Setup is scheduled independently for each task. The returned remote stages contain pending
/// locations; a consumer waits only for the task it reads. A plan sent to another worker must
/// first resolve the input addresses it includes.
pub(super) async fn prepare_static_plan(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the deletion of this doc comment intentional? I do think those docs are still relevant.

query_coordinator: &QueryCoordinator,
query_coordinator: &Arc<QueryCoordinator>,
base_plan: &Arc<dyn ExecutionPlan>,
) -> Result<PreparedPlan> {
let prepared = Arc::clone(base_plan).transform_up_async(async |plan| {
Expand All @@ -41,20 +45,7 @@ pub(super) async fn prepare_static_plan(
vec![]
};

let mut stage_coordinator = query_coordinator.stage_coordinator(stage);
let mut futures = Vec::with_capacity(stage.tasks);
for task_i in 0..stage.tasks {
futures.push(stage_coordinator.init_bidirectional_stream(task_i));
}
let results = futures::future::try_join_all(futures).await?;

let mut workers = Vec::with_capacity(stage.tasks);
for (task_i, (url, worker_tx, worker_stream)) in results.into_iter().enumerate() {
workers.push(url);
stage_coordinator.worker_to_coordinator_task(task_i, worker_stream);
stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?;
}
stage_coordinator.seal_dynamic_filter_stage()?;
let (workers, _) = query_coordinator.start_stage(stage);
Ok(Transformed::yes(plan.with_input_stage(Stage::Remote(
RemoteStage {
query_id: stage.query_id,
Expand Down
92 changes: 86 additions & 6 deletions datafusion/distributed/src/coordinator/query_coordinator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ use crate::work_unit_feed::{build_work_unit_batch_msg, set_work_unit_send_time};
use crate::{
CoordinatorToWorkerMsg, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedGetterExt,
DistributedTaskContext, DistributedWorkUnitFeedContext, LoadInfo, LocalWorkerContext,
MaybeEncoded, SetPlanRequest, TaskCompletedDynamicFilters, TaskKey, TaskMetrics,
WorkUnitFeedDeclaration, WorkerToCoordinatorMsg, get_distributed_channel_resolver,
MaybeEncoded, NetworkBoundaryExt, SetPlanRequest, Stage, TaskCompletedDynamicFilters, TaskKey,
TaskMetrics, WorkUnitFeedDeclaration, WorkerLocation, WorkerToCoordinatorMsg,
get_distributed_channel_resolver,
};
use datafusion::common::Result;
use datafusion::common::instant::Instant;
Expand All @@ -36,11 +37,12 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStreamBuilder;
use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions};
use datafusion::prelude::SessionConfig;
use futures::future::try_join_all;
use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use futures::stream::{BoxStream, empty};
use futures::{FutureExt, StreamExt, TryStreamExt};
use std::sync::{Arc, Mutex};
use tokio::select;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel};
use tokio::sync::oneshot;
use tokio_stream::wrappers::UnboundedReceiverStream;
use url::Url;
use uuid::Uuid;
Expand Down Expand Up @@ -110,6 +112,75 @@ impl QueryCoordinator {
pub(super) fn session_config(&self) -> &SessionConfig {
self.task_ctx.session_config()
}

/// Starts every task independently, using the same lifecycle for all stages.
///
/// Each reader drives its task's setup before collecting worker messages. Cancellation
/// stops pending setup, while connected streams drain their final reports as usual.
/// Locations retain only receivers, so cached plans cannot keep setup running.
/// Successful tasks can be consumed immediately, while stage sealing waits for all tasks.
/// Worker plans still wait for the input addresses they must serialize before routing.
pub(super) fn start_stage(
self: &Arc<Self>,
stage: &LocalStage,
) -> (Vec<WorkerLocation>, Vec<UnboundedReceiver<LoadInfo>>) {
let mut workers = Vec::with_capacity(stage.tasks);
let mut load_info = Vec::with_capacity(stage.tasks);
let mut stage_coordinator = self.stage_coordinator(stage);
for task in 0..stage.tasks {
let (location_tx, location_rx) = oneshot::channel();
workers.push(WorkerLocation::pending(location_rx));
let coordinator = Arc::clone(self);
let stage = stage.clone();
let finished = self.spawner.query_finished();
let stream = async move {
let mut stage_coordinator = coordinator.stage_coordinator(&stage);
let result = select! {
biased;
_ = finished.cancelled() => return empty().boxed(),
result = stage_coordinator.init_bidirectional_stream(task) => result,
};
match result {
Ok((url, tx, stream)) => {
let location = stage_coordinator
.coordinator_to_worker_task(task, tx.clone())
.map(|()| {
let _ = tx.send(CoordinatorToWorkerMsg::KickOffSampling);
url
});
let _ = location_tx.send(location.map_err(Arc::new));
stream
}
Err(error) => {
// The stage supervisor propagates the location error. End the reader
// normally so it completes any missing final reports.
let _ = location_tx.send(Err(Arc::new(error)));
empty().boxed()
}
}
}
.flatten_stream()
.boxed();
load_info.push(stage_coordinator.worker_to_coordinator_task(task, stream));
}

let coordinator = Arc::clone(self);
let stage = stage.clone();
let locations = workers.clone();
let finished = self.spawner.query_finished();
self.spawner.spawn(async move {
select! {
biased;
_ = finished.cancelled() => {}
result = try_join_all(locations.iter().map(WorkerLocation::resolve)) => {
result?;
coordinator.stage_coordinator(&stage).seal_dynamic_filter_stage()?;
}
}
Ok(())
});
(workers, load_info)
}
}

/// Manages all the coordinator->worker and worker->coordinator comms that happen during the
Expand Down Expand Up @@ -155,7 +226,7 @@ impl<'a> StageCoordinator<'a> {
plan,
work_unit_feed_declarations,
dynamic_filter_remote_producer_ids,
} = self.task_specialized_plan(task_i)?;
} = self.task_specialized_plan(task_i).await?;

let task_key = TaskKey {
query_id: self.query_id,
Expand Down Expand Up @@ -421,7 +492,7 @@ impl<'a> StageCoordinator<'a> {
/// trimming down any unnecessary information that the specific `task_i` task is not going to
/// need, like unexecuted branches in [ChildrenIsolatorUnionExec], or unexecuted variants of
/// [DistributedLeafExec].
fn task_specialized_plan(&self, task_i: usize) -> Result<TaskSpecializedPlan> {
async fn task_specialized_plan(&self, task_i: usize) -> Result<TaskSpecializedPlan> {
let session_config = self.task_ctx.session_config();
let wuf_registry = session_config
.get_extension::<WorkUnitFeedRegistry>()
Expand All @@ -436,7 +507,13 @@ impl<'a> StageCoordinator<'a> {
};

let plan = Arc::clone(self.plan);
let mut input_locations = vec![];
let transformed = plan.transform_down_with_dt_ctx(d_ctx, |plan, d_ctx| {
if let Some(boundary) = plan.as_network_boundary()
&& let Stage::Remote(stage) = boundary.input_stage()
{
input_locations.extend(stage.workers.iter().cloned());
}
if let Some(wuf) = wuf_registry.get_work_unit_feed(&plan) {
work_unit_feed_declarations.push(WorkUnitFeedDeclaration {
id: wuf.id(),
Expand Down Expand Up @@ -477,6 +554,9 @@ impl<'a> StageCoordinator<'a> {
// we are explicitly not retransforming the entire plan. if other operators cause shared state errors they will error out.
Ok(Transformed::no(plan))
})?;
// Worker plans carry concrete input addresses. Resolve them during the existing
// specialization pass, after unused task branches have been removed.
try_join_all(input_locations.iter().map(WorkerLocation::resolve)).await?;
let plan = if dynamic_filtering_enabled {
maybe_roundtrip_plan_to_sever_in_memory_dynamic_filter_relationships(
Arc::clone(&transformed.data),
Expand Down
2 changes: 1 addition & 1 deletion datafusion/distributed/src/events/defaults/routing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ impl RouteTaskHandler for SingleTaskChildUrlRouteTaskHandler {
if let Stage::Remote(remote) = nb.input_stage()
&& remote.workers.len() == 1
{
single_stage_url = Some(remote.workers[0].clone());
single_stage_url = remote.workers[0].ready_url().cloned();
return Ok(TreeNodeRecursion::Stop);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,12 @@ impl ShuffleFixture {
let input_stage = Stage::Remote(RemoteStage {
query_id,
num: 0,
workers: self.input_stage_workers.clone(),
workers: self
.input_stage_workers
.iter()
.cloned()
.map(Into::into)
.collect(),
runtime_stats: None,
dynamic_filter_anchors: vec![],
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,12 @@ impl TransportFixture {
let input_stage = Stage::Remote(RemoteStage {
query_id,
num: 0,
workers: self.input_stage_tasks.clone(),
workers: self
.input_stage_tasks
.iter()
.cloned()
.map(Into::into)
.collect(),
runtime_stats: None,
dynamic_filter_anchors: vec![],
});
Expand Down
3 changes: 2 additions & 1 deletion datafusion/distributed/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ pub use protocol::{
get_distributed_channel_resolver,
};
pub use stage::{
DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze,
DistributedTaskContext, Stage, WorkerLocation, display_plan_ascii, display_plan_graphviz,
explain_analyze,
};
pub use work_unit_feed::{
DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, WorkUnitFeedProvider,
Expand Down
55 changes: 53 additions & 2 deletions datafusion/distributed/src/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use crate::coordinator::DistributedExec;
use crate::execution_plans::{DistributedLeafExec, NetworkCoalesceExec};
use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL;
use datafusion::common::{HashMap, Statistics, config_err};
use datafusion::common::{exec_err, plan_err};
use datafusion::common::{exec_datafusion_err, exec_err, plan_err};
use datafusion::error::Result;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::physical_expr::PhysicalExpr;
Expand All @@ -12,12 +12,62 @@ use datafusion::physical_plan::{
ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext,
displayable,
};
use futures::FutureExt;
use futures::future::Shared;
use itertools::Either;
use std::collections::VecDeque;
use std::sync::Arc;
use tokio::sync::oneshot;
use url::Url;
use uuid::Uuid;

/// The eventual committed address of a worker whose task may still be setting up.
///
/// Routing may try several workers before committing to one. Consumers await only the
/// location they need; plans sent to another worker must first resolve all their inputs.
#[derive(Debug, Clone)]
pub enum WorkerLocation {
/// An address obtained from a decoded plan or supplied by the caller.
Ready(Url),
/// The result of query-owned routing and plan delivery.
Pending(Shared<oneshot::Receiver<Result<Url, Arc<DataFusionError>>>>),
}

impl WorkerLocation {
pub(crate) fn pending(rx: oneshot::Receiver<Result<Url, Arc<DataFusionError>>>) -> Self {
Self::Pending(rx.shared())
}

/// Waits for successful routing, retaining the original setup error on failure.
pub async fn resolve(&self) -> Result<Url> {
match self {
Self::Ready(url) => Ok(url.clone()),
Self::Pending(url) => url
.clone()
.await
.map_err(|_| exec_datafusion_err!("Task setup was cancelled"))?
.map_err(DataFusionError::Shared),
}
}

/// Returns a location that has already been resolved, without waiting for setup.
pub fn ready_url(&self) -> Option<&Url> {
match self {
Self::Ready(url) => Some(url),
Self::Pending(url) => match url.peek() {
Some(Ok(Ok(url))) => Some(url),
_ => None,
},
}
}
}

impl From<Url> for WorkerLocation {
fn from(url: Url) -> Self {
Self::Ready(url)
}
}

/// A unit of isolation for a portion of a physical execution plan
/// that can be executed independently and across a network boundary.
/// It implements [`ExecutionPlan`] and can be executed to produce a
Expand Down Expand Up @@ -112,7 +162,8 @@ pub struct RemoteStage {
/// Our stage number
pub num: usize,
/// The worker URLs to which queries should be issued.
pub workers: Vec<Url>,
/// Locations are in task-number order and may still be awaiting setup.
pub workers: Vec<WorkerLocation>,
/// Statistics collected at runtime, if any.
pub runtime_stats: Option<Arc<Statistics>>,
/// Dynamic-filter consumers retained after the stage's plan is moved to its workers.
Expand Down
Loading
Loading