Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 28 additions & 12 deletions datafusion/distributed/src/coordinator/query_coordinator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,12 @@ impl<'a> StageCoordinator<'a> {
stage_id: self.stage_id,
task_number: task_i,
};
let mut task_metrics = self.metrics_store.clone();
let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone();
// Completes this task's report waits however the task below ends, even if it's never polled.
let mut reports = TaskReports {
task_key,
metrics_store: self.metrics_store.clone(),
completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(),
};
let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry);
let task_ctx = Arc::clone(self.task_ctx);
let (load_info_tx, load_info_rx) = unbounded_channel();
Expand All @@ -302,12 +306,12 @@ impl<'a> StageCoordinator<'a> {
while let Some(msg) = worker_to_coordinator_stream.try_next().await? {
match msg {
WorkerToCoordinatorMsg::TaskMetrics(metrics) => {
if let Some(task_metrics) = task_metrics.take() {
task_metrics.insert(task_key, metrics);
if let Some(store) = reports.metrics_store.take() {
store.insert(task_key, metrics);
}
}
WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => {
if let Some(store) = completed_dynamic_filter_store.take() {
if let Some(store) = reports.completed_dynamic_filter_store.take() {
store.insert(task_key, filters);
}
}
Expand All @@ -325,13 +329,6 @@ impl<'a> StageCoordinator<'a> {
}
};
}
// An unexecuted task sends no final reports; still complete its waits.
if let Some(store) = task_metrics {
store.insert(task_key, TaskMetrics::default());
}
if let Some(store) = completed_dynamic_filter_store {
store.insert(task_key, TaskCompletedDynamicFilters::default());
}
let _ = load_info_tx_opt.take();
Ok::<_, DataFusionError>(())
});
Expand Down Expand Up @@ -499,6 +496,25 @@ impl<'a> StageCoordinator<'a> {
}
}

/// Completes missing report waits when the worker-to-coordinator task ends, including when it
/// fails or when the task is dropped before its first poll.
struct TaskReports {
task_key: TaskKey,
metrics_store: Option<Arc<Store<TaskMetrics>>>,
completed_dynamic_filter_store: Option<Arc<Store<TaskCompletedDynamicFilters>>>,
}

impl Drop for TaskReports {
fn drop(&mut self) {
if let Some(store) = self.metrics_store.take() {
store.insert(self.task_key, TaskMetrics::default());
}
if let Some(store) = self.completed_dynamic_filter_store.take() {
store.insert(self.task_key, TaskCompletedDynamicFilters::default());
}
}
}

struct TaskSpecializedPlan {
plan: Arc<dyn ExecutionPlan>,
work_unit_feed_declarations: Vec<WorkUnitFeedDeclaration>,
Expand Down
72 changes: 68 additions & 4 deletions datafusion/distributed/tests/metrics_collection.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#[cfg(all(feature = "integration", test))]
mod tests {
use async_trait::async_trait;
use datafusion::catalog::memory::DataSourceExec;
use datafusion::common::assert_not_contains;
use datafusion::common::{HashMap, Result, assert_contains};
use datafusion::common::{HashMap, Result, assert_contains, exec_err};
use datafusion::execution::SessionState;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
Expand All @@ -17,14 +18,16 @@ mod tests {
};
use datafusion_distributed::{
DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec,
DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics,
WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters,
DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, RouteTaskEvent,
RouteTaskEventResponse, RouteTaskHandler, TaskKey, TaskMetrics, WorkerQueryContext,
display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters,
rewrite_distributed_plan_with_metrics,
};
use futures::TryStreamExt;
use futures::{StreamExt, TryStreamExt, stream};
use std::sync::Arc;
use std::time::Duration;
use test_case::test_case;
use tokio_util::sync::CancellationToken;

const COLLECTION_TIMEOUT: Duration = Duration::from_secs(10);

Expand Down Expand Up @@ -477,6 +480,35 @@ mod tests {
Ok(())
}

/// Tasks whose worker->coordinator stream fails before their final reports still complete
/// their metrics wait.
#[tokio::test]
async fn metrics_collected_after_worker_stream_error() -> Result<(), Box<dyn std::error::Error>>
{
let (ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await;
let fail = FailTaskZeroWorkerStream::default();
let ctx = ctx
.with_distributed_dynamic_task_count(false)?
.with_distributed_route_task_handler(fail.clone());
register_parquet_tables(&ctx).await?;

let query = r#"SELECT "FL_DATE", count(*) FROM flights_1m GROUP BY "FL_DATE""#;
let plan = ctx.sql(query).await?.create_physical_plan().await?;
let mut stream = execute_stream(plan.clone(), ctx.task_ctx())?;
assert!(stream.try_next().await?.is_some());
// Workers send their final reports when the query ends, so the error arrives before them.
fail.release.cancel();
let result = stream.try_collect::<Vec<_>>().await;
assert_contains!(
result.unwrap_err().to_string(),
"injected worker stream error"
);

let task_metrics = wait_for_task_metrics(&plan).await;
assert!(!task_metrics.is_empty());
Ok(())
}

/// Waits until every task of the executed `plan` has a final metrics report.
async fn wait_for_task_metrics(plan: &Arc<dyn ExecutionPlan>) -> HashMap<TaskKey, TaskMetrics> {
let d_exec = plan
Expand Down Expand Up @@ -533,4 +565,36 @@ mod tests {
)
.expect("static dynamic-filter setting should be valid");
}

/// Routes tasks like the default handler, but fails the worker->coordinator stream of every
/// task 0 once `release` is cancelled.
#[derive(Clone, Default)]
struct FailTaskZeroWorkerStream {
release: CancellationToken,
}

#[async_trait]
impl RouteTaskHandler for FailTaskZeroWorkerStream {
async fn handle(&self, ev: RouteTaskEvent<'_>) -> Option<Result<RouteTaskEventResponse>> {
let urls = match ev.worker_resolver.get_urls() {
Ok(urls) => urls,
Err(err) => return Some(Err(err)),
};
let url = urls[ev.task_key.task_number % urls.len()].clone();
let mut response = match ev.dialer.dial(url).await {
Ok(response) => response,
Err(err) => return Some(Err(err)),
};
if ev.task_key.task_number == 0 {
let release = self.release.clone();
let error = stream::once(async move {
release.cancelled().await;
exec_err!("injected worker stream error")
});
response.worker_to_coordinator_stream =
error.chain(response.worker_to_coordinator_stream).boxed();
}
Some(Ok(response))
}
}
}
Loading