diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index 32e472172..5e6debab7 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -290,8 +290,12 @@ impl<'a> StageCoordinator<'a> { stage_id: self.stage_id, task_number: task_i, }; - let mut task_metrics = self.metrics_store.clone(); - let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone(); + // Completes this task's report waits however the task below ends, even if it's never polled. + let mut reports = TaskReports { + task_key, + metrics_store: self.metrics_store.clone(), + completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), + }; let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); let task_ctx = Arc::clone(self.task_ctx); let (load_info_tx, load_info_rx) = unbounded_channel(); @@ -302,12 +306,12 @@ impl<'a> StageCoordinator<'a> { while let Some(msg) = worker_to_coordinator_stream.try_next().await? { match msg { WorkerToCoordinatorMsg::TaskMetrics(metrics) => { - if let Some(task_metrics) = task_metrics.take() { - task_metrics.insert(task_key, metrics); + if let Some(store) = reports.metrics_store.take() { + store.insert(task_key, metrics); } } WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { - if let Some(store) = completed_dynamic_filter_store.take() { + if let Some(store) = reports.completed_dynamic_filter_store.take() { store.insert(task_key, filters); } } @@ -325,13 +329,6 @@ impl<'a> StageCoordinator<'a> { } }; } - // An unexecuted task sends no final reports; still complete its waits. - if let Some(store) = task_metrics { - store.insert(task_key, TaskMetrics::default()); - } - if let Some(store) = completed_dynamic_filter_store { - store.insert(task_key, TaskCompletedDynamicFilters::default()); - } let _ = load_info_tx_opt.take(); Ok::<_, DataFusionError>(()) }); @@ -499,6 +496,25 @@ impl<'a> StageCoordinator<'a> { } } +/// Completes missing report waits when the worker-to-coordinator task ends, including when it +/// fails or when the task is dropped before its first poll. +struct TaskReports { + task_key: TaskKey, + metrics_store: Option>>, + completed_dynamic_filter_store: Option>>, +} + +impl Drop for TaskReports { + fn drop(&mut self) { + if let Some(store) = self.metrics_store.take() { + store.insert(self.task_key, TaskMetrics::default()); + } + if let Some(store) = self.completed_dynamic_filter_store.take() { + store.insert(self.task_key, TaskCompletedDynamicFilters::default()); + } + } +} + struct TaskSpecializedPlan { plan: Arc, work_unit_feed_declarations: Vec, diff --git a/datafusion/distributed/tests/metrics_collection.rs b/datafusion/distributed/tests/metrics_collection.rs index 63b19428f..6de6743ef 100644 --- a/datafusion/distributed/tests/metrics_collection.rs +++ b/datafusion/distributed/tests/metrics_collection.rs @@ -1,8 +1,9 @@ #[cfg(all(feature = "integration", test))] mod tests { + use async_trait::async_trait; use datafusion::catalog::memory::DataSourceExec; use datafusion::common::assert_not_contains; - use datafusion::common::{HashMap, Result, assert_contains}; + use datafusion::common::{HashMap, Result, assert_contains, exec_err}; use datafusion::execution::SessionState; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; @@ -17,14 +18,16 @@ mod tests { }; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec, - DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics, - WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters, + DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, RouteTaskEvent, + RouteTaskEventResponse, RouteTaskHandler, TaskKey, TaskMetrics, WorkerQueryContext, + display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters, rewrite_distributed_plan_with_metrics, }; - use futures::TryStreamExt; + use futures::{StreamExt, TryStreamExt, stream}; use std::sync::Arc; use std::time::Duration; use test_case::test_case; + use tokio_util::sync::CancellationToken; const COLLECTION_TIMEOUT: Duration = Duration::from_secs(10); @@ -477,6 +480,35 @@ mod tests { Ok(()) } + /// Tasks whose worker->coordinator stream fails before their final reports still complete + /// their metrics wait. + #[tokio::test] + async fn metrics_collected_after_worker_stream_error() -> Result<(), Box> + { + let (ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + let fail = FailTaskZeroWorkerStream::default(); + let ctx = ctx + .with_distributed_dynamic_task_count(false)? + .with_distributed_route_task_handler(fail.clone()); + register_parquet_tables(&ctx).await?; + + let query = r#"SELECT "FL_DATE", count(*) FROM flights_1m GROUP BY "FL_DATE""#; + let plan = ctx.sql(query).await?.create_physical_plan().await?; + let mut stream = execute_stream(plan.clone(), ctx.task_ctx())?; + assert!(stream.try_next().await?.is_some()); + // Workers send their final reports when the query ends, so the error arrives before them. + fail.release.cancel(); + let result = stream.try_collect::>().await; + assert_contains!( + result.unwrap_err().to_string(), + "injected worker stream error" + ); + + let task_metrics = wait_for_task_metrics(&plan).await; + assert!(!task_metrics.is_empty()); + Ok(()) + } + /// Waits until every task of the executed `plan` has a final metrics report. async fn wait_for_task_metrics(plan: &Arc) -> HashMap { let d_exec = plan @@ -533,4 +565,36 @@ mod tests { ) .expect("static dynamic-filter setting should be valid"); } + + /// Routes tasks like the default handler, but fails the worker->coordinator stream of every + /// task 0 once `release` is cancelled. + #[derive(Clone, Default)] + struct FailTaskZeroWorkerStream { + release: CancellationToken, + } + + #[async_trait] + impl RouteTaskHandler for FailTaskZeroWorkerStream { + async fn handle(&self, ev: RouteTaskEvent<'_>) -> Option> { + let urls = match ev.worker_resolver.get_urls() { + Ok(urls) => urls, + Err(err) => return Some(Err(err)), + }; + let url = urls[ev.task_key.task_number % urls.len()].clone(); + let mut response = match ev.dialer.dial(url).await { + Ok(response) => response, + Err(err) => return Some(Err(err)), + }; + if ev.task_key.task_number == 0 { + let release = self.release.clone(); + let error = stream::once(async move { + release.cancelled().await; + exec_err!("injected worker stream error") + }); + response.worker_to_coordinator_stream = + error.chain(response.worker_to_coordinator_stream).boxed(); + } + Some(Ok(response)) + } + } }