From 60f7279569f0e3a3f7303638117fc4c77148dbbc Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 19 Nov 2026 13:49:36 -0500 Subject: [PATCH] fix: complete task reports after worker channel errors --- .../src/coordinator/query_coordinator.rs | 36 ++++++--- .../distributed/tests/metrics_collection.rs | 76 ++++++++++++++++++- 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/datafusion/distributed/src/coordinator/query_coordinator.rs b/datafusion/distributed/src/coordinator/query_coordinator.rs index 8274259ec..3b113d6ca 100644 --- a/datafusion/distributed/src/coordinator/query_coordinator.rs +++ b/datafusion/distributed/src/coordinator/query_coordinator.rs @@ -290,8 +290,11 @@ impl<'a> StageCoordinator<'a> { stage_id: self.stage_id, task_number: task_i, }; - let mut task_metrics = self.metrics_store.clone(); - let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone(); + let mut reports = TaskReports { + task_key, + metrics_store: self.metrics_store.clone(), + completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(), + }; let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry); let task_ctx = Arc::clone(self.task_ctx); let (load_info_tx, load_info_rx) = unbounded_channel(); @@ -302,12 +305,12 @@ impl<'a> StageCoordinator<'a> { while let Some(msg) = worker_to_coordinator_stream.try_next().await? { match msg { WorkerToCoordinatorMsg::TaskMetrics(metrics) => { - if let Some(task_metrics) = task_metrics.take() { + if let Some(task_metrics) = reports.metrics_store.take() { task_metrics.insert(task_key, metrics); } } WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => { - if let Some(store) = completed_dynamic_filter_store.take() { + if let Some(store) = reports.completed_dynamic_filter_store.take() { store.insert(task_key, filters); } } @@ -325,13 +328,6 @@ impl<'a> StageCoordinator<'a> { } }; } - // An unexecuted task sends no final reports; still complete its waits. - if let Some(store) = task_metrics { - store.insert(task_key, TaskMetrics::default()); - } - if let Some(store) = completed_dynamic_filter_store { - store.insert(task_key, TaskCompletedDynamicFilters::default()); - } let _ = load_info_tx_opt.take(); Ok::<_, DataFusionError>(()) }); @@ -499,6 +495,24 @@ impl<'a> StageCoordinator<'a> { } } +/// Completes missing report waits when the worker response reader ends. +struct TaskReports { + task_key: TaskKey, + metrics_store: Option>>, + completed_dynamic_filter_store: Option>>, +} + +impl Drop for TaskReports { + fn drop(&mut self) { + if let Some(store) = self.metrics_store.take() { + store.insert(self.task_key, TaskMetrics::default()); + } + if let Some(store) = self.completed_dynamic_filter_store.take() { + store.insert(self.task_key, TaskCompletedDynamicFilters::default()); + } + } +} + struct TaskSpecializedPlan { plan: Arc, work_unit_feed_declarations: Vec, diff --git a/datafusion/distributed/tests/metrics_collection.rs b/datafusion/distributed/tests/metrics_collection.rs index 63b19428f..27915feab 100644 --- a/datafusion/distributed/tests/metrics_collection.rs +++ b/datafusion/distributed/tests/metrics_collection.rs @@ -1,8 +1,9 @@ #[cfg(all(feature = "integration", test))] mod tests { + use async_trait::async_trait; use datafusion::catalog::memory::DataSourceExec; use datafusion::common::assert_not_contains; - use datafusion::common::{HashMap, Result, assert_contains}; + use datafusion::common::{HashMap, Result, assert_contains, exec_err}; use datafusion::execution::SessionState; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; @@ -11,20 +12,23 @@ mod tests { use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::metrics::{assert_metrics_equal, node_metrics}; use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::test_utils::routing::UrlEmitterRouteTaskHandler; use datafusion_distributed::test_utils::test_work_unit_feed::{ RowGeneratorExec, TestWorkUnitFeedExecCodec, TestWorkUnitFeedFunction, row_generator_desired_task_count_handler, row_generator_scale_up_leaf_node_handler, }; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec, - DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics, - WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters, + DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, RouteTaskEvent, + RouteTaskEventResponse, RouteTaskHandler, TaskKey, TaskMetrics, WorkerQueryContext, + display_plan_ascii, ok_or_some_err, rewrite_distributed_plan_with_dynamic_filters, rewrite_distributed_plan_with_metrics, }; - use futures::TryStreamExt; + use futures::{StreamExt, TryStreamExt, stream}; use std::sync::Arc; use std::time::Duration; use test_case::test_case; + use tokio_util::sync::CancellationToken; const COLLECTION_TIMEOUT: Duration = Duration::from_secs(10); @@ -477,6 +481,70 @@ mod tests { Ok(()) } + #[tokio::test] + async fn report_collection_completes_after_worker_channel_error() + -> Result<(), Box> { + let fail_channel = CancellationToken::new(); + let (ctx, _guard, _) = start_localhost_context(2, DefaultSessionBuilder).await; + let ctx = ctx + .with_distributed_dynamic_task_count(false)? + .with_distributed_desired_task_count_handler(2usize) + .with_distributed_dynamic_filter_collection(true)? + .with_distributed_route_task_handler(FailingReportChannel(fail_channel.clone())); + register_parquet_tables(&ctx).await?; + let plan = ctx + .sql(r#"SELECT "MinTemp" FROM weather WHERE "MinTemp" > 20.0"#) + .await? + .create_physical_plan() + .await?; + let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?; + + // Wait for execution to start so report collection sees the prepared plan. + assert!( + tokio::time::timeout(COLLECTION_TIMEOUT, output.try_next()) + .await?? + .is_some() + ); + fail_channel.cancel(); + let error = tokio::time::timeout(COLLECTION_TIMEOUT, output.try_collect::>()) + .await? + .expect_err("worker channel error must fail the query"); + assert_contains!(error.to_string(), "injected report channel error"); + + let task_metrics = wait_for_task_metrics(&plan).await; + assert_eq!(task_metrics.len(), 2); + for metrics in task_metrics.values() { + assert!(metrics.pre_order_plan_metrics.is_empty()); + assert!(metrics.task_metrics.iter().next().is_none()); + } + tokio::time::timeout( + COLLECTION_TIMEOUT, + rewrite_distributed_plan_with_dynamic_filters(plan, &ctx.task_ctx()), + ) + .await??; + Ok(()) + } + + struct FailingReportChannel(CancellationToken); + + #[async_trait] + impl RouteTaskHandler for FailingReportChannel { + async fn handle( + &self, + event: RouteTaskEvent<'_>, + ) -> Option> { + let mut response = ok_or_some_err!(UrlEmitterRouteTaskHandler.handle(event).await?); + let fail_channel = self.0.clone(); + response.worker_to_coordinator_stream = stream::once(async move { + fail_channel.cancelled().await; + exec_err!("injected report channel error") + }) + .chain(response.worker_to_coordinator_stream) + .boxed(); + Some(Ok(response)) + } + } + /// Waits until every task of the executed `plan` has a final metrics report. async fn wait_for_task_metrics(plan: &Arc) -> HashMap { let d_exec = plan