Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 25 additions & 11 deletions datafusion/distributed/src/coordinator/query_coordinator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,11 @@ impl<'a> StageCoordinator<'a> {
stage_id: self.stage_id,
task_number: task_i,
};
let mut task_metrics = self.metrics_store.clone();
let mut completed_dynamic_filter_store = self.completed_dynamic_filter_store.clone();
let mut reports = TaskReports {
task_key,
metrics_store: self.metrics_store.clone(),
completed_dynamic_filter_store: self.completed_dynamic_filter_store.clone(),
};
let dynamic_filter_registry = Arc::clone(self.dynamic_filter_registry);
let task_ctx = Arc::clone(self.task_ctx);
let (load_info_tx, load_info_rx) = unbounded_channel();
Expand All @@ -302,12 +305,12 @@ impl<'a> StageCoordinator<'a> {
while let Some(msg) = worker_to_coordinator_stream.try_next().await? {
match msg {
WorkerToCoordinatorMsg::TaskMetrics(metrics) => {
if let Some(task_metrics) = task_metrics.take() {
if let Some(task_metrics) = reports.metrics_store.take() {
task_metrics.insert(task_key, metrics);
}
}
WorkerToCoordinatorMsg::TaskCompletedDynamicFilters(filters) => {
if let Some(store) = completed_dynamic_filter_store.take() {
if let Some(store) = reports.completed_dynamic_filter_store.take() {
store.insert(task_key, filters);
}
}
Expand All @@ -325,13 +328,6 @@ impl<'a> StageCoordinator<'a> {
}
};
}
// An unexecuted task sends no final reports; still complete its waits.
if let Some(store) = task_metrics {
store.insert(task_key, TaskMetrics::default());
}
if let Some(store) = completed_dynamic_filter_store {
store.insert(task_key, TaskCompletedDynamicFilters::default());
}
let _ = load_info_tx_opt.take();
Ok::<_, DataFusionError>(())
});
Expand Down Expand Up @@ -499,6 +495,24 @@ impl<'a> StageCoordinator<'a> {
}
}

/// Completes missing report waits when the worker response reader ends.
struct TaskReports {
task_key: TaskKey,
metrics_store: Option<Arc<Store<TaskMetrics>>>,
completed_dynamic_filter_store: Option<Arc<Store<TaskCompletedDynamicFilters>>>,
}

impl Drop for TaskReports {
fn drop(&mut self) {
if let Some(store) = self.metrics_store.take() {
store.insert(self.task_key, TaskMetrics::default());
}
if let Some(store) = self.completed_dynamic_filter_store.take() {
store.insert(self.task_key, TaskCompletedDynamicFilters::default());
}
}
}

struct TaskSpecializedPlan {
plan: Arc<dyn ExecutionPlan>,
work_unit_feed_declarations: Vec<WorkUnitFeedDeclaration>,
Expand Down
76 changes: 72 additions & 4 deletions datafusion/distributed/tests/metrics_collection.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#[cfg(all(feature = "integration", test))]
mod tests {
use async_trait::async_trait;
use datafusion::catalog::memory::DataSourceExec;
use datafusion::common::assert_not_contains;
use datafusion::common::{HashMap, Result, assert_contains};
use datafusion::common::{HashMap, Result, assert_contains, exec_err};
use datafusion::execution::SessionState;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
Expand All @@ -11,20 +12,23 @@ mod tests {
use datafusion_distributed::test_utils::localhost::start_localhost_context;
use datafusion_distributed::test_utils::metrics::{assert_metrics_equal, node_metrics};
use datafusion_distributed::test_utils::parquet::register_parquet_tables;
use datafusion_distributed::test_utils::routing::UrlEmitterRouteTaskHandler;
use datafusion_distributed::test_utils::test_work_unit_feed::{
RowGeneratorExec, TestWorkUnitFeedExecCodec, TestWorkUnitFeedFunction,
row_generator_desired_task_count_handler, row_generator_scale_up_leaf_node_handler,
};
use datafusion_distributed::{
DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec,
DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics,
WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters,
DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, RouteTaskEvent,
RouteTaskEventResponse, RouteTaskHandler, TaskKey, TaskMetrics, WorkerQueryContext,
display_plan_ascii, ok_or_some_err, rewrite_distributed_plan_with_dynamic_filters,
rewrite_distributed_plan_with_metrics,
};
use futures::TryStreamExt;
use futures::{StreamExt, TryStreamExt, stream};
use std::sync::Arc;
use std::time::Duration;
use test_case::test_case;
use tokio_util::sync::CancellationToken;

const COLLECTION_TIMEOUT: Duration = Duration::from_secs(10);

Expand Down Expand Up @@ -477,6 +481,70 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn report_collection_completes_after_worker_channel_error()
-> Result<(), Box<dyn std::error::Error>> {
let fail_channel = CancellationToken::new();
let (ctx, _guard, _) = start_localhost_context(2, DefaultSessionBuilder).await;
let ctx = ctx
.with_distributed_dynamic_task_count(false)?
.with_distributed_desired_task_count_handler(2usize)
.with_distributed_dynamic_filter_collection(true)?
.with_distributed_route_task_handler(FailingReportChannel(fail_channel.clone()));
register_parquet_tables(&ctx).await?;
let plan = ctx
.sql(r#"SELECT "MinTemp" FROM weather WHERE "MinTemp" > 20.0"#)
.await?
.create_physical_plan()
.await?;
let mut output = execute_stream(Arc::clone(&plan), ctx.task_ctx())?;

// Wait for execution to start so report collection sees the prepared plan.
assert!(
tokio::time::timeout(COLLECTION_TIMEOUT, output.try_next())
.await??
.is_some()
);
fail_channel.cancel();
let error = tokio::time::timeout(COLLECTION_TIMEOUT, output.try_collect::<Vec<_>>())
.await?
.expect_err("worker channel error must fail the query");
assert_contains!(error.to_string(), "injected report channel error");

let task_metrics = wait_for_task_metrics(&plan).await;
assert_eq!(task_metrics.len(), 2);
for metrics in task_metrics.values() {
assert!(metrics.pre_order_plan_metrics.is_empty());
assert!(metrics.task_metrics.iter().next().is_none());
}
tokio::time::timeout(
COLLECTION_TIMEOUT,
rewrite_distributed_plan_with_dynamic_filters(plan, &ctx.task_ctx()),
)
.await??;
Ok(())
}

struct FailingReportChannel(CancellationToken);

#[async_trait]
impl RouteTaskHandler for FailingReportChannel {
async fn handle(
&self,
event: RouteTaskEvent<'_>,
) -> Option<Result<RouteTaskEventResponse>> {
let mut response = ok_or_some_err!(UrlEmitterRouteTaskHandler.handle(event).await?);
let fail_channel = self.0.clone();
response.worker_to_coordinator_stream = stream::once(async move {
fail_channel.cancelled().await;
exec_err!("injected report channel error")
})
.chain(response.worker_to_coordinator_stream)
.boxed();
Some(Ok(response))
}
}

/// Waits until every task of the executed `plan` has a final metrics report.
async fn wait_for_task_metrics(plan: &Arc<dyn ExecutionPlan>) -> HashMap<TaskKey, TaskMetrics> {
let d_exec = plan
Expand Down
Loading