diff --git a/datafusion/distributed/src/execution_plans/broadcast.rs b/datafusion/distributed/src/execution_plans/broadcast.rs index 6ef64df43..80f055ca3 100644 --- a/datafusion/distributed/src/execution_plans/broadcast.rs +++ b/datafusion/distributed/src/execution_plans/broadcast.rs @@ -1,4 +1,5 @@ use crate::common::{OnceLockResult, require_one_child}; +use crate::execution_plans::children_isolator_union::ObservedStream; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::runtime::SpawnedTask; @@ -7,6 +8,9 @@ use datafusion::error::{DataFusionError, Result}; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::metrics::{ + BaselineMetrics, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricValue, MetricsSet, Time, +}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, internal_err, @@ -15,6 +19,7 @@ use futures::{Stream, StreamExt}; use std::collections::VecDeque; use std::fmt::Formatter; use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::task::{Context, Poll}; use tokio_stream::wrappers::WatchStream; @@ -84,10 +89,17 @@ pub struct BroadcastExec { consumer_task_count: usize, properties: Arc, queues: Vec>, + metrics: ExecutionPlanMetricsSet, + /// Time spent buffering input batches. It is not tied to an output partition, as each input + /// partition is buffered once for all of them. + producer_compute: Time, + /// Peak bytes buffered across all input partitions, waiting for slower consumer tasks. + max_mem_used: Gauge, + buffered_bytes: Arc, } type BroadcastMessage = - std::result::Result<(RecordBatch, Arc), Arc>; + std::result::Result<(RecordBatch, Arc), Arc>; type StreamAndTask = (BroadcastReaders, Arc>); impl BroadcastExec { @@ -102,11 +114,24 @@ impl BroadcastExec { .map(|_| OnceLock::new()) .collect(); + let metrics = ExecutionPlanMetricsSet::new(); + let producer_compute = Time::new(); + MetricBuilder::new(&metrics).build(MetricValue::ElapsedCompute(producer_compute.clone())); + let max_mem_used = Gauge::new(); + MetricBuilder::new(&metrics).build(MetricValue::PeakMemoryUsage { + name: "max_mem_used".into(), + gauge: max_mem_used.clone(), + }); + Self { input, consumer_task_count, properties: Arc::new(properties), queues, + metrics, + producer_compute, + max_mem_used, + buffered_bytes: Arc::default(), } } @@ -178,6 +203,10 @@ impl ExecutionPlan for BroadcastExec { ))) } + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + fn execute( &self, partition: usize, @@ -188,6 +217,7 @@ impl ExecutionPlan for BroadcastExec { let consumer_task = partition / input_partition_count; let input = Arc::clone(&self.input); + let baseline_metrics = BaselineMetrics::new(&self.metrics, partition); let queue_or_err = self.queues[real_partition].get_or_init(|| { let queue = BroadcastQueue::new(self.consumer_task_count); @@ -196,6 +226,9 @@ impl ExecutionPlan for BroadcastExec { let pool = Arc::clone(context.memory_pool()); let mut stream = input.execute(real_partition, context).map_err(Arc::new)?; let cancel = queue.shared.cancel.clone(); + let elapsed_compute = self.producer_compute.clone(); + let max_mem_used = self.max_mem_used.clone(); + let buffered_bytes = Arc::clone(&self.buffered_bytes); let task = SpawnedTask::spawn(async move { let mem_consumer = MemoryConsumer::new(format!("BroadcastExec[{real_partition}]")); @@ -207,12 +240,20 @@ impl ExecutionPlan for BroadcastExec { let Some(msg) = msg else { break }; match msg { Ok(record_batch) => { + let _timer = elapsed_compute.timer(); + let size = record_batch.get_array_memory_size(); let reservation = mem_consumer.clone_with_new_id().register(&pool); - reservation.grow(record_batch.get_array_memory_size()); - if !queue.push(Ok((record_batch, Arc::new(reservation)))) { + reservation.grow(size); + let buffered = buffered_bytes.fetch_add(size, Ordering::Relaxed) + size; + let batch = BufferedBatch { + reservation, + buffered_bytes: Arc::clone(&buffered_bytes), + }; + if !queue.push(Ok((record_batch, Arc::new(batch)))) { // If there are no remaining readers, short-circuit. break; } + max_mem_used.set_max(buffered); } Err(err) => { let _ = queue.push(Err(Arc::new(err))); @@ -224,20 +265,30 @@ impl ExecutionPlan for BroadcastExec { Ok::<_, Arc>((readers, Arc::new(task))) }); - let (consumer, task) = match queue_or_err { + let (mut consumer, task) = match queue_or_err { Ok((readers, task)) => (readers.claim(consumer_task)?, Arc::clone(task)), Err(err) => return Err(DataFusionError::Shared(Arc::clone(err))), }; - Ok(Box::pin(RecordBatchStreamAdapter::new( + let elapsed_compute = baseline_metrics.elapsed_compute().clone(); + let consumer = futures::stream::poll_fn(move |cx| { + let _timer = elapsed_compute.timer(); + consumer.poll_next_unpin(cx) + }); + let stream = Box::pin(RecordBatchStreamAdapter::new( self.schema(), consumer .map(|msg| match msg { - Ok((batch, _reservation)) => Ok(batch), + Ok((batch, _buffered)) => Ok(batch), Err(e) => Err(DataFusionError::Shared(e)), }) .inspect(move |_| { let _ = &task; }), + )); + Ok(Box::pin(ObservedStream::new( + stream, + baseline_metrics, + None, ))) } @@ -246,6 +297,21 @@ impl ExecutionPlan for BroadcastExec { } } +/// Memory held by a batch while it waits in a [BroadcastQueue]. It is released once every reader +/// has read the batch and the entry is evicted. +#[derive(Debug)] +struct BufferedBatch { + reservation: MemoryReservation, + buffered_bytes: Arc, +} + +impl Drop for BufferedBatch { + fn drop(&mut self) { + self.buffered_bytes + .fetch_sub(self.reservation.size(), Ordering::Relaxed); + } +} + /// Buffered output of a single [BroadcastExec] input partition. /// /// [BroadcastExec] executes each input partition only once, but every consumer task reads it @@ -799,6 +865,37 @@ mod tests { Ok(()) } + #[tokio::test] + async fn broadcast_exec_tracks_buffered_memory() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0, 1, 2]))], + )?; + let batch_size = batch.get_array_memory_size(); + let input = Arc::new(MockExec::new_partitioned( + vec![vec![Ok(batch.clone()), Ok(batch)]], + Arc::clone(&schema), + )); + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + let task_ctx = SessionContext::new().task_ctx(); + + for partition in 0..2 { + datafusion::physical_plan::common::collect( + broadcast.execute(partition, Arc::clone(&task_ctx))?, + ) + .await?; + } + + // Each batch is buffered once, no matter how many consumers read it. + assert!(broadcast.max_mem_used.value() <= 2 * batch_size); + assert!(broadcast.max_mem_used.value() >= batch_size); + assert_eq!(broadcast.buffered_bytes.load(Ordering::SeqCst), 0); + let output_rows = broadcast.metrics().unwrap().output_rows().unwrap(); + assert_eq!(output_rows, 12); + Ok(()) + } + #[tokio::test] async fn broadcast_exec_maps_partitions_by_modulo() -> Result<()> { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); diff --git a/datafusion/distributed/tests/metrics_collection.rs b/datafusion/distributed/tests/metrics_collection.rs index 63b19428f..0212b1eac 100644 --- a/datafusion/distributed/tests/metrics_collection.rs +++ b/datafusion/distributed/tests/metrics_collection.rs @@ -5,6 +5,7 @@ mod tests { use datafusion::common::{HashMap, Result, assert_contains}; use datafusion::execution::SessionState; use datafusion::physical_plan::display::DisplayableExecutionPlan; + use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ExecutionPlan, execute_stream}; use datafusion::prelude::SessionContext; @@ -16,7 +17,7 @@ mod tests { row_generator_desired_task_count_handler, row_generator_scale_up_leaf_node_handler, }; use datafusion_distributed::{ - DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec, + BroadcastExec, DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec, DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics, WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters, rewrite_distributed_plan_with_metrics, @@ -206,6 +207,41 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_metrics_collection_in_broadcast_join() -> Result<(), Box> { + let (mut d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + d_ctx.set_distributed_broadcast_joins(true)?; + + let query = r#" + SELECT count(*) + FROM weather a + JOIN weather b ON a."RainToday" = b."RainToday" + WHERE b."MinTemp" > 15.0 + "#; + + let s_ctx = SessionContext::default(); + let (_, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + d_physical = + rewrite_with_metrics(d_physical.clone(), DistributedMetricsFormat::Aggregated).await; + println!("{}", display_plan_ascii(d_physical.as_ref(), true)); + + // The filtered side is the smaller one, so it is the build side under BroadcastExec, and + // every consumer task reads all of it. + let input_rows = node_metrics::(&d_physical, "output_rows", 0); + let output_rows = node_metrics::(&d_physical, "output_rows", 0); + assert_eq!(output_rows, 3 * input_rows); + let input_bytes = node_metrics::(&d_physical, "output_bytes", 0); + let output_bytes = node_metrics::(&d_physical, "output_bytes", 0); + assert_eq!(output_bytes, 3 * input_bytes); + // Batches are buffered once per task, not once per consumer. + let max_mem_used = node_metrics::(&d_physical, "max_mem_used", 0); + assert!(max_mem_used < output_bytes); + let value = node_metrics::(&d_physical, "elapsed_compute", 0); + assert!(value > 0); + + Ok(()) + } + #[tokio::test] async fn test_stage_level_metric_collection() -> Result<(), Box> { let format = DistributedMetricsFormat::PerTask;