Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 103 additions & 6 deletions datafusion/distributed/src/execution_plans/broadcast.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::common::{OnceLockResult, require_one_child};
use crate::execution_plans::children_isolator_union::ObservedStream;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::common::runtime::SpawnedTask;
Expand All @@ -7,6 +8,9 @@ use datafusion::error::{DataFusionError, Result};
use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::metrics::{
BaselineMetrics, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricValue, MetricsSet, Time,
};
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, internal_err,
Expand All @@ -15,6 +19,7 @@ use futures::{Stream, StreamExt};
use std::collections::VecDeque;
use std::fmt::Formatter;
use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::task::{Context, Poll};
use tokio_stream::wrappers::WatchStream;
Expand Down Expand Up @@ -84,10 +89,17 @@ pub struct BroadcastExec {
consumer_task_count: usize,
properties: Arc<PlanProperties>,
queues: Vec<OnceLockResult<StreamAndTask>>,
metrics: ExecutionPlanMetricsSet,
/// Time spent buffering input batches. It is not tied to an output partition, as each input
/// partition is buffered once for all of them.
producer_compute: Time,
/// Peak bytes buffered across all input partitions, waiting for slower consumer tasks.
max_mem_used: Gauge,
buffered_bytes: Arc<AtomicUsize>,
}

type BroadcastMessage =
std::result::Result<(RecordBatch, Arc<MemoryReservation>), Arc<DataFusionError>>;
std::result::Result<(RecordBatch, Arc<BufferedBatch>), Arc<DataFusionError>>;
type StreamAndTask = (BroadcastReaders<BroadcastMessage>, Arc<SpawnedTask<()>>);

impl BroadcastExec {
Expand All @@ -102,11 +114,24 @@ impl BroadcastExec {
.map(|_| OnceLock::new())
.collect();

let metrics = ExecutionPlanMetricsSet::new();
let producer_compute = Time::new();
MetricBuilder::new(&metrics).build(MetricValue::ElapsedCompute(producer_compute.clone()));
let max_mem_used = Gauge::new();
MetricBuilder::new(&metrics).build(MetricValue::PeakMemoryUsage {
name: "max_mem_used".into(),
gauge: max_mem_used.clone(),
});

Self {
input,
consumer_task_count,
properties: Arc::new(properties),
queues,
metrics,
producer_compute,
max_mem_used,
buffered_bytes: Arc::default(),
}
}

Expand Down Expand Up @@ -178,6 +203,10 @@ impl ExecutionPlan for BroadcastExec {
)))
}

fn metrics(&self) -> Option<MetricsSet> {
Some(self.metrics.clone_inner())
}

fn execute(
&self,
partition: usize,
Expand All @@ -188,6 +217,7 @@ impl ExecutionPlan for BroadcastExec {
let consumer_task = partition / input_partition_count;

let input = Arc::clone(&self.input);
let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);

let queue_or_err = self.queues[real_partition].get_or_init(|| {
let queue = BroadcastQueue::new(self.consumer_task_count);
Expand All @@ -196,6 +226,9 @@ impl ExecutionPlan for BroadcastExec {
let pool = Arc::clone(context.memory_pool());
let mut stream = input.execute(real_partition, context).map_err(Arc::new)?;
let cancel = queue.shared.cancel.clone();
let elapsed_compute = self.producer_compute.clone();
let max_mem_used = self.max_mem_used.clone();
let buffered_bytes = Arc::clone(&self.buffered_bytes);
let task = SpawnedTask::spawn(async move {
let mem_consumer = MemoryConsumer::new(format!("BroadcastExec[{real_partition}]"));

Expand All @@ -207,12 +240,20 @@ impl ExecutionPlan for BroadcastExec {
let Some(msg) = msg else { break };
match msg {
Ok(record_batch) => {
let _timer = elapsed_compute.timer();
let size = record_batch.get_array_memory_size();
let reservation = mem_consumer.clone_with_new_id().register(&pool);
reservation.grow(record_batch.get_array_memory_size());
if !queue.push(Ok((record_batch, Arc::new(reservation)))) {
reservation.grow(size);
let buffered = buffered_bytes.fetch_add(size, Ordering::Relaxed) + size;
let batch = BufferedBatch {
reservation,
buffered_bytes: Arc::clone(&buffered_bytes),
};
if !queue.push(Ok((record_batch, Arc::new(batch)))) {
// If there are no remaining readers, short-circuit.
break;
}
max_mem_used.set_max(buffered);
}
Err(err) => {
let _ = queue.push(Err(Arc::new(err)));
Expand All @@ -224,20 +265,30 @@ impl ExecutionPlan for BroadcastExec {

Ok::<_, Arc<DataFusionError>>((readers, Arc::new(task)))
});
let (consumer, task) = match queue_or_err {
let (mut consumer, task) = match queue_or_err {
Ok((readers, task)) => (readers.claim(consumer_task)?, Arc::clone(task)),
Err(err) => return Err(DataFusionError::Shared(Arc::clone(err))),
};
Ok(Box::pin(RecordBatchStreamAdapter::new(
let elapsed_compute = baseline_metrics.elapsed_compute().clone();
let consumer = futures::stream::poll_fn(move |cx| {
let _timer = elapsed_compute.timer();
consumer.poll_next_unpin(cx)
});
let stream = Box::pin(RecordBatchStreamAdapter::new(
self.schema(),
consumer
.map(|msg| match msg {
Ok((batch, _reservation)) => Ok(batch),
Ok((batch, _buffered)) => Ok(batch),
Err(e) => Err(DataFusionError::Shared(e)),
})
.inspect(move |_| {
let _ = &task;
}),
));
Ok(Box::pin(ObservedStream::new(
stream,
baseline_metrics,
None,
)))
}

Expand All @@ -246,6 +297,21 @@ impl ExecutionPlan for BroadcastExec {
}
}

/// Memory held by a batch while it waits in a [BroadcastQueue]. It is released once every reader
/// has read the batch and the entry is evicted.
#[derive(Debug)]
struct BufferedBatch {
reservation: MemoryReservation,
buffered_bytes: Arc<AtomicUsize>,
}

impl Drop for BufferedBatch {
fn drop(&mut self) {
self.buffered_bytes
.fetch_sub(self.reservation.size(), Ordering::Relaxed);
}
}

/// Buffered output of a single [BroadcastExec] input partition.
///
/// [BroadcastExec] executes each input partition only once, but every consumer task reads it
Expand Down Expand Up @@ -799,6 +865,37 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn broadcast_exec_tracks_buffered_memory() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(Int32Array::from(vec![0, 1, 2]))],
)?;
let batch_size = batch.get_array_memory_size();
let input = Arc::new(MockExec::new_partitioned(
vec![vec![Ok(batch.clone()), Ok(batch)]],
Arc::clone(&schema),
));
let broadcast = Arc::new(BroadcastExec::new(input, 2));
let task_ctx = SessionContext::new().task_ctx();

for partition in 0..2 {
datafusion::physical_plan::common::collect(
broadcast.execute(partition, Arc::clone(&task_ctx))?,
)
.await?;
}

// Each batch is buffered once, no matter how many consumers read it.
assert!(broadcast.max_mem_used.value() <= 2 * batch_size);
assert!(broadcast.max_mem_used.value() >= batch_size);
assert_eq!(broadcast.buffered_bytes.load(Ordering::SeqCst), 0);
let output_rows = broadcast.metrics().unwrap().output_rows().unwrap();
assert_eq!(output_rows, 12);
Ok(())
}

#[tokio::test]
async fn broadcast_exec_maps_partitions_by_modulo() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
Expand Down
38 changes: 37 additions & 1 deletion datafusion/distributed/tests/metrics_collection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ mod tests {
use datafusion::common::{HashMap, Result, assert_contains};
use datafusion::execution::SessionState;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::filter::FilterExec;
use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use datafusion::physical_plan::{ExecutionPlan, execute_stream};
use datafusion::prelude::SessionContext;
Expand All @@ -16,7 +17,7 @@ mod tests {
row_generator_desired_task_count_handler, row_generator_scale_up_leaf_node_handler,
};
use datafusion_distributed::{
DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec,
BroadcastExec, DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedLeafExec,
DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, TaskKey, TaskMetrics,
WorkerQueryContext, display_plan_ascii, rewrite_distributed_plan_with_dynamic_filters,
rewrite_distributed_plan_with_metrics,
Expand Down Expand Up @@ -206,6 +207,41 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn test_metrics_collection_in_broadcast_join() -> Result<(), Box<dyn std::error::Error>> {
let (mut d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await;
d_ctx.set_distributed_broadcast_joins(true)?;

let query = r#"
SELECT count(*)
FROM weather a
JOIN weather b ON a."RainToday" = b."RainToday"
WHERE b."MinTemp" > 15.0
"#;

let s_ctx = SessionContext::default();
let (_, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?;
d_physical =
rewrite_with_metrics(d_physical.clone(), DistributedMetricsFormat::Aggregated).await;
println!("{}", display_plan_ascii(d_physical.as_ref(), true));

// The filtered side is the smaller one, so it is the build side under BroadcastExec, and
// every consumer task reads all of it.
let input_rows = node_metrics::<FilterExec>(&d_physical, "output_rows", 0);
let output_rows = node_metrics::<BroadcastExec>(&d_physical, "output_rows", 0);
assert_eq!(output_rows, 3 * input_rows);
let input_bytes = node_metrics::<FilterExec>(&d_physical, "output_bytes", 0);
let output_bytes = node_metrics::<BroadcastExec>(&d_physical, "output_bytes", 0);
assert_eq!(output_bytes, 3 * input_bytes);
// Batches are buffered once per task, not once per consumer.
let max_mem_used = node_metrics::<BroadcastExec>(&d_physical, "max_mem_used", 0);
assert!(max_mem_used < output_bytes);
let value = node_metrics::<BroadcastExec>(&d_physical, "elapsed_compute", 0);
assert!(value > 0);

Ok(())
}

#[tokio::test]
async fn test_stage_level_metric_collection() -> Result<(), Box<dyn std::error::Error>> {
let format = DistributedMetricsFormat::PerTask;
Expand Down