Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions datafusion/distributed/src/codec/distributed_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,8 @@ fn new_network_broadcast_exec(
#[cfg(test)]
mod tests {
use super::super::physical_plan::{
new_proto_converter as default_proto_converter, roundtrip_pb,
decode_execution_plan, encode_execution_plan,
new_proto_converter as default_proto_converter,
};
use super::*;

Expand Down Expand Up @@ -971,7 +972,7 @@ mod tests {

let plan: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(dynamic_filter, network)?);

let decoded = roundtrip_pb(plan, &ctx)?;
let decoded = decode_execution_plan(&encode_execution_plan(plan, &ctx)?, &ctx)?;
let filter = decoded.downcast_ref::<FilterExec>().unwrap();
let predicate = filter
.predicate()
Expand Down
5 changes: 3 additions & 2 deletions datafusion/distributed/src/codec/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ mod user_codec;

pub use distributed_codec::DistributedCodec;
pub(crate) use physical_plan::{
decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan,
encode_partitioning, encode_physical_expr, roundtrip_pb,
RoundtripProtoConverter, decode_execution_plan, decode_partitioning, decode_physical_expr,
encode_execution_plan, encode_partitioning, encode_physical_expr, roundtrip_expr_pb,
roundtrip_pb,
};
pub(crate) use user_codec::{
get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc,
Expand Down
130 changes: 116 additions & 14 deletions datafusion/distributed/src/codec/physical_plan.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,29 @@
use super::DistributedCodec;
use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::common::Result;
use datafusion::common::{HashMap, Result};
use datafusion::execution::TaskContext;
use datafusion::physical_expr::{Partitioning, PhysicalExpr};
use datafusion::physical_plan::ExecutionPlan;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::{
ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties, ReplaceChildrenOptions,
};
use datafusion_proto::bytes::{
physical_plan_from_bytes_with_proto_converter, physical_plan_to_bytes_with_proto_converter,
};
use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning;
use datafusion_proto::physical_plan::to_proto::serialize_partitioning;
use datafusion_proto::physical_plan::from_proto::{
parse_physical_expr_with_converter, parse_protobuf_partitioning,
};
use datafusion_proto::physical_plan::to_proto::{
serialize_partitioning, serialize_physical_expr_with_converter,
};
use datafusion_proto::physical_plan::{
DeduplicatingProtoConverter, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,
DeduplicatingProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,
PhysicalPlanNodeExt, PhysicalProtoConverterExtension,
};
use datafusion_proto::protobuf;
use datafusion_proto::protobuf::proto_error;
use prost::Message;
use std::cell::RefCell;
use std::sync::Arc;

/// Creates the converter used by all top-level physical protobuf operations.
Expand Down Expand Up @@ -44,18 +53,111 @@ pub(crate) fn decode_execution_plan(
physical_plan_from_bytes_with_proto_converter(encoded, task_ctx, &codec, &converter)
}

/// Round-trips a plan through protobuf to produce an independent [`ExecutionPlan`].
/// Round-trips a single node through protobuf to produce an independent [`ExecutionPlan`].
///
/// Only `node` is serialized: its children are swapped for [`EmptyExec`] placeholders during
/// the round-trip and put back afterwards.
///
/// ```text
/// node node node
/// ...children -> EmptyExec -> ...children
/// ```
///
/// Nodes round-tripped with the same `converter` keep sharing the [`PhysicalExpr`]s (e.g.
/// dynamic filters) they shared before the round-trip.
pub(crate) fn roundtrip_pb(
plan: Arc<dyn ExecutionPlan>,
node: Arc<dyn ExecutionPlan>,
task_ctx: &TaskContext,
converter: &RoundtripProtoConverter,
) -> Result<Arc<dyn ExecutionPlan>> {
let encode_codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let encode_converter = new_proto_converter();
let proto = encode_converter.execution_plan_to_proto(&plan, &encode_codec)?;
let decode_codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &decode_codec);
let decode_converter = new_proto_converter();
decode_converter.proto_to_execution_plan(&proto, &decode_ctx)
let children = node.children().into_iter().cloned().collect::<Vec<_>>();
let placeholders = children
.iter()
.map(|child| {
let partitions = child.output_partitioning().partition_count();
Arc::new(EmptyExec::new(child.schema()).with_partitions(partitions)) as _
})
.collect();
let recompute = ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute);
let node = node.replace_children(placeholders, recompute)?;

let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let proto = converter.execution_plan_to_proto(&node, &codec)?;
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &codec);
let node = converter.proto_to_execution_plan(&proto, &decode_ctx)?;

node.replace_children(children, recompute)
}

/// Round-trips a single expression through protobuf. See [`roundtrip_pb`].
pub(crate) fn roundtrip_expr_pb(
expr: &Arc<dyn PhysicalExpr>,
input_schema: &Schema,
task_ctx: &TaskContext,
converter: &RoundtripProtoConverter,
) -> Result<Arc<dyn PhysicalExpr>> {
let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let proto = converter.physical_expr_to_proto(expr, &codec)?;
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &codec);
converter.proto_to_physical_expr(&proto, input_schema, &decode_ctx)
}

/// Same as [`DeduplicatingProtoConverter`], but expressions are deduplicated across every
/// round-trip performed with the same instance instead of within a single one.
#[derive(Default)]
pub(crate) struct RoundtripProtoConverter {
Comment on lines +107 to +108

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A bit annoying that we need this committed here...

/// Decoded expressions keyed by `expr_id`.
decoded: RefCell<HashMap<u64, Arc<dyn PhysicalExpr>>>,
}

impl PhysicalProtoConverterExtension for RoundtripProtoConverter {
fn proto_to_execution_plan(
&self,
proto: &protobuf::PhysicalPlanNode,
ctx: &PhysicalPlanDecodeContext<'_>,
) -> Result<Arc<dyn ExecutionPlan>> {
proto.try_into_physical_plan_with_context(ctx, self)
}

fn execution_plan_to_proto(
&self,
plan: &Arc<dyn ExecutionPlan>,
codec: &dyn PhysicalExtensionCodec,
) -> Result<protobuf::PhysicalPlanNode> {
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
Arc::clone(plan),
codec,
self,
)
}

fn proto_to_physical_expr(
&self,
proto: &protobuf::PhysicalExprNode,
input_schema: &Schema,
ctx: &PhysicalPlanDecodeContext<'_>,
) -> Result<Arc<dyn PhysicalExpr>> {
let parsed = parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;
let Some(id) = proto.expr_id else {
return Ok(parsed);
};
if let Some(cached) = self.decoded.borrow().get(&id) {
// Same as DeduplicatingProtoConverter: `with_new_children` keeps the state shared
// with the cached expression.
let children = parsed.children().into_iter().cloned().collect();
return Arc::clone(cached).with_new_children(children);
}
self.decoded.borrow_mut().insert(id, Arc::clone(&parsed));
Ok(parsed)
}

fn physical_expr_to_proto(
&self,
expr: &Arc<dyn PhysicalExpr>,
codec: &dyn PhysicalExtensionCodec,
) -> Result<protobuf::PhysicalExprNode> {
serialize_physical_expr_with_converter(expr, codec, self)
}
}

pub(crate) fn encode_physical_expr(
Expand Down
4 changes: 2 additions & 2 deletions datafusion/distributed/src/coordinator/query_coordinator.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::codec::roundtrip_pb;
use crate::codec::{RoundtripProtoConverter, roundtrip_pb};
use crate::common::{TreeNodeExt, now_ns, task_ctx_with_extension};
use crate::config_extension_ext::get_config_extension_propagation_headers;
use crate::coordinator::DynamicFilterRegistry;
Expand Down Expand Up @@ -449,7 +449,7 @@ impl<'a> StageCoordinator<'a> {
//
// Doing this roundtrip here is not super clean, but it transitions the feed with
// very little specialized code.
let plan = roundtrip_pb(plan, self.task_ctx)?;
let plan = roundtrip_pb(plan, self.task_ctx, &RoundtripProtoConverter::default())?;
return Ok(Transformed::yes(plan));
};

Expand Down
45 changes: 16 additions & 29 deletions datafusion/distributed/src/dynamic_filtering/display.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::codec::roundtrip_pb;
use crate::codec::{RoundtripProtoConverter, roundtrip_pb};
use crate::common::TreeNodeExt;
use crate::coordinator::DistributedExec;
use crate::dynamic_filtering::{
Expand All @@ -13,11 +13,8 @@ use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion::common::{HashMap, Result, internal_err};
use datafusion::execution::TaskContext;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::sorts::sort::SortExec;
use datafusion::physical_plan::{
ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties, ReplaceChildrenOptions,
};
use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions};
use std::sync::Arc;

/// Rewrites an executed distributed plan with dynamic filters from the coordinator and its
Expand Down Expand Up @@ -157,7 +154,11 @@ pub(crate) fn sever_dynamic_filter_relationships_in_plan_for_display(
.variants()
.iter()
.map(|variant| {
let variant = roundtrip_pb(Arc::clone(variant), task_ctx)?;
let variant = roundtrip_pb(
Arc::clone(variant),
task_ctx,
&RoundtripProtoConverter::default(),
)?;
// The variant can have a SortExec
isolate_sort_dynamic_filters_for_display(variant, task_ctx)
})
Expand All @@ -177,38 +178,24 @@ pub(crate) fn sever_dynamic_filter_relationships_in_plan_for_display(
/// displaying different dynamic filters for each task containing a [`SortExec`]. For now, we
/// just clear the dynamic filter and don't display it.
///
/// To avoid serializing an entire subtree, we swap in an [`EmptyExec`]:
///
/// ```text
/// SortExec SortExec SortExec
/// ...children -> EmptyExec -> ...children
/// ```
/// Only the [`SortExec`] node is round-tripped, its children are left untouched.
///
/// TODO(#677): display producer dynamic filters
fn isolate_sort_dynamic_filters_for_display(
plan: Arc<dyn ExecutionPlan>,
task_ctx: &Arc<TaskContext>,
) -> Result<Arc<dyn ExecutionPlan>> {
plan.transform_up(|node| {
let Some(sort) = node.downcast_ref::<SortExec>() else {
return Ok(Transformed::no(node));
};
if node.dynamic_expressions_produced().is_empty() {
if node.downcast_ref::<SortExec>().is_none()
|| node.dynamic_expressions_produced().is_empty()
{
return Ok(Transformed::no(node));
}

let input = Arc::clone(sort.input());
let placeholder = Arc::new(
EmptyExec::new(input.schema())
.with_partitions(input.output_partitioning().partition_count()),
) as Arc<dyn ExecutionPlan>;
let recompute = ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute);
let sort_with_placeholder = node.replace_children(vec![placeholder], recompute)?;

let isolated = roundtrip_pb(sort_with_placeholder, task_ctx)?;
let isolated = isolated.replace_children(vec![input], recompute)?;

Ok(Transformed::yes(isolated))
Ok(Transformed::yes(roundtrip_pb(
node,
task_ctx,
&RoundtripProtoConverter::default(),
)?))
})
.map(|transformed| transformed.data)
}
Expand Down
54 changes: 43 additions & 11 deletions datafusion/distributed/src/dynamic_filtering/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ mod apply;
mod discovery;
mod display;

use crate::DistributedConfig;
use crate::codec::roundtrip_pb;
use crate::codec::{RoundtripProtoConverter, roundtrip_expr_pb, roundtrip_pb};
use crate::stage::RemoteStage;
use crate::{DistributedConfig, NetworkBoundaryExt, Stage};
use datafusion::common::Result;
use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion::execution::TaskContext;
use datafusion::execution::config::SessionConfig;
use datafusion::physical_expr::expressions::DynamicFilterPhysicalExpr;
use datafusion::physical_plan::ExecutionPlan;
use std::sync::Arc;

Expand Down Expand Up @@ -178,14 +181,43 @@ pub(crate) fn maybe_roundtrip_plan_to_sever_in_memory_dynamic_filter_relationshi
plan: Arc<dyn ExecutionPlan>,
task_ctx: &Arc<TaskContext>,
) -> Result<Arc<dyn ExecutionPlan>> {
let has_producers = !discover_dynamic_filter_producers(&plan)?.is_empty();
let has_consumers = !discover_dynamic_filter_consumers(&plan)?
.consumers
.is_empty();
// Only the nodes holding dynamic filters are round-tripped, all through the same converter,
// so nodes in the plan that shared a dynamic filter before still share it after.
let converter = RoundtripProtoConverter::default();
plan.transform_up(|node| {
if let Some(boundary) = node.as_network_boundary() {
// Network boundaries are not round-tripped as a whole, as their input stage might
// reference workers that are not known yet. Only their anchors are.
let Stage::Remote(stage) = boundary.input_stage() else {
return Ok(Transformed::no(node));
};
if stage.dynamic_filter_anchors.is_empty() {
return Ok(Transformed::no(node));
}
let schema = node.schema();
let anchors = stage
.dynamic_filter_anchors
.iter()
.map(|anchor| roundtrip_expr_pb(anchor, &schema, task_ctx, &converter))
.collect::<Result<_>>()?;
let stage = Stage::Remote(RemoteStage {
dynamic_filter_anchors: anchors,
..stage.clone()
});
return Ok(Transformed::yes(boundary.with_input_stage(stage)?));
}

if has_producers || has_consumers {
roundtrip_pb(plan, task_ctx)
} else {
Ok(plan)
}
let mut has_dynamic_filters = !node.dynamic_expressions_produced().is_empty();
node.apply_expressions(&mut |root| {
root.apply(|expr| {
has_dynamic_filters |= expr.downcast_ref::<DynamicFilterPhysicalExpr>().is_some();
Ok(TreeNodeRecursion::Continue)
})
})?;
if !has_dynamic_filters {
return Ok(Transformed::no(node));
}
Ok(Transformed::yes(roundtrip_pb(node, task_ctx, &converter)?))
})
.map(|transformed| transformed.data)
}
Loading