From b0c46c48b5e5198b0ab42d0ecd430122666a899c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 18 Nov 2026 11:51:46 +0100 Subject: [PATCH 1/2] Add coalescing producer head to NetworkCoalesceExec --- .../src/proto/generated/iceberg.rs | 17 +- .../src/codec/distributed_codec.rs | 19 +- datafusion/distributed/src/codec/mod.rs | 5 +- .../distributed/src/codec/physical_plan.rs | 45 +++- .../distributed/src/common/maybe_encoded.rs | 25 +- .../distributed_query_planner.rs | 3 - .../inject_network_boundaries.rs | 10 +- .../src/distributed_planner/mod.rs | 1 - .../distributed_planner/network_boundary.rs | 32 ++- .../push_fetch_into_network_coalesce.rs | 243 ------------------ .../src/execution_plans/network_coalesce.rs | 79 ++---- .../src/protocol/grpc/generated/worker.rs | 22 +- .../src/protocol/grpc/worker.proto | 15 ++ .../src/protocol/grpc/worker_client.rs | 17 ++ .../src/protocol/grpc/worker_service.rs | 12 + 15 files changed, 209 insertions(+), 336 deletions(-) delete mode 100644 datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs diff --git a/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs b/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs index 8e512a298..75f0de515 100644 --- a/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs +++ b/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct IcebergDataSource { #[prost(message, optional, tag = "1")] @@ -11,8 +10,10 @@ pub struct IcebergDataSource { #[prost(uint64, optional, tag = "4")] pub fetch: ::core::option::Option, #[prost(map = "string, string", tag = "5")] - pub storage_properties: - ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + pub storage_properties: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, #[prost(bytes = "vec", optional, tag = "6")] pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec>, /// Filters pushed into the scan. A row must pass all of them. @@ -23,13 +24,11 @@ pub struct IcebergDataSource { #[prost(bool, tag = "8")] pub dynamic_filtering_enabled: bool, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WorkUnitFeed { #[prost(bytes = "vec", tag = "1")] pub id: ::prost::alloc::vec::Vec, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ScanConfig { #[prost(message, optional, tag = "1")] @@ -39,8 +38,10 @@ pub struct ScanConfig { #[prost(uint64, optional, tag = "3")] pub fetch: ::core::option::Option, #[prost(map = "string, string", tag = "4")] - pub storage_properties: - ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + pub storage_properties: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, #[prost(bytes = "vec", optional, tag = "5")] pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec>, /// Filters pushed into the scan. A row must pass all of them. @@ -51,7 +52,6 @@ pub struct ScanConfig { #[prost(bool, tag = "7")] pub dynamic_filtering_enabled: bool, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AssignedScan { #[prost(message, optional, tag = "1")] @@ -59,7 +59,6 @@ pub struct AssignedScan { #[prost(message, repeated, tag = "2")] pub partitions: ::prost::alloc::vec::Vec, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FileGroup { #[prost(bytes = "vec", repeated, tag = "1")] diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 2e53861d9..3d11debe2 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -178,6 +178,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning, input_stage, equivalence_classes, + fetch, }) => { let schema: Schema = schema .as_ref() @@ -208,11 +209,14 @@ impl PhysicalExtensionCodec for DistributedCodec { proto_converter, )?; - Ok(Arc::new(new_network_coalesce_tasks_exec( - partitioning, - equivalence_properties, - parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?, - ))) + Ok(Arc::new( + new_network_coalesce_tasks_exec( + partitioning, + equivalence_properties, + parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?, + ) + .with_fetch(fetch.map(|v| v as usize)), + )) } DistributedExecNode::NetworkBroadcast(NetworkBroadcastExecProto { schema, @@ -420,6 +424,7 @@ impl PhysicalExtensionCodec for DistributedCodec { self, proto_converter, )?, + fetch: node.fetch.map(|v| v as u64), }; let wrapper = DistributedExecProto { @@ -787,6 +792,9 @@ pub struct NetworkCoalesceExecProto { input_stage: Option, #[prost(message, repeated, tag = "4")] equivalence_classes: Vec, + /// Row limit the producer head applies per input task. + #[prost(uint64, optional, tag = "5")] + fetch: Option, } fn new_network_coalesce_tasks_exec( @@ -803,6 +811,7 @@ fn new_network_coalesce_tasks_exec( )), worker_connections: WorkerConnectionPool::new(input_stage.task_count()), input_stage, + fetch: None, } } diff --git a/datafusion/distributed/src/codec/mod.rs b/datafusion/distributed/src/codec/mod.rs index fc03f09fb..cb1a36843 100644 --- a/datafusion/distributed/src/codec/mod.rs +++ b/datafusion/distributed/src/codec/mod.rs @@ -4,8 +4,9 @@ mod user_codec; pub use distributed_codec::DistributedCodec; pub(crate) use physical_plan::{ - decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan, - encode_partitioning, encode_physical_expr, roundtrip_pb, + decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr, + encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr, + roundtrip_pb, }; pub(crate) use user_codec::{ get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc, diff --git a/datafusion/distributed/src/codec/physical_plan.rs b/datafusion/distributed/src/codec/physical_plan.rs index f1a970ace..700cdad9b 100644 --- a/datafusion/distributed/src/codec/physical_plan.rs +++ b/datafusion/distributed/src/codec/physical_plan.rs @@ -1,14 +1,18 @@ use super::DistributedCodec; use datafusion::arrow::datatypes::{Schema, SchemaRef}; -use datafusion::common::Result; +use datafusion::common::{Result, internal_datafusion_err}; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{Partitioning, PhysicalExpr}; +use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr}; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::bytes::{ physical_plan_from_bytes_with_proto_converter, physical_plan_to_bytes_with_proto_converter, }; -use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; -use datafusion_proto::physical_plan::to_proto::serialize_partitioning; +use datafusion_proto::physical_plan::from_proto::{ + parse_physical_sort_exprs, parse_protobuf_partitioning, +}; +use datafusion_proto::physical_plan::to_proto::{ + serialize_partitioning, serialize_physical_sort_exprs, +}; use datafusion_proto::physical_plan::{ DeduplicatingProtoConverter, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension, }; @@ -100,3 +104,36 @@ pub(crate) fn decode_partitioning( parse_protobuf_partitioning(Some(&proto_partitioning), &decode_ctx, &schema, &converter)? .ok_or_else(|| proto_error("Could not parse partitioning")) } + +/// Protobuf wrapper for a [LexOrdering], as upstream has no message for a list of +/// sort expressions. +#[derive(Clone, PartialEq, ::prost::Message)] +struct LexOrderingProto { + #[prost(message, repeated, tag = "1")] + sort_exprs: Vec, +} + +pub(crate) fn encode_lex_ordering( + ordering: &LexOrdering, + task_ctx: &TaskContext, +) -> Result> { + let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config()); + let converter = new_proto_converter(); + let sort_exprs = serialize_physical_sort_exprs(ordering.iter().cloned(), &codec, &converter)?; + Ok(LexOrderingProto { sort_exprs }.encode_to_vec()) +} + +pub(crate) fn decode_lex_ordering( + encoded: &[u8], + input_schema: &Schema, + task_ctx: &TaskContext, +) -> Result { + let proto = LexOrderingProto::decode(encoded).map_err(|err| proto_error(err.to_string()))?; + let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config()); + let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &codec); + let converter = new_proto_converter(); + let sort_exprs = + parse_physical_sort_exprs(&proto.sort_exprs, &decode_ctx, input_schema, &converter)?; + LexOrdering::new(sort_exprs) + .ok_or_else(|| internal_datafusion_err!("Cannot decode an empty LexOrdering")) +} diff --git a/datafusion/distributed/src/common/maybe_encoded.rs b/datafusion/distributed/src/common/maybe_encoded.rs index 7e01ad2b3..c28d19e1d 100644 --- a/datafusion/distributed/src/common/maybe_encoded.rs +++ b/datafusion/distributed/src/common/maybe_encoded.rs @@ -1,11 +1,11 @@ use crate::codec::{ - decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan, - encode_partitioning, encode_physical_expr, + decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr, + encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr, }; use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::common::{Result, internal_err}; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{Partitioning, PhysicalExpr}; +use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr}; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::protobuf::PhysicalExprNode; use datafusion_proto::protobuf::proto_error; @@ -124,6 +124,25 @@ impl MaybeEncoded> { } } +impl MaybeEncoded { + /// Returns the encoded [`LexOrdering`] as protobuf bytes: + /// - If in `Decoded` state, it encodes it using the codecs registered in the [`TaskContext`]. + /// - If in `Encoded` state, it passes through the existing bytes. + pub fn encode(self, ctx: &Arc) -> Result> { + match self { + Self::Encoded(encoded) => Ok(encoded), + Self::Decoded(lex_ordering) => encode_lex_ordering(&lex_ordering, ctx), + } + } + + /// Returns the decoded [`LexOrdering`]. + /// - If in `Decoded` state, it passes through the ordering. + /// - If in `Encoded` state, it decodes it using the provided schema and task context. + pub fn decode(self, input_schema: &Schema, task_ctx: &TaskContext) -> Result { + self.decode_with(|encoded| decode_lex_ordering(&encoded, input_schema, task_ctx)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs index 8ecbab25c..092dacf85 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs @@ -8,7 +8,6 @@ use crate::distributed_planner::insert_children_isolator_union::insert_children_ use crate::distributed_planner::normalize_collect_joins::normalize_collect_joins; use crate::distributed_planner::partial_reduce_below_network_shuffles::partial_reduce_below_network_shuffles; use crate::distributed_planner::prepare_network_boundaries::prepare_network_boundaries; -use crate::distributed_planner::push_fetch_into_network_coalesce::push_fetch_into_network_coalesce; use crate::events::{ScaleUpLeafNodeEvent, ScaleUpLeafNodeHandlers}; use crate::explain_analyze::DistributedAnalyzeExec; use crate::{DistributedConfig, DistributedExec, NetworkBoundaryExt}; @@ -135,7 +134,6 @@ pub fn create_distributed_plan( if !plan.exists(|plan| Ok(plan.is_network_boundary()))? { return Ok(plan); } - let plan = push_fetch_into_network_coalesce(plan)?; return Ok(create_distributed_exec(Arc::clone(&plan), d_cfg)); } @@ -166,7 +164,6 @@ pub fn create_distributed_plan( } let plan = partial_reduce_below_network_shuffles(plan, cfg)?; - let plan = push_fetch_into_network_coalesce(plan)?; Ok(create_distributed_exec(Arc::clone(&plan), d_cfg)) }) diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index e796a7cca..b176e2fab 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -425,12 +425,10 @@ async fn _inject_network_boundaries( // The parent that triggered this branch is a `CoalescePartitionsExec` or // `SortPreservingMergeExec`, both of which fold all partitions into one — so the // stage above this boundary must run in exactly one task. - let nb = Arc::new(NetworkCoalesceExec::try_from_stage( - result.input_stage, - result.input_properties, - 1, - )?); - Ok(nb_ctx.plan_with_task_count(nb, result.consumer_task_count)) + let nb = + NetworkCoalesceExec::try_from_stage(result.input_stage, result.input_properties, 1)? + .with_fetch(parent.fetch()); + Ok(nb_ctx.plan_with_task_count(Arc::new(nb), result.consumer_task_count)) } else if parent.is_none() { // We've just finished walking the head stage's subplan. Run a final propagation so // every node in the head stage (which never crossed a stage boundary on the way up) diff --git a/datafusion/distributed/src/distributed_planner/mod.rs b/datafusion/distributed/src/distributed_planner/mod.rs index ff32d02f7..a4e8ae5dc 100644 --- a/datafusion/distributed/src/distributed_planner/mod.rs +++ b/datafusion/distributed/src/distributed_planner/mod.rs @@ -8,7 +8,6 @@ mod network_boundary; mod normalize_collect_joins; mod partial_reduce_below_network_shuffles; mod prepare_network_boundaries; -mod push_fetch_into_network_coalesce; mod session_state_builder_ext; mod statistics; diff --git a/datafusion/distributed/src/distributed_planner/network_boundary.rs b/datafusion/distributed/src/distributed_planner/network_boundary.rs index 259c802b2..4f4d63cee 100644 --- a/datafusion/distributed/src/distributed_planner/network_boundary.rs +++ b/datafusion/distributed/src/distributed_planner/network_boundary.rs @@ -6,8 +6,10 @@ use crate::{ use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::Result; use datafusion::execution::TaskContext; -use datafusion::physical_expr::Partitioning; +use datafusion::physical_expr::{LexOrdering, Partitioning}; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties, ReplaceChildrenOptions, }; @@ -47,6 +49,13 @@ pub enum ProducerHead { /// Whether the producer must preserve its input's output ordering. preserve_order: bool, }, + /// The head node should be a [CoalescePartitionsExec]. + CoalescePartitionsExec { fetch: Option }, + /// The head node should be a [SortPreservingMergeExec]. + SortPreservingMergeExec { + fetch: Option, + lex_ordering: MaybeEncoded, + }, } /// Extension trait for downcasting dynamic types to [NetworkBoundary]. @@ -83,6 +92,13 @@ impl ProducerHead { partitioning: MaybeEncoded::Decoded(partitioning.decode(schema, ctx)?), preserve_order, }, + Self::SortPreservingMergeExec { + fetch, + lex_ordering, + } => Self::SortPreservingMergeExec { + fetch, + lex_ordering: MaybeEncoded::Decoded(lex_ordering.decode(&schema, ctx)?), + }, v => v, }) } @@ -94,6 +110,10 @@ impl ProducerHead { Arc::clone(r_exec.input()) } else if let Some(b_exec) = input.downcast_ref::() { Arc::clone(b_exec.input()) + } else if let Some(c_exec) = input.downcast_ref::() { + Arc::clone(c_exec.input()) + } else if let Some(s_exec) = input.downcast_ref::() { + Arc::clone(s_exec.input()) } else { input }; @@ -114,6 +134,16 @@ impl ProducerHead { repartition }) } + ProducerHead::CoalescePartitionsExec { fetch } => { + Arc::new(CoalescePartitionsExec::new(input).with_fetch(fetch)) + } + ProducerHead::SortPreservingMergeExec { + fetch, + lex_ordering, + } => { + let lex_ordering = lex_ordering.try_decoded()?; + Arc::new(SortPreservingMergeExec::new(lex_ordering, input).with_fetch(fetch)) + } }; Ok(plan) } diff --git a/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs b/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs deleted file mode 100644 index a9130b356..000000000 --- a/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs +++ /dev/null @@ -1,243 +0,0 @@ -use std::sync::Arc; - -use datafusion::common::Result; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::physical_plan::limit::GlobalLimitExec; -use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; - -use crate::NetworkCoalesceExec; - -/// Pushes fetch limits through [NetworkCoalesceExec] stage boundaries. -/// -/// DataFusion's normal `LimitPushdown` rule runs before the distributed planner inserts network -/// boundaries, so a fetch-bearing parent can end up directly above a [NetworkCoalesceExec] whose -/// producer stage is still unbounded. -/// -/// This pass rewrites: -/// -/// ```text -/// ┌─────────────────────────────┐ -/// │ CoalescePartitions(fetch=10)│ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkCoalesceExec │ -/// └──────────────▲──────────────┘ -/// │ input_stage.plan -/// ┌──────────────┴──────────────┐ -/// │ AggregateExec │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkShuffleExec │ -/// └─────────────────────────────┘ -/// ``` -/// -/// into: -/// -/// ```text -/// ┌─────────────────────────────┐ -/// │ CoalescePartitions(fetch=10)│ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkCoalesceExec │ -/// └──────────────▲──────────────┘ -/// │ input_stage.plan -/// ┌──────────────┴──────────────┐ -/// │ LocalLimitExec(fetch=10) │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ AggregateExec │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkShuffleExec │ -/// └─────────────────────────────┘ -/// ``` -/// -/// The parent fetch remains in place because it enforces the global result limit. The producer-stage -/// fetch is only a per-task bound that lets remote work stop earlier. -pub(crate) fn push_fetch_into_network_coalesce( - plan: Arc, -) -> Result> { - plan.transform_down(|node| { - let Some(fetch) = fetch_required_from_child(&node) else { - return Ok(Transformed::no(node)); - }; - - let mut changed = false; - let new_children = node - .children() - .into_iter() - .map(|child| { - if let Some(network_coalesce) = child.downcast_ref::() { - changed = true; - network_coalesce.with_fetch_on_input_stage(fetch) - } else { - Ok(Arc::clone(child)) - } - }) - .collect::>>()?; - - if changed { - node.replace_children( - new_children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - ) - .map(Transformed::yes) - } else { - Ok(Transformed::no(node)) - } - }) - .map(|t| t.data) -} - -fn fetch_required_from_child(plan: &Arc) -> Option { - if let Some(global_limit) = plan.downcast_ref::() { - return global_limit - .fetch() - .and_then(|fetch| fetch.checked_add(global_limit.skip())); - } - - plan.fetch() -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::physical_expr::expressions::Column; - use datafusion::physical_plan::ExecutionPlan; - use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; - use datafusion::physical_plan::empty::EmptyExec; - use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; - use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; - - use crate::{NetworkBoundary, Stage}; - - use super::*; - - #[test] - fn fetch_pushed_into_network_coalesce_input_stage() -> Result<()> { - let input: Arc = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let network_coalesce: Arc = - Arc::new(NetworkCoalesceExec::try_new(input, 1, 1)?); - let plan: Arc = - Arc::new(CoalescePartitionsExec::new(network_coalesce).with_fetch(Some(7))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - assert_eq!(rewritten.fetch(), Some(7)); - - let parent = rewritten - .downcast_ref::() - .expect("root should remain CoalescePartitionsExec"); - let network_coalesce = parent.children()[0] - .downcast_ref::() - .expect("child should remain NetworkCoalesceExec"); - let Stage::Local(local_stage) = network_coalesce.input_stage() else { - panic!("test network boundary should still hold a local stage"); - }; - let local_limit = local_stage - .plan - .downcast_ref::() - .expect("input stage should be bounded by LocalLimitExec"); - - assert_eq!(local_limit.fetch(), 7); - Ok(()) - } - - #[test] - fn global_limit_with_offset_pushes_skip_plus_fetch_into_network_coalesce_input_stage() - -> Result<()> { - let input: Arc = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let network_coalesce: Arc = - Arc::new(NetworkCoalesceExec::try_new(input, 2, 1)?); - let plan: Arc = - Arc::new(GlobalLimitExec::new(network_coalesce, 100, Some(10))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - let global_limit = rewritten - .downcast_ref::() - .expect("root should remain GlobalLimitExec"); - assert_eq!(global_limit.skip(), 100); - assert_eq!(global_limit.fetch(), Some(10)); - - let network_coalesce = global_limit - .input() - .downcast_ref::() - .expect("child should remain NetworkCoalesceExec"); - let Stage::Local(local_stage) = network_coalesce.input_stage() else { - panic!("test network boundary should still hold a local stage"); - }; - let local_limit = local_stage - .plan - .downcast_ref::() - .expect("input stage should be bounded by LocalLimitExec"); - - assert_eq!(local_limit.fetch(), 110); - Ok(()) - } - - #[test] - fn fetch_separated_by_projection_does_not_add_local_limit() -> Result<()> { - let inner = Arc::new(CoalescePartitionsExec::new(empty_plan()).with_fetch(Some(7))); - let network_coalesce: Arc = Arc::new(NetworkCoalesceExec::try_new( - identity_projection(inner)?, - 1, - 1, - )?); - let plan: Arc = - Arc::new(CoalescePartitionsExec::new(network_coalesce).with_fetch(Some(7))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - let input_stage = local_input_stage_plan(&rewritten); - - assert!( - input_stage.downcast_ref::().is_none(), - "ProjectionExec preserves row count, so a LocalLimitExec above it is redundant" - ); - let projection = input_stage - .downcast_ref::() - .expect("input stage root should remain ProjectionExec"); - assert_eq!(projection.input().fetch(), Some(7)); - Ok(()) - } - - fn empty_plan() -> Arc { - Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new( - "c", - DataType::Int32, - true, - )])))) - } - - fn identity_projection(input: Arc) -> Result> { - let schema = input.schema(); - let exprs = schema - .fields() - .iter() - .enumerate() - .map(|(i, field)| ProjectionExpr { - expr: Arc::new(Column::new(field.name(), i)), - alias: field.name().to_string(), - }) - .collect::>(); - Ok(Arc::new(ProjectionExec::try_new(exprs, input)?)) - } - - fn local_input_stage_plan(plan: &Arc) -> Arc { - fn find(node: &Arc) -> Option> { - if let Some(network_coalesce) = node.downcast_ref::() { - let Stage::Local(local) = network_coalesce.input_stage() else { - return None; - }; - return Some(Arc::clone(&local.plan)); - } - node.children().into_iter().find_map(find) - } - find(plan).expect("plan should contain a local NetworkCoalesceExec stage") - } -} diff --git a/datafusion/distributed/src/execution_plans/network_coalesce.rs b/datafusion/distributed/src/execution_plans/network_coalesce.rs index db9b964a8..3122f91ac 100644 --- a/datafusion/distributed/src/execution_plans/network_coalesce.rs +++ b/datafusion/distributed/src/execution_plans/network_coalesce.rs @@ -1,22 +1,19 @@ -use crate::DistributedTaskContext; use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::execution_plans::common::scale_partitioning_props; use crate::stage::{LocalStage, Stage}; use crate::worker::WorkerConnectionPool; +use crate::{DistributedTaskContext, MaybeEncoded}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{exec_err, not_impl_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr_common::metrics::MetricsSet; -use datafusion::physical_plan::limit::LocalLimitExec; -use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - ChildrenPropertiesMode, DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, - PlanProperties, ReplaceChildrenOptions, Statistics, StatisticsArgs, apply_expression_roots, - internal_err, + DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, PlanProperties, + Statistics, StatisticsArgs, apply_expression_roots, internal_err, }; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -81,6 +78,7 @@ pub struct NetworkCoalesceExec { pub(crate) properties: Arc, pub(crate) input_stage: Stage, pub(crate) worker_connections: WorkerConnectionPool, + pub(crate) fetch: Option, } impl NetworkCoalesceExec { @@ -93,12 +91,13 @@ impl NetworkCoalesceExec { // per output task based on the maximum group size, returning empty streams for tasks with // smaller groups. let max_input_task_count = input_stage.task_count().div_ceil(consumer_tasks).max(1); - let props = scale_partitioning_props(&input_properties, |p| p * max_input_task_count)?; + let props = scale_partitioning_props(&input_properties, |_| max_input_task_count)?; Ok(Self { properties: props, worker_connections: WorkerConnectionPool::new(input_stage.task_count()), input_stage, + fetch: None, }) } @@ -142,56 +141,12 @@ impl NetworkCoalesceExec { ) } - pub(crate) fn with_fetch_on_input_stage(&self, fetch: usize) -> Result> { - let Stage::Local(local) = &self.input_stage else { - return Ok(Arc::new(self.clone())); - }; - - let input_with_fetch = apply_fetch_through_projection(Arc::clone(&local.plan), fetch)?; - - let mut self_clone = self.clone(); - self_clone.input_stage = Stage::Local(LocalStage { - query_id: local.query_id, - num: local.num, - plan: input_with_fetch, - tasks: local.tasks, - metrics_set: Default::default(), - }); - Ok(Arc::new(self_clone)) + pub fn with_fetch(mut self, fetch: Option) -> Self { + self.fetch = fetch; + self } } -/// Applies `fetch` to `plan`, looking through a [ProjectionExec] that preserves row count. -/// -/// If the projection child already carries a sufficient fetch, the plan is left unchanged instead -/// of wrapping a redundant [LocalLimitExec]. -fn apply_fetch_through_projection( - plan: Arc, - fetch: usize, -) -> Result> { - if let Some(projection) = plan.downcast_ref::() { - let input = Arc::clone(projection.input()); - if input.fetch().is_some_and(|existing| existing <= fetch) { - return Ok(plan); - } - let new_input = input - .with_fetch(Some(fetch)) - .unwrap_or_else(|| Arc::new(LocalLimitExec::new(input, fetch))); - return plan.replace_children( - vec![new_input], - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - ); - } - - if plan.fetch().is_some_and(|existing| existing <= fetch) { - return Ok(plan); - } - - Ok(plan - .with_fetch(Some(fetch)) - .unwrap_or_else(|| Arc::new(LocalLimitExec::new(plan, fetch)))) -} - impl NetworkBoundary for NetworkCoalesceExec { fn input_stage(&self) -> &Stage { &self.input_stage @@ -208,7 +163,13 @@ impl NetworkBoundary for NetworkCoalesceExec { } fn producer_head(&self, _consumer_task_count: usize) -> Result { - Ok(ProducerHead::None) + match self.properties.output_ordering() { + Some(lex_ordering) => Ok(ProducerHead::SortPreservingMergeExec { + fetch: self.fetch, + lex_ordering: MaybeEncoded::Decoded(lex_ordering.clone()), + }), + None => Ok(ProducerHead::CoalescePartitionsExec { fetch: self.fetch }), + } } } @@ -251,18 +212,20 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/protocol/grpc/generated/worker.rs b/datafusion/distributed/src/protocol/grpc/generated/worker.rs index cd2d58d43..10c2252ba 100644 --- a/datafusion/distributed/src/protocol/grpc/generated/worker.rs +++ b/datafusion/distributed/src/protocol/grpc/generated/worker.rs @@ -240,7 +240,7 @@ pub struct ExecuteTaskRequest { /// * A RepartitionExecHead implies a RepartitionExec at the head of the task. /// * A BroadcastExecHead implies a BroadcastExec at the head of the task. /// * A NoneHead does not need any specific head. - #[prost(oneof = "execute_task_request::ProducerHead", tags = "6, 7, 8")] + #[prost(oneof = "execute_task_request::ProducerHead", tags = "6, 7, 8, 9, 10")] pub producer_head: ::core::option::Option, } /// Nested message and enum types in `ExecuteTaskRequest`. @@ -262,6 +262,12 @@ pub mod execute_task_request { /// The boundary executing the task is a NetworkShuffleExec. #[prost(message, tag = "8")] Repartition(super::RepartitionExecHead), + /// The boundary executing the task is a NetworkCoalesceExec with no ordering requirements. + #[prost(message, tag = "9")] + CoalescePartitions(super::CoalescePartitionsExecHead), + /// The boundary executing the task is a NetworkCoalesceExec with specific requirements. + #[prost(message, tag = "10")] + SortPreservingMerge(super::SortPreservingMergeExecHead), } } /// Head needed by a NetworkCoalesceExec. @@ -284,6 +290,20 @@ pub struct RepartitionExecHead { #[prost(bool, tag = "2")] pub preserve_order: bool, } +/// Head needed by a NetworkCoalesceExec in case no ordering is specified. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CoalescePartitionsExecHead { + #[prost(uint64, optional, tag = "1")] + pub fetch: ::core::option::Option, +} +/// Head needed by a NetworkCoalesceExec in case a specific ordering is specified. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SortPreservingMergeExecHead { + #[prost(uint64, optional, tag = "1")] + pub fetch: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub lex_ordering: ::prost::alloc::vec::Vec, +} /// A key that uniquely identifies a task in a query. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TaskKey { diff --git a/datafusion/distributed/src/protocol/grpc/worker.proto b/datafusion/distributed/src/protocol/grpc/worker.proto index 62aadebf4..d9419b8f2 100644 --- a/datafusion/distributed/src/protocol/grpc/worker.proto +++ b/datafusion/distributed/src/protocol/grpc/worker.proto @@ -204,6 +204,10 @@ message ExecuteTaskRequest { BroadcastExecHead broadcast = 7; // The boundary executing the task is a NetworkShuffleExec. RepartitionExecHead repartition = 8; + // The boundary executing the task is a NetworkCoalesceExec with no ordering requirements. + CoalescePartitionsExecHead coalesce_partitions = 9; + // The boundary executing the task is a NetworkCoalesceExec with specific requirements. + SortPreservingMergeExecHead sort_preserving_merge = 10; } } @@ -224,6 +228,17 @@ message RepartitionExecHead { bool preserve_order = 2; } +// Head needed by a NetworkCoalesceExec in case no ordering is specified. +message CoalescePartitionsExecHead { + optional uint64 fetch = 1; +} + +// Head needed by a NetworkCoalesceExec in case a specific ordering is specified. +message SortPreservingMergeExecHead { + optional uint64 fetch = 1; + bytes lex_ordering = 2; +} + // A key that uniquely identifies a task in a query. message TaskKey { // Our query id. diff --git a/datafusion/distributed/src/protocol/grpc/worker_client.rs b/datafusion/distributed/src/protocol/grpc/worker_client.rs index 4afbbc898..02cca4471 100644 --- a/datafusion/distributed/src/protocol/grpc/worker_client.rs +++ b/datafusion/distributed/src/protocol/grpc/worker_client.rs @@ -31,6 +31,7 @@ use futures::future::{Either, pending, ready}; use futures::stream::{BoxStream, select}; use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use http::{Extensions, HeaderMap}; +use num_traits::AsPrimitive; use pin_project::{pin_project, pinned_drop}; use prost::Message; use std::borrow::Cow; @@ -496,6 +497,22 @@ pub(super) fn encode_producer_head( partitioning: partitioning.encode(ctx)?, preserve_order, }), + ProducerHead::CoalescePartitionsExec { fetch } => { + pb::execute_task_request::ProducerHead::CoalescePartitions( + pb::CoalescePartitionsExecHead { + fetch: fetch.map(|v| v.as_()), + }, + ) + } + ProducerHead::SortPreservingMergeExec { + fetch, + lex_ordering, + } => pb::execute_task_request::ProducerHead::SortPreservingMerge( + pb::SortPreservingMergeExecHead { + fetch: fetch.map(|v| v.as_()), + lex_ordering: lex_ordering.encode(ctx)?, + }, + ), }) } diff --git a/datafusion/distributed/src/protocol/grpc/worker_service.rs b/datafusion/distributed/src/protocol/grpc/worker_service.rs index 27b5fc9aa..a2d90b9b0 100644 --- a/datafusion/distributed/src/protocol/grpc/worker_service.rs +++ b/datafusion/distributed/src/protocol/grpc/worker_service.rs @@ -28,6 +28,7 @@ use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion_proto::protobuf; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; +use num_traits::AsPrimitive; use prost::Message; use std::sync::Arc; use tonic::metadata::MetadataMap; @@ -281,6 +282,17 @@ pub(super) fn decode_producer_head(proto: pb::execute_task_request::ProducerHead partitioning: MaybeEncoded::Encoded(v.partitioning), preserve_order: v.preserve_order, }, + pb::execute_task_request::ProducerHead::CoalescePartitions(v) => { + ProducerHead::CoalescePartitionsExec { + fetch: v.fetch.map(|v| v.as_()), + } + } + pb::execute_task_request::ProducerHead::SortPreservingMerge(v) => { + ProducerHead::SortPreservingMergeExec { + fetch: v.fetch.map(|v| v.as_()), + lex_ordering: MaybeEncoded::Encoded(v.lex_ordering), + } + } } } From 7bea8c443170c9ad5274f68bc1ede3db0b7981a2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 18 Nov 2026 13:55:36 +0100 Subject: [PATCH 2/2] Update plan snapshots --- .../distributed_query_planner.rs | 414 +-- .../distributed_planner/insert_broadcast.rs | 5 +- .../partial_reduce_below_network_shuffles.rs | 13 +- .../src/dynamic_filtering/discovery.rs | 33 +- .../metrics/plan_dump/plan_dump_display.rs | 84 +- .../tests/clickbench_plans_test.rs | 481 ++-- .../tests/distributed_aggregation.rs | 22 +- .../distributed/tests/distributed_unions.rs | 243 +- .../tests/dynamic_filtering/aggregates.rs | 30 +- .../dynamic_filtering/collect_left_join.rs | 125 +- .../tests/dynamic_filtering/config.rs | 17 +- .../dynamic_filtering/partitioned_join.rs | 208 +- .../tests/dynamic_filtering/sorts.rs | 28 +- .../tests/empty_build_partitions.rs | 13 +- datafusion/distributed/tests/join.rs | 65 +- .../distributed/tests/metrics_collection.rs | 2 +- .../tests/multi_task_collect_join_repros.rs | 232 +- datafusion/distributed/tests/shuffle.rs | 9 +- .../distributed/tests/sorted_data_source.rs | 96 +- .../tests/stateful_execution_plan.rs | 21 +- .../distributed/tests/task_estimator_test.rs | 111 +- .../distributed/tests/tpcds_plans_test.rs | 2260 +++++++++-------- .../distributed/tests/tpch_plans_test.rs | 377 +-- .../tests/tpch_sorted_plans_test.rs | 377 +-- datafusion/distributed/tests/udfs.rs | 11 +- .../distributed/tests/work_unit_feed.rs | 262 +- 26 files changed, 2925 insertions(+), 2614 deletions(-) diff --git a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs index 092dacf85..8c2bbad7d 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs @@ -222,13 +222,14 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet └────────────────────────────────────────────────── "); } @@ -242,16 +243,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -273,16 +275,17 @@ mod tests { .num_workers(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=2, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -371,16 +374,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -401,15 +405,16 @@ mod tests { let physical_plan_ascii = TestPlanBuilder::default() .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=3, partitions=4 - │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 - │ [Stage 2] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 3 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + │ [Stage 2] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=12 │ RepartitionExec: partitioning=Hash([RainToday@1], 12), input_partitions=3 @@ -510,14 +515,15 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [MinTemp@0 DESC] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + ┌───── Stage 1 ── tasks=3, partitions=3 + │ SortPreservingMergeExec: [MinTemp@0 DESC] + │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true └────────────────────────────────────────────────── "); } @@ -531,16 +537,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] │ CoalescePartitionsExec: fetch=10 - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ LocalLimitExec: fetch=10 - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec: fetch=10 + │ LocalLimitExec: fetch=10 + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -562,14 +569,15 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=3 @@ -610,17 +618,18 @@ mod tests { .num_workers(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=2, partitions=4 - │ DistributedUnionExec: t0:[c0] t1:[c1] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + ┌───── Stage 1 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -636,17 +645,18 @@ mod tests { .distributed_max_tasks_per_stage(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=2, partitions=8 - │ DistributedUnionExec: t0:[c0] t1:[c1] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + ┌───── Stage 1 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -665,23 +675,24 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=6 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=6 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=6, partitions=24 - │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 + ┌───── Stage 1 ── tasks=6, partitions=6 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -699,20 +710,21 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=12 - │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -732,24 +744,25 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=12 - │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp9am@0 > 15 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -773,34 +786,35 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=24 - │ DistributedUnionExec: t0:[c0, c3] t1:[c1, c4] t2:[c2] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp9am@0 > 15 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] - │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp3pm@0 < 25 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0, c3] t1:[c1, c4] t2:[c2] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] - │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Rainfall@0 > 5 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp3pm@0 < 25 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Rainfall@0 > 5 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -816,19 +830,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -852,19 +867,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@2, RainToday@1)], projection=[MinTemp@0, MaxTemp@1, Rainfall@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 3 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@2, RainToday@1)], projection=[MinTemp@0, MaxTemp@1, Rainfall@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -901,15 +917,16 @@ mod tests { │ CoalescePartitionsExec │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet └────────────────────────────────────────────────── "); @@ -917,19 +934,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -960,28 +978,29 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=3, partitions=9 - │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 4 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 @@ -1019,19 +1038,20 @@ mod tests { .desired_task_count_handler(build_side_one_desired_task_count_handler) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=1, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 diff --git a/datafusion/distributed/src/distributed_planner/insert_broadcast.rs b/datafusion/distributed/src/distributed_planner/insert_broadcast.rs index de3f18185..25d4e323d 100644 --- a/datafusion/distributed/src/distributed_planner/insert_broadcast.rs +++ b/datafusion/distributed/src/distributed_planner/insert_broadcast.rs @@ -212,8 +212,9 @@ mod tests { CoalescePartitionsExec HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec - [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 - DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=4 + CoalescePartitionsExec + DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); let test_plan = TestPlanBuilder::new() diff --git a/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs b/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs index 338088837..08076da12 100644 --- a/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs +++ b/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs @@ -117,15 +117,16 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ AggregateExec: mode=PartialReduce, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] diff --git a/datafusion/distributed/src/dynamic_filtering/discovery.rs b/datafusion/distributed/src/dynamic_filtering/discovery.rs index 0bf1ce2f7..ae78a05d8 100644 --- a/datafusion/distributed/src/dynamic_filtering/discovery.rs +++ b/datafusion/distributed/src/dynamic_filtering/discovery.rs @@ -251,11 +251,12 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 5 remote_producers=[1] - AggregateExec - HashJoinExec producers=[1] - NetworkShuffleExec - AggregateExec - NetworkShuffleExec anchors=[1] + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] + NetworkShuffleExec + AggregateExec + NetworkShuffleExec anchors=[1] Stage 4 RepartitionExec AggregateExec @@ -302,13 +303,14 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 4 remote_producers=[1] - AggregateExec - HashJoinExec producers=[1] - AggregateExec - NetworkShuffleExec - ProjectionExec + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] AggregateExec - NetworkShuffleExec anchors=[1] + NetworkShuffleExec + ProjectionExec + AggregateExec + NetworkShuffleExec anchors=[1] Stage 3 RepartitionExec AggregateExec @@ -350,11 +352,12 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 5 - AggregateExec - HashJoinExec producers=[1] - NetworkShuffleExec - AggregateExec + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] NetworkShuffleExec + AggregateExec + NetworkShuffleExec Stage 4 RepartitionExec AggregateExec diff --git a/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs index c95f66fb2..7a69551e1 100644 --- a/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs +++ b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs @@ -606,7 +606,7 @@ mod tests { let display = display_plan_dump(dump, DisplayPlanDumpOptions::default())?; assert_snapshot!(display, @r" DistributedExec - └─ Stage 2 (Coalesce) tasks=3 partitions=4 + └─ Stage 2 (Coalesce) tasks=3 partitions=3 └─ Stage 1 (Shuffle) tasks=3 partitions=12 "); Ok(()) @@ -622,10 +622,11 @@ mod tests { let opts = DisplayPlanDumpOptions::default().with_stage_id(2); let display = display_plan_dump(dump, opts)?; assert_snapshot!(display, @r" - Stage 2 (Coalesce) tasks=3 partitions=4 - └─ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] - └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - └─ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + Stage 2 (Coalesce) tasks=3 partitions=3 + └─ CoalescePartitionsExec + └─ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + └─ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 "); Ok(()) } @@ -640,13 +641,14 @@ mod tests { let opts = DisplayPlanDumpOptions::default().with_stage_id(1); let display = display_plan_dump(dump, opts)?; assert_snapshot!(display, @r" - Stage 1 (Coalesce) tasks=3 partitions=12 - └─ FilterExec: Rainfall@1 > 0, projection=[MinTemp@0, RainToday@2] - └─ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - └─ DistributedLeafExec: - ├─ t0: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] - ├─ t1: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] - └─ t2: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + Stage 1 (Coalesce) tasks=3 partitions=3 + └─ CoalescePartitionsExec + └─ FilterExec: Rainfall@1 > 0, projection=[MinTemp@0, RainToday@2] + └─ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + └─ DistributedLeafExec: + ├─ t0: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + ├─ t1: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + └─ t2: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] "); Ok(()) } @@ -674,7 +676,7 @@ mod tests { │ plan_send_latency_avg=