diff --git a/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs b/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs index 8e512a298..75f0de515 100644 --- a/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs +++ b/datafusion/distributed-iceberg/src/proto/generated/iceberg.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct IcebergDataSource { #[prost(message, optional, tag = "1")] @@ -11,8 +10,10 @@ pub struct IcebergDataSource { #[prost(uint64, optional, tag = "4")] pub fetch: ::core::option::Option, #[prost(map = "string, string", tag = "5")] - pub storage_properties: - ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + pub storage_properties: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, #[prost(bytes = "vec", optional, tag = "6")] pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec>, /// Filters pushed into the scan. A row must pass all of them. @@ -23,13 +24,11 @@ pub struct IcebergDataSource { #[prost(bool, tag = "8")] pub dynamic_filtering_enabled: bool, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WorkUnitFeed { #[prost(bytes = "vec", tag = "1")] pub id: ::prost::alloc::vec::Vec, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ScanConfig { #[prost(message, optional, tag = "1")] @@ -39,8 +38,10 @@ pub struct ScanConfig { #[prost(uint64, optional, tag = "3")] pub fetch: ::core::option::Option, #[prost(map = "string, string", tag = "4")] - pub storage_properties: - ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + pub storage_properties: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, #[prost(bytes = "vec", optional, tag = "5")] pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec>, /// Filters pushed into the scan. A row must pass all of them. @@ -51,7 +52,6 @@ pub struct ScanConfig { #[prost(bool, tag = "7")] pub dynamic_filtering_enabled: bool, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AssignedScan { #[prost(message, optional, tag = "1")] @@ -59,7 +59,6 @@ pub struct AssignedScan { #[prost(message, repeated, tag = "2")] pub partitions: ::prost::alloc::vec::Vec, } -#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FileGroup { #[prost(bytes = "vec", repeated, tag = "1")] diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 2e53861d9..3d11debe2 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -178,6 +178,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning, input_stage, equivalence_classes, + fetch, }) => { let schema: Schema = schema .as_ref() @@ -208,11 +209,14 @@ impl PhysicalExtensionCodec for DistributedCodec { proto_converter, )?; - Ok(Arc::new(new_network_coalesce_tasks_exec( - partitioning, - equivalence_properties, - parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?, - ))) + Ok(Arc::new( + new_network_coalesce_tasks_exec( + partitioning, + equivalence_properties, + parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?, + ) + .with_fetch(fetch.map(|v| v as usize)), + )) } DistributedExecNode::NetworkBroadcast(NetworkBroadcastExecProto { schema, @@ -420,6 +424,7 @@ impl PhysicalExtensionCodec for DistributedCodec { self, proto_converter, )?, + fetch: node.fetch.map(|v| v as u64), }; let wrapper = DistributedExecProto { @@ -787,6 +792,9 @@ pub struct NetworkCoalesceExecProto { input_stage: Option, #[prost(message, repeated, tag = "4")] equivalence_classes: Vec, + /// Row limit the producer head applies per input task. + #[prost(uint64, optional, tag = "5")] + fetch: Option, } fn new_network_coalesce_tasks_exec( @@ -803,6 +811,7 @@ fn new_network_coalesce_tasks_exec( )), worker_connections: WorkerConnectionPool::new(input_stage.task_count()), input_stage, + fetch: None, } } diff --git a/datafusion/distributed/src/codec/mod.rs b/datafusion/distributed/src/codec/mod.rs index fc03f09fb..cb1a36843 100644 --- a/datafusion/distributed/src/codec/mod.rs +++ b/datafusion/distributed/src/codec/mod.rs @@ -4,8 +4,9 @@ mod user_codec; pub use distributed_codec::DistributedCodec; pub(crate) use physical_plan::{ - decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan, - encode_partitioning, encode_physical_expr, roundtrip_pb, + decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr, + encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr, + roundtrip_pb, }; pub(crate) use user_codec::{ get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc, diff --git a/datafusion/distributed/src/codec/physical_plan.rs b/datafusion/distributed/src/codec/physical_plan.rs index f1a970ace..700cdad9b 100644 --- a/datafusion/distributed/src/codec/physical_plan.rs +++ b/datafusion/distributed/src/codec/physical_plan.rs @@ -1,14 +1,18 @@ use super::DistributedCodec; use datafusion::arrow::datatypes::{Schema, SchemaRef}; -use datafusion::common::Result; +use datafusion::common::{Result, internal_datafusion_err}; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{Partitioning, PhysicalExpr}; +use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr}; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::bytes::{ physical_plan_from_bytes_with_proto_converter, physical_plan_to_bytes_with_proto_converter, }; -use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; -use datafusion_proto::physical_plan::to_proto::serialize_partitioning; +use datafusion_proto::physical_plan::from_proto::{ + parse_physical_sort_exprs, parse_protobuf_partitioning, +}; +use datafusion_proto::physical_plan::to_proto::{ + serialize_partitioning, serialize_physical_sort_exprs, +}; use datafusion_proto::physical_plan::{ DeduplicatingProtoConverter, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension, }; @@ -100,3 +104,36 @@ pub(crate) fn decode_partitioning( parse_protobuf_partitioning(Some(&proto_partitioning), &decode_ctx, &schema, &converter)? .ok_or_else(|| proto_error("Could not parse partitioning")) } + +/// Protobuf wrapper for a [LexOrdering], as upstream has no message for a list of +/// sort expressions. +#[derive(Clone, PartialEq, ::prost::Message)] +struct LexOrderingProto { + #[prost(message, repeated, tag = "1")] + sort_exprs: Vec, +} + +pub(crate) fn encode_lex_ordering( + ordering: &LexOrdering, + task_ctx: &TaskContext, +) -> Result> { + let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config()); + let converter = new_proto_converter(); + let sort_exprs = serialize_physical_sort_exprs(ordering.iter().cloned(), &codec, &converter)?; + Ok(LexOrderingProto { sort_exprs }.encode_to_vec()) +} + +pub(crate) fn decode_lex_ordering( + encoded: &[u8], + input_schema: &Schema, + task_ctx: &TaskContext, +) -> Result { + let proto = LexOrderingProto::decode(encoded).map_err(|err| proto_error(err.to_string()))?; + let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config()); + let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &codec); + let converter = new_proto_converter(); + let sort_exprs = + parse_physical_sort_exprs(&proto.sort_exprs, &decode_ctx, input_schema, &converter)?; + LexOrdering::new(sort_exprs) + .ok_or_else(|| internal_datafusion_err!("Cannot decode an empty LexOrdering")) +} diff --git a/datafusion/distributed/src/common/maybe_encoded.rs b/datafusion/distributed/src/common/maybe_encoded.rs index 7e01ad2b3..c28d19e1d 100644 --- a/datafusion/distributed/src/common/maybe_encoded.rs +++ b/datafusion/distributed/src/common/maybe_encoded.rs @@ -1,11 +1,11 @@ use crate::codec::{ - decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan, - encode_partitioning, encode_physical_expr, + decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr, + encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr, }; use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::common::{Result, internal_err}; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{Partitioning, PhysicalExpr}; +use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr}; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::protobuf::PhysicalExprNode; use datafusion_proto::protobuf::proto_error; @@ -124,6 +124,25 @@ impl MaybeEncoded> { } } +impl MaybeEncoded { + /// Returns the encoded [`LexOrdering`] as protobuf bytes: + /// - If in `Decoded` state, it encodes it using the codecs registered in the [`TaskContext`]. + /// - If in `Encoded` state, it passes through the existing bytes. + pub fn encode(self, ctx: &Arc) -> Result> { + match self { + Self::Encoded(encoded) => Ok(encoded), + Self::Decoded(lex_ordering) => encode_lex_ordering(&lex_ordering, ctx), + } + } + + /// Returns the decoded [`LexOrdering`]. + /// - If in `Decoded` state, it passes through the ordering. + /// - If in `Encoded` state, it decodes it using the provided schema and task context. + pub fn decode(self, input_schema: &Schema, task_ctx: &TaskContext) -> Result { + self.decode_with(|encoded| decode_lex_ordering(&encoded, input_schema, task_ctx)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs index 8ecbab25c..8c2bbad7d 100644 --- a/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs +++ b/datafusion/distributed/src/distributed_planner/distributed_query_planner.rs @@ -8,7 +8,6 @@ use crate::distributed_planner::insert_children_isolator_union::insert_children_ use crate::distributed_planner::normalize_collect_joins::normalize_collect_joins; use crate::distributed_planner::partial_reduce_below_network_shuffles::partial_reduce_below_network_shuffles; use crate::distributed_planner::prepare_network_boundaries::prepare_network_boundaries; -use crate::distributed_planner::push_fetch_into_network_coalesce::push_fetch_into_network_coalesce; use crate::events::{ScaleUpLeafNodeEvent, ScaleUpLeafNodeHandlers}; use crate::explain_analyze::DistributedAnalyzeExec; use crate::{DistributedConfig, DistributedExec, NetworkBoundaryExt}; @@ -135,7 +134,6 @@ pub fn create_distributed_plan( if !plan.exists(|plan| Ok(plan.is_network_boundary()))? { return Ok(plan); } - let plan = push_fetch_into_network_coalesce(plan)?; return Ok(create_distributed_exec(Arc::clone(&plan), d_cfg)); } @@ -166,7 +164,6 @@ pub fn create_distributed_plan( } let plan = partial_reduce_below_network_shuffles(plan, cfg)?; - let plan = push_fetch_into_network_coalesce(plan)?; Ok(create_distributed_exec(Arc::clone(&plan), d_cfg)) }) @@ -225,13 +222,14 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet └────────────────────────────────────────────────── "); } @@ -245,16 +243,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -276,16 +275,17 @@ mod tests { .num_workers(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=2, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -374,16 +374,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] - │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -404,15 +405,16 @@ mod tests { let physical_plan_ascii = TestPlanBuilder::default() .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=3, partitions=4 - │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 - │ [Stage 2] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 3 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + │ [Stage 2] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=12 │ RepartitionExec: partitioning=Hash([RainToday@1], 12), input_partitions=3 @@ -513,14 +515,15 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [MinTemp@0 DESC] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + ┌───── Stage 1 ── tasks=3, partitions=3 + │ SortPreservingMergeExec: [MinTemp@0 DESC] + │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, sort_order_for_reorder=[MinTemp@0 DESC], reverse_row_groups=true └────────────────────────────────────────────────── "); } @@ -534,16 +537,17 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] │ CoalescePartitionsExec: fetch=10 - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ LocalLimitExec: fetch=10 - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec: fetch=10 + │ LocalLimitExec: fetch=10 + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=3 @@ -565,14 +569,15 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=3 @@ -613,17 +618,18 @@ mod tests { .num_workers(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=2, partitions=4 - │ DistributedUnionExec: t0:[c0] t1:[c1] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + ┌───── Stage 1 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -639,17 +645,18 @@ mod tests { .distributed_max_tasks_per_stage(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=2, partitions=8 - │ DistributedUnionExec: t0:[c0] t1:[c1] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + ┌───── Stage 1 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -668,23 +675,24 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=6 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=6 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=6, partitions=24 - │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 + ┌───── Stage 1 ── tasks=6, partitions=6 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -702,20 +710,21 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=12 - │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -735,24 +744,25 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=12 - │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp9am@0 > 15 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -776,34 +786,35 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=24 - │ DistributedUnionExec: t0:[c0, c3] t1:[c1, c4] t2:[c2] - │ FilterExec: MinTemp@0 > 10 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: MaxTemp@0 < 30 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp9am@0 > 15 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] - │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Temp3pm@0 < 25 + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0, c3] t1:[c1, c4] t2:[c2] + │ FilterExec: MinTemp@0 > 10 │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] - │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] - │ FilterExec: Rainfall@0 > 5 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp3pm@0 < 25 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Rainfall@0 > 5 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] └────────────────────────────────────────────────── "); } @@ -819,19 +830,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -855,19 +867,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@2, RainToday@1)], projection=[MinTemp@0, MaxTemp@1, Rainfall@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 3 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@2, RainToday@1)], projection=[MinTemp@0, MaxTemp@1, Rainfall@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -904,15 +917,16 @@ mod tests { │ CoalescePartitionsExec │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=3, partitions=9 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + ┌───── Stage 1 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet └────────────────────────────────────────────────── "); @@ -920,19 +934,20 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -963,28 +978,29 @@ mod tests { .broadcast_joins(true) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=3, partitions=9 - │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 4 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 @@ -1022,19 +1038,20 @@ mod tests { .desired_task_count_handler(build_side_one_desired_task_count_handler) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=3, partitions=9 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + ┌───── Stage 2 ── tasks=3, partitions=3 + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=1, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 diff --git a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs index e796a7cca..b176e2fab 100644 --- a/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs +++ b/datafusion/distributed/src/distributed_planner/inject_network_boundaries.rs @@ -425,12 +425,10 @@ async fn _inject_network_boundaries( // The parent that triggered this branch is a `CoalescePartitionsExec` or // `SortPreservingMergeExec`, both of which fold all partitions into one — so the // stage above this boundary must run in exactly one task. - let nb = Arc::new(NetworkCoalesceExec::try_from_stage( - result.input_stage, - result.input_properties, - 1, - )?); - Ok(nb_ctx.plan_with_task_count(nb, result.consumer_task_count)) + let nb = + NetworkCoalesceExec::try_from_stage(result.input_stage, result.input_properties, 1)? + .with_fetch(parent.fetch()); + Ok(nb_ctx.plan_with_task_count(Arc::new(nb), result.consumer_task_count)) } else if parent.is_none() { // We've just finished walking the head stage's subplan. Run a final propagation so // every node in the head stage (which never crossed a stage boundary on the way up) diff --git a/datafusion/distributed/src/distributed_planner/insert_broadcast.rs b/datafusion/distributed/src/distributed_planner/insert_broadcast.rs index de3f18185..25d4e323d 100644 --- a/datafusion/distributed/src/distributed_planner/insert_broadcast.rs +++ b/datafusion/distributed/src/distributed_planner/insert_broadcast.rs @@ -212,8 +212,9 @@ mod tests { CoalescePartitionsExec HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec - [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 - DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=4 + CoalescePartitionsExec + DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet DistributedLeafExec: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); let test_plan = TestPlanBuilder::new() diff --git a/datafusion/distributed/src/distributed_planner/mod.rs b/datafusion/distributed/src/distributed_planner/mod.rs index ff32d02f7..a4e8ae5dc 100644 --- a/datafusion/distributed/src/distributed_planner/mod.rs +++ b/datafusion/distributed/src/distributed_planner/mod.rs @@ -8,7 +8,6 @@ mod network_boundary; mod normalize_collect_joins; mod partial_reduce_below_network_shuffles; mod prepare_network_boundaries; -mod push_fetch_into_network_coalesce; mod session_state_builder_ext; mod statistics; diff --git a/datafusion/distributed/src/distributed_planner/network_boundary.rs b/datafusion/distributed/src/distributed_planner/network_boundary.rs index 259c802b2..4f4d63cee 100644 --- a/datafusion/distributed/src/distributed_planner/network_boundary.rs +++ b/datafusion/distributed/src/distributed_planner/network_boundary.rs @@ -6,8 +6,10 @@ use crate::{ use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::Result; use datafusion::execution::TaskContext; -use datafusion::physical_expr::Partitioning; +use datafusion::physical_expr::{LexOrdering, Partitioning}; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties, ReplaceChildrenOptions, }; @@ -47,6 +49,13 @@ pub enum ProducerHead { /// Whether the producer must preserve its input's output ordering. preserve_order: bool, }, + /// The head node should be a [CoalescePartitionsExec]. + CoalescePartitionsExec { fetch: Option }, + /// The head node should be a [SortPreservingMergeExec]. + SortPreservingMergeExec { + fetch: Option, + lex_ordering: MaybeEncoded, + }, } /// Extension trait for downcasting dynamic types to [NetworkBoundary]. @@ -83,6 +92,13 @@ impl ProducerHead { partitioning: MaybeEncoded::Decoded(partitioning.decode(schema, ctx)?), preserve_order, }, + Self::SortPreservingMergeExec { + fetch, + lex_ordering, + } => Self::SortPreservingMergeExec { + fetch, + lex_ordering: MaybeEncoded::Decoded(lex_ordering.decode(&schema, ctx)?), + }, v => v, }) } @@ -94,6 +110,10 @@ impl ProducerHead { Arc::clone(r_exec.input()) } else if let Some(b_exec) = input.downcast_ref::() { Arc::clone(b_exec.input()) + } else if let Some(c_exec) = input.downcast_ref::() { + Arc::clone(c_exec.input()) + } else if let Some(s_exec) = input.downcast_ref::() { + Arc::clone(s_exec.input()) } else { input }; @@ -114,6 +134,16 @@ impl ProducerHead { repartition }) } + ProducerHead::CoalescePartitionsExec { fetch } => { + Arc::new(CoalescePartitionsExec::new(input).with_fetch(fetch)) + } + ProducerHead::SortPreservingMergeExec { + fetch, + lex_ordering, + } => { + let lex_ordering = lex_ordering.try_decoded()?; + Arc::new(SortPreservingMergeExec::new(lex_ordering, input).with_fetch(fetch)) + } }; Ok(plan) } diff --git a/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs b/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs index 338088837..08076da12 100644 --- a/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs +++ b/datafusion/distributed/src/distributed_planner/partial_reduce_below_network_shuffles.rs @@ -117,15 +117,16 @@ mod tests { .distributed_cardinality_effect_task_scale_factor(1.5) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @" + assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=4 - │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + ┌───── Stage 2 ── tasks=2, partitions=2 + │ CoalescePartitionsExec + │ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=8 │ AggregateExec: mode=PartialReduce, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] diff --git a/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs b/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs deleted file mode 100644 index a9130b356..000000000 --- a/datafusion/distributed/src/distributed_planner/push_fetch_into_network_coalesce.rs +++ /dev/null @@ -1,243 +0,0 @@ -use std::sync::Arc; - -use datafusion::common::Result; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::physical_plan::limit::GlobalLimitExec; -use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; - -use crate::NetworkCoalesceExec; - -/// Pushes fetch limits through [NetworkCoalesceExec] stage boundaries. -/// -/// DataFusion's normal `LimitPushdown` rule runs before the distributed planner inserts network -/// boundaries, so a fetch-bearing parent can end up directly above a [NetworkCoalesceExec] whose -/// producer stage is still unbounded. -/// -/// This pass rewrites: -/// -/// ```text -/// ┌─────────────────────────────┐ -/// │ CoalescePartitions(fetch=10)│ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkCoalesceExec │ -/// └──────────────▲──────────────┘ -/// │ input_stage.plan -/// ┌──────────────┴──────────────┐ -/// │ AggregateExec │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkShuffleExec │ -/// └─────────────────────────────┘ -/// ``` -/// -/// into: -/// -/// ```text -/// ┌─────────────────────────────┐ -/// │ CoalescePartitions(fetch=10)│ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkCoalesceExec │ -/// └──────────────▲──────────────┘ -/// │ input_stage.plan -/// ┌──────────────┴──────────────┐ -/// │ LocalLimitExec(fetch=10) │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ AggregateExec │ -/// └──────────────▲──────────────┘ -/// │ -/// ┌──────────────┴──────────────┐ -/// │ NetworkShuffleExec │ -/// └─────────────────────────────┘ -/// ``` -/// -/// The parent fetch remains in place because it enforces the global result limit. The producer-stage -/// fetch is only a per-task bound that lets remote work stop earlier. -pub(crate) fn push_fetch_into_network_coalesce( - plan: Arc, -) -> Result> { - plan.transform_down(|node| { - let Some(fetch) = fetch_required_from_child(&node) else { - return Ok(Transformed::no(node)); - }; - - let mut changed = false; - let new_children = node - .children() - .into_iter() - .map(|child| { - if let Some(network_coalesce) = child.downcast_ref::() { - changed = true; - network_coalesce.with_fetch_on_input_stage(fetch) - } else { - Ok(Arc::clone(child)) - } - }) - .collect::>>()?; - - if changed { - node.replace_children( - new_children, - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - ) - .map(Transformed::yes) - } else { - Ok(Transformed::no(node)) - } - }) - .map(|t| t.data) -} - -fn fetch_required_from_child(plan: &Arc) -> Option { - if let Some(global_limit) = plan.downcast_ref::() { - return global_limit - .fetch() - .and_then(|fetch| fetch.checked_add(global_limit.skip())); - } - - plan.fetch() -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::physical_expr::expressions::Column; - use datafusion::physical_plan::ExecutionPlan; - use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; - use datafusion::physical_plan::empty::EmptyExec; - use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; - use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; - - use crate::{NetworkBoundary, Stage}; - - use super::*; - - #[test] - fn fetch_pushed_into_network_coalesce_input_stage() -> Result<()> { - let input: Arc = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let network_coalesce: Arc = - Arc::new(NetworkCoalesceExec::try_new(input, 1, 1)?); - let plan: Arc = - Arc::new(CoalescePartitionsExec::new(network_coalesce).with_fetch(Some(7))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - assert_eq!(rewritten.fetch(), Some(7)); - - let parent = rewritten - .downcast_ref::() - .expect("root should remain CoalescePartitionsExec"); - let network_coalesce = parent.children()[0] - .downcast_ref::() - .expect("child should remain NetworkCoalesceExec"); - let Stage::Local(local_stage) = network_coalesce.input_stage() else { - panic!("test network boundary should still hold a local stage"); - }; - let local_limit = local_stage - .plan - .downcast_ref::() - .expect("input stage should be bounded by LocalLimitExec"); - - assert_eq!(local_limit.fetch(), 7); - Ok(()) - } - - #[test] - fn global_limit_with_offset_pushes_skip_plus_fetch_into_network_coalesce_input_stage() - -> Result<()> { - let input: Arc = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let network_coalesce: Arc = - Arc::new(NetworkCoalesceExec::try_new(input, 2, 1)?); - let plan: Arc = - Arc::new(GlobalLimitExec::new(network_coalesce, 100, Some(10))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - let global_limit = rewritten - .downcast_ref::() - .expect("root should remain GlobalLimitExec"); - assert_eq!(global_limit.skip(), 100); - assert_eq!(global_limit.fetch(), Some(10)); - - let network_coalesce = global_limit - .input() - .downcast_ref::() - .expect("child should remain NetworkCoalesceExec"); - let Stage::Local(local_stage) = network_coalesce.input_stage() else { - panic!("test network boundary should still hold a local stage"); - }; - let local_limit = local_stage - .plan - .downcast_ref::() - .expect("input stage should be bounded by LocalLimitExec"); - - assert_eq!(local_limit.fetch(), 110); - Ok(()) - } - - #[test] - fn fetch_separated_by_projection_does_not_add_local_limit() -> Result<()> { - let inner = Arc::new(CoalescePartitionsExec::new(empty_plan()).with_fetch(Some(7))); - let network_coalesce: Arc = Arc::new(NetworkCoalesceExec::try_new( - identity_projection(inner)?, - 1, - 1, - )?); - let plan: Arc = - Arc::new(CoalescePartitionsExec::new(network_coalesce).with_fetch(Some(7))); - - let rewritten = push_fetch_into_network_coalesce(plan)?; - let input_stage = local_input_stage_plan(&rewritten); - - assert!( - input_stage.downcast_ref::().is_none(), - "ProjectionExec preserves row count, so a LocalLimitExec above it is redundant" - ); - let projection = input_stage - .downcast_ref::() - .expect("input stage root should remain ProjectionExec"); - assert_eq!(projection.input().fetch(), Some(7)); - Ok(()) - } - - fn empty_plan() -> Arc { - Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new( - "c", - DataType::Int32, - true, - )])))) - } - - fn identity_projection(input: Arc) -> Result> { - let schema = input.schema(); - let exprs = schema - .fields() - .iter() - .enumerate() - .map(|(i, field)| ProjectionExpr { - expr: Arc::new(Column::new(field.name(), i)), - alias: field.name().to_string(), - }) - .collect::>(); - Ok(Arc::new(ProjectionExec::try_new(exprs, input)?)) - } - - fn local_input_stage_plan(plan: &Arc) -> Arc { - fn find(node: &Arc) -> Option> { - if let Some(network_coalesce) = node.downcast_ref::() { - let Stage::Local(local) = network_coalesce.input_stage() else { - return None; - }; - return Some(Arc::clone(&local.plan)); - } - node.children().into_iter().find_map(find) - } - find(plan).expect("plan should contain a local NetworkCoalesceExec stage") - } -} diff --git a/datafusion/distributed/src/dynamic_filtering/discovery.rs b/datafusion/distributed/src/dynamic_filtering/discovery.rs index 0bf1ce2f7..ae78a05d8 100644 --- a/datafusion/distributed/src/dynamic_filtering/discovery.rs +++ b/datafusion/distributed/src/dynamic_filtering/discovery.rs @@ -251,11 +251,12 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 5 remote_producers=[1] - AggregateExec - HashJoinExec producers=[1] - NetworkShuffleExec - AggregateExec - NetworkShuffleExec anchors=[1] + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] + NetworkShuffleExec + AggregateExec + NetworkShuffleExec anchors=[1] Stage 4 RepartitionExec AggregateExec @@ -302,13 +303,14 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 4 remote_producers=[1] - AggregateExec - HashJoinExec producers=[1] - AggregateExec - NetworkShuffleExec - ProjectionExec + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] AggregateExec - NetworkShuffleExec anchors=[1] + NetworkShuffleExec + ProjectionExec + AggregateExec + NetworkShuffleExec anchors=[1] Stage 3 RepartitionExec AggregateExec @@ -350,11 +352,12 @@ mod tests { .await?; assert_snapshot!(display, @r" Stage 5 - AggregateExec - HashJoinExec producers=[1] - NetworkShuffleExec - AggregateExec + CoalescePartitionsExec + AggregateExec + HashJoinExec producers=[1] NetworkShuffleExec + AggregateExec + NetworkShuffleExec Stage 4 RepartitionExec AggregateExec diff --git a/datafusion/distributed/src/execution_plans/network_coalesce.rs b/datafusion/distributed/src/execution_plans/network_coalesce.rs index db9b964a8..3122f91ac 100644 --- a/datafusion/distributed/src/execution_plans/network_coalesce.rs +++ b/datafusion/distributed/src/execution_plans/network_coalesce.rs @@ -1,22 +1,19 @@ -use crate::DistributedTaskContext; use crate::common::require_one_child; use crate::distributed_planner::{NetworkBoundary, ProducerHead}; use crate::execution_plans::common::scale_partitioning_props; use crate::stage::{LocalStage, Stage}; use crate::worker::WorkerConnectionPool; +use crate::{DistributedTaskContext, MaybeEncoded}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{exec_err, not_impl_err, plan_err}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr_common::metrics::MetricsSet; -use datafusion::physical_plan::limit::LocalLimitExec; -use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - ChildrenPropertiesMode, DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, - PlanProperties, ReplaceChildrenOptions, Statistics, StatisticsArgs, apply_expression_roots, - internal_err, + DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, PlanProperties, + Statistics, StatisticsArgs, apply_expression_roots, internal_err, }; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -81,6 +78,7 @@ pub struct NetworkCoalesceExec { pub(crate) properties: Arc, pub(crate) input_stage: Stage, pub(crate) worker_connections: WorkerConnectionPool, + pub(crate) fetch: Option, } impl NetworkCoalesceExec { @@ -93,12 +91,13 @@ impl NetworkCoalesceExec { // per output task based on the maximum group size, returning empty streams for tasks with // smaller groups. let max_input_task_count = input_stage.task_count().div_ceil(consumer_tasks).max(1); - let props = scale_partitioning_props(&input_properties, |p| p * max_input_task_count)?; + let props = scale_partitioning_props(&input_properties, |_| max_input_task_count)?; Ok(Self { properties: props, worker_connections: WorkerConnectionPool::new(input_stage.task_count()), input_stage, + fetch: None, }) } @@ -142,56 +141,12 @@ impl NetworkCoalesceExec { ) } - pub(crate) fn with_fetch_on_input_stage(&self, fetch: usize) -> Result> { - let Stage::Local(local) = &self.input_stage else { - return Ok(Arc::new(self.clone())); - }; - - let input_with_fetch = apply_fetch_through_projection(Arc::clone(&local.plan), fetch)?; - - let mut self_clone = self.clone(); - self_clone.input_stage = Stage::Local(LocalStage { - query_id: local.query_id, - num: local.num, - plan: input_with_fetch, - tasks: local.tasks, - metrics_set: Default::default(), - }); - Ok(Arc::new(self_clone)) + pub fn with_fetch(mut self, fetch: Option) -> Self { + self.fetch = fetch; + self } } -/// Applies `fetch` to `plan`, looking through a [ProjectionExec] that preserves row count. -/// -/// If the projection child already carries a sufficient fetch, the plan is left unchanged instead -/// of wrapping a redundant [LocalLimitExec]. -fn apply_fetch_through_projection( - plan: Arc, - fetch: usize, -) -> Result> { - if let Some(projection) = plan.downcast_ref::() { - let input = Arc::clone(projection.input()); - if input.fetch().is_some_and(|existing| existing <= fetch) { - return Ok(plan); - } - let new_input = input - .with_fetch(Some(fetch)) - .unwrap_or_else(|| Arc::new(LocalLimitExec::new(input, fetch))); - return plan.replace_children( - vec![new_input], - ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), - ); - } - - if plan.fetch().is_some_and(|existing| existing <= fetch) { - return Ok(plan); - } - - Ok(plan - .with_fetch(Some(fetch)) - .unwrap_or_else(|| Arc::new(LocalLimitExec::new(plan, fetch)))) -} - impl NetworkBoundary for NetworkCoalesceExec { fn input_stage(&self) -> &Stage { &self.input_stage @@ -208,7 +163,13 @@ impl NetworkBoundary for NetworkCoalesceExec { } fn producer_head(&self, _consumer_task_count: usize) -> Result { - Ok(ProducerHead::None) + match self.properties.output_ordering() { + Some(lex_ordering) => Ok(ProducerHead::SortPreservingMergeExec { + fetch: self.fetch, + lex_ordering: MaybeEncoded::Decoded(lex_ordering.clone()), + }), + None => Ok(ProducerHead::CoalescePartitionsExec { fetch: self.fetch }), + } } } @@ -251,18 +212,20 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result> { - let mut self_clone = self.as_ref().clone(); - match &mut self_clone.input_stage { + match &self.input_stage { Stage::Local(local) => { + let mut local = local.clone(); local.plan = require_one_child(children)?; + self.with_input_stage(Stage::Local(local)) + .map(|node| node as Arc) } Stage::Remote(_) => { if !children.is_empty() { not_impl_err!("NetworkBoundary cannot accept children")? } + Ok(self) } } - Ok(Arc::new(self_clone)) } fn execute( diff --git a/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs index c95f66fb2..7a69551e1 100644 --- a/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs +++ b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs @@ -606,7 +606,7 @@ mod tests { let display = display_plan_dump(dump, DisplayPlanDumpOptions::default())?; assert_snapshot!(display, @r" DistributedExec - └─ Stage 2 (Coalesce) tasks=3 partitions=4 + └─ Stage 2 (Coalesce) tasks=3 partitions=3 └─ Stage 1 (Shuffle) tasks=3 partitions=12 "); Ok(()) @@ -622,10 +622,11 @@ mod tests { let opts = DisplayPlanDumpOptions::default().with_stage_id(2); let display = display_plan_dump(dump, opts)?; assert_snapshot!(display, @r" - Stage 2 (Coalesce) tasks=3 partitions=4 - └─ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] - └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - └─ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + Stage 2 (Coalesce) tasks=3 partitions=3 + └─ CoalescePartitionsExec + └─ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + └─ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 "); Ok(()) } @@ -640,13 +641,14 @@ mod tests { let opts = DisplayPlanDumpOptions::default().with_stage_id(1); let display = display_plan_dump(dump, opts)?; assert_snapshot!(display, @r" - Stage 1 (Coalesce) tasks=3 partitions=12 - └─ FilterExec: Rainfall@1 > 0, projection=[MinTemp@0, RainToday@2] - └─ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 - └─ DistributedLeafExec: - ├─ t0: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] - ├─ t1: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] - └─ t2: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + Stage 1 (Coalesce) tasks=3 partitions=3 + └─ CoalescePartitionsExec + └─ FilterExec: Rainfall@1 > 0, projection=[MinTemp@0, RainToday@2] + └─ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + └─ DistributedLeafExec: + ├─ t0: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + ├─ t1: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + └─ t2: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] "); Ok(()) } @@ -674,7 +676,7 @@ mod tests { │ plan_send_latency_avg=