Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions datafusion/distributed-iceberg/src/proto/generated/iceberg.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
// This file is @generated by prost-build.
#[allow(clippy::all)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct IcebergDataSource {
#[prost(message, optional, tag = "1")]
Expand All @@ -11,8 +10,10 @@ pub struct IcebergDataSource {
#[prost(uint64, optional, tag = "4")]
pub fetch: ::core::option::Option<u64>,
#[prost(map = "string, string", tag = "5")]
pub storage_properties:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
pub storage_properties: ::std::collections::HashMap<
::prost::alloc::string::String,
::prost::alloc::string::String,
>,
#[prost(bytes = "vec", optional, tag = "6")]
pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
/// Filters pushed into the scan. A row must pass all of them.
Expand All @@ -23,13 +24,11 @@ pub struct IcebergDataSource {
#[prost(bool, tag = "8")]
pub dynamic_filtering_enabled: bool,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct WorkUnitFeed {
#[prost(bytes = "vec", tag = "1")]
pub id: ::prost::alloc::vec::Vec<u8>,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ScanConfig {
#[prost(message, optional, tag = "1")]
Expand All @@ -39,8 +38,10 @@ pub struct ScanConfig {
#[prost(uint64, optional, tag = "3")]
pub fetch: ::core::option::Option<u64>,
#[prost(map = "string, string", tag = "4")]
pub storage_properties:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
pub storage_properties: ::std::collections::HashMap<
::prost::alloc::string::String,
::prost::alloc::string::String,
>,
#[prost(bytes = "vec", optional, tag = "5")]
pub custom_file_io: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
/// Filters pushed into the scan. A row must pass all of them.
Expand All @@ -51,15 +52,13 @@ pub struct ScanConfig {
#[prost(bool, tag = "7")]
pub dynamic_filtering_enabled: bool,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct AssignedScan {
#[prost(message, optional, tag = "1")]
pub config: ::core::option::Option<ScanConfig>,
#[prost(message, repeated, tag = "2")]
pub partitions: ::prost::alloc::vec::Vec<FileGroup>,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FileGroup {
#[prost(bytes = "vec", repeated, tag = "1")]
Expand Down
19 changes: 14 additions & 5 deletions datafusion/distributed/src/codec/distributed_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ impl PhysicalExtensionCodec for DistributedCodec {
partitioning,
input_stage,
equivalence_classes,
fetch,
}) => {
let schema: Schema = schema
.as_ref()
Expand Down Expand Up @@ -208,11 +209,14 @@ impl PhysicalExtensionCodec for DistributedCodec {
proto_converter,
)?;

Ok(Arc::new(new_network_coalesce_tasks_exec(
partitioning,
equivalence_properties,
parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?,
)))
Ok(Arc::new(
new_network_coalesce_tasks_exec(
partitioning,
equivalence_properties,
parse_stage_proto(input_stage, inputs, dynamic_filter_anchors)?,
)
.with_fetch(fetch.map(|v| v as usize)),
))
}
DistributedExecNode::NetworkBroadcast(NetworkBroadcastExecProto {
schema,
Expand Down Expand Up @@ -420,6 +424,7 @@ impl PhysicalExtensionCodec for DistributedCodec {
self,
proto_converter,
)?,
fetch: node.fetch.map(|v| v as u64),
};

let wrapper = DistributedExecProto {
Expand Down Expand Up @@ -787,6 +792,9 @@ pub struct NetworkCoalesceExecProto {
input_stage: Option<StageProto>,
#[prost(message, repeated, tag = "4")]
equivalence_classes: Vec<EquivalenceClassProto>,
/// Row limit the producer head applies per input task.
#[prost(uint64, optional, tag = "5")]
fetch: Option<u64>,
}

fn new_network_coalesce_tasks_exec(
Expand All @@ -803,6 +811,7 @@ fn new_network_coalesce_tasks_exec(
)),
worker_connections: WorkerConnectionPool::new(input_stage.task_count()),
input_stage,
fetch: None,
}
}

Expand Down
5 changes: 3 additions & 2 deletions datafusion/distributed/src/codec/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ mod user_codec;

pub use distributed_codec::DistributedCodec;
pub(crate) use physical_plan::{
decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan,
encode_partitioning, encode_physical_expr, roundtrip_pb,
decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr,
encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr,
roundtrip_pb,
};
pub(crate) use user_codec::{
get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc,
Expand Down
45 changes: 41 additions & 4 deletions datafusion/distributed/src/codec/physical_plan.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
use super::DistributedCodec;
use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::common::Result;
use datafusion::common::{Result, internal_datafusion_err};
use datafusion::execution::TaskContext;
use datafusion::physical_expr::{Partitioning, PhysicalExpr};
use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr};
use datafusion::physical_plan::ExecutionPlan;
use datafusion_proto::bytes::{
physical_plan_from_bytes_with_proto_converter, physical_plan_to_bytes_with_proto_converter,
};
use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning;
use datafusion_proto::physical_plan::to_proto::serialize_partitioning;
use datafusion_proto::physical_plan::from_proto::{
parse_physical_sort_exprs, parse_protobuf_partitioning,
};
use datafusion_proto::physical_plan::to_proto::{
serialize_partitioning, serialize_physical_sort_exprs,
};
use datafusion_proto::physical_plan::{
DeduplicatingProtoConverter, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,
};
Expand Down Expand Up @@ -100,3 +104,36 @@ pub(crate) fn decode_partitioning(
parse_protobuf_partitioning(Some(&proto_partitioning), &decode_ctx, &schema, &converter)?
.ok_or_else(|| proto_error("Could not parse partitioning"))
}

/// Protobuf wrapper for a [LexOrdering], as upstream has no message for a list of
/// sort expressions.
#[derive(Clone, PartialEq, ::prost::Message)]
struct LexOrderingProto {
#[prost(message, repeated, tag = "1")]
sort_exprs: Vec<protobuf::PhysicalSortExprNode>,
}

pub(crate) fn encode_lex_ordering(
ordering: &LexOrdering,
task_ctx: &TaskContext,
) -> Result<Vec<u8>> {
let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let converter = new_proto_converter();
let sort_exprs = serialize_physical_sort_exprs(ordering.iter().cloned(), &codec, &converter)?;
Ok(LexOrderingProto { sort_exprs }.encode_to_vec())
}

pub(crate) fn decode_lex_ordering(
encoded: &[u8],
input_schema: &Schema,
task_ctx: &TaskContext,
) -> Result<LexOrdering> {
let proto = LexOrderingProto::decode(encoded).map_err(|err| proto_error(err.to_string()))?;
let codec = DistributedCodec::new_combined_with_user(task_ctx.session_config());
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx, &codec);
let converter = new_proto_converter();
let sort_exprs =
parse_physical_sort_exprs(&proto.sort_exprs, &decode_ctx, input_schema, &converter)?;
LexOrdering::new(sort_exprs)
.ok_or_else(|| internal_datafusion_err!("Cannot decode an empty LexOrdering"))
}
25 changes: 22 additions & 3 deletions datafusion/distributed/src/common/maybe_encoded.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use crate::codec::{
decode_execution_plan, decode_partitioning, decode_physical_expr, encode_execution_plan,
encode_partitioning, encode_physical_expr,
decode_execution_plan, decode_lex_ordering, decode_partitioning, decode_physical_expr,
encode_execution_plan, encode_lex_ordering, encode_partitioning, encode_physical_expr,
};
use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::common::{Result, internal_err};
use datafusion::execution::TaskContext;
use datafusion::physical_expr::{Partitioning, PhysicalExpr};
use datafusion::physical_expr::{LexOrdering, Partitioning, PhysicalExpr};
use datafusion::physical_plan::ExecutionPlan;
use datafusion_proto::protobuf::PhysicalExprNode;
use datafusion_proto::protobuf::proto_error;
Expand Down Expand Up @@ -124,6 +124,25 @@ impl MaybeEncoded<Arc<dyn PhysicalExpr>> {
}
}

impl MaybeEncoded<LexOrdering> {
/// Returns the encoded [`LexOrdering`] as protobuf bytes:
/// - If in `Decoded` state, it encodes it using the codecs registered in the [`TaskContext`].
/// - If in `Encoded` state, it passes through the existing bytes.
pub fn encode(self, ctx: &Arc<TaskContext>) -> Result<Vec<u8>> {
match self {
Self::Encoded(encoded) => Ok(encoded),
Self::Decoded(lex_ordering) => encode_lex_ordering(&lex_ordering, ctx),
}
}

/// Returns the decoded [`LexOrdering`].
/// - If in `Decoded` state, it passes through the ordering.
/// - If in `Encoded` state, it decodes it using the provided schema and task context.
pub fn decode(self, input_schema: &Schema, task_ctx: &TaskContext) -> Result<LexOrdering> {
self.decode_with(|encoded| decode_lex_ordering(&encoded, input_schema, task_ctx))
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading
Loading