Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,7 @@ jobs:
org.apache.comet.exec.CometExecSuite
org.apache.comet.exec.CometMergeRowsSuite
org.apache.comet.exec.CometTopKSuite
org.apache.comet.exec.CometMinMaxSuite
org.apache.comet.exec.CometEmptyRelationExecSuite
org.apache.comet.exec.CometRangeExecSuite
org.apache.comet.exec.CometInMemoryCacheSuite
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/pr_build_macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ jobs:
org.apache.comet.exec.CometExecSuite
org.apache.comet.exec.CometMergeRowsSuite
org.apache.comet.exec.CometTopKSuite
org.apache.comet.exec.CometMinMaxSuite
org.apache.comet.exec.CometEmptyRelationExecSuite
org.apache.comet.exec.CometRangeExecSuite
org.apache.comet.exec.CometInMemoryCacheSuite
Expand Down
1 change: 1 addition & 0 deletions dev/release/rat_exclude_files.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ spark/src/test/resources/tpch-query-results/*.out
spark/src/test/resources/tpch-extended/q*.sql
spark/src/test/resources/test-data/*.csv
spark/src/test/resources/test-data/*.ndjson
spark/src/test/resources/test-data/text-test-1.txt
spark/src/test/resources/abfs-auth-parity/*.json
spark/inspections/CometTPC*results.txt
benchmarks/tpc/queries/**/*.sql
Expand Down
2 changes: 0 additions & 2 deletions native/core/src/execution/operators/dynamic_filter/min_max.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@ struct RuntimeAggregate {
}

impl MinMaxReaderFilterExec {
// Spark planner integration is a separate step of apache/datafusion-comet#5774.
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn try_new(
aggregate: &AggregateExec,
config: &ConfigOptions,
Expand Down
1 change: 1 addition & 0 deletions native/core/src/execution/operators/dynamic_filter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ mod topk;
mod test_utils;

pub(crate) use join::DynamicFilterJoinExec;
pub(crate) use min_max::MinMaxReaderFilterExec;
pub(crate) use topk::TopKReaderFilterExec;

use std::fmt::Formatter;
Expand Down
4 changes: 3 additions & 1 deletion native/core/src/execution/operators/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ mod arrow_python_udf;
#[cfg(feature = "python-udf")]
pub use arrow_python_udf::{ArrowPythonUdfExec, ArrowPythonUdfSpec};
mod dynamic_filter;
pub(crate) use dynamic_filter::{DynamicFilterJoinExec, TopKReaderFilterExec};
pub(crate) use dynamic_filter::{
DynamicFilterJoinExec, MinMaxReaderFilterExec, TopKReaderFilterExec,
};
pub(crate) mod iceberg_common;
pub use iceberg_common::clear_file_io_cache;
mod iceberg_dictionary;
Expand Down
2 changes: 2 additions & 0 deletions native/core/src/execution/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3171,6 +3171,7 @@ fn needs_fields_coercion(sig: &TypeSignature) -> bool {
#[cfg(test)]
mod tests {
mod empty_native_scan;
mod min_max_reader_filter;
mod multi_store_scan;
mod semi_join_reduction;

Expand Down Expand Up @@ -4647,6 +4648,7 @@ mod tests {
expr_modes: vec![],
initial_input_buffer_offset: 0,
ordered_by_grouping_keys: false,
dynamic_filter_enabled: false,
})),
};
let projection = Operator {
Expand Down
59 changes: 45 additions & 14 deletions native/core/src/execution/planner/aggregate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ use super::{
PlanCreationResult,
};
use crate::{
execution::{operators::ExecutionError, serde::to_arrow_datatype, spark_plan::SparkPlan},
execution::{
operators::{ExecutionError, MinMaxReaderFilterExec},
serde::to_arrow_datatype,
spark_plan::SparkPlan,
},
extract_op,
};

Expand Down Expand Up @@ -175,7 +179,9 @@ impl OperatorBuilder for HashAggBuilder {
let agg_exprs: PhyAggResult = agg
.agg_exprs
.iter()
.map(|expr| planner.create_agg_expr(expr, Arc::clone(&schema)))
.map(|expr| {
planner.create_agg_expr(expr, Arc::clone(&schema), agg.dynamic_filter_enabled)
})
.collect();

let aggr_expr: Vec<Arc<AggregateFunctionExpr>> = if has_partial_merge {
Expand Down Expand Up @@ -276,16 +282,25 @@ impl OperatorBuilder for HashAggBuilder {
mode
};

let aggregate: Arc<dyn ExecutionPlan> = Arc::new(
datafusion::physical_plan::aggregates::AggregateExec::try_new(
mode,
group_by,
aggr_expr,
filter_exprs?,
Arc::clone(&child.native_plan),
Arc::clone(&schema),
)?,
);
let aggregate = datafusion::physical_plan::aggregates::AggregateExec::try_new(
mode,
group_by,
aggr_expr,
filter_exprs?,
Arc::clone(&child.native_plan),
Arc::clone(&schema),
)?;
let aggregate: Arc<dyn ExecutionPlan> = if agg.dynamic_filter_enabled {
match MinMaxReaderFilterExec::try_new(
&aggregate,
planner.session_ctx.copied_config().options(),
)? {
Some(wrapper) => Arc::new(wrapper),
None => Arc::new(aggregate),
}
} else {
Arc::new(aggregate)
};

let mut output = aggregate;
let mut additional_native_plans = vec![];
Expand Down Expand Up @@ -375,6 +390,7 @@ impl PhysicalPlanner {
&self,
spark_expr: &AggExpr,
schema: SchemaRef,
dynamic_filter_enabled: bool,
) -> Result<AggregateFunctionExpr, ExecutionError> {
self.register_query_context(spark_expr.expr_id, spark_expr.query_context.as_ref());

Expand All @@ -399,7 +415,8 @@ impl PhysicalPlanner {
let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?;
let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap());
let func = min_max_udaf(&datatype, false);
let child = Self::cast_min_max_child(child, &datatype, &schema)?;
let child =
Self::cast_min_max_child(child, &datatype, &schema, dynamic_filter_enabled)?;

AggregateExprBuilder::new(func, vec![child])
.schema(schema)
Expand All @@ -413,7 +430,8 @@ impl PhysicalPlanner {
let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?;
let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap());
let func = min_max_udaf(&datatype, true);
let child = Self::cast_min_max_child(child, &datatype, &schema)?;
let child =
Self::cast_min_max_child(child, &datatype, &schema, dynamic_filter_enabled)?;

AggregateExprBuilder::new(func, vec![child])
.schema(schema)
Expand Down Expand Up @@ -877,7 +895,20 @@ impl PhysicalPlanner {
child: Arc<dyn PhysicalExpr>,
datatype: &DataType,
schema: &SchemaRef,
dynamic_filter_enabled: bool,
) -> Result<Arc<dyn PhysicalExpr>, ExecutionError> {
// Keep the identity cast when disabled: exposing the column makes DataFusion
// maintain per-batch bounds even without an attached reader filter.
if dynamic_filter_enabled
&& child.is::<Column>()
&& matches!(
datatype,
DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64
)
&& child.data_type(schema.as_ref())? == *datatype
{
return Ok(child);
}
if !matches!(datatype, DataType::Struct(_)) {
return Ok(Arc::new(CastExpr::new(child, datatype.clone(), None)));
}
Expand Down
Loading