diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 0437ed9dbff..2006b5ceb06 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1641,10 +1641,10 @@ fn execute_plan( exec_context.stream = Some(stream); } exec_context.root_op = Some(root_op); - } else { - // Pull input batches - pull_input_batches(exec_context)?; } + // JVM input is pulled only when the stream waits for it, in `next_batch` below. An operator + // above a scan, such as a `BatchSplitExec`, can still hold rows of the last batch it took, + // and a limit can finish on them without the JVM producing another batch. // Wait no later than when metrics are due, so a plan that runs long before its first batch, // such as a native shuffle write, still publishes on the interval. A publish while waiting diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests.rs b/native/core/src/execution/operators/dynamic_filter/join/tests.rs index 54ae6737edf..ee1594d8603 100644 --- a/native/core/src/execution/operators/dynamic_filter/join/tests.rs +++ b/native/core/src/execution/operators/dynamic_filter/join/tests.rs @@ -46,7 +46,7 @@ use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::statistics::{StatisticsArgs, StatisticsContext}; use datafusion::physical_plan::{ChildrenPropertiesMode, Distribution, ReplaceChildrenOptions}; use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_comet_operators::CometFilterExec; +use datafusion_comet_operators::{BatchSplitExec, CometFilterExec}; use datafusion_comet_spark_expr::RandExpr; use datafusion_datasource::file::FileSource; use parquet::arrow::ArrowWriter; @@ -1108,6 +1108,54 @@ async fn run_parquet_join(values: Vec, enabled: bool) -> (usize, usize, usi ) } +#[tokio::test] +async fn reader_filter_attaches_through_a_batch_split() { + // The planner puts a CometBatchSplitExec under a filter whose predicate expands a string + // scalar. It only slices batches, so a reader filter below it discards the same rows. + let mut config = SessionConfig::new() + .with_target_partitions(1) + .with_parquet_page_index_pruning(false); + config.options_mut().execution.parquet.pushdown_filters = false; + let session = Arc::new(SessionContext::new_with_config(config)); + let (_file, scan) = parquet_probe((0..400).collect(), &session, 100); + let split: Arc = Arc::new(BatchSplitExec::new( + Arc::clone(&scan) as Arc, + 7, + )); + let predicate: Arc = + Arc::new(IsNotNullExpr::new(Arc::new(Column::new("key", 0)))); + let probe = Arc::new(CometFilterExec::from_datafusion( + FilterExec::try_new(predicate, split).unwrap(), + )); + let join = single_key_join_plans(two_batch_build(), probe, PartitionMode::Partitioned); + let wrapper = + DynamicFilterJoinExec::new(&join, session.copied_config().options().as_ref().clone()) + .unwrap(); + let runtime = wrapper.build_runtime_join().unwrap(); + assert!(runtime.reader_filter_attached); + let consumer = runtime + .join + .right() + .downcast_ref::() + .unwrap(); + let filter = consumer.children()[0] + .downcast_ref::() + .unwrap(); + let split = filter.input().downcast_ref::().unwrap(); + assert_eq!(split.batch_size(), 7); + let reader = split.children()[0] + .downcast_ref::() + .unwrap(); + let (_, source) = reader.downcast_to_file_source::().unwrap(); + let reader_filter = source.filter().unwrap(); + let reader_filter = find_dynamic_filter(&reader_filter).unwrap(); + let join_filter = produced_join_filter(&runtime.join); + assert_eq!( + reader_filter.expression_id().unwrap(), + join_filter.expression_id().unwrap() + ); +} + #[tokio::test] async fn broadcast_filter_reaches_parquet_reader_after_complete_build() { let mut config = SessionConfig::new() diff --git a/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs b/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs index 6afd49cb226..dfe6d3777a6 100644 --- a/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs +++ b/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs @@ -32,7 +32,7 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ExecutionPlan; use crate::parquet::file_error_context::ParquetErrorContext; -use datafusion_comet_operators::CometFilterExec; +use datafusion_comet_operators::{BatchSplitExec, CometFilterExec}; mod schema_adapter; @@ -98,6 +98,15 @@ pub(super) fn try_attach_parquet_reader_filter( .map(|reader| filter.with_execution_input(reader)), ); } + // The batch split that bounds a scalar expansion only slices batches, so a + // reader filter below it discards the same rows. + if let Some(split) = input.downcast_ref::() { + return Ok( + try_attach_parquet_reader_filter(split.input(), predicate, config)?.map(|reader| { + Arc::new(BatchSplitExec::new(reader, split.batch_size())) as Arc + }), + ); + } // Spark inserts IS NOT NULL residuals above equijoin inputs, including AND // chains of inferred null checks. A reader predicate can cross those direct // checks because both operations only discard rows. Broader deterministic diff --git a/native/core/src/execution/operators/projection.rs b/native/core/src/execution/operators/projection.rs index f22efc53c8e..7933c0dbfda 100644 --- a/native/core/src/execution/operators/projection.rs +++ b/native/core/src/execution/operators/projection.rs @@ -80,10 +80,11 @@ impl OperatorBuilder for ProjectionBuilder { } } } - let projection = Arc::new(ProjectionExec::try_new( - exprs.iter().cloned(), + let input = planner.bound_scalar_expansion( Arc::clone(&child.native_plan), - )?); + exprs.iter().map(|expr| &expr.expr), + ); + let projection = Arc::new(ProjectionExec::try_new(exprs.iter().cloned(), input)?); let spark_plan = if child.plan_id == spark_plan.plan_id { let mut additional_native_plans = vec![Arc::clone(&child.native_plan)]; diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 72071d9dd19..5665fa56129 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -25,6 +25,7 @@ mod literal; pub mod macros; mod merge_rows; pub mod operator_registry; +mod scalar_expansion; mod shuffle; mod window; mod write; @@ -1759,10 +1760,12 @@ impl PhysicalPlanner { let predicate = self.create_expr(filter.predicate.as_ref().unwrap(), child.schema())?; + let input = + self.bound_scalar_expansion(Arc::clone(&child.native_plan), [&predicate]); // FilterExec merges its output up to its own batch size, 8192 rows by default. // Use the session batch size so it doesn't build batches bigger than that. let filter: Arc = Arc::new( - FilterExecBuilder::new(predicate, Arc::clone(&child.native_plan)) + FilterExecBuilder::new(predicate, input) .with_batch_size(self.session_ctx.copied_config().batch_size()) .build()?, ); @@ -1853,19 +1856,28 @@ impl PhysicalPlanner { let (scans, shuffle_scans, child) = self.create_plan(&children[0], inputs, partition_count)?; - let exprs: Result, ExecutionError> = sort + let exprs: Vec = sort .sort_orders .iter() .map(|expr| self.create_sort_expr(expr, child.schema())) - .collect(); + .collect::>()?; let fetch = sort.fetch.map(|num| num as usize); - let sort_plan = SortExec::new( - LexOrdering::new(exprs?).unwrap(), - Arc::clone(&child.native_plan), - ) - .with_fetch(fetch); + // A TopK evaluates its keys on each batch it receives. A full sort concatenates + // its buffered input before evaluating them, so a split would not bound it, and + // DataFusion's `ExternalSorter` reserves each slice of a shared buffer as if it + // owned the buffer (apache/datafusion#25804). + let input = if fetch.is_some() { + self.bound_scalar_expansion( + Arc::clone(&child.native_plan), + exprs.iter().map(|sort| &sort.expr), + ) + } else { + Arc::clone(&child.native_plan) + }; + let sort_plan = + SortExec::new(LexOrdering::new(exprs).unwrap(), input).with_fetch(fetch); let mut sort_exec: Arc = if sort.dynamic_filter_enabled { match TopKReaderFilterExec::try_new( &sort_plan, @@ -3238,7 +3250,7 @@ mod tests { use crate::execution::shuffle::CometPartitioning; use crate::parquet::parquet_support::SparkParquetOptions; use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory; - use datafusion_comet_operators::{PartitionedRankLimitExec, WindowFnKind}; + use datafusion_comet_operators::{BatchSplitExec, PartitionedRankLimitExec, WindowFnKind}; use datafusion_comet_proto::spark_expression::expr::ExprStruct; use datafusion_comet_proto::spark_expression::ListLiteral; use datafusion_comet_proto::{ @@ -4368,6 +4380,82 @@ mod tests { ); } + #[test] + fn sort_bounds_the_scalar_its_keys_expand() { + // A TopK evaluates its keys on each batch it inserts, expanding a string scalar in a key + // to the batch's length, so the keys get the bound a projection's expressions get. A full + // sort gets no split: it evaluates its keys after concatenating its buffered input. + let string_type = spark_expression::DataType { + type_id: spark_expression::data_type::DataTypeId::String as i32, + type_info: None, + }; + let scan = Operator { + op_struct: Some(OpStruct::Scan(spark_operator::Scan { + fields: vec![string_type.clone()], + source: String::new(), + })), + ..Default::default() + }; + let wide = Expr { + expr_struct: Some(Literal(spark_expression::Literal { + value: Some(literal::Value::StringVal("x".repeat(256 * 1024))), + datatype: Some(string_type.clone()), + is_null: false, + })), + ..Default::default() + }; + let column = Expr { + expr_struct: Some(Bound(spark_expression::BoundReference { + index: 0, + datatype: Some(string_type.clone()), + })), + ..Default::default() + }; + let key = Expr { + expr_struct: Some(ScalarFunc(spark_expression::ScalarFunc { + func: "concat".to_string(), + args: vec![wide, column], + return_type: Some(string_type), + ..Default::default() + })), + ..Default::default() + }; + for fetch in [None, Some(1)] { + let sort = Operator { + children: vec![scan.clone()], + op_struct: Some(OpStruct::Sort(spark_operator::Sort { + sort_orders: vec![Expr { + expr_struct: Some(SortOrder(Box::new(spark_expression::SortOrder { + child: Some(Box::new(key.clone())), + direction: spark_expression::SortDirection::Ascending as i32, + null_ordering: spark_expression::NullOrdering::NullsFirst as i32, + }))), + ..Default::default() + }], + fetch, + ..Default::default() + })), + ..Default::default() + }; + let (_, _, planned) = PhysicalPlanner::default() + .create_plan(&sort, &mut vec![], 1) + .unwrap(); + let sort = planned.native_plan.downcast_ref::().unwrap(); + assert_eq!(fetch.map(|n| n as usize), sort.fetch()); + match fetch { + None => assert!(!sort.input().is::()), + Some(_) => { + let split = sort + .input() + .downcast_ref::() + .expect("a split under the TopK"); + // A 256 KiB scalar expanded to 4096 rows takes 1 GiB. + assert_eq!(4096, split.batch_size()); + } + } + } + } + #[test] fn declared_order_reaches_a_scan_of_one_file_group() { // A scan of files reads the files of its one file group one after the other, so the order diff --git a/native/core/src/execution/planner/aggregate.rs b/native/core/src/execution/planner/aggregate.rs index bdc7dacad8a..6a3df632ca0 100644 --- a/native/core/src/execution/planner/aggregate.rs +++ b/native/core/src/execution/planner/aggregate.rs @@ -282,12 +282,20 @@ impl OperatorBuilder for HashAggBuilder { mode }; + let filter_exprs = filter_exprs?; + let input_exprs: Vec> = group_by + .input_exprs() + .into_iter() + .chain(aggr_expr.iter().flat_map(|expr| expr.expressions())) + .chain(filter_exprs.iter().flatten().cloned()) + .collect(); + let input = planner.bound_scalar_expansion(Arc::clone(&child.native_plan), &input_exprs); let aggregate = datafusion::physical_plan::aggregates::AggregateExec::try_new( mode, group_by, aggr_expr, - filter_exprs?, - Arc::clone(&child.native_plan), + filter_exprs, + input, Arc::clone(&schema), )?; let aggregate: Arc = if agg.dynamic_filter_enabled { diff --git a/native/core/src/execution/planner/join.rs b/native/core/src/execution/planner/join.rs index f135c6afa70..cddd52a35af 100644 --- a/native/core/src/execution/planner/join.rs +++ b/native/core/src/execution/planner/join.rs @@ -47,7 +47,7 @@ use datafusion::{ }, }; use datafusion_comet_operators::{ - CometFilterExec, CometProjectionExec, MembershipKeyReductionExec, + BatchSplitExec, CometFilterExec, CometProjectionExec, MembershipKeyReductionExec, }; use datafusion_comet_proto::{ spark_expression::Expr, @@ -422,6 +422,16 @@ impl OperatorBuilder for BroadcastNestedLoopJoinBuilder { } } +/// `plan`, or its input when it is the batch split that bounds a scalar expansion. The +/// planner puts that split between an operator and its child, and the Spark tree has no +/// node for it. +fn through_batch_split(plan: &Arc) -> &Arc { + match plan.downcast_ref::() { + Some(split) => split.input(), + None => plan, + } +} + /// Builder for `CrossJoin` (Spark's `CartesianProductExec`). pub struct CrossJoinBuilder; @@ -473,22 +483,22 @@ impl PhysicalPlanner { ) -> Result, ExecutionError> { // Walk the serialized and Spark trees together. A mismatch must not lend // another filter's permission to this node or any of its descendants. - let source = source.filter(|source| { - source.plan_id == plan.plan_id - && source.children.len() == plan.children.len() - && plan.children.len() == plan.native_plan.children().len() - && plan - .native_plan - .children() - .iter() - .zip(&plan.children) - .all(|(native, spark)| Arc::ptr_eq(native, &spark.native_plan)) - && match &source.op_struct { - Some(OpStruct::Filter(_)) => plan.native_plan.is::(), - Some(OpStruct::Projection(_)) => plan.native_plan.is::(), - _ => false, - } - }); + let source = + source.filter(|source| { + source.plan_id == plan.plan_id + && source.children.len() == plan.children.len() + && plan.children.len() == plan.native_plan.children().len() + && plan.native_plan.children().iter().zip(&plan.children).all( + |(native, spark)| { + Arc::ptr_eq(through_batch_split(native), &spark.native_plan) + }, + ) + && match &source.op_struct { + Some(OpStruct::Filter(_)) => plan.native_plan.is::(), + Some(OpStruct::Projection(_)) => plan.native_plan.is::(), + _ => false, + } + }); let mut native: Arc = if let Some(filter) = plan.native_plan.downcast_ref::() { let allowed = matches!( @@ -506,13 +516,23 @@ impl PhysicalPlanner { }; let mut prepared = plan.as_ref().clone(); if let ([child], [only]) = (plan.children.as_slice(), native.children().as_slice()) { - if Arc::ptr_eq(only, &child.native_plan) { + if Arc::ptr_eq(through_batch_split(only), &child.native_plan) { + let split_size = only + .downcast_ref::() + .map(|split| split.batch_size()); let child = Self::prepare_probe_for_runtime_filters( Arc::clone(child), source.and_then(|source| source.children.first()), )?; + let input = match split_size { + Some(size) => { + Arc::new(BatchSplitExec::new(Arc::clone(&child.native_plan), size)) + as Arc + } + None => Arc::clone(&child.native_plan), + }; native = native.replace_children( - vec![Arc::clone(&child.native_plan)], + vec![input], ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), )?; prepared.children = vec![child]; @@ -943,6 +963,7 @@ fn rewrite_physical_expr( #[cfg(test)] mod tests { use super::*; + use datafusion_comet_proto::spark_expression::expr::ExprStruct; use datafusion_comet_proto::{spark_expression, spark_operator}; fn column() -> Expr { @@ -979,6 +1000,69 @@ mod tests { } } + fn string_type() -> spark_expression::DataType { + spark_expression::DataType { + type_id: spark_expression::data_type::DataTypeId::String as i32, + type_info: None, + } + } + + fn string_scan() -> Operator { + Operator { + op_struct: Some(OpStruct::Scan(spark_operator::Scan { + fields: vec![string_type()], + ..Default::default() + })), + ..Default::default() + } + } + + /// A filter on `concat(column, 'x') = 'yx'`: the scalar inside the concatenation puts a + /// batch split under the filter. A scalar compared directly is never expanded. + fn string_filter(child: Operator, allowed: bool) -> Operator { + let column = Expr { + expr_struct: Some(ExprStruct::Bound(spark_expression::BoundReference { + index: 0, + datatype: Some(string_type()), + })), + ..Default::default() + }; + let literal = |value: &str| Expr { + expr_struct: Some(ExprStruct::Literal(spark_expression::Literal { + value: Some(spark_expression::literal::Value::StringVal( + value.to_string(), + )), + datatype: Some(string_type()), + is_null: false, + })), + ..Default::default() + }; + let concatenated = Expr { + expr_struct: Some(ExprStruct::ScalarFunc(spark_expression::ScalarFunc { + func: "concat".to_string(), + args: vec![column, literal("x")], + return_type: Some(string_type()), + ..Default::default() + })), + ..Default::default() + }; + Operator { + plan_id: child.plan_id + 1, + children: vec![child], + op_struct: Some(OpStruct::Filter(spark_operator::Filter { + predicate: Some(Expr { + expr_struct: Some(ExprStruct::Eq(Box::new(spark_expression::BinaryExpr { + left: Some(Box::new(concatenated)), + right: Some(Box::new(literal("yx"))), + }))), + ..Default::default() + }), + allow_runtime_filter_pushdown: allowed, + })), + ..Default::default() + } + } + fn projection(child: Operator) -> Operator { Operator { plan_id: child.plan_id + 1, @@ -1020,6 +1104,25 @@ mod tests { } } + #[test] + fn probe_filter_permission_survives_a_scalar_batch_split() { + for permission in [false, true] { + let source = string_filter(string_scan(), permission); + let planned = plan(&source); + assert!(planned.native_plan.children()[0].is::()); + let prepared = + PhysicalPlanner::prepare_probe_for_runtime_filters(planned, Some(&source)).unwrap(); + assert_eq!(allowed(&prepared), permission); + let split = prepared.native_plan.children()[0] + .downcast_ref::() + .unwrap(); + assert!(Arc::ptr_eq( + split.children()[0], + &prepared.children[0].native_plan + )); + } + } + #[test] fn stacked_probe_filters_keep_independent_permissions() { for outer in [false, true] { diff --git a/native/core/src/execution/planner/scalar_expansion.rs b/native/core/src/execution/planner/scalar_expansion.rs new file mode 100644 index 00000000000..84fb28babb7 --- /dev/null +++ b/native/core/src/execution/planner/scalar_expansion.rs @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Bounds the rows per batch under an operator that expands a large string scalar. +//! +//! An aggregate, a hash partitioner and most string kernels expand a scalar argument to the +//! batch's length: a 256 KiB string over the default 8192-row batch needs 2 GiB of value bytes, +//! past the 32-bit offsets of Arrow's `Utf8` and `Binary` arrays. Spark evaluates such scalars +//! row by row and never materializes the column. See +//! . +//! +//! The bound is enforced right under the operator that evaluates the expressions, with a +//! [`BatchSplitExec`], because the operators below do not keep the batch size they were +//! handed: a filter coalesces its output back up to the session's batch size, a window emits +//! its whole partition and a broadcast arrives as a few wide batches. No width can be assumed, +//! so the split is installed whenever the operator expands a string or binary scalar at all. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion::common::ScalarValue; +use datafusion::logical_expr::{ColumnarValue, Operator, Volatility}; +use datafusion::physical_expr::expressions::{BinaryExpr, Column, InListExpr}; +use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_comet_operators::BatchSplitExec; +use datafusion_comet_spark_expr::jvm_udf::JvmScalarUdfExpr; +use datafusion_comet_spark_expr::monotonically_increasing_id::MonotonicallyIncreasingId; +use datafusion_comet_spark_expr::{ + RandExpr, RandStrExpr, RandnExpr, ShuffleExpr, Subquery, UuidExpr, +}; + +use super::{PhysicalPlanner, TEST_EXEC_CONTEXT_ID}; + +/// The most value bytes the string and binary scalars of one operator may take once expanded to +/// a batch's length. Arrow's 32-bit offsets make 2 GiB the hard limit; half of it leaves room for +/// kernels that append to the expanded value, such as `concat`, and matches what an Arrow Java +/// vector can hold before its reallocation fails. +const SCALAR_EXPANSION_BYTES: usize = 1 << 30; + +/// The most rows per batch at which `scalar_bytes`, expanded to the batch's length, stay within +/// `SCALAR_EXPANSION_BYTES`; unbounded when there is nothing to expand. +fn scalar_rows_cap(scalar_bytes: usize) -> usize { + if scalar_bytes == 0 { + return usize::MAX; + } + (SCALAR_EXPANSION_BYTES / scalar_bytes).max(1) +} + +impl PhysicalPlanner { + /// Wraps `input` so that the batches it emits keep the string and binary scalars in `exprs`, + /// expanded to a batch's length, within `SCALAR_EXPANSION_BYTES` and the session's batch + /// size. Returns `input` as is when `exprs` expand no scalar. + pub(crate) fn bound_scalar_expansion<'a>( + &self, + input: Arc, + exprs: impl IntoIterator>, + ) -> Arc { + let schema = input.schema(); + let bytes: usize = exprs + .into_iter() + .map(|expr| self.expanded_scalar_bytes(expr, &schema)) + .sum(); + if bytes == 0 { + return input; + } + let batch_size = self.session_ctx.copied_config().batch_size(); + Arc::new(BatchSplitExec::new( + input, + scalar_rows_cap(bytes).min(batch_size), + )) + } + + /// The bytes of the string and binary scalars that evaluating `expr` expands to the batch's + /// length, added up, so that a scalar used twice counts twice. + /// + /// A scalar is any subtree without a column reference or a volatile function, evaluated + /// here on a one-row batch: `repeat(, 8)` is wider than the subquery result it is + /// built from. A subtree that is not a scalar is evaluated per row, so only the scalars + /// among its children count. A subtree that fails to evaluate counts nothing, so that its + /// error surfaces where Spark's would, at execution. A scalar that a comparison or an `IN` + /// list takes as a constant is compared and never expanded; an operand computed per row + /// counts the scalars it contains. + fn expanded_scalar_bytes(&self, expr: &Arc, schema: &Schema) -> usize { + if let Some(in_list) = expr.downcast_ref::() { + // The tested value is expanded to the batch's length. A member that IN compares as + // a constant is not; one computed per row is, along with the scalars it contains. + return self.expanded_scalar_bytes(in_list.expr(), schema) + + self.per_row_operand_bytes(in_list.list().iter(), schema); + } + if let Some(comparison) = expr + .downcast_ref::() + .filter(|binary| compares_as_datum(binary.op())) + { + return self.per_row_operand_bytes( + [comparison.left(), comparison.right()].into_iter(), + schema, + ); + } + if !self.is_scalar(expr) { + return expr + .children() + .into_iter() + .map(|child| self.expanded_scalar_bytes(child, schema)) + .sum(); + } + if !matches!( + expr.data_type(schema), + Ok(DataType::Utf8 | DataType::Binary) + ) { + return 0; + } + // One row, so that a kernel that expands its scalar arguments still yields one value. + let Ok(one_row) = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(1)), + ) else { + return 0; + }; + let value = match expr.evaluate(&one_row) { + Ok(ColumnarValue::Scalar(value)) => value, + Ok(ColumnarValue::Array(array)) if array.len() == 1 => { + match ScalarValue::try_from_array(&array, 0) { + Ok(value) => value, + Err(_) => return 0, + } + } + _ => return 0, + }; + match value { + ScalarValue::Utf8(Some(s)) => s.len(), + ScalarValue::Binary(Some(b)) => b.len(), + _ => 0, + } + } + + /// The bytes of the scalars inside the `operands` computed per row. An operand that is a + /// scalar is compared as it is. + fn per_row_operand_bytes<'a>( + &self, + operands: impl Iterator>, + schema: &Schema, + ) -> usize { + operands + .filter(|operand| !self.is_scalar(operand)) + .map(|operand| self.expanded_scalar_bytes(operand, schema)) + .sum() + } +} + +/// Whether DataFusion's `BinaryExpr` evaluates `op` with `apply_cmp`, which takes a scalar +/// operand as a `Datum` instead of expanding it to the batch's length. +fn compares_as_datum(op: &Operator) -> bool { + matches!( + op, + Operator::Eq + | Operator::NotEq + | Operator::Lt + | Operator::LtEq + | Operator::Gt + | Operator::GtEq + | Operator::IsDistinctFrom + | Operator::IsNotDistinctFrom + | Operator::LikeMatch + | Operator::ILikeMatch + | Operator::NotLikeMatch + | Operator::NotILikeMatch + ) +} + +impl PhysicalPlanner { + /// Whether `expr` yields one value for a batch that can be computed at plan time. A column + /// reference and a volatile function yield a value per row, and evaluating a volatile + /// function advances its seeded generator. A JVM UDF is the user's code, which may have + /// side effects or be slow, and a planner unit test has no JVM to read a subquery result + /// from. + fn is_scalar(&self, expr: &Arc) -> bool { + let per_row = expr.is::() + || is_volatile(expr) + || expr.is::() + || (self.exec_context_id == TEST_EXEC_CONTEXT_ID && expr.is::()); + !per_row && expr.children().iter().all(|child| self.is_scalar(child)) + } +} + +fn is_volatile(expr: &Arc) -> bool { + expr.is::() + || expr.is::() + || expr.is::() + || expr.is::() + || expr.is::() + || expr.is::() + || expr + .downcast_ref::() + .is_some_and(|f| f.fun().signature().volatility == Volatility::Volatile) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::datatypes::Field; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{binary, col, in_list, lit}; + + fn schema() -> Schema { + Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + } + + fn bytes(expr: Arc) -> usize { + PhysicalPlanner::default().expanded_scalar_bytes(&expr, &schema()) + } + + #[test] + fn scalar_rows_cap_bounds_the_rows_by_the_expanded_bytes() { + assert_eq!(scalar_rows_cap(0), usize::MAX); + assert_eq!(scalar_rows_cap(1024), 1024 * 1024); + // A 256 KiB scalar expanded to 8192 rows needs 2 GiB; 4096 rows keep it at 1 GiB. + assert_eq!(scalar_rows_cap(256 * 1024), 4096); + // A scalar past 1 GiB still gets one row per batch. + assert_eq!(scalar_rows_cap(2 * 1024 * 1024 * 1024), 1); + } + + #[test] + fn counts_string_scalars_and_not_columns() { + assert_eq!(bytes(lit("abcd")), 4); + assert_eq!(bytes(lit(7)), 0); + assert_eq!(bytes(col("s", &schema()).unwrap()), 0); + } + + #[test] + fn counts_a_scalar_built_by_an_expression_at_its_final_width() { + let schema = schema(); + let built = binary(lit("abcd"), Operator::StringConcat, lit("efgh"), &schema).unwrap(); + assert_eq!(bytes(Arc::clone(&built)), 8); + // Under a column reference the scalar subtree still counts, once per use. + let over_column = binary( + Arc::clone(&built), + Operator::StringConcat, + col("s", &schema).unwrap(), + &schema, + ) + .unwrap(); + assert_eq!(bytes(Arc::clone(&over_column)), 8); + let twice = binary(over_column, Operator::StringConcat, built, &schema).unwrap(); + assert_eq!(bytes(twice), 16); + } + + #[test] + fn measuring_a_seeded_generator_does_not_advance_it() { + use datafusion_comet_spark_expr::UuidExpr; + let one_row = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(1)), + ) + .unwrap(); + let first_draw = + |expr: &Arc| expr.evaluate(&one_row).unwrap().into_array(1).unwrap(); + let measured: Arc = Arc::new(UuidExpr::new(0)); + assert_eq!(bytes(Arc::clone(&measured)), 0); + let fresh: Arc = Arc::new(UuidExpr::new(0)); + assert_eq!(*first_draw(&measured), *first_draw(&fresh)); + } + + #[test] + fn wraps_the_input_at_a_session_batch_size_the_scalar_fits() { + use datafusion::execution::context::SessionContext; + use datafusion::physical_plan::empty::EmptyExec; + use datafusion::prelude::SessionConfig; + let session = SessionContext::new_with_config(SessionConfig::new().with_batch_size(256)); + let planner = PhysicalPlanner::new(Arc::new(session), 0); + let input: Arc = Arc::new(EmptyExec::new(Arc::new(schema()))); + // A 256 KiB scalar allows 4096 rows, more than the session's 256, which the operators + // below do not keep to. + let wide = lit("x".repeat(256 * 1024)); + let bounded = planner.bound_scalar_expansion(input, [&wide]); + let split = bounded + .downcast_ref::() + .expect("a split under the expanding operator"); + assert_eq!(256, split.batch_size()); + } + + #[test] + fn counts_a_scalar_beside_a_volatile_sibling_without_evaluating_the_sibling() { + use datafusion_comet_spark_expr::UuidExpr; + let schema = schema(); + let one_row = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(1)), + ) + .unwrap(); + let first_draw = + |expr: &Arc| expr.evaluate(&one_row).unwrap().into_array(1).unwrap(); + let uuid: Arc = Arc::new(UuidExpr::new(0)); + let mixed = binary( + lit("x".repeat(256 * 1024)), + Operator::StringConcat, + Arc::clone(&uuid), + &schema, + ) + .unwrap(); + assert_eq!(bytes(mixed), 256 * 1024); + let fresh: Arc = Arc::new(UuidExpr::new(0)); + assert_eq!(*first_draw(&uuid), *first_draw(&fresh)); + } + + #[test] + fn skips_the_members_of_an_in_list() { + let schema = schema(); + let expr = in_list( + col("s", &schema).unwrap(), + vec![lit("abcd"), lit("efgh")], + &false, + &schema, + ) + .unwrap(); + assert_eq!(bytes(expr), 0); + } + + #[test] + fn counts_the_scalar_in_a_list_member_computed_per_row() { + use datafusion_comet_spark_expr::spark_in_list; + let schema = schema(); + let per_row = binary( + lit("x".repeat(256 * 1024)), + Operator::StringConcat, + col("s", &schema).unwrap(), + &schema, + ) + .unwrap(); + // The per-row member expands its scalar for every input row; the literal is compared. + let expr = spark_in_list( + col("s", &schema).unwrap(), + vec![per_row, lit("foo")], + false, + &schema, + ) + .unwrap(); + assert!(expr.is::()); + assert_eq!(bytes(expr), 256 * 1024); + } + + #[test] + fn wraps_the_input_at_the_session_batch_size_or_the_scalar_cap() { + use datafusion::physical_plan::empty::EmptyExec; + let planner = PhysicalPlanner::default(); + let input: Arc = Arc::new(EmptyExec::new(Arc::new(schema()))); + let none = planner.bound_scalar_expansion(Arc::clone(&input), [&lit(7)]); + assert_eq!("EmptyExec", none.name()); + let split_at = |exprs: &[Arc]| { + planner + .bound_scalar_expansion(Arc::clone(&input), exprs) + .downcast_ref::() + .expect("a split under the expanding operator") + .batch_size() + }; + // A small scalar is bounded by the session's batch size, whatever arrives from below. + assert_eq!(8192, split_at(&[lit("abcd")])); + let wide = lit("x".repeat(256 * 1024)); + assert_eq!(2048, split_at(&[Arc::clone(&wide), wide])); + } + + #[test] + fn skips_a_scalar_that_a_comparison_takes_as_a_datum() { + let schema = schema(); + let wide = lit("x".repeat(256 * 1024)); + for op in [ + Operator::Eq, + Operator::NotEq, + Operator::Lt, + Operator::IsNotDistinctFrom, + ] { + let direct = + binary(col("s", &schema).unwrap(), op, Arc::clone(&wide), &schema).unwrap(); + assert_eq!(bytes(direct), 0, "{op}"); + // An operand computed per row still expands the scalar it contains. + let per_row = binary( + Arc::clone(&wide), + Operator::StringConcat, + col("s", &schema).unwrap(), + &schema, + ) + .unwrap(); + let compared = binary(col("s", &schema).unwrap(), op, per_row, &schema).unwrap(); + assert_eq!(bytes(compared), 256 * 1024, "{op}"); + } + } +} diff --git a/native/core/src/execution/planner/shuffle.rs b/native/core/src/execution/planner/shuffle.rs index 9d3b073879d..3646d304182 100644 --- a/native/core/src/execution/planner/shuffle.rs +++ b/native/core/src/execution/planner/shuffle.rs @@ -85,6 +85,15 @@ impl OperatorBuilder for ShuffleWriterBuilder { let partitioning = planner .create_partitioning(writer.partitioning.as_ref().unwrap(), writer_input.schema())?; + let writer_input = match &partitioning { + CometPartitioning::Hash(exprs, _) => { + planner.bound_scalar_expansion(writer_input, exprs) + } + CometPartitioning::RangePartitioning(ordering, ..) => { + planner.bound_scalar_expansion(writer_input, ordering.iter().map(|sort| &sort.expr)) + } + _ => writer_input, + }; let codec = match writer.codec.try_into() { Ok(SparkCompressionCodec::None) => Ok(CompressionCodec::None), diff --git a/native/operators/src/batch_split.rs b/native/operators/src/batch_split.rs new file mode 100644 index 00000000000..ec8ff7b528d --- /dev/null +++ b/native/operators/src/batch_split.rs @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! An operator that bounds the rows per batch its parent receives. + +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::Result; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet, SplitMetrics}; +use datafusion::physical_plan::stream::BatchSplitStream; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SendableRecordBatchStream, +}; +use std::fmt::Formatter; +use std::sync::Arc; + +/// Passes its input through in batches of at most `batch_size` rows, slicing wider ones without +/// copying. The planner puts it under an operator whose expressions expand a large string or +/// binary scalar to the batch's length, so that the expanded value fits Arrow's 32-bit offsets +/// whatever batch size the operators below settle on. +#[derive(Debug)] +pub struct BatchSplitExec { + input: Arc, + batch_size: usize, + metrics: ExecutionPlanMetricsSet, + cache: Arc, +} + +impl BatchSplitExec { + pub fn new(input: Arc, batch_size: usize) -> Self { + // Slicing keeps every row in place, so the input's ordering and partitioning stand. + let cache = Arc::new(PlanProperties::new( + input.equivalence_properties().clone(), + input.output_partitioning().clone(), + input.pipeline_behavior(), + input.boundedness(), + )); + Self { + input, + batch_size: batch_size.max(1), + metrics: ExecutionPlanMetricsSet::new(), + cache, + } + } + + pub fn input(&self) -> &Arc { + &self.input + } + + pub fn batch_size(&self) -> usize { + self.batch_size + } +} + +impl DisplayAs for BatchSplitExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "CometBatchSplitExec: batch_size={}", self.batch_size) + } + DisplayFormatType::TreeRender => unimplemented!(), + } + } +} + +impl ExecutionPlan for BatchSplitExec { + fn name(&self) -> &str { + "CometBatchSplitExec" + } + + fn properties(&self) -> &Arc { + &self.cache + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn maintains_input_order(&self) -> Vec { + vec![true] + } + + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(BatchSplitExec::new( + Arc::clone(&children[0]), + self.batch_size, + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let input = self.input.execute(partition, context)?; + Ok(Box::pin(BatchSplitStream::new( + input, + self.batch_size, + SplitMetrics::new(&self.metrics, partition), + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Int32Array, RecordBatch}; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::datasource::source::DataSourceExec; + use datafusion::prelude::SessionContext; + use futures::StreamExt; + + fn batch(values: Vec) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap() + } + + /// The rows of each batch the operator emits for `batches`, split at `batch_size`. + async fn split(batches: Vec, batch_size: usize) -> Vec> { + let schema = batches[0].schema(); + let config = MemorySourceConfig::try_new(&[batches], schema, None).unwrap(); + let input = Arc::new(DataSourceExec::new(Arc::new(config))); + let exec = Arc::new(BatchSplitExec::new(input, batch_size)); + let ctx = SessionContext::new(); + let mut stream = exec.execute(0, ctx.task_ctx()).unwrap(); + let mut out = Vec::new(); + while let Some(batch) = stream.next().await { + let batch = batch.unwrap(); + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + out.push(column.values().to_vec()); + } + out + } + + #[tokio::test] + async fn splits_a_wider_batch_and_passes_narrower_ones_through() { + let out = split(vec![batch((0..10).collect()), batch(vec![10, 11])], 4).await; + assert_eq!( + out, + vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7], vec![8, 9], vec![10, 11]] + ); + } +} diff --git a/native/operators/src/lib.rs b/native/operators/src/lib.rs index 05a0324567c..25fdf6e1ece 100644 --- a/native/operators/src/lib.rs +++ b/native/operators/src/lib.rs @@ -25,6 +25,7 @@ // The lint makes easier for code reader/reviewer separate references clones from more heavyweight ones #![deny(clippy::clone_on_ref_ptr)] +mod batch_split; mod expand; mod explode; mod filter; @@ -35,6 +36,7 @@ mod range; mod rank_limit; mod sample; +pub use batch_split::BatchSplitExec; pub use expand::ExpandExec; pub use explode::ExplodeExec; pub use filter::CometFilterExec; diff --git a/spark/src/test/resources/sql-tests/operators/large_scalar_string.sql b/spark/src/test/resources/sql-tests/operators/large_scalar_string.sql new file mode 100644 index 00000000000..1c8851ac85a --- /dev/null +++ b/spark/src/test/resources/sql-tests/operators/large_scalar_string.sql @@ -0,0 +1,91 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- A string scalar, from a scalar subquery or a literal, is expanded to the batch length by the +-- aggregate, the hash partitioner and string kernels. A 256 KiB string over the default 8192-row +-- batch needs 2 GiB of value bytes, past Arrow's 32-bit offsets. Spark evaluates row by row. +-- The ids are written in one task so they arrive in one batch. Constant folding is on so that +-- `repeat('x', 262144)` reaches Comet as the literal it is in a Spark session. Null propagation +-- is off so that the grouped count keeps the literal as its argument instead of `count(1)`. +-- See https://github.com/apache/datafusion-comet/issues/7247. +-- Config: spark.sql.adaptive.enabled=false +-- Config: spark.comet.exec.range.enabled=true +-- ConstantFolding: enabled +-- ExcludeRules: org.apache.spark.sql.catalyst.optimizer.NullPropagation + +statement +CREATE TABLE test_large_scalar_wide(s string) USING parquet + +statement +INSERT INTO test_large_scalar_wide SELECT repeat('x', 262144) + +statement +CREATE TABLE test_large_scalar_ids(k int) USING parquet + +statement +INSERT INTO test_large_scalar_ids SELECT /*+ COALESCE(1) */ CAST(id % 2 AS INT) FROM range(8192) + +-- count of the scalar in a grouped aggregate, which expands it to the batch length. A literal +-- rather than the subquery: the harness's missingInput check trips on a grouped count of a +-- scalar subquery whatever the string's size, which is a separate plan issue. +query +SELECT k, count(repeat('x', 262144)) FROM test_large_scalar_ids GROUP BY k ORDER BY k + +-- the scalar as a hash partitioning key +query +SELECT count(*) FROM (SELECT k FROM test_large_scalar_ids DISTRIBUTE BY k, (SELECT max(s) FROM test_large_scalar_wide)) + +-- a string kernel whose output exceeds 2 GiB per 8192-row batch +query +SELECT sum(length(c)) FROM (SELECT concat((SELECT max(s) FROM test_large_scalar_wide), CAST(k AS STRING)) AS c FROM test_large_scalar_ids) + +-- the same with a literal instead of a subquery +query +SELECT sum(length(c)) FROM (SELECT concat(repeat('x', 262144), CAST(k AS STRING)) AS c FROM test_large_scalar_ids) + +-- the literal over the native range source, which sizes its own batches, in one slice +query +SELECT sum(length(concat(repeat('x', 262144), CAST(id AS STRING)))) FROM range(0, 8192, 1, 1) + +-- a filter under the projection coalesces its output to its own batch size +query +SELECT sum(length(c)) FROM (SELECT concat((SELECT max(s) FROM test_large_scalar_wide), CAST(k AS STRING)) AS c FROM test_large_scalar_ids WHERE k >= 0) + +-- a window under the aggregate emits its whole partition +query +SELECT sum(length(concat(repeat('x', 262144), CAST(tile AS STRING)))) FROM (SELECT ntile(2) OVER (ORDER BY id) AS tile FROM range(0, 8192, 1, 1)) + +-- a partition wider than the session batch size, with a scalar that fits the session batch size +query +SELECT sum(length(concat(repeat('x', 65536), CAST(tile AS STRING)))) FROM (SELECT ntile(2) OVER (ORDER BY id) AS tile FROM range(0, 32768, 1, 1)) + +-- a scalar beside a volatile expression, which is not evaluated at plan time +query +SELECT sum(length(c)) FROM (SELECT concat(repeat('x', 262144), uuid()) AS c FROM range(0, 8192, 1, 1)) + +-- a scalar built from the subquery result is wider than the result itself +query +SELECT sum(length(c)) FROM (SELECT concat(repeat((SELECT max(s) FROM test_large_scalar_wide), 2), CAST(k AS STRING)) AS c FROM test_large_scalar_ids) + +-- A list member computed per row expands the scalar it contains; the literal member does not +query +SELECT sum(CASE WHEN CAST(id AS STRING) IN (concat(repeat('x', 262144), CAST(id AS STRING)), 'foo') THEN 1 ELSE 0 END) +FROM range(0, 8192, 1, 1) + +-- a sort key evaluated per batch by the TopK under the limit +query +SELECT id FROM range(0, 8192, 1, 1) ORDER BY concat(repeat('x', 262144), CAST(id AS STRING)) LIMIT 1 diff --git a/spark/src/test/resources/sql-tests/operators/large_scalar_string_small_batch.sql b/spark/src/test/resources/sql-tests/operators/large_scalar_string_small_batch.sql new file mode 100644 index 00000000000..49f3963901f --- /dev/null +++ b/spark/src/test/resources/sql-tests/operators/large_scalar_string_small_batch.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- The filter case of large_scalar_string.sql with a session batch size that a 256 KiB scalar +-- fits as is. The projection above the filter still bounds its input, whatever the filter +-- emits. +-- See https://github.com/apache/datafusion-comet/issues/7247. +-- Config: spark.sql.adaptive.enabled=false +-- Config: spark.comet.batchSize=256 +-- ConstantFolding: enabled + +statement +CREATE TABLE test_large_scalar_small_batch(k int) USING parquet + +statement +INSERT INTO test_large_scalar_small_batch SELECT /*+ COALESCE(1) */ CAST(id % 2 AS INT) FROM range(8192) + +query +SELECT sum(length(c)) FROM (SELECT concat(repeat('x', 262144), CAST(k AS STRING)) AS c FROM test_large_scalar_small_batch WHERE k >= 0) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index e57d2eee864..432ae81ec2d 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -678,42 +678,53 @@ class CometJoinSuite extends CometTestBase { } else { "dynamic_residual_probe p JOIN dynamic_residual_build b" } - val query = "SELECT /*+ BROADCAST(b) */ p.probe_key, p.payload FROM " + from + - " ON p.probe_key = b._1 WHERE pmod(p.payload, 7) > 0" - var unfilteredBytes = 0L - for (allowed <- Seq(false, true)) { - withSQLConf( - CometConf.COMET_EXEC_JOIN_DYNAMIC_FILTER_ALLOW_DETERMINISTIC_FILTER_PUSHDOWN.key -> allowed.toString) { - val (_, plan) = checkSparkAnswerAndOperator( - sql(query), - Seq(classOf[CometBroadcastHashJoinExec], classOf[CometNativeScanExec])) - checkAnswer(sql(query), Seq(Row(2500, 2500L))) - val joins = collect(plan) { case join: CometBroadcastHashJoinExec => join } - assert(joins.size == 1) - val filters = collect(plan) { - case filter: CometFilterExec if filter.output.exists(_.name == "probe_key") => - filter - } - assert(filters.size == 1, s"Expected one native residual filter:\n$plan") - assert(filters.head.condition.deterministic) - assert(filters.head.nativeOp.getFilter.getAllowRuntimeFilterPushdown == allowed) - val scans = collect(plan) { - case scan: CometNativeScanExec if scan.output.exists(_.name == "probe_key") => - scan - } - assert(scans.size == 1) - val metrics = scans.head.metrics - val bytes = metrics("bytes_scanned").value - assert(joins.head.metrics("output_rows").value == 1L) - assert(filters.head.metrics("output_rows").value > 0L) - if (allowed) { - assert(joins.head.metrics("dynamic_filter_join_filters_attached").value > 0L) - assert(metrics("row_groups_pruned_statistics").value > 0L) - assert(bytes < unfilteredBytes) - } else { - unfilteredBytes = bytes - assert(joins.head.metrics("dynamic_filter_join_filters_attached").value == 0L) - assert(joins.head.metrics("dynamic_filter_join_filters_skipped").value > 0L) + // A string scalar that the residual expands puts a batch split under the native + // filter, which must keep the filter's permission and let the reader filter through. + for (residual <- Seq( + "pmod(p.payload, 7) > 0", + "concat(CAST(p.payload AS STRING), '!') <> '2506!'")) { + val query = "SELECT /*+ BROADCAST(b) */ p.probe_key, p.payload FROM " + from + + s" ON p.probe_key = b._1 WHERE $residual" + var unfilteredBytes = 0L + for (allowed <- Seq(false, true)) { + withSQLConf( + CometConf.COMET_EXEC_JOIN_DYNAMIC_FILTER_ALLOW_DETERMINISTIC_FILTER_PUSHDOWN.key -> allowed.toString) { + val (_, plan) = checkSparkAnswerAndOperator( + sql(query), + Seq(classOf[CometBroadcastHashJoinExec], classOf[CometNativeScanExec])) + checkAnswer(sql(query), Seq(Row(2500, 2500L))) + val joins = collect(plan) { case join: CometBroadcastHashJoinExec => join } + assert(joins.size == 1) + val filters = collect(plan) { + case filter: CometFilterExec + if filter.output.exists(_.name == "probe_key") => + filter + } + assert(filters.size == 1, s"Expected one native residual filter:\n$plan") + assert(filters.head.condition.deterministic) + assert( + filters.head.nativeOp.getFilter.getAllowRuntimeFilterPushdown == allowed) + val scans = collect(plan) { + case scan: CometNativeScanExec + if scan.output.exists(_.name == "probe_key") => + scan + } + assert(scans.size == 1) + val metrics = scans.head.metrics + val bytes = metrics("bytes_scanned").value + assert(joins.head.metrics("output_rows").value == 1L) + assert(filters.head.metrics("output_rows").value > 0L) + if (allowed) { + assert( + joins.head.metrics("dynamic_filter_join_filters_attached").value > 0L) + assert(metrics("row_groups_pruned_statistics").value > 0L) + assert(bytes < unfilteredBytes) + } else { + unfilteredBytes = bytes + assert( + joins.head.metrics("dynamic_filter_join_filters_attached").value == 0L) + assert(joins.head.metrics("dynamic_filter_join_filters_skipped").value > 0L) + } } } } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index f3ced4e3012..2a995a1f195 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -50,7 +50,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, LocalTableScanExec, S import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.functions.{broadcast, col, count, countDistinct, spark_partition_id, sum} +import org.apache.spark.sql.functions.{broadcast, col, count, countDistinct, expr, spark_partition_id, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, MapType, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} @@ -534,6 +534,23 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper } } + test("range shuffle bounds a large string scalar in its ordering expression") { + // A 256 KiB scalar expanded to an 8192-row batch needs 2 GiB of value bytes, past Arrow's + // 32-bit offsets. The expression is evaluated by the exchange itself, with no projection + // between it and the range source. See https://github.com/apache/datafusion-comet/issues/7247. + withSQLConf( + CometConf.COMET_SHUFFLE_MODE.key -> "native", + CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true", + CometConf.COMET_EXEC_RANGE_ENABLED.key -> "true", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val df = spark + .range(0, 8192, 1, 1) + .repartitionByRange(2, expr("concat(repeat('x', 262144), cast(id AS string))").asc) + .toDF() + checkSparkAnswerAndOperator(df, Seq(classOf[CometShuffleExchangeExec])) + } + } + test("range shuffle reports current child failure reasons instead of stale sort-order tags") { withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true") { val attr = AttributeReference("key", LongType)() diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index 1caaec887fa..a9f3d4b5cc7 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -29,18 +29,18 @@ import scala.reflect.ClassTag import org.apache.arrow.c.ArrowArrayStream import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.BigIntVector +import org.apache.arrow.vector.{BigIntVector, VarCharVector} import org.apache.arrow.vector.ipc.ArrowReader import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} import org.apache.spark.executor.TaskMetrics import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager} import org.apache.spark.rdd.RDD import org.apache.spark.sql.CometTestBase -import org.apache.spark.sql.catalyst.expressions.PrettyAttribute +import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Concat, Literal, PrettyAttribute} import org.apache.spark.sql.comet.{CometExec, CometExecUtils, CometMetricNode, CometNativeScanExec, CometProjectExec, CometSortExec} import org.apache.spark.sql.comet.execution.arrow.CometArrowStream import org.apache.spark.sql.functions.{col, udf} -import org.apache.spark.sql.types.{LongType, StructField, StructType} +import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.JsonProtocol @@ -293,6 +293,58 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { } } + test("a limit finishes on the rows a split holds without pulling the next input batch") { + // `concat(s, '!')` expands a string scalar, so the planner splits the scan's batches to the + // batch size of 2 under the projection. The limit of 3 is met by the first input batch's + // second slice, so native must not pull the input again, which here would throw. + withSQLConf(CometConf.COMET_BATCH_SIZE.key -> "2") { + val allocator = CometArrowAllocator.newChildAllocator("split-limit", 0, Long.MaxValue) + try { + withTaskContext(4650000L) { + val vector = new VarCharVector("s", allocator) + vector.allocateNew(10) + (0 until 10).foreach(i => vector.setSafe(i, s"r$i".getBytes("UTF-8"))) + vector.setValueCount(10) + val first = new ColumnarBatch(Array[ColumnVector](new CometPlainVector(vector)), 10) + val input = Iterator.single(first) ++ new Iterator[ColumnarBatch] { + override def hasNext: Boolean = throw new IllegalStateException("read ahead") + override def next(): ColumnarBatch = throw new IllegalStateException("read ahead") + } + val stream = CometArrowStream.fromColumnarBatchIter( + input, + StructType(Seq(StructField("s", StringType, nullable = false))), + CometArrowStream.NATIVE_TIMEZONE, + "split-limit") + val s = AttributeReference("s", StringType, nullable = false)() + val limit = CometExecUtils.getLimitNativePlan(Seq(s), 3).get + val projection = CometExecUtils + .getProjectionNativePlan( + Seq(Alias(Concat(Seq(s, Literal("!"))), "c")()), + Seq(s), + limit.getChildren(0)) + .get + val iter = CometExec.getCometIterator( + Array(stream.asInstanceOf[Object]), + 1, + limit.toBuilder.setChildren(0, projection).build(), + CometMetricNode(Map.empty), + 1, + 0, + None, + Seq.empty, + None) + var rows = 0 + while (iter.hasNext) { + rows += iter.next().numRows() + } + assert(rows == 3) + } + } finally { + allocator.close() + } + } + } + test("a plan fed only by native scans returns its memory when its consumer stops early") { // See issue #2453. The sort keeps its sorted runs in memory, and the merge that reads them // spawns a Tokio task for each run.