diff --git a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs index 2b50b40013d..12ed0d978e2 100644 --- a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs +++ b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs @@ -190,9 +190,13 @@ impl Accumulator for SparkBloomFilter { "Expect one element in 'states' but found {}", states.len() ); - assert_eq!(states[0].len(), 1); - let state_sv = downcast_value!(states[0], BinaryArray); - self.merge_filter(state_sv.value_data()) + let states = downcast_value!(states[0], BinaryArray); + // A final aggregate can pass several partial states in one batch. Merge each + // row's byte slice because the backing buffer can also contain other groups' states. + for state in states.iter().flatten() { + self.merge_filter(state)?; + } + Ok(()) } } @@ -200,6 +204,69 @@ impl Accumulator for SparkBloomFilter { mod tests { use super::*; + fn filter_with_values(version: SparkBloomFilterVersion, values: &[i64]) -> SparkBloomFilter { + let mut filter = SparkBloomFilter::new(version, 3, 1024, 0); + for &value in values { + filter.put_long(value); + } + filter + } + + #[test] + fn merge_batch_merges_each_row() { + for version in [SparkBloomFilterVersion::V1, SparkBloomFilterVersion::V2] { + let first = filter_with_values(version, &[11]).state_as_bytes(); + let second = filter_with_values(version, &[22]).state_as_bytes(); + let states = + BinaryArray::from(vec![Some(first.as_slice()), None, Some(second.as_slice())]); + let mut acc = filter_with_values(version, &[33]); + acc.merge_batch(&[Arc::new(states)]).unwrap(); + assert_eq!( + acc.state_as_bytes(), + filter_with_values(version, &[11, 22, 33]).state_as_bytes() + ); + } + } + + #[test] + fn merge_batch_respects_sliced_row_offsets() { + for version in [SparkBloomFilterVersion::V1, SparkBloomFilterVersion::V2] { + let excluded = filter_with_values(version, &[99]).state_as_bytes(); + let included = filter_with_values(version, &[11]).state_as_bytes(); + let states = BinaryArray::from(vec![ + excluded.as_slice(), + included.as_slice(), + excluded.as_slice(), + ]); + let mut acc = filter_with_values(version, &[]); + acc.merge_batch(&[Arc::new(states.slice(1, 1))]).unwrap(); + assert_eq!(acc.state_as_bytes(), included); + } + } + + #[test] + fn merge_batch_ignores_empty_and_null_states() { + let mut acc = filter_with_values(SparkBloomFilterVersion::V1, &[11]); + let expected = acc.state_as_bytes(); + for states in [ + BinaryArray::from(Vec::>::new()), + BinaryArray::from(vec![None::<&[u8]>]), + ] { + acc.merge_batch(&[Arc::new(states)]).unwrap(); + assert_eq!(acc.state_as_bytes(), expected); + } + } + + #[test] + fn merge_batch_propagates_incompatible_state_error() { + let first = filter_with_values(SparkBloomFilterVersion::V1, &[11]).state_as_bytes(); + let second = filter_with_values(SparkBloomFilterVersion::V2, &[22]).state_as_bytes(); + let states = BinaryArray::from(vec![first.as_slice(), second.as_slice()]); + let mut acc = filter_with_values(SparkBloomFilterVersion::V1, &[]); + let error = acc.merge_batch(&[Arc::new(states)]).unwrap_err(); + assert!(error.to_string().contains("version mismatch")); + } + /// Spark's BloomFilterAggregate.eval returns NULL when the filter saw no /// non-null input (cardinality 0); an untouched accumulator must match. #[test] diff --git a/native/spark-expr/tests/bloom_filter_merge.rs b/native/spark-expr/tests/bloom_filter_merge.rs new file mode 100644 index 00000000000..993b6664003 --- /dev/null +++ b/native/spark-expr/tests/bloom_filter_merge.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use arrow::array::{AsArray, BinaryArray, Int64Array, RecordBatch}; +use arrow::datatypes::{DataType, Field, Int64Type, Schema}; +use datafusion::common::{Result, ScalarValue}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::AggregateUDF; +use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr}; +use datafusion::physical_expr::expressions::{Column, Literal}; +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; +use datafusion::physical_plan::{common::collect, ExecutionPlan}; +use datafusion_comet_spark_expr::{BloomFilterAgg, SparkBloomFilterVersion}; + +fn bloom_state(aggregate: &AggregateFunctionExpr, values: &[i64]) -> Result> { + let mut accumulator = aggregate.create_accumulator()?; + accumulator.update_batch(&[Arc::new(Int64Array::from_iter_values( + values.iter().copied(), + ))])?; + let ScalarValue::Binary(Some(state)) = accumulator.state()?.remove(0) else { + panic!("a nonempty Bloom filter must have a binary state"); + }; + Ok(state) +} + +#[tokio::test] +async fn grouped_final_bloom_filter_merges_batched_states() -> Result<()> { + let value_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ])); + let aggregate_udf = Arc::new(AggregateUDF::new_from_impl(BloomFilterAgg::new( + Arc::new(Literal::new(ScalarValue::Int64(Some(100)))), + Arc::new(Literal::new(ScalarValue::Int64(Some(1024)))), + DataType::Binary, + SparkBloomFilterVersion::V1, + ))); + let aggregate_expr = Arc::new( + AggregateExprBuilder::new(aggregate_udf, vec![Arc::new(Column::new("value", 1))]) + .schema(Arc::clone(&value_schema)) + .alias("bloom") + .build()?, + ); + let state_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int64, false), + Field::new("bits", DataType::Binary, false), + ])); + + // Interleave the keys so one group merges two states and the other uses a + // separate slice of the same binary array. + let states = BinaryArray::from_iter_values([ + bloom_state(&aggregate_expr, &[11])?, + bloom_state(&aggregate_expr, &[99])?, + bloom_state(&aggregate_expr, &[22])?, + ]); + let batch = RecordBatch::try_new( + Arc::clone(&state_schema), + vec![Arc::new(Int64Array::from(vec![1, 2, 1])), Arc::new(states)], + )?; + let scan = MemorySourceConfig::try_new_exec(&[vec![batch]], state_schema, None)?; + let aggregate = AggregateExec::try_new( + AggregateMode::Final, + PhysicalGroupBy::new_single(vec![(Arc::new(Column::new("key", 0)), "key".into())]), + vec![Arc::clone(&aggregate_expr)], + vec![None], + scan, + value_schema, + )?; + let output = collect(aggregate.execute(0, Arc::new(TaskContext::default()))?).await?; + + let mut actual = Vec::new(); + for batch in output { + let keys = batch.column(0).as_primitive::(); + let filters = batch.column(1).as_binary::(); + for (key, filter) in keys.iter().zip(filters.iter()) { + actual.push((key.unwrap(), filter.unwrap().to_vec())); + } + } + actual.sort_by_key(|(key, _)| *key); + assert_eq!( + actual, + vec![ + (1, bloom_state(&aggregate_expr, &[11, 22])?), + (2, bloom_state(&aggregate_expr, &[99])?), + ] + ); + Ok(()) +}