From ab4b5e0d964dd2b76a68ba515764d186bb476298 Mon Sep 17 00:00:00 2001 From: Geoffrey Claude Date: Wed, 5 Aug 2026 13:58:36 +0200 Subject: [PATCH 1/2] Add FixedSizeBinary IN LIST benchmarks --- .../physical-expr/benches/in_list_strategy.rs | 191 ++++++++++++++---- 1 file changed, 152 insertions(+), 39 deletions(-) diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs index 762c2c97a1115..00a7c476f5877 100644 --- a/datafusion/physical-expr/benches/in_list_strategy.rs +++ b/datafusion/physical-expr/benches/in_list_strategy.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -//! Focused benchmarks for `InList` cases. +//! Benchmarks for static `IN LIST` filters. //! -//! This benchmark file adds targeted coverage for representative `IN LIST` -//! workloads with controlled parameters: +//! The cases control match rate and list size across several value types and +//! string layouts: //! //! - **Controlled match rates**: Exercises both hit-heavy and miss-heavy paths //! - **List size scaling**: Measures behavior across small and large `IN` lists @@ -27,7 +27,7 @@ //! - **Shared-prefix strings**: Adds collision-heavy string cases where values //! only differ late in the string //! - **Mixed-length strings**: Covers inputs that combine short and long values -//! - **Null handling**: Includes representative `NULL` and `NOT IN` cases +//! - **Null handling**: Covers `NULL` and `NOT IN` cases //! //! # Case Coverage //! @@ -45,14 +45,19 @@ //! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 | //! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 | //! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 | -//! | Fixed-size binary cases | FixedSizeBinary(16) | fixed-width binary values | 4, 64, 256, 10000 | +//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 | +//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 | +//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 | +//! | Fixed-size binary hash-set cases | FixedSizeBinary(16) | hash-set scaling | 64, 256, 10000 | +//! | Fixed-size binary unaligned case | FixedSizeBinary(16) | per-evaluation alignment copy | 64 | use arrow::array::types::IntervalMonthDayNano; use arrow::array::*; +use arrow::buffer::{Buffer, MutableBuffer}; use arrow::datatypes::{Field, Int32Type, IntervalMonthDayNanoType, Schema}; use arrow::record_batch::RecordBatch; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use datafusion_common::ScalarValue; +use datafusion_common::{HashSet, ScalarValue}; use datafusion_physical_expr::expressions::{col, in_list, lit}; use half::f16; use rand::distr::Alphanumeric; @@ -870,7 +875,7 @@ fn bench_dictionary(c: &mut Criterion) { // NULL HANDLING BENCHMARKS // ============================================================================= // -// Tests representative null-containing inputs across primitive and string cases. +// Null-containing primitive and string cases. fn bench_nulls(c: &mut Criterion) { // ========================================================================= @@ -1014,72 +1019,180 @@ fn bench_nulls(c: &mut Criterion) { } // ============================================================================= -// FIXED SIZE BINARY BENCHMARKS (FixedSizeBinary<16>, e.g. UUIDs) +// FIXED SIZE BINARY BENCHMARKS // ============================================================================= -/// Generates a random 16-byte value (UUID-sized). -fn random_fixed_binary_16(rng: &mut StdRng) -> Vec { - let mut buf = vec![0u8; 16]; +fn random_fixed_binary(rng: &mut StdRng, width: i32) -> Vec { + let mut buf = vec![0u8; width as usize]; rng.fill(&mut buf[..]); buf } -/// Benchmarks FixedSizeBinary(16) IN list evaluation. -/// FixedSizeBinary doesn't use the generic numeric helpers since its array -/// construction differs from primitive types. -fn bench_fixed_size_binary_inner( - c: &mut Criterion, - name: &str, +#[derive(Clone, Copy)] +enum InputLayout { + Aligned, + UnalignedI128, +} + +#[derive(Clone, Copy)] +struct FixedSizeBinaryBenchConfig { + width: i32, + list_size: usize, + input_layout: InputLayout, +} + +impl FixedSizeBinaryBenchConfig { + const fn aligned(width: i32, list_size: usize) -> Self { + Self { + width, + list_size, + input_layout: InputLayout::Aligned, + } + } + + const fn unaligned_i128(list_size: usize) -> Self { + Self { + width: 16, + list_size, + input_layout: InputLayout::UnalignedI128, + } + } +} + +const FIXED_SIZE_BINARY_CASES: [FixedSizeBinaryBenchConfig; 7] = [ + FixedSizeBinaryBenchConfig::aligned(1, 16), + FixedSizeBinaryBenchConfig::aligned(2, 64), + FixedSizeBinaryBenchConfig::aligned(16, 4), + FixedSizeBinaryBenchConfig::aligned(16, 64), + FixedSizeBinaryBenchConfig::aligned(16, 256), + FixedSizeBinaryBenchConfig::aligned(16, 10000), + // 8,192 rows at 16 bytes each copy 128 KiB per evaluation. + FixedSizeBinaryBenchConfig::unaligned_i128(64), +]; + +fn generate_fixed_size_binary_data( + rng: &mut StdRng, + width: i32, list_size: usize, match_rate: f64, -) { - let seed = 0xF1ED_B1A7_u64.wrapping_add(list_size as u64 * 0x6666); - let mut rng = StdRng::seed_from_u64(seed); +) -> (Vec>, Vec>) { + if let Some(domain_size) = match width { + 1 => Some(1_usize << 8), + 2 => Some(1_usize << 16), + _ => None, + } { + // The value generator needs at least one value outside the haystack. + assert!(list_size < domain_size); + } - // Generate IN list values (16-byte each) - let haystack: Vec> = (0..list_size) - .map(|_| random_fixed_binary_16(&mut rng)) - .collect(); + // Keep the number of distinct haystack values equal to the configured list size. + let mut haystack_set = HashSet::with_capacity(list_size); + let mut haystack = Vec::with_capacity(list_size); + while haystack.len() < list_size { + let value = random_fixed_binary(rng, width); + if haystack_set.insert(value.clone()) { + haystack.push(value); + } + } - // Generate array with controlled match rate - let values: Vec> = (0..ARRAY_SIZE) + // Generate values with the configured match rate. + let values = (0..ARRAY_SIZE) .map(|_| { if !haystack.is_empty() && rng.random_bool(match_rate) { - haystack.choose(&mut rng).unwrap().clone() + haystack.choose(rng).unwrap().clone() } else { - random_fixed_binary_16(&mut rng) + loop { + let value = random_fixed_binary(rng, width); + if !haystack_set.contains(&value) { + break value; + } + } } }) .collect(); - let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect(); - let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap(); + (haystack, values) +} + +fn unaligned_fixed_size_binary_16(values: &[Vec]) -> FixedSizeBinaryArray { + const WIDTH: usize = 16; + let payload_len = values.len() * WIDTH; + let mut bytes = MutableBuffer::with_capacity(payload_len + 1); + bytes.push(0_u8); + for value in values { + assert_eq!(value.len(), WIDTH); + bytes.extend_from_slice(value); + } + + // MutableBuffer starts at an Arrow-aligned address. Fixed-size binary + // values only require byte alignment, so slicing off this padding byte + // creates a valid Arrow buffer that models unaligned external input. + let buffer = Buffer::from(bytes).slice(1); + assert!( + !buffer.as_ptr().cast::().is_aligned(), + "benchmark input must be unaligned" + ); + FixedSizeBinaryArray::new(WIDTH as i32, buffer, None) +} + +/// FixedSizeBinary doesn't use the generic numeric helpers since its array +/// construction differs from primitive types. +fn bench_fixed_size_binary_inner( + c: &mut Criterion, + config: FixedSizeBinaryBenchConfig, + match_pct: u32, +) { + assert!(match_pct <= 100); + let match_rate = f64::from(match_pct) / 100.0; + + let seed = 0xF1ED_B1A7_u64 + .wrapping_add(config.list_size as u64 * 0x6666) + .wrapping_add(config.width as u64 * 0x7777); + let mut rng = StdRng::seed_from_u64(seed); + + let (haystack, values) = generate_fixed_size_binary_data( + &mut rng, + config.width, + config.list_size, + match_rate, + ); + + let array = match config.input_layout { + InputLayout::Aligned => { + FixedSizeBinaryArray::try_from_iter(values.iter().map(Vec::as_slice)).unwrap() + } + InputLayout::UnalignedI128 => unaligned_fixed_size_binary_16(&values), + }; let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]); let exprs: Vec<_> = haystack .iter() - .map(|v| lit(ScalarValue::FixedSizeBinary(16, Some(v.clone())))) + .map(|v| lit(ScalarValue::FixedSizeBinary(config.width, Some(v.clone())))) .collect(); let expr = in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap(); let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array) as ArrayRef]) .unwrap(); c.bench_with_input( - BenchmarkId::new("fixed_size_binary", name), + BenchmarkId::new("fixed_size_binary", { + let name = format!( + "fsb{}/list={}/match={match_pct}%", + config.width, config.list_size + ); + match config.input_layout { + InputLayout::Aligned => name, + InputLayout::UnalignedI128 => format!("{name}/input=unaligned"), + } + }), &batch, |b, batch| b.iter(|| expr.evaluate(batch).unwrap()), ); } fn bench_fixed_size_binary(c: &mut Criterion) { - for list_size in [4, 64, 256, 10000] { + for config in FIXED_SIZE_BINARY_CASES { for match_pct in MATCH_RATES { - bench_fixed_size_binary_inner( - c, - &format!("fsb16/list={list_size}/match={match_pct}%"), - list_size, - match_pct as f64 / 100.0, - ); + bench_fixed_size_binary_inner(c, config, match_pct); } } } From 1217f5ec322ae31940a7d53235780fa24734eb3e Mon Sep 17 00:00:00 2001 From: Geoffrey Claude Date: Sun, 9 Aug 2026 09:22:46 +0200 Subject: [PATCH 2/2] Optimize IN LIST for fixed-size binary arrays --- .../physical-expr/src/expressions/in_list.rs | 33 ++ .../in_list/fixed_size_binary_filter.rs | 371 ++++++++++++++++++ .../src/expressions/in_list/strategy.rs | 5 + 3 files changed, 409 insertions(+) create mode 100644 datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 0fb978cd0bafe..154decfd8bb89 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -38,6 +38,7 @@ use datafusion_expr::{ColumnarValue, expr_vec_fmt}; mod array_static_filter; mod branchless_filter; +mod fixed_size_binary_filter; mod primitive_filter; mod result; mod static_filter; @@ -3548,6 +3549,38 @@ mod tests { ); } + // FixedSizeBinary in_array, FixedSizeBinary and Dictionary needles + let fsb_in = Arc::new(FixedSizeBinaryArray::try_from_iter( + [ + [1, 2, 3, 4].as_slice(), + [5, 6, 7, 8].as_slice(), + [9, 10, 11, 12].as_slice(), + ] + .into_iter(), + )?) as ArrayRef; + let fsb_needle = Arc::new(FixedSizeBinaryArray::try_from_iter( + [ + [1, 2, 3, 4].as_slice(), + [13, 14, 15, 16].as_slice(), + [5, 6, 7, 8].as_slice(), + ] + .into_iter(), + )?) as ArrayRef; + assert_eq!( + expected, + eval_in_list_from_array(Arc::clone(&fsb_needle), Arc::clone(&fsb_in))? + ); + // The dictionary does not reference its second value, so that value + // must not become a member of the flattened list. + let dict_fsb_in = Arc::new(DictionaryArray::new( + Int32Array::from(vec![0, 2]), + Arc::clone(&fsb_in), + )); + assert_eq!( + BooleanArray::from(vec![Some(true), Some(false), Some(false)]), + eval_in_list_from_array(wrap_in_dict(fsb_needle), dict_fsb_in)? + ); + // Utf8 (falls through to ArrayStaticFilter) let utf8_in = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef; let utf8_needle = Arc::new(StringArray::from(vec!["a", "d", "b"])) as ArrayRef; diff --git a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs new file mode 100644 index 0000000000000..421c02ea1f2d5 --- /dev/null +++ b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs @@ -0,0 +1,371 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Optimized filters for fixed-size binary `IN` lists. +//! +//! Supported widths use an Arrow primitive representation with the same +//! in-memory size: +//! +//! | Width | Primitive representation | +//! |------:|--------------------------| +//! | 1 | `UInt8` | +//! | 2 | `UInt16` | +//! | 4 | `UInt32` | +//! | 8 | `UInt64` | +//! | 16 | `Decimal128` | +//! +//! The shared primitive selector applies the native primitive branchless cutoffs +//! and chooses the bitmap or hash-set fallback. +//! +//! The list and input bytes are read the same way, so each primitive value is +//! an exact key for comparison, bitmap lookup, or hashing. No arithmetic, +//! ordering, or decimal operations are used. +//! +//! Reinterpreting an aligned Arrow buffer is zero-copy. An unaligned buffer is +//! copied into aligned primitive storage before filter construction or probing. + +use std::marker::PhantomData; +use std::mem::size_of; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray, PrimitiveArray, +}; +use arrow::buffer::{Buffer, ScalarBuffer}; +use arrow::datatypes::{ + ArrowPrimitiveType, DataType, Decimal128Type, UInt8Type, UInt16Type, UInt32Type, + UInt64Type, +}; +use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err}; + +use super::primitive_filter::instantiate_primitive_filter; +use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary}; + +/// Reinterpret fixed-size binary values as same-width primitive values. +/// +/// Arrow buffers are normally sufficiently aligned, making this zero-copy. A +/// valid Arrow array can still be constructed from a sliced, unaligned buffer; +/// in that case, copy each value into aligned primitive storage. +fn reinterpret_as_primitive(array: &FixedSizeBinaryArray) -> Result> +where + T: ArrowPrimitiveType, +{ + let width = size_of::(); + if array.value_size() != width { + return Err(internal_datafusion_err!( + "FixedSizeBinary filter: expected {width}-byte values, got {}", + array.value_size() + )); + } + + let source = array.values(); + let values = if source.as_ptr().cast::().is_aligned() { + ScalarBuffer::new(source.clone(), 0, array.len()) + } else { + // `Buffer::from(&[u8])` copies into Arrow-aligned storage. + ScalarBuffer::new(Buffer::from(source.as_slice()), 0, array.len()) + }; + + Ok(PrimitiveArray::::new(values, array.nulls().cloned())) +} + +/// Adapts a primitive filter to concrete, same-width `FixedSizeBinary` arrays. +struct FixedSizeBinaryFilter { + data_type: DataType, + inner: StaticFilterRef, + _marker: PhantomData, +} + +impl StaticFilter for FixedSizeBinaryFilter +where + T: ArrowPrimitiveType + Send + Sync + 'static, +{ + fn null_count(&self) -> usize { + self.inner.null_count() + } + + fn contains(&self, v: &dyn Array, negated: bool) -> Result { + handle_dictionary!(self, v, negated); + + if v.data_type() != &self.data_type { + return Err(exec_datafusion_err!( + "FixedSizeBinary filter: expected {} array, got {}", + self.data_type, + v.data_type() + )); + } + let array = v.as_fixed_size_binary_opt().ok_or_else(|| { + exec_datafusion_err!( + "FixedSizeBinary filter: expected concrete {} array", + self.data_type + ) + })?; + let primitive = reinterpret_as_primitive::(array)?; + self.inner.contains(&primitive, negated) + } +} + +fn instantiate_for_primitive(array: &FixedSizeBinaryArray) -> Result +where + T: ArrowPrimitiveType + Send + Sync + 'static, +{ + let primitive: ArrayRef = Arc::new(reinterpret_as_primitive::(array)?); + let inner = instantiate_primitive_filter(&primitive)?.ok_or_else(|| { + internal_datafusion_err!( + "FixedSizeBinary filter: no primitive filter for {}", + primitive.data_type() + ) + })?; + Ok(Arc::new(FixedSizeBinaryFilter:: { + data_type: array.data_type().clone(), + inner, + _marker: PhantomData, + })) +} + +/// Creates an optimized filter for supported concrete `FixedSizeBinary` arrays. +pub(super) fn instantiate_fixed_size_binary_filter( + in_array: &ArrayRef, +) -> Result> { + let DataType::FixedSizeBinary(width) = in_array.data_type() else { + return Ok(None); + }; + let Some(array) = in_array.as_fixed_size_binary_opt() else { + return Ok(None); + }; + + let filter = match width { + 1 => instantiate_for_primitive::(array)?, + 2 => instantiate_for_primitive::(array)?, + 4 => instantiate_for_primitive::(array)?, + 8 => instantiate_for_primitive::(array)?, + 16 => instantiate_for_primitive::(array)?, + _ => return Ok(None), + }; + Ok(Some(filter)) +} + +#[cfg(test)] +mod tests { + use arrow::array::{DictionaryArray, Int8Array, StringArray}; + use arrow::buffer::{Buffer, MutableBuffer, NullBuffer}; + use arrow::datatypes::Int8Type; + + use super::*; + + fn value(width: i32, index: usize, miss: bool) -> Vec { + let mut value = (index as u128).to_le_bytes()[..width as usize].to_vec(); + let last = value.last_mut().unwrap(); + if miss { + *last |= 0x80; + } else { + *last &= 0x7f; + } + value + } + + fn array(width: i32, values: &[Option>]) -> FixedSizeBinaryArray { + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + values.iter().map(|value| value.as_deref()), + width, + ) + .unwrap() + } + + fn make_filter(width: i32, values: &[Option>]) -> Result { + let in_array: ArrayRef = Arc::new(array(width, values)); + Ok(instantiate_fixed_size_binary_filter(&in_array)?.unwrap()) + } + + #[test] + fn filters_supported_widths_across_strategy_thresholds() -> Result<()> { + for (width, list_len) in [ + (1, 16), + (1, 17), + (2, 8), + (2, 9), + (4, 32), + (4, 33), + (8, 16), + (8, 17), + (16, 4), + (16, 5), + ] { + let mut hit = vec![0x80; width as usize]; + hit[width as usize - 1] = 0xff; + let mut miss = hit.clone(); + miss[width as usize - 1] ^= 1; + + let mut haystack = (0..list_len - 1) + .map(|index| Some(value(width, index, false))) + .collect::>(); + haystack.push(Some(hit.clone())); + let filter = make_filter(width, &haystack)?; + let needles = array(width, &[Some(hit), Some(miss), None]); + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(true), Some(false), None]), + "width={width}, list_len={list_len}" + ); + } + Ok(()) + } + + #[test] + fn handles_slices_nulls_and_not_in() -> Result<()> { + let width = 16; + let parent = array( + width, + &[ + Some(value(width, 0, false)), + Some(value(width, 1, false)), + None, + Some(value(width, 2, false)), + Some(value(width, 3, false)), + Some(value(width, 4, false)), + Some(value(width, 5, false)), + Some(value(width, 6, false)), + ], + ); + // Five non-null values select the hash-set path. + let in_array: ArrayRef = Arc::new(parent.slice(1, 6)); + let filter = instantiate_fixed_size_binary_filter(&in_array)?.unwrap(); + let needles = array( + width, + &[ + Some(value(width, 2, false)), + Some(value(width, 0, false)), + Some(value(width, 6, false)), + Some(value(width, 7, false)), + None, + ], + ); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(true), None, None, None, None]) + ); + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), None, None, None, None]) + ); + Ok(()) + } + + #[test] + fn handles_dictionary_needles() -> Result<()> { + let filter = make_filter(4, &[Some(value(4, 7, false))])?; + let dictionary_values: ArrayRef = Arc::new(array( + 4, + &[Some(value(4, 7, false)), Some(value(4, 8, false))], + )); + let keys = Int8Array::from(vec![Some(0), Some(1), None]); + let needles = + DictionaryArray::::try_new(keys, dictionary_values).unwrap(); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(true), Some(false), None]) + ); + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), Some(true), None]) + ); + Ok(()) + } + + #[test] + fn rejects_unsupported_arrays() -> Result<()> { + let filter = make_filter(4, &[Some(value(4, 1, false))])?; + let wrong_width = array(8, &[Some(value(8, 1, false))]); + let error = filter + .contains(&wrong_width, false) + .unwrap_err() + .to_string(); + assert!( + error.contains("expected FixedSizeBinary(4) array, got FixedSizeBinary(8)"), + "{error}" + ); + + let wrong_type = StringArray::from(vec!["one"]); + let error = filter.contains(&wrong_type, false).unwrap_err().to_string(); + assert!( + error.contains("expected FixedSizeBinary(4) array, got Utf8"), + "{error}" + ); + + for width in [0, 3, 5, 15, 17] { + let unsupported: ArrayRef = + Arc::new(FixedSizeBinaryArray::new_null(width, 1)); + assert!( + instantiate_fixed_size_binary_filter(&unsupported)?.is_none(), + "width={width}" + ); + } + + Ok(()) + } + + fn unaligned_i128_array( + values: &[Vec], + nulls: Option, + ) -> FixedSizeBinaryArray { + let width = size_of::(); + let mut bytes = MutableBuffer::with_capacity(1 + width * values.len()); + bytes.push(0_u8); + for value in values { + assert_eq!(value.len(), width); + bytes.extend_from_slice(value); + } + let buffer = Buffer::from(bytes).slice(1); + assert!( + !buffer.as_ptr().cast::().is_aligned(), + "test buffer must be unaligned" + ); + FixedSizeBinaryArray::new(width as i32, buffer, nulls) + } + + #[test] + fn handles_aligned_and_unaligned_buffers() -> Result<()> { + let buffer = Buffer::from_vec(vec![1_u64, 2, 3]); + let source_ptr = buffer.as_ptr(); + let array = FixedSizeBinaryArray::new(8, buffer, None); + let primitive = reinterpret_as_primitive::(&array)?; + assert_eq!(primitive.values().inner().as_ptr(), source_ptr); + + let width = 16; + let haystack_values = (0..5) + .map(|index| value(width, index, false)) + .collect::>(); + let haystack: ArrayRef = Arc::new(unaligned_i128_array(&haystack_values, None)); + let needles = unaligned_i128_array( + &[ + value(width, 3, false), + value(width, 8, true), + value(width, 9, true), + ], + Some(NullBuffer::from(vec![true, false, true])), + ); + let filter = instantiate_fixed_size_binary_filter(&haystack)?.unwrap(); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(true), None, Some(false)]) + ); + Ok(()) + } +} diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs index d008217ee19fa..98bd698507844 100644 --- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs +++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs @@ -23,12 +23,17 @@ use arrow::datatypes::DataType; use datafusion_common::Result; use super::array_static_filter::ArrayStaticFilter; +use super::fixed_size_binary_filter::instantiate_fixed_size_binary_filter; use super::primitive_filter::instantiate_primitive_filter; use super::static_filter::StaticFilterRef; pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result { let in_array = flatten_dictionary_haystack(in_array)?; + if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? { + return Ok(filter); + } + if let Some(filter) = instantiate_primitive_filter(&in_array)? { return Ok(filter); }