From ad2950cc6d9543d5e8f25790ce14469e231968c7 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Fri, 30 Oct 2026 23:30:53 -0400 Subject: [PATCH 1/6] Bound dictionary merging by referenced keys --- arrow-select/src/concat.rs | 31 ++++- arrow-select/src/dictionary.rs | 233 ++++++++++++++++++++++++--------- arrow-select/src/interleave.rs | 29 +++- 3 files changed, 231 insertions(+), 62 deletions(-) diff --git a/arrow-select/src/concat.rs b/arrow-select/src/concat.rs index 7a9664457845..1806043b71e5 100644 --- a/arrow-select/src/concat.rs +++ b/arrow-select/src/concat.rs @@ -118,7 +118,7 @@ fn concat_dictionaries( has_nulls |= d.null_count() != 0; for key in d.keys().values() { // Use get to safely handle nulls - key_values.push(mapping.get(key.as_usize()).copied().unwrap_or_default()) + key_values.push(mapping.get(key.as_usize()).unwrap_or_default()) } } @@ -1567,6 +1567,35 @@ mod tests { assert!((30..40).contains(&values_len), "{values_len}") } + #[test] + fn test_concat_oversized_dictionary_slices() { + let make_dictionary = |prefix| { + let values = StringArray::from_iter_values( + (0..1_000).map(|idx| format!("{prefix}_value_{idx}")), + ); + let keys = Int32Array::from_iter_values(0..1_000); + DictionaryArray::::new(keys, Arc::new(values)) + }; + let a = make_dictionary("a").slice(997, 3); + let b = make_dictionary("b").slice(1, 3); + + let result = concat(&[&a, &b]).unwrap(); + let actual = collect_string_dictionary(result.as_dictionary::()); + + assert_eq!( + actual, + [ + "a_value_997", + "a_value_998", + "a_value_999", + "b_value_1", + "b_value_2", + "b_value_3", + ] + .map(Some) + ); + } + #[test] fn test_primitive_dictionary_merge() { // Same value repeated 5 times. diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs index 47e3a13bf6a8..3c1c3be54d4f 100644 --- a/arrow-select/src/dictionary.rs +++ b/arrow-select/src/dictionary.rs @@ -4,7 +4,7 @@ //! Dictionary utilities for Arrow arrays -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use crate::filter::filter; use crate::interleave::interleave; @@ -127,9 +127,46 @@ impl<'a, V> Interner<'a, V> { } } +// Avoid dense mappings when the dictionary is much larger than the selected rows. +const MAX_DENSE_MAPPING_VALUES_PER_ROW: usize = 16; + +pub(crate) enum KeyMapping { + Dense(Vec), + Sparse(HashMap), +} + +impl KeyMapping { + fn new_dense(len: usize) -> Self { + Self::Dense(vec![K::Native::ZERO; len]) + } + + fn new_sparse(capacity: usize) -> Self { + Self::Sparse(HashMap::with_capacity_and_hasher( + capacity, + RandomState::with_seeds(0, 0, 0, 0), + )) + } + + fn insert(&mut self, old_key: usize, new_key: K::Native) { + match self { + Self::Dense(mapping) => mapping[old_key] = new_key, + Self::Sparse(mapping) => { + mapping.insert(old_key, new_key); + } + } + } + + pub fn get(&self, old_key: usize) -> Option { + match self { + Self::Dense(mapping) => mapping.get(old_key).copied(), + Self::Sparse(mapping) => mapping.get(&old_key).copied(), + } + } +} + pub(crate) struct MergedDictionaries { - /// Provides `key_mappings[`array_idx`][`old_key`] -> new_key` - pub key_mappings: Vec>, + /// Maps each input dictionary key to its merged key. + pub key_mappings: Vec>, /// The new values pub values: ArrayRef, } @@ -260,11 +297,24 @@ pub(crate) fn merge_dictionary_values( }; let keys = dictionary.keys().values(); let values = dictionary.values().as_ref(); - let values_mask = compute_values_mask(keys, key_mask, values.len()); - - let masked_values = get_masked_values(values, &values_mask); - num_values += masked_values.len(); - value_slices.push(masked_values); + let selected_rows = key_mask + .map(BooleanBuffer::count_set_bits) + .unwrap_or(keys.len()); + let use_sparse_mapping = + values.len() > selected_rows.saturating_mul(MAX_DENSE_MAPPING_VALUES_PER_ROW); + + let referenced_keys; + let values_mask; + let selection = if use_sparse_mapping { + referenced_keys = compute_referenced_keys(keys, key_mask); + ValueSelection::Indices(&referenced_keys) + } else { + values_mask = compute_values_mask(keys, key_mask, values.len()); + ValueSelection::Mask(&values_mask) + }; + let selected_values = get_selected_values(values, selection); + num_values += selected_values.len(); + value_slices.push((selected_values, use_sparse_mapping)); values_arrays.push(values) } @@ -278,22 +328,27 @@ pub(crate) fn merge_dictionary_values( .iter() .enumerate() .zip(value_slices) - .map(|((dictionary_idx, dictionary), values)| { - let zero = K::Native::from_usize(0).unwrap(); - let mut mapping = vec![zero; dictionary.values().len()]; - - for (value_idx, value) in values { - mapping[value_idx] = - *interner.intern(value, || match K::Native::from_usize(indices.len()) { - Some(idx) => { - indices.push((dictionary_idx, value_idx)); - Ok(idx) - } - None => Err(ArrowError::DictionaryKeyOverflowError), - })?; - } - Ok(mapping) - }) + .map( + |((dictionary_idx, dictionary), (values, use_sparse_mapping))| { + let mut mapping = match use_sparse_mapping { + true => KeyMapping::new_sparse(values.len()), + false => KeyMapping::new_dense(dictionary.values().len()), + }; + + for (value_idx, value) in values { + let new_key = + *interner.intern(value, || match K::Native::from_usize(indices.len()) { + Some(idx) => { + indices.push((dictionary_idx, value_idx)); + Ok(idx) + } + None => Err(ArrowError::DictionaryKeyOverflowError), + })?; + mapping.insert(value_idx, new_key); + } + Ok(mapping) + }, + ) .collect::, ArrowError>>()?; Ok(MergedDictionaries { @@ -334,7 +389,7 @@ pub(crate) fn merge_dictionary_values_with_masks( mapping[value_idx] = K::Native::from_usize(indices.len()).unwrap(); indices.push((dictionary_idx, value_idx)); } - mapping + KeyMapping::Dense(mapping) }) .collect(); return Ok(MergedDictionaries { @@ -369,7 +424,7 @@ pub(crate) fn merge_dictionary_values_with_masks( None => Err(ArrowError::DictionaryKeyOverflowError), })?; } - Ok(mapping) + Ok(KeyMapping::Dense(mapping)) }) .collect::, ArrowError>>()?; @@ -379,6 +434,22 @@ pub(crate) fn merge_dictionary_values_with_masks( }) } +fn compute_referenced_keys( + keys: &ScalarBuffer, + mask: Option<&BooleanBuffer>, +) -> Vec { + let mut indices = match mask { + Some(mask) => mask + .set_indices() + .map(|idx| keys[idx].as_usize()) + .collect::>(), + None => keys.iter().map(|key| key.as_usize()).collect(), + }; + indices.sort_unstable(); + indices.dedup(); + indices +} + /// Return a mask identifying the values that are referenced by keys in `dictionary` /// at the positions indicated by `selection` fn compute_values_mask( @@ -400,60 +471,77 @@ fn compute_values_mask( builder.finish() } -/// Process primitive array values to bytes -fn masked_primitives_to_bytes<'a, T: ArrowPrimitiveType>( +#[derive(Clone, Copy)] +enum ValueSelection<'a> { + Mask(&'a BooleanBuffer), + Indices(&'a [usize]), +} + +impl ValueSelection<'_> { + fn len(&self) -> usize { + match self { + Self::Mask(mask) => mask.count_set_bits(), + Self::Indices(indices) => indices.len(), + } + } + + fn for_each(&self, f: impl FnMut(usize)) { + match self { + Self::Mask(mask) => mask.set_indices().for_each(f), + Self::Indices(indices) => indices.iter().copied().for_each(f), + } + } +} + +fn selected_primitives_to_bytes<'a, T: ArrowPrimitiveType>( array: &'a PrimitiveArray, - mask: &BooleanBuffer, + selection: ValueSelection<'_>, ) -> Vec<(usize, Option<&'a [u8]>)> where T::Native: ToByteSlice, { - let mut out = Vec::with_capacity(mask.count_set_bits()); + let mut out = Vec::with_capacity(selection.len()); let values = array.values(); - for idx in mask.set_indices() { + selection.for_each(|idx| { out.push(( idx, array.is_valid(idx).then_some(values[idx].to_byte_slice()), )) - } + }); out } -macro_rules! masked_primitive_to_bytes_helper { - ($t:ty, $array:expr, $mask:expr) => { - masked_primitives_to_bytes::<$t>($array.as_primitive(), $mask) +macro_rules! selected_primitive_to_bytes_helper { + ($t:ty, $array:expr, $selection:expr) => { + selected_primitives_to_bytes::<$t>($array.as_primitive(), $selection) }; } -/// Return a Vec containing for each set index in `mask`, the index and byte value of that index -fn get_masked_values<'a>( +fn get_selected_values<'a>( array: &'a dyn Array, - mask: &BooleanBuffer, + selection: ValueSelection<'_>, ) -> Vec<(usize, Option<&'a [u8]>)> { downcast_primitive! { - array.data_type() => (masked_primitive_to_bytes_helper, array, mask), - DataType::Utf8 => masked_bytes(array.as_string::(), mask), - DataType::LargeUtf8 => masked_bytes(array.as_string::(), mask), - DataType::Binary => masked_bytes(array.as_binary::(), mask), - DataType::LargeBinary => masked_bytes(array.as_binary::(), mask), + array.data_type() => (selected_primitive_to_bytes_helper, array, selection), + DataType::Utf8 => selected_bytes(array.as_string::(), selection), + DataType::LargeUtf8 => selected_bytes(array.as_string::(), selection), + DataType::Binary => selected_bytes(array.as_binary::(), selection), + DataType::LargeBinary => selected_bytes(array.as_binary::(), selection), _ => unimplemented!("Dictionary merging for type {} is not implemented", array.data_type()), } } -/// Compute [`get_masked_values`] for a [`GenericByteArray`] -/// -/// Note: this does not check the null mask and will return values contained in null slots -fn masked_bytes<'a, T: ByteArrayType>( +fn selected_bytes<'a, T: ByteArrayType>( array: &'a GenericByteArray, - mask: &BooleanBuffer, + selection: ValueSelection<'_>, ) -> Vec<(usize, Option<&'a [u8]>)> { - let mut out = Vec::with_capacity(mask.count_set_bits()); - for idx in mask.set_indices() { + let mut out = Vec::with_capacity(selection.len()); + selection.for_each(|idx| { out.push(( idx, array.is_valid(idx).then_some(array.value(idx).as_ref()), )) - } + }); out } @@ -472,6 +560,13 @@ mod tests { use arrow_schema::Field; use std::sync::Arc; + fn assert_mapping(mapping: &KeyMapping, expected: &[i32]) { + let actual = (0..expected.len()) + .map(|idx| mapping.get(idx).unwrap_or_default()) + .collect::>(); + assert_eq!(actual, expected); + } + #[test] fn test_child_values_ptr_eq_accepts_reusable_value_mapping() { let dictionary = DictionaryArray::::new( @@ -659,8 +754,8 @@ mod tests { assert_eq!(&actual, &["a", "b", "d", "c", "e", "f"]); assert_eq!(merged.key_mappings.len(), 2); - assert_eq!(&merged.key_mappings[0], &[0, 1, 2, 3, 4]); - assert_eq!(&merged.key_mappings[1], &[3, 5, 2, 0]); + assert_mapping(&merged.key_mappings[0], &[0, 1, 2, 3, 4]); + assert_mapping(&merged.key_mappings[1], &[3, 5, 2, 0]); let a_slice = a.slice(1, 4); let merged = merge_dictionary_values(&[&a_slice, &b], None).unwrap(); @@ -670,8 +765,8 @@ mod tests { assert_eq!(&actual, &["a", "b", "d", "c", "f"]); assert_eq!(merged.key_mappings.len(), 2); - assert_eq!(&merged.key_mappings[0], &[0, 1, 2, 0, 0]); - assert_eq!(&merged.key_mappings[1], &[3, 4, 2, 0]); + assert_mapping(&merged.key_mappings[0], &[0, 1, 2, 0, 0]); + assert_mapping(&merged.key_mappings[1], &[3, 4, 2, 0]); // Mask out only ["b", "b", "d"] from a let a_mask = BooleanBuffer::from_iter([false, true, false, true, true, false, false]); @@ -683,8 +778,8 @@ mod tests { assert_eq!(&actual, &["b", "d", "c", "f", "a"]); assert_eq!(merged.key_mappings.len(), 2); - assert_eq!(&merged.key_mappings[0], &[0, 0, 1, 0, 0]); - assert_eq!(&merged.key_mappings[1], &[2, 3, 1, 4]); + assert_mapping(&merged.key_mappings[0], &[0, 0, 1, 0, 0]); + assert_mapping(&merged.key_mappings[1], &[2, 3, 1, 4]); } #[test] @@ -701,7 +796,8 @@ mod tests { let expected = StringArray::from_iter_values(["a", "shared", "shared", "b"]); assert_eq!(merged.values.as_ref(), &expected); - assert_eq!(merged.key_mappings, vec![vec![0, 0, 1], vec![2, 3, 0]]); + assert_mapping(&merged.key_mappings[0], &[0, 0, 1]); + assert_mapping(&merged.key_mappings[1], &[2, 3, 0]); } #[test] @@ -723,8 +819,8 @@ mod tests { let expected = StringArray::from(vec![None, Some("bingo"), Some("hello")]); assert_eq!(merged.values.as_ref(), &expected); assert_eq!(merged.key_mappings.len(), 2); - assert_eq!(&merged.key_mappings[0], &[0, 0, 1, 2, 0]); - assert_eq!(&merged.key_mappings[1], &[] as &[i32; 0]); + assert_mapping(&merged.key_mappings[0], &[0, 0, 1, 2, 0]); + assert_mapping(&merged.key_mappings[1], &[]); } #[test] @@ -737,4 +833,21 @@ mod tests { let expected = StringArray::from(vec!["b"]); assert_eq!(merged.values.as_ref(), &expected); } + + #[test] + fn test_merge_oversized_dictionary_slice() { + let values = StringArray::from_iter_values((0..1_000).map(|idx| idx.to_string())); + let keys = Int32Array::from(vec![Some(999), None, Some(1), Some(999)]); + let dictionary = DictionaryArray::new(keys, Arc::new(values)); + + let merged = merge_dictionary_values(&[&dictionary], None).unwrap(); + let values = as_string_array(merged.values.as_ref()); + let actual = values.iter().collect::>(); + + assert_eq!(actual, vec![Some("1"), Some("999")]); + assert!(matches!(merged.key_mappings[0], KeyMapping::Sparse(_))); + assert_eq!(merged.key_mappings[0].get(1), Some(0)); + assert_eq!(merged.key_mappings[0].get(999), Some(1)); + assert_eq!(merged.key_mappings[0].get(500), None); + } } diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index a9b88867cf64..bd8f9b471f77 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -297,7 +297,11 @@ fn interleave_dictionaries( match old_keys.is_valid(*b) { true => { let old_key = old_keys.values()[*b]; - keys.append_value(merged.key_mappings[*a][old_key.as_usize()]) + keys.append_value( + merged.key_mappings[*a] + .get(old_key.as_usize()) + .expect("referenced dictionary key should be mapped"), + ) } false => keys.append_null(), } @@ -1130,6 +1134,29 @@ mod tests { assert_eq!(&collected, &["c", "c", "c"]); } + #[test] + fn test_interleave_oversized_dictionaries() { + let make_dictionary = |prefix| { + let values = StringArray::from_iter_values( + (0..1_000).map(|idx| format!("{prefix}_value_{idx}")), + ); + let keys = Int32Array::from_iter_values([1, 999]); + DictionaryArray::::new(keys, Arc::new(values)) + }; + let a = make_dictionary("a"); + let b = make_dictionary("b"); + + let result = interleave(&[&a, &b], &[(0, 1), (1, 0), (0, 0)]).unwrap(); + let actual = result + .as_dictionary::() + .downcast_dict::() + .unwrap() + .into_iter() + .collect::>(); + + assert_eq!(actual, ["a_value_999", "b_value_1", "a_value_1"].map(Some)); + } + #[test] fn test_interleave_dictionary_nulls() { let input_1_keys = Int32Array::from_iter_values([0, 2, 1, 3]); From b70fc1406c5117ac537d57068ff20fd3b19fc00d Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 22:19:45 -0500 Subject: [PATCH 2/6] Document sparse dictionary key ordering Sorting selected keys both enables deduplication and preserves the dense path's dictionary-value order. Add a focused test that keeps this ordering invariant explicit. --- arrow-select/src/dictionary.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs index 3c1c3be54d4f..6f66202795b8 100644 --- a/arrow-select/src/dictionary.rs +++ b/arrow-select/src/dictionary.rs @@ -434,6 +434,11 @@ pub(crate) fn merge_dictionary_values_with_masks( }) } +/// Returns the unique dictionary keys referenced by the selected rows. +/// +/// Sorting makes repeated keys adjacent for deduplication and preserves the +/// ascending dictionary-key order of the dense mask path. This keeps the sparse +/// and dense paths' merged values ordered consistently. fn compute_referenced_keys( keys: &ScalarBuffer, mask: Option<&BooleanBuffer>, @@ -834,6 +839,15 @@ mod tests { assert_eq!(merged.values.as_ref(), &expected); } + #[test] + fn test_compute_referenced_keys_preserves_dense_order() { + let keys = Int32Array::from_iter_values([999, 1, 500, 1, 999]); + + let actual = compute_referenced_keys(keys.values(), None); + + assert_eq!(actual, vec![1, 500, 999]); + } + #[test] fn test_merge_oversized_dictionary_slice() { let values = StringArray::from_iter_values((0..1_000).map(|idx| idx.to_string())); From fe0fa01db0a8a7555f1be87487948241c7265455 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 22:55:09 -0500 Subject: [PATCH 3/6] Tune sparse dictionary merge crossover The sparse path only wins once retained values substantially outnumber selected rows. Raise the strict cutoff to 128x and keep dense iteration monomorphic so ordinary dictionaries avoid mapping dispatch overhead. --- arrow-select/src/concat.rs | 18 +++++-- arrow-select/src/dictionary.rs | 95 ++++++++++++++-------------------- 2 files changed, 52 insertions(+), 61 deletions(-) diff --git a/arrow-select/src/concat.rs b/arrow-select/src/concat.rs index 1806043b71e5..c5d8c85f2184 100644 --- a/arrow-select/src/concat.rs +++ b/arrow-select/src/concat.rs @@ -17,7 +17,7 @@ //! assert_eq!(arr.len(), 3); //! ``` -use crate::dictionary::{merge_dictionary_values, should_merge_dictionary_values}; +use crate::dictionary::{KeyMapping, merge_dictionary_values, should_merge_dictionary_values}; use arrow_array::builder::{ BooleanBuilder, GenericByteBuilder, GenericByteViewBuilder, PrimitiveBuilder, }; @@ -116,9 +116,19 @@ fn concat_dictionaries( let mut has_nulls = false; for (d, mapping) in dictionaries.iter().zip(merged.key_mappings) { has_nulls |= d.null_count() != 0; - for key in d.keys().values() { - // Use get to safely handle nulls - key_values.push(mapping.get(key.as_usize()).unwrap_or_default()) + match mapping { + KeyMapping::Dense(mapping) => key_values.extend( + d.keys() + .values() + .iter() + .map(|key| mapping.get(key.as_usize()).copied().unwrap_or_default()), + ), + KeyMapping::Sparse(mapping) => key_values.extend( + d.keys() + .values() + .iter() + .map(|key| mapping.get(&key.as_usize()).copied().unwrap_or_default()), + ), } } diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs index 6f66202795b8..0c589bc57406 100644 --- a/arrow-select/src/dictionary.rs +++ b/arrow-select/src/dictionary.rs @@ -127,8 +127,8 @@ impl<'a, V> Interner<'a, V> { } } -// Avoid dense mappings when the dictionary is much larger than the selected rows. -const MAX_DENSE_MAPPING_VALUES_PER_ROW: usize = 16; +// Sparse hashing pays off only when retained values greatly outnumber selected rows. +const MAX_DENSE_MAPPING_VALUES_PER_ROW: usize = 128; pub(crate) enum KeyMapping { Dense(Vec), @@ -156,6 +156,7 @@ impl KeyMapping { } } + #[inline] pub fn get(&self, old_key: usize) -> Option { match self { Self::Dense(mapping) => mapping.get(old_key).copied(), @@ -303,16 +304,15 @@ pub(crate) fn merge_dictionary_values( let use_sparse_mapping = values.len() > selected_rows.saturating_mul(MAX_DENSE_MAPPING_VALUES_PER_ROW); - let referenced_keys; - let values_mask; - let selection = if use_sparse_mapping { - referenced_keys = compute_referenced_keys(keys, key_mask); - ValueSelection::Indices(&referenced_keys) + let selected_values = if use_sparse_mapping { + let referenced_keys = compute_referenced_keys(keys, key_mask); + let len = referenced_keys.len(); + get_selected_values(values, referenced_keys.into_iter(), len) } else { - values_mask = compute_values_mask(keys, key_mask, values.len()); - ValueSelection::Mask(&values_mask) + let values_mask = compute_values_mask(keys, key_mask, values.len()); + let len = values_mask.count_set_bits(); + get_selected_values(values, values_mask.set_indices(), len) }; - let selected_values = get_selected_values(values, selection); num_values += selected_values.len(); value_slices.push((selected_values, use_sparse_mapping)); values_arrays.push(values) @@ -476,77 +476,58 @@ fn compute_values_mask( builder.finish() } -#[derive(Clone, Copy)] -enum ValueSelection<'a> { - Mask(&'a BooleanBuffer), - Indices(&'a [usize]), -} - -impl ValueSelection<'_> { - fn len(&self) -> usize { - match self { - Self::Mask(mask) => mask.count_set_bits(), - Self::Indices(indices) => indices.len(), - } - } - - fn for_each(&self, f: impl FnMut(usize)) { - match self { - Self::Mask(mask) => mask.set_indices().for_each(f), - Self::Indices(indices) => indices.iter().copied().for_each(f), - } - } -} - -fn selected_primitives_to_bytes<'a, T: ArrowPrimitiveType>( - array: &'a PrimitiveArray, - selection: ValueSelection<'_>, -) -> Vec<(usize, Option<&'a [u8]>)> +fn selected_primitives_to_bytes>( + array: &PrimitiveArray, + indices: I, + len: usize, +) -> Vec<(usize, Option<&[u8]>)> where T::Native: ToByteSlice, { - let mut out = Vec::with_capacity(selection.len()); + let mut out = Vec::with_capacity(len); let values = array.values(); - selection.for_each(|idx| { + for idx in indices { out.push(( idx, array.is_valid(idx).then_some(values[idx].to_byte_slice()), )) - }); + } out } macro_rules! selected_primitive_to_bytes_helper { - ($t:ty, $array:expr, $selection:expr) => { - selected_primitives_to_bytes::<$t>($array.as_primitive(), $selection) + ($t:ty, $array:expr, $indices:expr, $len:expr) => { + selected_primitives_to_bytes::<$t, _>($array.as_primitive(), $indices, $len) }; } -fn get_selected_values<'a>( - array: &'a dyn Array, - selection: ValueSelection<'_>, -) -> Vec<(usize, Option<&'a [u8]>)> { +fn get_selected_values>( + array: &dyn Array, + indices: I, + len: usize, +) -> Vec<(usize, Option<&[u8]>)> { downcast_primitive! { - array.data_type() => (selected_primitive_to_bytes_helper, array, selection), - DataType::Utf8 => selected_bytes(array.as_string::(), selection), - DataType::LargeUtf8 => selected_bytes(array.as_string::(), selection), - DataType::Binary => selected_bytes(array.as_binary::(), selection), - DataType::LargeBinary => selected_bytes(array.as_binary::(), selection), + array.data_type() => (selected_primitive_to_bytes_helper, array, indices, len), + DataType::Utf8 => selected_bytes(array.as_string::(), indices, len), + DataType::LargeUtf8 => selected_bytes(array.as_string::(), indices, len), + DataType::Binary => selected_bytes(array.as_binary::(), indices, len), + DataType::LargeBinary => selected_bytes(array.as_binary::(), indices, len), _ => unimplemented!("Dictionary merging for type {} is not implemented", array.data_type()), } } -fn selected_bytes<'a, T: ByteArrayType>( - array: &'a GenericByteArray, - selection: ValueSelection<'_>, -) -> Vec<(usize, Option<&'a [u8]>)> { - let mut out = Vec::with_capacity(selection.len()); - selection.for_each(|idx| { +fn selected_bytes>( + array: &GenericByteArray, + indices: I, + len: usize, +) -> Vec<(usize, Option<&[u8]>)> { + let mut out = Vec::with_capacity(len); + for idx in indices { out.push(( idx, array.is_valid(idx).then_some(array.value(idx).as_ref()), )) - }); + } out } From 7d39e41c6bd73b9bcafbe2e5dbbc10f8c7ab9fe3 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Wed, 4 Nov 2026 10:18:55 -0500 Subject: [PATCH 4/6] Test sparse dictionary key overflow The sparse mapping path must preserve the existing overflow contract. Build 257 UInt8 dictionaries with one unique referenced value each and verify the merge returns DictionaryKeyOverflowError. --- arrow-select/src/dictionary.rs | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs index 0c589bc57406..f5c73ed99996 100644 --- a/arrow-select/src/dictionary.rs +++ b/arrow-select/src/dictionary.rs @@ -536,11 +536,10 @@ mod tests { use super::*; use arrow_array::cast::as_string_array; - use arrow_array::types::Int8Type; - use arrow_array::types::Int32Type; + use arrow_array::types::{Int8Type, Int32Type, UInt8Type}; use arrow_array::{ ArrayRef, DictionaryArray, Int8Array, Int32Array, ListArray, NullArray, RunArray, - StringArray, StringViewArray, + StringArray, StringViewArray, UInt8Array, }; use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer, OffsetBuffer}; use arrow_schema::Field; @@ -845,4 +844,29 @@ mod tests { assert_eq!(merged.key_mappings[0].get(999), Some(1)); assert_eq!(merged.key_mappings[0].get(500), None); } + + #[test] + fn test_merge_sparse_dictionary_overflow() { + const UNIQUE_VALUES: usize = u8::MAX as usize + 2; + + // One sparse value per dictionary exceeds the 256 values representable by UInt8. + let dictionaries = (0..UNIQUE_VALUES) + .map(|dictionary_idx| { + let values = StringArray::from_iter_values( + (0..=MAX_DENSE_MAPPING_VALUES_PER_ROW) + .map(|value_idx| format!("{dictionary_idx}-{value_idx}")), + ); + let keys = UInt8Array::from_iter_values([0]); + DictionaryArray::::new(keys, Arc::new(values)) + }) + .collect::>(); + let dictionaries = dictionaries.iter().collect::>(); + + let result = merge_dictionary_values(&dictionaries, None); + + assert!(matches!( + result, + Err(ArrowError::DictionaryKeyOverflowError) + )); + } } From bdd44b6963e6d8401edca55040c00332c120cd7b Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Sat, 7 Nov 2026 13:43:16 -0500 Subject: [PATCH 5/6] Address sparse dictionary review feedback --- arrow-select/src/coalesce/dictionary.rs | 10 ++- arrow-select/src/concat.rs | 44 +++++++------ arrow-select/src/dictionary.rs | 82 ++++++++++++++++++------- arrow-select/src/interleave.rs | 46 +++++++++----- arrow/benches/concatenate_kernel.rs | 54 +++++++++------- 5 files changed, 156 insertions(+), 80 deletions(-) diff --git a/arrow-select/src/coalesce/dictionary.rs b/arrow-select/src/coalesce/dictionary.rs index 99c126ea97c9..5a5e1e1e087c 100644 --- a/arrow-select/src/coalesce/dictionary.rs +++ b/arrow-select/src/coalesce/dictionary.rs @@ -153,12 +153,18 @@ impl InProgressDictionaryArray { match &chunk.nulls { Some(nulls) => keys.extend(chunk.keys.iter().enumerate().map(|(offset, key)| { if nulls.is_valid(offset) { - mapping[key.as_usize()] + mapping + .get(key.as_usize()) + .expect("referenced dictionary key should be mapped") } else { K::Native::default() } })), - None => keys.extend(chunk.keys.iter().map(|key| mapping[key.as_usize()])), + None => keys.extend(chunk.keys.iter().map(|key| { + mapping + .get(key.as_usize()) + .expect("referenced dictionary key should be mapped") + })), } } Ok((keys, merged.values)) diff --git a/arrow-select/src/concat.rs b/arrow-select/src/concat.rs index c5d8c85f2184..9ae29f6d6d7c 100644 --- a/arrow-select/src/concat.rs +++ b/arrow-select/src/concat.rs @@ -108,7 +108,7 @@ fn concat_dictionaries( return concat_fallback(arrays, Capacities::Array(output_len)); } - let merged = merge_dictionary_values(&dictionaries, None)?; + let merged = merge_dictionary_values(&dictionaries, None, None)?; // Recompute keys let mut key_values = Vec::with_capacity(output_len); @@ -1578,31 +1578,37 @@ mod tests { } #[test] - fn test_concat_oversized_dictionary_slices() { - let make_dictionary = |prefix| { - let values = StringArray::from_iter_values( - (0..1_000).map(|idx| format!("{prefix}_value_{idx}")), + fn test_concat_sparse_primitive_dictionaries_with_nulls() { + let values_len = 3 * crate::dictionary::MAX_DENSE_MAPPING_VALUES_PER_ROW + 1; + let last = values_len as i32 - 1; + let make_dictionary = |offset, null_value, keys| { + let values = Int32Array::from_iter( + (0..values_len).map(|idx| (idx != null_value).then_some(offset + idx as i32)), ); - let keys = Int32Array::from_iter_values(0..1_000); - DictionaryArray::::new(keys, Arc::new(values)) + DictionaryArray::::new(Int32Array::from(keys), Arc::new(values)) }; - let a = make_dictionary("a").slice(997, 3); - let b = make_dictionary("b").slice(1, 3); + let a = make_dictionary(0, 1, vec![Some(last), None, Some(1)]); + let b = make_dictionary(10_000, 2, vec![Some(2), Some(last - 1), None]); + + let merged = merge_dictionary_values(&[&a, &b], None, None).unwrap(); + assert!( + merged + .key_mappings + .iter() + .all(|mapping| matches!(mapping, KeyMapping::Sparse(_))) + ); let result = concat(&[&a, &b]).unwrap(); - let actual = collect_string_dictionary(result.as_dictionary::()); + let actual = result + .as_dictionary::() + .downcast_dict::() + .unwrap() + .into_iter() + .collect::>(); assert_eq!( actual, - [ - "a_value_997", - "a_value_998", - "a_value_999", - "b_value_1", - "b_value_2", - "b_value_3", - ] - .map(Some) + [Some(last), None, None, None, Some(10_000 + last - 1), None] ); } diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs index f5c73ed99996..2a4f0a8503f8 100644 --- a/arrow-select/src/dictionary.rs +++ b/arrow-select/src/dictionary.rs @@ -127,8 +127,8 @@ impl<'a, V> Interner<'a, V> { } } -// Sparse hashing pays off only when retained values greatly outnumber selected rows. -const MAX_DENSE_MAPPING_VALUES_PER_ROW: usize = 128; +// Sparse hashing pays off only when retained values greatly outnumber key lookups. +pub(crate) const MAX_DENSE_MAPPING_VALUES_PER_ROW: usize = 256; pub(crate) enum KeyMapping { Dense(Vec), @@ -274,10 +274,12 @@ pub(crate) fn should_merge_dictionary_values( /// /// This method is meant to be very fast and the output dictionary values /// may not be unique, unlike `GenericByteDictionaryBuilder` which is slower -/// but produces unique values +/// but produces unique values. `key_lookups` is the number of times each +/// input's keys will be remapped, and defaults to the input length. pub(crate) fn merge_dictionary_values( dictionaries: &[&DictionaryArray], masks: Option<&[BooleanBuffer]>, + key_lookups: Option<&[usize]>, ) -> Result, ArrowError> { let mut num_values = 0; @@ -298,11 +300,9 @@ pub(crate) fn merge_dictionary_values( }; let keys = dictionary.keys().values(); let values = dictionary.values().as_ref(); - let selected_rows = key_mask - .map(BooleanBuffer::count_set_bits) - .unwrap_or(keys.len()); + let lookups = key_lookups.map_or(keys.len(), |lookups| lookups[idx]); let use_sparse_mapping = - values.len() > selected_rows.saturating_mul(MAX_DENSE_MAPPING_VALUES_PER_ROW); + values.len() > lookups.saturating_mul(MAX_DENSE_MAPPING_VALUES_PER_ROW); let selected_values = if use_sparse_mapping { let referenced_keys = compute_referenced_keys(keys, key_mask); @@ -401,7 +401,10 @@ pub(crate) fn merge_dictionary_values_with_masks( let value_slices = values_arrays .iter() .zip(values_masks) - .map(|(values, mask)| get_masked_values(*values, mask)) + .map(|(values, mask)| { + let len = mask.count_set_bits(); + get_selected_values(*values, mask.set_indices(), len) + }) .collect::>(); let mut interner = Interner::new(num_values); let mut indices = Vec::with_capacity(num_values); @@ -546,9 +549,9 @@ mod tests { use std::sync::Arc; fn assert_mapping(mapping: &KeyMapping, expected: &[i32]) { - let actual = (0..expected.len()) - .map(|idx| mapping.get(idx).unwrap_or_default()) - .collect::>(); + let KeyMapping::Dense(actual) = mapping else { + panic!("expected a dense mapping"); + }; assert_eq!(actual, expected); } @@ -732,7 +735,7 @@ mod tests { fn test_merge_strings() { let a = DictionaryArray::::from_iter(["a", "b", "a", "b", "d", "c", "e"]); let b = DictionaryArray::::from_iter(["c", "f", "c", "d", "a", "d"]); - let merged = merge_dictionary_values(&[&a, &b], None).unwrap(); + let merged = merge_dictionary_values(&[&a, &b], None, None).unwrap(); let values = as_string_array(merged.values.as_ref()); let actual: Vec<_> = values.iter().map(Option::unwrap).collect(); @@ -743,7 +746,7 @@ mod tests { assert_mapping(&merged.key_mappings[1], &[3, 5, 2, 0]); let a_slice = a.slice(1, 4); - let merged = merge_dictionary_values(&[&a_slice, &b], None).unwrap(); + let merged = merge_dictionary_values(&[&a_slice, &b], None, None).unwrap(); let values = as_string_array(merged.values.as_ref()); let actual: Vec<_> = values.iter().map(Option::unwrap).collect(); @@ -756,7 +759,7 @@ mod tests { // Mask out only ["b", "b", "d"] from a let a_mask = BooleanBuffer::from_iter([false, true, false, true, true, false, false]); let b_mask = BooleanBuffer::new_set(b.len()); - let merged = merge_dictionary_values(&[&a, &b], Some(&[a_mask, b_mask])).unwrap(); + let merged = merge_dictionary_values(&[&a, &b], Some(&[a_mask, b_mask]), None).unwrap(); let values = as_string_array(merged.values.as_ref()); let actual: Vec<_> = values.iter().map(Option::unwrap).collect(); @@ -800,7 +803,7 @@ mod tests { let b = DictionaryArray::new(Int32Array::new_null(10), Arc::new(StringArray::new_null(0))); - let merged = merge_dictionary_values(&[&a, &b], None).unwrap(); + let merged = merge_dictionary_values(&[&a, &b], None, None).unwrap(); let expected = StringArray::from(vec![None, Some("bingo"), Some("hello")]); assert_eq!(merged.values.as_ref(), &expected); assert_eq!(merged.key_mappings.len(), 2); @@ -814,7 +817,7 @@ mod tests { let keys = Int32Array::from_iter_values([1]); let a = DictionaryArray::new(keys, Arc::new(values)); - let merged = merge_dictionary_values(&[&a], None).unwrap(); + let merged = merge_dictionary_values(&[&a], None, None).unwrap(); let expected = StringArray::from(vec!["b"]); assert_eq!(merged.values.as_ref(), &expected); } @@ -828,20 +831,55 @@ mod tests { assert_eq!(actual, vec![1, 500, 999]); } + #[test] + fn test_sparse_gate_counts_key_lookups() { + let values: ArrayRef = Arc::new(StringArray::from_iter_values( + (0..8 * MAX_DENSE_MAPPING_VALUES_PER_ROW).map(|idx| idx.to_string()), + )); + let is_sparse = |dictionary: &DictionaryArray, + masks: Option<&[BooleanBuffer]>, + lookups: Option<&[usize]>| { + let merged = merge_dictionary_values(&[dictionary], masks, lookups).unwrap(); + matches!(merged.key_mappings[0], KeyMapping::Sparse(_)) + }; + + // One valid row of ten is over the gate by selected rows, but concat remaps all ten. + let keys = Int32Array::from_iter((0..10).map(|idx| (idx == 0).then_some(999))); + let mostly_null = DictionaryArray::new(keys, Arc::clone(&values)); + assert!(!is_sparse(&mostly_null, None, None)); + + // Seven lookups stay under the gate, keeping the sparse path. + let keys = Int32Array::from_iter_values([999; 7]); + let short = DictionaryArray::new(keys, Arc::clone(&values)); + assert!(is_sparse(&short, None, None)); + + // Interleave may look up the selected row more than once. + let mask = BooleanBuffer::collect_bool(7, |idx| idx == 0); + assert!(!is_sparse( + &short, + Some(std::slice::from_ref(&mask)), + Some(&[10]), + )); + assert!(is_sparse(&short, Some(&[mask]), Some(&[7]))); + } + #[test] fn test_merge_oversized_dictionary_slice() { - let values = StringArray::from_iter_values((0..1_000).map(|idx| idx.to_string())); - let keys = Int32Array::from(vec![Some(999), None, Some(1), Some(999)]); + let values_len = 4 * MAX_DENSE_MAPPING_VALUES_PER_ROW + 1; + let values = StringArray::from_iter_values((0..values_len).map(|idx| idx.to_string())); + let last = values_len as i32 - 1; + let keys = Int32Array::from(vec![Some(last), None, Some(1), Some(last)]); let dictionary = DictionaryArray::new(keys, Arc::new(values)); - let merged = merge_dictionary_values(&[&dictionary], None).unwrap(); + let merged = merge_dictionary_values(&[&dictionary], None, None).unwrap(); let values = as_string_array(merged.values.as_ref()); let actual = values.iter().collect::>(); + let last_value = last.to_string(); - assert_eq!(actual, vec![Some("1"), Some("999")]); + assert_eq!(actual, vec![Some("1"), Some(last_value.as_str())]); assert!(matches!(merged.key_mappings[0], KeyMapping::Sparse(_))); assert_eq!(merged.key_mappings[0].get(1), Some(0)); - assert_eq!(merged.key_mappings[0].get(999), Some(1)); + assert_eq!(merged.key_mappings[0].get(last as usize), Some(1)); assert_eq!(merged.key_mappings[0].get(500), None); } @@ -862,7 +900,7 @@ mod tests { .collect::>(); let dictionaries = dictionaries.iter().collect::>(); - let result = merge_dictionary_values(&dictionaries, None); + let result = merge_dictionary_values(&dictionaries, None, None); assert!(matches!( result, diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index bd8f9b471f77..4a81d8df91fb 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -272,7 +272,7 @@ fn interleave_dictionaries( }; } - let masks: Result, ArrowError> = dictionaries + let masks_and_lookups: Result, ArrowError> = dictionaries .iter() .enumerate() .map(|(a_idx, dictionary)| { @@ -280,15 +280,17 @@ fn interleave_dictionaries( .map_err(|e| ArrowError::MemoryError(e.to_string()))?; let mut key_mask = BooleanBufferBuilder::new_from_buffer(buffer, dictionary.len()); + let mut lookups = 0; for (_, key_idx) in indices.iter().filter(|(a, _)| *a == a_idx) { key_mask.set_bit(*key_idx, true); + lookups += 1; } - Ok(key_mask.finish()) + Ok((key_mask.finish(), lookups)) }) .collect(); - let masks = masks?; + let (masks, key_lookups): (Vec<_>, Vec<_>) = masks_and_lookups?.into_iter().unzip(); - let merged = merge_dictionary_values(&dictionaries, Some(&masks))?; + let merged = merge_dictionary_values(&dictionaries, Some(&masks), Some(&key_lookups))?; // Recompute keys let mut keys = PrimitiveBuilder::::with_capacity(indices.len()); @@ -1135,26 +1137,40 @@ mod tests { } #[test] - fn test_interleave_oversized_dictionaries() { - let make_dictionary = |prefix| { - let values = StringArray::from_iter_values( - (0..1_000).map(|idx| format!("{prefix}_value_{idx}")), + fn test_interleave_sparse_primitive_dictionaries_with_nulls() { + let values_len = 2 * crate::dictionary::MAX_DENSE_MAPPING_VALUES_PER_ROW + 1; + let last = values_len as i32 - 1; + let make_dictionary = |offset, null_value, keys| { + let values = Int32Array::from_iter( + (0..values_len).map(|idx| (idx != null_value).then_some(offset + idx as i32)), ); - let keys = Int32Array::from_iter_values([1, 999]); - DictionaryArray::::new(keys, Arc::new(values)) + DictionaryArray::::new(Int32Array::from(keys), Arc::new(values)) }; - let a = make_dictionary("a"); - let b = make_dictionary("b"); + let a = make_dictionary(0, 1, vec![Some(last), None, Some(1)]); + let b = make_dictionary(10_000, 2, vec![Some(2), Some(last - 1), None]); + let indices = [(0, 2), (1, 0), (0, 0), (1, 2)]; + let masks = [ + BooleanBuffer::from_iter([true, false, true]), + BooleanBuffer::from_iter([true, false, true]), + ]; - let result = interleave(&[&a, &b], &[(0, 1), (1, 0), (0, 0)]).unwrap(); + let merged = merge_dictionary_values(&[&a, &b], Some(&masks), Some(&[2, 2])).unwrap(); + assert!( + merged + .key_mappings + .iter() + .all(|mapping| matches!(mapping, crate::dictionary::KeyMapping::Sparse(_))) + ); + + let result = interleave(&[&a, &b], &indices).unwrap(); let actual = result .as_dictionary::() - .downcast_dict::() + .downcast_dict::() .unwrap() .into_iter() .collect::>(); - assert_eq!(actual, ["a_value_999", "b_value_1", "a_value_1"].map(Some)); + assert_eq!(actual, [None, None, Some(last), None]); } #[test] diff --git a/arrow/benches/concatenate_kernel.rs b/arrow/benches/concatenate_kernel.rs index d5efcd20041f..461e8b73e6f5 100644 --- a/arrow/benches/concatenate_kernel.rs +++ b/arrow/benches/concatenate_kernel.rs @@ -187,28 +187,38 @@ fn add_benchmark(c: &mut Criterion) { const DICTIONARY_SLICE_ROWS: usize = 682; const DICTIONARY_SLICE_COUNT: usize = 12; - for values_per_key in [12, 16, 24] { - let values_len = DICTIONARY_SLICE_ROWS * values_per_key; - let inputs = (0..DICTIONARY_SLICE_COUNT) - .map(|dictionary_idx| { - let values = StringArray::from_iter_values( - (0..values_len).map(|value_idx| format!("{dictionary_idx:02}_{value_idx:017}")), - ); - let keys = Int32Array::from_iter_values(0..values_len as i32); - let dictionary = DictionaryArray::::new(keys, Arc::new(values)); - dictionary.slice(values_len - DICTIONARY_SLICE_ROWS, DICTIONARY_SLICE_ROWS) - }) - .collect::>(); - let arrays = inputs - .iter() - .map(|array| array as &dyn Array) - .collect::>(); - c.bench_function( - &format!( - "concat str_dict_slice rows={DICTIONARY_SLICE_ROWS} values_per_key={values_per_key} count={DICTIONARY_SLICE_COUNT}" - ), - |b| b.iter(|| bench_concat_arrays(&arrays)), - ); + for values_per_key in [12, 16, 24, 128, 256, 400, 1024] { + for (key_order, scattered) in [("contiguous", false), ("scattered", true)] { + let values_len = DICTIONARY_SLICE_ROWS * values_per_key; + let inputs = (0..DICTIONARY_SLICE_COUNT) + .map(|dictionary_idx| { + let values = StringArray::from_iter_values( + (0..values_len) + .map(|value_idx| format!("{dictionary_idx:02}_{value_idx:017}")), + ); + let mut keys = (0..values_len as i32).collect::>(); + if scattered { + for row_idx in 0..DICTIONARY_SLICE_ROWS { + keys[values_len - DICTIONARY_SLICE_ROWS + row_idx] = + (row_idx * values_per_key) as i32; + } + } + let dictionary = + DictionaryArray::::new(Int32Array::from(keys), Arc::new(values)); + dictionary.slice(values_len - DICTIONARY_SLICE_ROWS, DICTIONARY_SLICE_ROWS) + }) + .collect::>(); + let arrays = inputs + .iter() + .map(|array| array as &dyn Array) + .collect::>(); + c.bench_function( + &format!( + "concat str_dict_slice rows={DICTIONARY_SLICE_ROWS} values_per_key={values_per_key} key_order={key_order} count={DICTIONARY_SLICE_COUNT}" + ), + |b| b.iter(|| bench_concat_arrays(&arrays)), + ); + } } let v1 = FixedSizeListArray::try_new( From 84612fe6634d7039ac472ae7eefbba011534d3bc Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Sat, 7 Nov 2026 20:23:22 -0500 Subject: [PATCH 6/6] Address dictionary concat review feedback --- arrow-select/src/coalesce/dictionary.rs | 16 ++++++---------- arrow-select/src/concat.rs | 7 +++++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/arrow-select/src/coalesce/dictionary.rs b/arrow-select/src/coalesce/dictionary.rs index 5a5e1e1e087c..ec8e8cc12aeb 100644 --- a/arrow-select/src/coalesce/dictionary.rs +++ b/arrow-select/src/coalesce/dictionary.rs @@ -3,7 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use super::{InProgressArray, generic::GenericInProgressArray}; -use crate::dictionary::{child_values_ptr_eq, merge_dictionary_values_with_masks}; +use crate::dictionary::{KeyMapping, child_values_ptr_eq, merge_dictionary_values_with_masks}; use ahash::RandomState; use arrow_array::cast::AsArray; use arrow_array::types::ArrowDictionaryKeyType; @@ -149,22 +149,18 @@ impl InProgressDictionaryArray { let mut keys = Vec::with_capacity(len); for chunk in chunks { - let mapping = &merged.key_mappings[chunk.source]; + let KeyMapping::Dense(mapping) = &merged.key_mappings[chunk.source] else { + unreachable!("merge_dictionary_values_with_masks returns dense mappings"); + }; match &chunk.nulls { Some(nulls) => keys.extend(chunk.keys.iter().enumerate().map(|(offset, key)| { if nulls.is_valid(offset) { - mapping - .get(key.as_usize()) - .expect("referenced dictionary key should be mapped") + mapping[key.as_usize()] } else { K::Native::default() } })), - None => keys.extend(chunk.keys.iter().map(|key| { - mapping - .get(key.as_usize()) - .expect("referenced dictionary key should be mapped") - })), + None => keys.extend(chunk.keys.iter().map(|key| mapping[key.as_usize()])), } } Ok((keys, merged.values)) diff --git a/arrow-select/src/concat.rs b/arrow-select/src/concat.rs index 9ae29f6d6d7c..450660aea906 100644 --- a/arrow-select/src/concat.rs +++ b/arrow-select/src/concat.rs @@ -1587,8 +1587,11 @@ mod tests { ); DictionaryArray::::new(Int32Array::from(keys), Arc::new(values)) }; - let a = make_dictionary(0, 1, vec![Some(last), None, Some(1)]); - let b = make_dictionary(10_000, 2, vec![Some(2), Some(last - 1), None]); + // Slicing moves the keys and their nulls to a non-zero offset; the unsliced + // arrays have 4 keys each and would take the dense path. + let a = make_dictionary(0, 1, vec![Some(0), Some(last), None, Some(1)]).slice(1, 3); + let b = + make_dictionary(10_000, 2, vec![Some(0), Some(2), Some(last - 1), None]).slice(1, 3); let merged = merge_dictionary_values(&[&a, &b], None, None).unwrap(); assert!(