From 17cc21fd3d4f0852c5629bcff21ffd4c2488e5d1 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 10:24:09 -0500 Subject: [PATCH 1/5] Add dictionary slice concat ratio benchmarks --- arrow/benches/concatenate_kernel.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arrow/benches/concatenate_kernel.rs b/arrow/benches/concatenate_kernel.rs index fbc2e87c7245..e1d588fc8b7b 100644 --- a/arrow/benches/concatenate_kernel.rs +++ b/arrow/benches/concatenate_kernel.rs @@ -198,6 +198,32 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| bench_concat(&v1, &v2)) }); + const DICTIONARY_SLICE_ROWS: usize = 682; + const DICTIONARY_SLICE_COUNT: usize = 12; + for values_per_key in [4, 8, 12, 16, 20, 24, 32, 64] { + let values_len = DICTIONARY_SLICE_ROWS * values_per_key; + let inputs = (0..DICTIONARY_SLICE_COUNT) + .map(|dictionary_idx| { + let values = StringArray::from_iter_values( + (0..values_len).map(|value_idx| format!("{dictionary_idx:02}_{value_idx:017}")), + ); + let keys = Int32Array::from_iter_values(0..values_len as i32); + let dictionary = DictionaryArray::::new(keys, Arc::new(values)); + dictionary.slice(values_len - DICTIONARY_SLICE_ROWS, DICTIONARY_SLICE_ROWS) + }) + .collect::>(); + let arrays = inputs + .iter() + .map(|array| array as &dyn Array) + .collect::>(); + c.bench_function( + &format!( + "concat str_dict_slice rows={DICTIONARY_SLICE_ROWS} values_per_key={values_per_key} count={DICTIONARY_SLICE_COUNT}" + ), + |b| b.iter(|| bench_concat_arrays(&arrays)), + ); + } + let v1 = FixedSizeListArray::try_new( Arc::new(Field::new_list_field(DataType::Int32, true)), 1024, From 73d9a6551bc6bb62952fa478a0c2bb195a9bdcd5 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 10:28:56 -0500 Subject: [PATCH 2/5] Narrow dictionary concat ratio sweep --- arrow/benches/concatenate_kernel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow/benches/concatenate_kernel.rs b/arrow/benches/concatenate_kernel.rs index e1d588fc8b7b..fe76ce32aa7f 100644 --- a/arrow/benches/concatenate_kernel.rs +++ b/arrow/benches/concatenate_kernel.rs @@ -200,7 +200,7 @@ fn add_benchmark(c: &mut Criterion) { const DICTIONARY_SLICE_ROWS: usize = 682; const DICTIONARY_SLICE_COUNT: usize = 12; - for values_per_key in [4, 8, 12, 16, 20, 24, 32, 64] { + for values_per_key in [4, 8, 12, 16, 24] { let values_len = DICTIONARY_SLICE_ROWS * values_per_key; let inputs = (0..DICTIONARY_SLICE_COUNT) .map(|dictionary_idx| { From efbc5b166bdd01cf26f31c164518317da43625b3 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Tue, 3 Nov 2026 10:31:50 -0500 Subject: [PATCH 3/5] Focus dictionary ratio sweep on cutoff --- arrow/benches/concatenate_kernel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow/benches/concatenate_kernel.rs b/arrow/benches/concatenate_kernel.rs index fe76ce32aa7f..7545ad650335 100644 --- a/arrow/benches/concatenate_kernel.rs +++ b/arrow/benches/concatenate_kernel.rs @@ -200,7 +200,7 @@ fn add_benchmark(c: &mut Criterion) { const DICTIONARY_SLICE_ROWS: usize = 682; const DICTIONARY_SLICE_COUNT: usize = 12; - for values_per_key in [4, 8, 12, 16, 24] { + for values_per_key in [12, 16, 24] { let values_len = DICTIONARY_SLICE_ROWS * values_per_key; let inputs = (0..DICTIONARY_SLICE_COUNT) .map(|dictionary_idx| { From 6d334f03ec14a1b83b70902e3b041b028c11a653 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Wed, 28 Oct 2026 19:07:34 -0400 Subject: [PATCH 4/5] Add Boolean filter kernel benchmarks --- arrow/benches/filter_kernels.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arrow/benches/filter_kernels.rs b/arrow/benches/filter_kernels.rs index 8b2d45423356..dd2ce443d4fe 100644 --- a/arrow/benches/filter_kernels.rs +++ b/arrow/benches/filter_kernels.rs @@ -374,7 +374,7 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| bench_built_filter(&data_array, &sparse_filter)) }); - let mut add_benchmark_for_nested = |name: &str, data_array: &dyn Array| { + let mut add_benchmark_for_array = |name: &str, data_array: &dyn Array| { c.bench_function(&format!("filter {name} (kept 1/2)"), |b| { b.iter(|| bench_filter(data_array, &filter_array)) }); @@ -400,20 +400,26 @@ fn add_benchmark(c: &mut Criterion) { ); }; + let data_array = create_boolean_array(size, 0.0, 0.5); + add_benchmark_for_array("boolean", &data_array); + + let data_array = create_boolean_array(size, 0.1, 0.5); + add_benchmark_for_array("boolean w NULLs", &data_array); + let data_array = create_primitive_list_array::(size, 0.1, 0.1, 16); - add_benchmark_for_nested("list w NULLs", &data_array); + add_benchmark_for_array("list w NULLs", &data_array); let data_array = create_struct_list_array(size, 0.1, 16); - add_benchmark_for_nested("list> w NULLs", &data_array); + add_benchmark_for_array("list> w NULLs", &data_array); let data_array = create_primitive_fixed_size_list_array::(size, 0.1, 0.1, 8); - add_benchmark_for_nested("fixed_size_list w NULLs", &data_array); + add_benchmark_for_array("fixed_size_list w NULLs", &data_array); let data_array = create_struct_fixed_size_list_array(size, 8); - add_benchmark_for_nested("fixed_size_list>", &data_array); + add_benchmark_for_array("fixed_size_list>", &data_array); let data_array = create_string_map_array::(size, 0.1, 16, 8); - add_benchmark_for_nested("map w NULLs", &data_array); + add_benchmark_for_array("map w NULLs", &data_array); } criterion_group!(benches, add_benchmark); From 37da83467e7aa7777be3288040675ed71aad2f42 Mon Sep 17 00:00:00 2001 From: Richard Baah Date: Mon, 2 Nov 2026 23:00:50 -0500 Subject: [PATCH 5/5] bench(row): cover oversized dictionaries --- arrow/benches/row_format.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/arrow/benches/row_format.rs b/arrow/benches/row_format.rs index 09d3800fae0b..04205b57a363 100644 --- a/arrow/benches/row_format.rs +++ b/arrow/benches/row_format.rs @@ -18,7 +18,7 @@ #[macro_use] extern crate criterion; -use arrow::array::ArrayRef; +use arrow::array::{ArrayRef, DictionaryArray, Int32Array, StringArray}; use arrow::datatypes::{Int64Type, UInt64Type}; use arrow::row::{RowConverter, SortField}; use arrow::util::bench_util::{ @@ -282,6 +282,34 @@ fn row_bench(c: &mut Criterion) { let cols = vec![Arc::new(dict) as ArrayRef]; do_bench(c, "4096 string_dictionary_low_cardinality(100, 0)", cols); + const ROWS: usize = 8_192; + for values_len in [1_000_000, 4_000_000] { + let values = Arc::new(StringArray::from_iter_values( + (0..values_len).map(|idx| format!("value_{idx}")), + )); + let chunks = (0..values_len / ROWS) + .map(|chunk| { + let start = chunk * ROWS; + let keys = + Int32Array::from_iter_values((start..start + ROWS).map(|key| key as i32)); + let dictionary = + DictionaryArray::::try_new(keys, values.clone()).unwrap(); + vec![Arc::new(dictionary) as ArrayRef] + }) + .collect::>(); + let converter = + RowConverter::new(vec![SortField::new(chunks[0][0].data_type().clone())]).unwrap(); + let mut chunks = chunks.iter().cycle(); + c.bench_function( + &format!("convert_columns_prepared {ROWS} string_dictionary({values_len} values)"), + |b| { + b.iter(|| { + hint::black_box(converter.convert_columns(chunks.next().unwrap()).unwrap()) + }) + }, + ); + } + let cols = vec![ Arc::new(create_string_array_with_len::(4096, 0.5, 20)) as ArrayRef, Arc::new(create_string_array_with_len::(4096, 0., 30)) as ArrayRef,