diff --git a/arrow/benches/concatenate_kernel.rs b/arrow/benches/concatenate_kernel.rs index fbc2e87c7245..7545ad650335 100644 --- a/arrow/benches/concatenate_kernel.rs +++ b/arrow/benches/concatenate_kernel.rs @@ -198,6 +198,32 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| bench_concat(&v1, &v2)) }); + const DICTIONARY_SLICE_ROWS: usize = 682; + const DICTIONARY_SLICE_COUNT: usize = 12; + for values_per_key in [12, 16, 24] { + let values_len = DICTIONARY_SLICE_ROWS * values_per_key; + let inputs = (0..DICTIONARY_SLICE_COUNT) + .map(|dictionary_idx| { + let values = StringArray::from_iter_values( + (0..values_len).map(|value_idx| format!("{dictionary_idx:02}_{value_idx:017}")), + ); + let keys = Int32Array::from_iter_values(0..values_len as i32); + let dictionary = DictionaryArray::::new(keys, Arc::new(values)); + dictionary.slice(values_len - DICTIONARY_SLICE_ROWS, DICTIONARY_SLICE_ROWS) + }) + .collect::>(); + let arrays = inputs + .iter() + .map(|array| array as &dyn Array) + .collect::>(); + c.bench_function( + &format!( + "concat str_dict_slice rows={DICTIONARY_SLICE_ROWS} values_per_key={values_per_key} count={DICTIONARY_SLICE_COUNT}" + ), + |b| b.iter(|| bench_concat_arrays(&arrays)), + ); + } + let v1 = FixedSizeListArray::try_new( Arc::new(Field::new_list_field(DataType::Int32, true)), 1024, diff --git a/arrow/benches/filter_kernels.rs b/arrow/benches/filter_kernels.rs index 8b2d45423356..dd2ce443d4fe 100644 --- a/arrow/benches/filter_kernels.rs +++ b/arrow/benches/filter_kernels.rs @@ -374,7 +374,7 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| bench_built_filter(&data_array, &sparse_filter)) }); - let mut add_benchmark_for_nested = |name: &str, data_array: &dyn Array| { + let mut add_benchmark_for_array = |name: &str, data_array: &dyn Array| { c.bench_function(&format!("filter {name} (kept 1/2)"), |b| { b.iter(|| bench_filter(data_array, &filter_array)) }); @@ -400,20 +400,26 @@ fn add_benchmark(c: &mut Criterion) { ); }; + let data_array = create_boolean_array(size, 0.0, 0.5); + add_benchmark_for_array("boolean", &data_array); + + let data_array = create_boolean_array(size, 0.1, 0.5); + add_benchmark_for_array("boolean w NULLs", &data_array); + let data_array = create_primitive_list_array::(size, 0.1, 0.1, 16); - add_benchmark_for_nested("list w NULLs", &data_array); + add_benchmark_for_array("list w NULLs", &data_array); let data_array = create_struct_list_array(size, 0.1, 16); - add_benchmark_for_nested("list> w NULLs", &data_array); + add_benchmark_for_array("list> w NULLs", &data_array); let data_array = create_primitive_fixed_size_list_array::(size, 0.1, 0.1, 8); - add_benchmark_for_nested("fixed_size_list w NULLs", &data_array); + add_benchmark_for_array("fixed_size_list w NULLs", &data_array); let data_array = create_struct_fixed_size_list_array(size, 8); - add_benchmark_for_nested("fixed_size_list>", &data_array); + add_benchmark_for_array("fixed_size_list>", &data_array); let data_array = create_string_map_array::(size, 0.1, 16, 8); - add_benchmark_for_nested("map w NULLs", &data_array); + add_benchmark_for_array("map w NULLs", &data_array); } criterion_group!(benches, add_benchmark); diff --git a/arrow/benches/row_format.rs b/arrow/benches/row_format.rs index 09d3800fae0b..04205b57a363 100644 --- a/arrow/benches/row_format.rs +++ b/arrow/benches/row_format.rs @@ -18,7 +18,7 @@ #[macro_use] extern crate criterion; -use arrow::array::ArrayRef; +use arrow::array::{ArrayRef, DictionaryArray, Int32Array, StringArray}; use arrow::datatypes::{Int64Type, UInt64Type}; use arrow::row::{RowConverter, SortField}; use arrow::util::bench_util::{ @@ -282,6 +282,34 @@ fn row_bench(c: &mut Criterion) { let cols = vec![Arc::new(dict) as ArrayRef]; do_bench(c, "4096 string_dictionary_low_cardinality(100, 0)", cols); + const ROWS: usize = 8_192; + for values_len in [1_000_000, 4_000_000] { + let values = Arc::new(StringArray::from_iter_values( + (0..values_len).map(|idx| format!("value_{idx}")), + )); + let chunks = (0..values_len / ROWS) + .map(|chunk| { + let start = chunk * ROWS; + let keys = + Int32Array::from_iter_values((start..start + ROWS).map(|key| key as i32)); + let dictionary = + DictionaryArray::::try_new(keys, values.clone()).unwrap(); + vec![Arc::new(dictionary) as ArrayRef] + }) + .collect::>(); + let converter = + RowConverter::new(vec![SortField::new(chunks[0][0].data_type().clone())]).unwrap(); + let mut chunks = chunks.iter().cycle(); + c.bench_function( + &format!("convert_columns_prepared {ROWS} string_dictionary({values_len} values)"), + |b| { + b.iter(|| { + hint::black_box(converter.convert_columns(chunks.next().unwrap()).unwrap()) + }) + }, + ); + } + let cols = vec![ Arc::new(create_string_array_with_len::(4096, 0.5, 20)) as ArrayRef, Arc::new(create_string_array_with_len::(4096, 0., 30)) as ArrayRef,