From 29e4b1a6829b66af8555f0f38b575c76ab40c53d Mon Sep 17 00:00:00 2001 From: cetra3 Date: Sun, 11 Oct 2026 13:07:47 -0400 Subject: [PATCH] bench(variant): add per-row metadata variant_get and shred_variant benchmarks --- .../benches/variant_kernels.rs | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/parquet-variant-compute/benches/variant_kernels.rs b/parquet-variant-compute/benches/variant_kernels.rs index c1b5e6fda6bb..a28c7ddf889e 100644 --- a/parquet-variant-compute/benches/variant_kernels.rs +++ b/parquet-variant-compute/benches/variant_kernels.rs @@ -42,6 +42,7 @@ use std::sync::Arc; const VARIANT_GET_UNSHREDDED_OBJECT_ROWS: usize = 262_144; const VARIANT_ARRAY_BUILD_ROWS: usize = 262_144; const SHRED_VARIANT_OBJECT_ROWS: usize = 8_192; +const PER_ROW_METADATA_ROWS: usize = 65_536; fn variant_array_builder_build_bench(c: &mut Criterion) { c.bench_function("variant_array_builder_build_262k_small_values", |b| { @@ -299,6 +300,65 @@ pub fn variant_get_unshredded_object_path_bench(c: &mut Criterion) { }); } +/// Reads one field from unshredded objects where every row has its own metadata dictionary. +/// +/// Each row holds a different subset of field names, so no two rows share metadata bytes, as in +/// data written one record at a time. `present` reads a field that every row has, `sparse` one +/// that about 10% of rows have, and `absent` one that no row has. +pub fn variant_get_per_row_metadata_bench(c: &mut Criterion) { + let input = ArrayRef::from(create_per_row_metadata_object_variant_array( + PER_ROW_METADATA_ROWS, + )); + let rows = PER_ROW_METADATA_ROWS; + let cases = [ + ("present", PER_ROW_METADATA_PRESENT_FIELD, rows..=rows), + ( + "sparse", + PER_ROW_METADATA_SPARSE_FIELD, + rows / 20..=rows / 5, + ), + ("absent", "attr.missing", 0..=0), + ]; + for (name, field_name, expected_non_null) in cases { + let field: FieldRef = Arc::new(Field::new("typed_value", DataType::Int32, true)); + let path = VariantPath::from(vec![VariantPathElement::field(field_name)]); + let options = GetOptions::new_with_path(path).with_as_type(Some(field)); + + let result = variant_get(&input, options.clone()).unwrap(); + let non_null = result.len() - result.null_count(); + assert!( + expected_non_null.contains(&non_null), + "{name}: {non_null} non-null rows" + ); + + c.bench_function( + &format!("variant_get_per_row_metadata_{name}_64k_rows"), + |b| b.iter(|| variant_get(&input, options.clone()).unwrap()), + ); + } +} + +/// Shreds objects where every row has its own metadata dictionary. See +/// [`variant_get_per_row_metadata_bench`] for the input shape. +pub fn shred_variant_per_row_metadata_bench(c: &mut Criterion) { + let variant_array = create_per_row_metadata_object_variant_array(SHRED_VARIANT_OBJECT_ROWS); + let shredded_fields = [ + PER_ROW_METADATA_PRESENT_FIELD, + PER_ROW_METADATA_SPARSE_FIELD, + "attr.000", + "attr.100", + "attr.200", + ] + .into_iter() + .map(|name| Arc::new(Field::new(name, DataType::Int32, true))) + .collect::>(); + let as_type = DataType::Struct(Fields::from(shredded_fields)); + + c.bench_function("shred_variant_per_row_metadata_8k_rows", |b| { + b.iter(|| std::hint::black_box(shred_variant(&variant_array, &as_type).unwrap())) + }); +} + /// Shreds objects whose fields only partially match the requested shredding schema. /// /// Every field that is *not* covered by the schema is copied into the leftover `value` column, @@ -708,8 +768,10 @@ criterion_group!( variant_get_list_index_all_oob_int64_bench, variant_get_shredded_utf8_bench, variant_get_unshredded_object_path_bench, + variant_get_per_row_metadata_bench, shred_variant_partial_object_bench, shred_variant_unmatched_object_bench, + shred_variant_per_row_metadata_bench, variant_array_builder_build_bench, benchmark_batch_json_string_to_variant, variant_get_utf8_from_unshredded_string_bench, @@ -809,6 +871,44 @@ fn create_unshredded_object_variant_array(size: usize) -> VariantArray { .expect("created struct should be a valid unshredded variant") } +/// A field that every row of [`create_per_row_metadata_object_variant_array`] has. +const PER_ROW_METADATA_PRESENT_FIELD: &str = "attr.140"; +/// A field that about 10% of the rows of [`create_per_row_metadata_object_variant_array`] have. +const PER_ROW_METADATA_SPARSE_FIELD: &str = "attr.142"; + +/// Creates a `VariantArray` of objects in which every row has its own metadata dictionary. +/// +/// Each row holds [`PER_ROW_METADATA_PRESENT_FIELD`], [`PER_ROW_METADATA_SPARSE_FIELD`] in about 10% +/// of rows, and about 16 other fields drawn from a pool of 300 names. The fields are inserted in +/// name order, so each dictionary is sorted. Values alternate between integers and short strings. +fn create_per_row_metadata_object_variant_array(size: usize) -> VariantArray { + let field_names = (0..300) + .map(|index| format!("attr.{index:03}")) + .collect::>(); + let mut rng = StdRng::seed_from_u64(42); + let mut builder = VariantArrayBuilder::new(size); + for row in 0..size { + let mut object = builder.new_object(); + for (index, name) in field_names.iter().enumerate() { + let include = match name.as_str() { + PER_ROW_METADATA_PRESENT_FIELD => true, + PER_ROW_METADATA_SPARSE_FIELD => rng.random_range(0..10) == 0, + _ => rng.random_range(0..300) < 16, + }; + if !include { + continue; + } + if index % 2 == 0 { + object.insert(name, index as i32); + } else { + object.insert(name, format!("value_{row}_{index}").as_str()); + } + } + object.finish(); + } + builder.build() +} + /// Return an iterator off JSON strings, each representing a person /// with random first name, last name, and age. ///