From 71a5997f00f855ea4f987ae8348833b2601f6daf Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 2 Nov 2026 18:46:17 +0000 Subject: [PATCH] bench: measure projected Parquet scans across file boundaries --- datafusion/core/benches/sql_query_with_io.rs | 52 +++++++++-- .../tests/datasource/object_store_access.rs | 86 +++++++++++++++++++ 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rs index 88aa4d3c67612..c57fc7780c82c 100644 --- a/datafusion/core/benches/sql_query_with_io.rs +++ b/datafusion/core/benches/sql_query_with_io.rs @@ -16,6 +16,7 @@ use datafusion::{ prelude::{SessionConfig, SessionContext}, }; use datafusion_execution::runtime_env::RuntimeEnv; +use datafusion_physical_plan::{collect, execution_plan::reset_plan_states}; use itertools::Itertools; use object_store::{ ObjectStore, ObjectStoreExt, @@ -67,7 +68,7 @@ fn create_parquet_file(rng: &mut StdRng, id_offset: usize) -> Bytes { let mut data = Vec::::new(); let mut writer = ArrowWriter::try_new(&mut data, schema, Default::default()).unwrap(); writer.write(&batch).unwrap(); - writer.close().unwrap(); + assert_eq!(writer.close().unwrap().num_row_groups(), 1); data.into() } @@ -101,15 +102,19 @@ where Arc::new(ThrottledStore::new(object_store, config)) } -async fn setup_context(object_store: Arc) -> SessionContext { - let config = SessionConfig::new().with_target_partitions(THREADS); +async fn setup_context( + object_store: Arc, + config: SessionConfig, +) -> SessionContext { let rt = Arc::new(RuntimeEnv::default()); rt.register_object_store(&Url::parse("data://my_store").unwrap(), object_store); let context = SessionContext::new_with_config_rt(config, rt); for table_id in 0..TABLES { let table_name = table_name(table_id); - let file_format = ParquetFormat::default().with_enable_pruning(true); + let file_format = ParquetFormat::default() + .with_options(context.copied_table_options().parquet) + .with_enable_pruning(true); let options = ListingOptions::new(Arc::new(file_format)) .with_table_partition_cols(vec![( String::from("partition"), @@ -167,7 +172,8 @@ fn bench_query( fn criterion_benchmark(c: &mut Criterion) { env_logger::try_init().ok(); - let store = make_store_slow(InMemory::new()); + let store: Arc = Arc::new(InMemory::new()); + let slow_store = make_store_slow(Arc::clone(&store)); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(THREADS) .enable_time() @@ -175,7 +181,10 @@ fn criterion_benchmark(c: &mut Criterion) { .unwrap(); rt.block_on(setup_files(Arc::clone(&store))); - let ctx = rt.block_on(setup_context(store)); + let ctx = rt.block_on(setup_context( + Arc::clone(&slow_store), + SessionConfig::new().with_target_partitions(THREADS), + )); let table0_name = table_name(0); let select_query = format!("SELECT * FROM {table0_name}"); @@ -244,6 +253,37 @@ fn criterion_benchmark(c: &mut Criterion) { &format!("{join_query} WHERE {table0_name}.partition = 0"), PARTITION_FILES * FILE_ROWS, ); + + // Each file contains one row group. Using one scan partition makes crossing + // a file boundary the only opportunity to read ahead into another group. + let mut group = c.benchmark_group("parquet_cross_file_projection"); + group.sample_size(10); + group.sampling_mode(SamplingMode::Flat); + group.measurement_time(Duration::from_secs(25)); + for (name, store) in [("in_memory", store), ("100ms_get", slow_store)] { + let config = SessionConfig::from_env().unwrap().with_target_partitions(1); + let ctx = rt.block_on(setup_context(store, config)); + let plan = rt.block_on(async { + ctx.sql("SELECT sum(id) FROM table_0") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap() + }); + let rows_per_partition = (PARTITION_FILES * FILE_ROWS) as u64; + let expected = + TABLE_PARTITIONS as u64 * rows_per_partition * (rows_per_partition - 1) / 2; + let scan = || async { + let plan = reset_plan_states(Arc::clone(&plan)).unwrap(); + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let sum = + datafusion_common::cast::as_uint64_array(batches[0].column(0)).unwrap(); + assert_eq!(sum.value(0), expected); + }; + group.bench_function(name, |b| b.to_async(&rt).iter(scan)); + } + group.finish(); } criterion_group!(benches, criterion_benchmark); diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs index e44e14df213e8..d50806ee6d984 100644 --- a/datafusion/core/tests/datasource/object_store_access.rs +++ b/datafusion/core/tests/datasource/object_store_access.rs @@ -958,6 +958,92 @@ async fn query_single_parquet_file() { ); } +#[tokio::test] +async fn projected_parquet_ranges_across_files() -> datafusion_common::Result<()> { + // Keep both files in one scan partition, and read just the footer when + // looking up metadata so those requests are distinct from column chunks. + let mut test = Test::new().with_parquet_metadata_size_hint(Some(8)); + test.query("SET datafusion.execution.target_partitions = 1") + .await; + // Let the scan enforce LIMIT. Exact row counts would let the planner + // discard the second file before the scan starts. + test.query("SET datafusion.execution.collect_statistics = false") + .await; + let mut selected_chunks = vec![]; + for (path, value) in [("data/a.parquet", 1), ("data/b.parquet", 3)] { + let unused: ArrayRef = Arc::new(Int32Array::from(vec![0, 0])); + let selected: ArrayRef = Arc::new(Int32Array::from(vec![value, value + 1])); + let batch = + RecordBatch::try_from_iter([("unused", unused), ("selected", selected)])?; + let mut bytes = vec![]; + // A single data page gives the projected column a single payload range. + let props = parquet::file::properties::WriterProperties::builder() + .set_dictionary_enabled(false) + .build(); + let mut writer = parquet::arrow::ArrowWriter::try_new( + &mut bytes, + batch.schema(), + Some(props), + )?; + writer.write(&batch)?; + let metadata = writer.close()?; + let (start, len) = metadata.row_group(0).column(1).byte_range(); + selected_chunks.push((Path::from(path), bytes.len() as u64, start..start + len)); + test = test.with_bytes(path, bytes).await; + } + let test = test.register_parquet("t", "/data/").await; + + for (query, files_to_read) in [ + ("SELECT selected FROM t", 2), + ("SELECT selected FROM t LIMIT 1", 1), + ] { + test.object_store.clear_requests(); + let batches = test.session_context.sql(query).await?.collect().await?; + let mut values = vec![]; + for batch in &batches { + values.extend_from_slice( + datafusion_common::cast::as_int32_array(batch.column(0))?.values(), + ); + } + values.sort_unstable(); + if files_to_read == 2 { + assert_eq!(values, [1, 2, 3, 4]); + } else { + assert!(matches!(values.as_slice(), [1] | [3])); + } + + let requests = test.object_store.recorded_requests(); + let mut files_read = 0; + for (file, size, selected) in &selected_chunks { + let mut data_ranges = vec![]; + for request in &requests { + match request { + RequestDetails::GetRanges { path, ranges } if path == file => { + data_ranges.extend_from_slice(ranges); + } + RequestDetails::GetOpts { path, get_options } + if path == file && !get_options.head => + { + data_ranges.push(match &get_options.range { + Some(range) => range.as_range(*size).unwrap(), + None => 0..*size, + }); + } + _ => {} + } + } + // Ignore footer reads. The selected column is the last data chunk. + data_ranges.retain(|range| range.start < selected.end); + if !data_ranges.is_empty() { + assert_eq!(data_ranges, std::slice::from_ref(selected), "{file}"); + files_read += 1; + } + } + assert_eq!(files_read, files_to_read); + } + Ok(()) +} + #[tokio::test] async fn query_single_parquet_file_with_single_predicate() { let test = Test::new().with_single_file_parquet().await;