Skip to content
1 change: 1 addition & 0 deletions datafusion/datasource-parquet/src/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ mod metrics;
mod nested_schema_pruning;
mod opener;
mod page_filter;
pub mod prefetch;
mod projection_read_plan;
mod pruning;
mod push_decoder;
Expand Down
34 changes: 33 additions & 1 deletion datafusion/datasource-parquet/src/opener/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1928,6 +1928,7 @@ impl RowGroupsPrunedParquetOpen {
.file_metrics
.row_groups_pruned_dynamic_filter
.clone();
let prepared_memory_pool = prepared.memory_pool.clone();
let read_ahead = read_ahead_bytes.map(|window| {
let memory =
ReadAheadMemory::new(prepared.memory_pool, prepared.partition_index);
Expand All @@ -1938,11 +1939,42 @@ impl RowGroupsPrunedParquetOpen {
);
ReadAhead::new(window, decoder.scan_plan(), memory, metrics)
});
// PROTOTYPE: prefetch this file's scan plan in the background.
let reader: Box<dyn AsyncFileReader> = match (
std::env::var("DF_PREFETCH").is_ok(),
prepared_memory_pool.clone(),
) {
(true, Some(pool)) => {
let config = crate::prefetch::PrefetchConfig::from_env();
match crate::prefetch::PrefetchReader::spawn(
prepared.async_file_reader,
(0..config.concurrency)
.map(|_| {
prepared.parquet_file_reader_factory.create_reader(
prepared.partition_index,
prepared.partitioned_file.clone(),
prepared.metadata_size_hint,
&prepared.metrics,
)
})
.map(|r| r.map(|r| r as Box<dyn AsyncFileReader>))
.collect::<Result<Vec<_>>>()?,
decoder.scan_plan(),
pool,
prepared.partition_index,
config,
) {
Ok(r) => Box::new(r),
Err(inner) => inner,
}
}
_ => prepared.async_file_reader,
};
let stream = PushDecoderStreamState {
decoder: Some(decoder),
active_reader: None,
rg_plan,
reader: Some(prepared.async_file_reader),
reader: Some(reader),
decoder_projection,
arrow_reader_metrics,
predicate_cache_inner_records,
Expand Down
Loading
Loading