From 7cd1f3d5b4189b36178e066494c0e10999d71b67 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:22:05 -0500 Subject: [PATCH 1/9] PROTOTYPE: background scan-plan prefetch and file lookahead --- datafusion/datasource-parquet/src/mod.rs | 1 + .../datasource-parquet/src/opener/mod.rs | 22 +- datafusion/datasource-parquet/src/prefetch.rs | 283 ++++++++++++++++++ .../datasource/src/file_stream/scan_state.rs | 79 ++++- 4 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 datafusion/datasource-parquet/src/prefetch.rs diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs index 80d2205a1fab0..7fb4167794554 100644 --- a/datafusion/datasource-parquet/src/mod.rs +++ b/datafusion/datasource-parquet/src/mod.rs @@ -35,6 +35,7 @@ mod opener; mod page_filter; mod projection_read_plan; mod pruning; +mod prefetch; mod push_decoder; mod reader; mod row_filter; diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index 6cfafad459d31..3127a7e754616 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1928,6 +1928,7 @@ impl RowGroupsPrunedParquetOpen { .file_metrics .row_groups_pruned_dynamic_filter .clone(); + let prepared_memory_pool = prepared.memory_pool.clone(); let read_ahead = read_ahead_bytes.map(|window| { let memory = ReadAheadMemory::new(prepared.memory_pool, prepared.partition_index); @@ -1938,11 +1939,30 @@ impl RowGroupsPrunedParquetOpen { ); ReadAhead::new(window, decoder.scan_plan(), memory, metrics) }); + // PROTOTYPE: prefetch this file's scan plan in the background. + let reader: Box = match ( + std::env::var("DF_PREFETCH").is_ok(), + prepared_memory_pool.clone(), + ) { + (true, Some(pool)) => Box::new(crate::prefetch::PrefetchReader::spawn( + prepared.async_file_reader, + prepared.parquet_file_reader_factory.create_reader( + prepared.partition_index, + prepared.partitioned_file.clone(), + prepared.metadata_size_hint, + &prepared.metrics, + )?, + decoder.scan_plan(), + pool, + prepared.partition_index, + )), + _ => prepared.async_file_reader, + }; let stream = PushDecoderStreamState { decoder: Some(decoder), active_reader: None, rg_plan, - reader: Some(prepared.async_file_reader), + reader: Some(reader), decoder_projection, arrow_reader_metrics, predicate_cache_inner_records, diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs new file mode 100644 index 0000000000000..556045d5dda0c --- /dev/null +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -0,0 +1,283 @@ +// PROTOTYPE: background prefetch of a file's scan plan into the memory pool. +// +// A spawned task fetches the decoder's `scan_plan()` ranges in decode order, +// independent of whether the stream is being polled. The decoder's reader +// serves requests from the prefetched bytes when they cover the request. +// The task reserves every byte it holds from the memory pool, and stops +// fetching while the pool is above `DF_PREFETCH_FRACTION` of its limit, so +// other operators keep headroom. + +use std::collections::BTreeMap; +use std::ops::Range; +use std::sync::{Arc, Mutex, Weak}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +static HIT_BYTES: AtomicU64 = AtomicU64::new(0); +static MISS_BYTES: AtomicU64 = AtomicU64::new(0); +static WAIT_US: AtomicU64 = AtomicU64::new(0); + +use bytes::Bytes; +use datafusion_execution::memory_pool::{ + MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation, +}; +use futures::future::BoxFuture; +use futures::FutureExt; +use parquet::arrow::async_reader::AsyncFileReader; +use parquet::arrow::push_decoder::ScanPlan; +use parquet::errors::Result as ParquetResult; +use parquet::file::metadata::ParquetMetaData; +use tokio::sync::Notify; + +pub(crate) fn env_f64(name: &str) -> Option { + std::env::var(name).ok().and_then(|v| v.parse().ok()) +} + +pub(crate) fn env_usize(name: &str) -> Option { + std::env::var(name).ok().and_then(|v| v.parse().ok()) +} + +enum Entry { + InFlight, + Ready { bytes: Bytes, consumed: u64 }, +} + +struct Shared { + /// start -> (end, entry) + entries: Mutex>, + reservation: Mutex, + notify: Notify, +} + +impl Shared { + /// Find an entry that fully covers `range`. + fn covering(&self, range: &Range) -> Option { + let entries = self.entries.lock().unwrap(); + let (start, (end, _)) = entries.range(..=range.start).next_back()?; + (*end >= range.end).then_some(*start) + } +} + +pub(crate) struct PrefetchReader { + inner: Box, + shared: Arc, +} + +impl PrefetchReader { + /// Spawn a prefetch task for `plan`, using `prefetch_reader` for its I/O, + /// and wrap `inner` so it reads from the prefetched bytes. + pub(crate) fn spawn( + inner: Box, + prefetch_reader: Box, + plan: ScanPlan, + pool: Arc, + partition: usize, + ) -> Self { + let reservation = MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")) + .register(&pool); + let shared = Arc::new(Shared { + entries: Mutex::new(BTreeMap::new()), + reservation: Mutex::new(reservation), + notify: Notify::new(), + }); + let fraction = env_f64("DF_PREFETCH_FRACTION").unwrap_or(0.6); + let concurrency = env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4); + let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; + tokio::spawn(prefetch_task( + Arc::downgrade(&shared), + prefetch_reader, + coalesce(plan, chunk), + pool, + fraction, + concurrency, + )); + Self { inner, shared } + } +} + +impl Drop for PrefetchReader { + fn drop(&mut self) { + // The task holds only a `Weak`. Waking it lets it observe the drop. + self.shared.notify.notify_waiters(); + if std::env::var("DF_PREFETCH_STATS").is_ok() { + eprintln!( + "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={}", + HIT_BYTES.load(Ordering::Relaxed) >> 20, + MISS_BYTES.load(Ordering::Relaxed) >> 20, + WAIT_US.load(Ordering::Relaxed) / 1000 + ); + } + } +} + +/// Merge adjacent plan ranges into fetches of at most `chunk` bytes. +fn coalesce(plan: ScanPlan, chunk: u64) -> Vec> { + let mut out: Vec> = Vec::new(); + for planned in plan { + let r = planned.range; + if let Some(last) = out.last_mut() { + if r.start == last.end && last.end - last.start + (r.end - r.start) <= chunk { + last.end = r.end; + continue; + } + if r.start >= last.start && r.end <= last.end { + continue; + } + } + out.push(r); + } + out +} + +fn pool_has_room(pool: &dyn MemoryPool, fraction: f64, add: usize) -> bool { + match pool.memory_limit() { + MemoryLimit::Finite(limit) => { + (pool.reserved() + add) as f64 <= limit as f64 * fraction + } + _ => true, + } +} + +async fn prefetch_task( + shared: Weak, + mut reader: Box, + ranges: Vec>, + pool: Arc, + fraction: f64, + concurrency: usize, +) { + // One reader, so fetches run one at a time on it. Run `concurrency` + // requests by batching them into one `get_byte_ranges` call. + let mut next = 0; + while next < ranges.len() { + let Some(s) = shared.upgrade() else { return }; + let mut batch = Vec::new(); + let mut bytes = 0usize; + while next < ranges.len() && batch.len() < concurrency { + let r = ranges[next].clone(); + let len = (r.end - r.start) as usize; + if !pool_has_room(pool.as_ref(), fraction, bytes + len) { + break; + } + bytes += len; + batch.push(r); + next += 1; + } + if batch.is_empty() { + // No room. Wait for consumption, or poll the pool again soon, + // because other partitions also free memory. + let _ = tokio::time::timeout(Duration::from_millis(5), s.notify.notified()) + .await; + continue; + } + s.reservation.lock().unwrap().grow(bytes); + { + let mut entries = s.entries.lock().unwrap(); + for r in &batch { + entries.insert(r.start, (r.end, Entry::InFlight)); + } + } + drop(s); + let result = reader.get_byte_ranges(batch.clone()).await; + let Some(s) = shared.upgrade() else { return }; + let mut entries = s.entries.lock().unwrap(); + match result { + Ok(data) => { + for (r, b) in batch.iter().zip(data) { + entries.insert(r.start, (r.end, Entry::Ready { bytes: b, consumed: 0 })); + } + } + Err(_) => { + // Let the decoder fetch these itself. + for r in &batch { + entries.remove(&r.start); + } + s.reservation.lock().unwrap().shrink(bytes); + } + } + drop(entries); + s.notify.notify_waiters(); + } +} + +impl PrefetchReader { + async fn read(&mut self, range: Range) -> ParquetResult { + loop { + let Some(start) = self.shared.covering(&range) else { + MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + return self.inner.get_bytes(range).await; + }; + let notified = self.shared.notify.notified(); + { + let mut entries = self.shared.entries.lock().unwrap(); + match entries.get_mut(&start) { + Some((end, Entry::Ready { bytes, consumed })) => { + let off = (range.start - start) as usize; + let out = bytes.slice(off..off + (range.end - range.start) as usize); + *consumed += range.end - range.start; + HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + if *consumed >= *end - start { + let len = (*end - start) as usize; + entries.remove(&start); + self.shared.reservation.lock().unwrap().shrink(len); + self.shared.notify.notify_waiters(); + } + return Ok(out); + } + Some((_, Entry::InFlight)) => {} + None => continue, + } + } + let t = std::time::Instant::now(); + notified.await; + WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); + } + } +} + +impl AsyncFileReader for PrefetchReader { + fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, ParquetResult> { + self.read(range).boxed() + } + + fn get_byte_ranges( + &mut self, + ranges: Vec>, + ) -> BoxFuture<'_, ParquetResult>> { + async move { + let mut out = Vec::with_capacity(ranges.len()); + let mut misses = Vec::new(); + for (i, r) in ranges.iter().enumerate() { + if self.shared.covering(r).is_some() { + out.push(Some(self.read(r.clone()).await?)); + } else { + out.push(None); + misses.push(i); + } + } + if !misses.is_empty() { + let fetched = self + .inner + .get_byte_ranges(misses.iter().map(|i| ranges[*i].clone()).collect()) + .await?; + MISS_BYTES.fetch_add( + misses.iter().map(|i| ranges[*i].end - ranges[*i].start).sum(), + Ordering::Relaxed, + ); + for (i, b) in misses.into_iter().zip(fetched) { + out[i] = Some(b); + } + } + Ok(out.into_iter().map(Option::unwrap).collect()) + } + .boxed() + } + + fn get_metadata<'a>( + &'a mut self, + options: Option<&'a parquet::arrow::arrow_reader::ArrowReaderOptions>, + ) -> BoxFuture<'a, ParquetResult>> { + self.inner.get_metadata(options) + } +} + diff --git a/datafusion/datasource/src/file_stream/scan_state.rs b/datafusion/datasource/src/file_stream/scan_state.rs index d1e57604f971c..1ef90bd74887c 100644 --- a/datafusion/datasource/src/file_stream/scan_state.rs +++ b/datafusion/datasource/src/file_stream/scan_state.rs @@ -81,6 +81,10 @@ pub(super) struct ScanState { /// Once the I/O completes, yields the next planner and is pushed back /// onto `ready_planners`. pending_planner: Option, + /// PROTOTYPE: extra planners blocked on I/O, for files opened ahead. + pending_ahead: Vec, + /// PROTOTYPE: files to open ahead of the active reader (`DF_FILE_LOOKAHEAD`). + lookahead: usize, /// Metrics for the active scan queues. metrics: FileStreamMetrics, } @@ -102,6 +106,11 @@ impl ScanState { ready_morsels: Default::default(), reader: None, pending_planner: None, + pending_ahead: Vec::new(), + lookahead: std::env::var("DF_FILE_LOOKAHEAD") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0), metrics, } } @@ -123,8 +132,8 @@ impl ScanState { /// The return [`ScanAndReturn`] tells `poll_inner` how to update the /// outer `FileStreamState`. pub(super) fn poll_scan(&mut self, cx: &mut Context<'_>) -> ScanAndReturn { - let _processing_timer: ScopedTimerGuard<'_> = - self.metrics.time_processing.timer(); + let time_processing = self.metrics.time_processing.clone(); + let _processing_timer: ScopedTimerGuard<'_> = time_processing.timer(); // Try and resolve outstanding IO first. If it is still pending, check // the current reader or ready morsels before yielding. New planning @@ -154,6 +163,14 @@ impl ScanState { } } + // PROTOTYPE: open files ahead while a reader is active, so their + // metadata I/O and prefetch overlap the current file's decode. + if self.lookahead > 0 && self.reader.is_some() { + if let Err(err) = self.plan_ahead(cx) { + return ScanAndReturn::Error(err); + } + } + // Next try and get the next batch from the active reader, if any. if let Some(reader) = self.reader.as_mut() { match reader.poll_next_unpin(cx) { @@ -227,6 +244,15 @@ impl ScanState { if self.pending_planner.is_some() { return ScanAndReturn::Return(Poll::Pending); } + if !self.pending_ahead.is_empty() && self.ready_planners.is_empty() { + if let Err(err) = self.plan_ahead(cx) { + return ScanAndReturn::Error(err); + } + if self.ready_planners.is_empty() && self.ready_morsels.is_empty() { + return ScanAndReturn::Return(Poll::Pending); + } + return ScanAndReturn::Continue; + } // No reader or morsel, so try to produce more work via CPU planning. if let Some(planner) = self.ready_planners.pop_front() { @@ -290,6 +316,55 @@ impl ScanState { } } +impl ScanState { + /// PROTOTYPE: poll the look-ahead I/O, plan ready files, and open new + /// files until `lookahead` files are queued behind the active reader. + /// Errors are returned regardless of `on_error`. + fn plan_ahead(&mut self, cx: &mut Context<'_>) -> Result<()> { + let mut i = 0; + while i < self.pending_ahead.len() { + match self.pending_ahead[i].poll_unpin(cx) { + Poll::Pending => i += 1, + Poll::Ready(r) => { + self.pending_ahead.swap_remove(i); + self.ready_planners.push_back(r?); + } + } + } + while let Some(planner) = self.ready_planners.pop_front() { + if let Some(mut plan) = planner.plan()? { + self.ready_morsels.extend(plan.take_morsels()); + self.ready_planners.extend(plan.take_ready_planners()); + if let Some(mut p) = plan.take_pending_planner() { + // Poll once so its waker is registered. + match p.poll_unpin(cx) { + Poll::Pending => self.pending_ahead.push(p), + Poll::Ready(r) => self.ready_planners.push_back(r?), + } + } + } else { + self.metrics.files_processed.add(1); + } + } + while self.ready_morsels.len() + self.pending_ahead.len() < self.lookahead { + let Some(file) = self.work_source.pop_front() else { break }; + self.metrics.files_opened.add(1); + let planner = self.morselizer.plan_file(file)?; + if let Some(mut plan) = planner.plan()? { + self.ready_morsels.extend(plan.take_morsels()); + self.ready_planners.extend(plan.take_ready_planners()); + if let Some(mut p) = plan.take_pending_planner() { + match p.poll_unpin(cx) { + Poll::Pending => self.pending_ahead.push(p), + Poll::Ready(r) => self.ready_planners.push_back(r?), + } + } + } + } + Ok(()) + } +} + /// What should be done on the next iteration of [`ScanState::poll_scan`]? pub(super) enum ScanAndReturn { /// Poll again. From c540892f3e6c6e8f2a3e170cdc80122c37f443eb Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:59:14 -0500 Subject: [PATCH 2/9] PROTOTYPE: streamed prefetch workers, spanning reads, active-file priority --- datafusion/datasource-parquet/src/mod.rs | 2 +- .../datasource-parquet/src/opener/mod.rs | 17 +- datafusion/datasource-parquet/src/prefetch.rs | 303 +++++++++++------- 3 files changed, 201 insertions(+), 121 deletions(-) diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs index 7fb4167794554..2bf064fb0fb98 100644 --- a/datafusion/datasource-parquet/src/mod.rs +++ b/datafusion/datasource-parquet/src/mod.rs @@ -33,9 +33,9 @@ mod metrics; mod nested_schema_pruning; mod opener; mod page_filter; +mod prefetch; mod projection_read_plan; mod pruning; -mod prefetch; mod push_decoder; mod reader; mod row_filter; diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index 3127a7e754616..11ed27c8130cd 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1946,12 +1946,17 @@ impl RowGroupsPrunedParquetOpen { ) { (true, Some(pool)) => Box::new(crate::prefetch::PrefetchReader::spawn( prepared.async_file_reader, - prepared.parquet_file_reader_factory.create_reader( - prepared.partition_index, - prepared.partitioned_file.clone(), - prepared.metadata_size_hint, - &prepared.metrics, - )?, + (0..crate::prefetch::env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4)) + .map(|_| { + prepared.parquet_file_reader_factory.create_reader( + prepared.partition_index, + prepared.partitioned_file.clone(), + prepared.metadata_size_hint, + &prepared.metrics, + ) + }) + .map(|r| r.map(|r| r as Box)) + .collect::>>()?, decoder.scan_plan(), pool, prepared.partition_index, diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index 556045d5dda0c..48a78e5b07a98 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -9,8 +9,8 @@ use std::collections::BTreeMap; use std::ops::Range; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, Weak}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; static HIT_BYTES: AtomicU64 = AtomicU64::new(0); @@ -21,8 +21,8 @@ use bytes::Bytes; use datafusion_execution::memory_pool::{ MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation, }; -use futures::future::BoxFuture; use futures::FutureExt; +use futures::future::BoxFuture; use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::push_decoder::ScanPlan; use parquet::errors::Result as ParquetResult; @@ -49,55 +49,84 @@ struct Shared { notify: Notify, } -impl Shared { - /// Find an entry that fully covers `range`. - fn covering(&self, range: &Range) -> Option { - let entries = self.entries.lock().unwrap(); - let (start, (end, _)) = entries.range(..=range.start).next_back()?; - (*end >= range.end).then_some(*start) - } -} - pub(crate) struct PrefetchReader { inner: Box, shared: Arc, + partition: usize, + seq: u64, +} + +/// Per partition: file sequence number -> whether that file has issued all +/// its fetches. A file fetches only after every earlier open file of the +/// same partition has issued all its fetches, so the file being decoded +/// gets I/O and memory before look-ahead files. +static PRIORITY: Mutex> = Mutex::new(BTreeMap::new()); +static SEQ: AtomicU64 = AtomicU64::new(0); + +fn may_fetch(partition: usize, seq: u64) -> bool { + PRIORITY + .lock() + .unwrap() + .range((partition, 0)..(partition, seq)) + .all(|(_, done)| *done) +} + +fn set_done(partition: usize, seq: u64) { + if let Some(d) = PRIORITY.lock().unwrap().get_mut(&(partition, seq)) { + *d = true; + } } impl PrefetchReader { - /// Spawn a prefetch task for `plan`, using `prefetch_reader` for its I/O, - /// and wrap `inner` so it reads from the prefetched bytes. + /// Spawn one prefetch worker per reader in `prefetch_readers` for + /// `plan`, and wrap `inner` so it reads from the prefetched bytes. pub(crate) fn spawn( inner: Box, - prefetch_reader: Box, + prefetch_readers: Vec>, plan: ScanPlan, pool: Arc, partition: usize, ) -> Self { - let reservation = MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")) - .register(&pool); + let reservation = + MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")).register(&pool); let shared = Arc::new(Shared { entries: Mutex::new(BTreeMap::new()), reservation: Mutex::new(reservation), notify: Notify::new(), }); let fraction = env_f64("DF_PREFETCH_FRACTION").unwrap_or(0.6); - let concurrency = env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4); let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; - tokio::spawn(prefetch_task( - Arc::downgrade(&shared), - prefetch_reader, - coalesce(plan, chunk), - pool, - fraction, - concurrency, - )); - Self { inner, shared } + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + PRIORITY.lock().unwrap().insert((partition, seq), false); + let ranges = Arc::new(coalesce(plan, chunk)); + let next = Arc::new(AtomicUsize::new(0)); + let live = Arc::new(AtomicUsize::new(prefetch_readers.len())); + for reader in prefetch_readers { + tokio::spawn(prefetch_worker( + Arc::downgrade(&shared), + reader, + Arc::clone(&ranges), + Arc::clone(&next), + Arc::clone(&live), + Arc::clone(&pool), + fraction, + partition, + seq, + )); + } + Self { + inner, + shared, + partition, + seq, + } } } impl Drop for PrefetchReader { fn drop(&mut self) { - // The task holds only a `Weak`. Waking it lets it observe the drop. + PRIORITY.lock().unwrap().remove(&(self.partition, self.seq)); + // The workers hold only a `Weak`. Waking them lets them observe the drop. self.shared.notify.notify_waiters(); if std::env::var("DF_PREFETCH_STATS").is_ok() { eprintln!( @@ -138,99 +167,165 @@ fn pool_has_room(pool: &dyn MemoryPool, fraction: f64, add: usize) -> bool { } } -async fn prefetch_task( +/// One worker keeps one request in flight. Each worker takes the next range +/// in decode order as soon as its previous request completes, so a slow +/// range does not hold back the others. +#[allow(clippy::too_many_arguments)] +async fn prefetch_worker( shared: Weak, mut reader: Box, - ranges: Vec>, + ranges: Arc>>, + next: Arc, + live: Arc, pool: Arc, fraction: f64, - concurrency: usize, + partition: usize, + seq: u64, ) { - // One reader, so fetches run one at a time on it. Run `concurrency` - // requests by batching them into one `get_byte_ranges` call. - let mut next = 0; - while next < ranges.len() { + loop { let Some(s) = shared.upgrade() else { return }; - let mut batch = Vec::new(); - let mut bytes = 0usize; - while next < ranges.len() && batch.len() < concurrency { - let r = ranges[next].clone(); - let len = (r.end - r.start) as usize; - if !pool_has_room(pool.as_ref(), fraction, bytes + len) { - break; + let i = next.load(Ordering::Relaxed); + if i >= ranges.len() { + if live.fetch_sub(1, Ordering::Relaxed) == 1 { + set_done(partition, seq); } - bytes += len; - batch.push(r); - next += 1; + return; } - if batch.is_empty() { - // No room. Wait for consumption, or poll the pool again soon, - // because other partitions also free memory. - let _ = tokio::time::timeout(Duration::from_millis(5), s.notify.notified()) - .await; + let r = ranges[i].clone(); + let len = (r.end - r.start) as usize; + if !may_fetch(partition, seq) || !pool_has_room(pool.as_ref(), fraction, len) { + // Wait for consumption, or poll again soon, because other + // partitions and earlier files also free memory and I/O. + let _ = + tokio::time::timeout(Duration::from_millis(2), s.notify.notified()).await; continue; } - s.reservation.lock().unwrap().grow(bytes); + if next + .compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed) + .is_err() { - let mut entries = s.entries.lock().unwrap(); - for r in &batch { - entries.insert(r.start, (r.end, Entry::InFlight)); - } + continue; } + if i + 1 == ranges.len() { + set_done(partition, seq); + } + s.reservation.lock().unwrap().grow(len); + s.entries + .lock() + .unwrap() + .insert(r.start, (r.end, Entry::InFlight)); drop(s); - let result = reader.get_byte_ranges(batch.clone()).await; + let result = reader.get_bytes(r.clone()).await; let Some(s) = shared.upgrade() else { return }; - let mut entries = s.entries.lock().unwrap(); - match result { - Ok(data) => { - for (r, b) in batch.iter().zip(data) { - entries.insert(r.start, (r.end, Entry::Ready { bytes: b, consumed: 0 })); + { + let mut entries = s.entries.lock().unwrap(); + match result { + Ok(b) => { + entries.insert( + r.start, + ( + r.end, + Entry::Ready { + bytes: b, + consumed: 0, + }, + ), + ); } - } - Err(_) => { - // Let the decoder fetch these itself. - for r in &batch { + Err(_) => { + // Let the decoder fetch this range itself. entries.remove(&r.start); + s.reservation.lock().unwrap().shrink(len); } - s.reservation.lock().unwrap().shrink(bytes); } } - drop(entries); s.notify.notify_waiters(); } } +enum Lookup { + Miss, + Wait, + Hit(Bytes), +} + +impl Shared { + /// Serve `range` from one or more contiguous entries. + fn lookup(&self, range: &Range) -> Lookup { + let mut entries = self.entries.lock().unwrap(); + let Some((&first, _)) = entries.range(..=range.start).next_back() else { + return Lookup::Miss; + }; + let mut parts = Vec::new(); + let mut pos = range.start; + for (&start, (end, entry)) in entries.range(first..) { + if start > pos { + return Lookup::Miss; + } + if *end <= pos { + continue; + } + match entry { + Entry::InFlight => return Lookup::Wait, + Entry::Ready { .. } => parts.push(start), + } + pos = *end; + if pos >= range.end { + break; + } + } + if pos < range.end { + return Lookup::Miss; + } + let mut out = bytes::BytesMut::new(); + let mut single = None; + let mut freed = 0; + for start in &parts { + let (end, entry) = entries.get_mut(start).unwrap(); + let Entry::Ready { bytes, consumed } = entry else { + unreachable!() + }; + let lo = range.start.max(*start); + let hi = range.end.min(*end); + let piece = bytes.slice((lo - start) as usize..(hi - start) as usize); + if parts.len() == 1 { + single = Some(piece); + } else { + out.extend_from_slice(&piece); + } + *consumed += hi - lo; + if *consumed >= *end - start { + freed += (*end - start) as usize; + entries.remove(start); + } + } + if freed > 0 { + self.reservation.lock().unwrap().shrink(freed); + self.notify.notify_waiters(); + } + Lookup::Hit(single.unwrap_or_else(|| out.freeze())) + } +} + impl PrefetchReader { async fn read(&mut self, range: Range) -> ParquetResult { loop { - let Some(start) = self.shared.covering(&range) else { - MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); - return self.inner.get_bytes(range).await; - }; let notified = self.shared.notify.notified(); - { - let mut entries = self.shared.entries.lock().unwrap(); - match entries.get_mut(&start) { - Some((end, Entry::Ready { bytes, consumed })) => { - let off = (range.start - start) as usize; - let out = bytes.slice(off..off + (range.end - range.start) as usize); - *consumed += range.end - range.start; - HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); - if *consumed >= *end - start { - let len = (*end - start) as usize; - entries.remove(&start); - self.shared.reservation.lock().unwrap().shrink(len); - self.shared.notify.notify_waiters(); - } - return Ok(out); - } - Some((_, Entry::InFlight)) => {} - None => continue, + match self.shared.lookup(&range) { + Lookup::Hit(b) => { + HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + return Ok(b); + } + Lookup::Miss => { + MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + return self.inner.get_bytes(range).await; + } + Lookup::Wait => { + let t = std::time::Instant::now(); + notified.await; + WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); } } - let t = std::time::Instant::now(); - notified.await; - WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); } } } @@ -246,29 +341,10 @@ impl AsyncFileReader for PrefetchReader { ) -> BoxFuture<'_, ParquetResult>> { async move { let mut out = Vec::with_capacity(ranges.len()); - let mut misses = Vec::new(); - for (i, r) in ranges.iter().enumerate() { - if self.shared.covering(r).is_some() { - out.push(Some(self.read(r.clone()).await?)); - } else { - out.push(None); - misses.push(i); - } + for r in ranges { + out.push(self.read(r).await?); } - if !misses.is_empty() { - let fetched = self - .inner - .get_byte_ranges(misses.iter().map(|i| ranges[*i].clone()).collect()) - .await?; - MISS_BYTES.fetch_add( - misses.iter().map(|i| ranges[*i].end - ranges[*i].start).sum(), - Ordering::Relaxed, - ); - for (i, b) in misses.into_iter().zip(fetched) { - out[i] = Some(b); - } - } - Ok(out.into_iter().map(Option::unwrap).collect()) + Ok(out) } .boxed() } @@ -280,4 +356,3 @@ impl AsyncFileReader for PrefetchReader { self.inner.get_metadata(options) } } - From 40160e3027ac9fa000aea50bd81b4d180c4bde41 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:23:45 -0500 Subject: [PATCH 3/9] PROTOTYPE: fallible prefetch, pool reclaim, strict required reservations --- datafusion/datasource-parquet/src/mod.rs | 2 +- datafusion/datasource-parquet/src/prefetch.rs | 79 +++++++++++++++++-- .../datasource-parquet/src/push_decoder.rs | 24 +++++- datafusion/datasource-parquet/src/source.rs | 1 + 4 files changed, 94 insertions(+), 12 deletions(-) diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs index 2bf064fb0fb98..1b08314620f69 100644 --- a/datafusion/datasource-parquet/src/mod.rs +++ b/datafusion/datasource-parquet/src/mod.rs @@ -33,7 +33,7 @@ mod metrics; mod nested_schema_pruning; mod opener; mod page_filter; -mod prefetch; +pub mod prefetch; mod projection_read_plan; mod pruning; mod push_decoder; diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index 48a78e5b07a98..5f6d555a8f5c9 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -16,6 +16,54 @@ use std::time::Duration; static HIT_BYTES: AtomicU64 = AtomicU64::new(0); static MISS_BYTES: AtomicU64 = AtomicU64::new(0); static WAIT_US: AtomicU64 = AtomicU64::new(0); +static EVICT_BYTES: AtomicU64 = AtomicU64::new(0); +static DENIED: AtomicU64 = AtomicU64::new(0); + +/// Every live prefetch buffer, so a memory pool under pressure can reclaim +/// optional bytes. +static REGISTRY: Mutex>> = Mutex::new(Vec::new()); + +/// Drop up to `bytes` of prefetched data that the decoder has not started to +/// read, furthest from the decoder first. Returns the bytes freed. The +/// decoder fetches a dropped range itself when it reaches it. +pub fn reclaim_prefetch(bytes: usize) -> usize { + let shareds: Vec> = { + let Ok(mut reg) = REGISTRY.try_lock() else { return 0 }; + reg.retain(|w| w.strong_count() > 0); + reg.iter().filter_map(Weak::upgrade).collect() + }; + // Gather candidates across all buffers, then evict the furthest ahead of + // each buffer's read position first, round robin by distance. + let mut cands: Vec<(u64, usize, u64)> = Vec::new(); // (distance, shared idx, start) + for (i, s) in shareds.iter().enumerate() { + let Ok(entries) = s.entries.try_lock() else { continue }; + let pos = entries.keys().next().copied().unwrap_or(0); + for (start, (_, e)) in entries.iter() { + if let Entry::Ready { consumed: 0, .. } = e { + cands.push((start - pos, i, *start)); + } + } + } + cands.sort_by(|a, b| b.0.cmp(&a.0)); + let mut freed = 0; + for (_, i, start) in cands { + if freed >= bytes { + break; + } + let s = &shareds[i]; + let Ok(mut entries) = s.entries.try_lock() else { continue }; + if let Some((end, Entry::Ready { consumed: 0, .. })) = entries.get(&start) { + let len = (*end - start) as usize; + entries.remove(&start); + drop(entries); + s.reservation.shrink(len); + freed += len; + } + } + EVICT_BYTES.fetch_add(freed as u64, Ordering::Relaxed); + freed +} + use bytes::Bytes; use datafusion_execution::memory_pool::{ @@ -45,7 +93,7 @@ enum Entry { struct Shared { /// start -> (end, entry) entries: Mutex>, - reservation: Mutex, + reservation: MemoryReservation, notify: Notify, } @@ -91,11 +139,13 @@ impl PrefetchReader { MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")).register(&pool); let shared = Arc::new(Shared { entries: Mutex::new(BTreeMap::new()), - reservation: Mutex::new(reservation), + reservation, notify: Notify::new(), }); let fraction = env_f64("DF_PREFETCH_FRACTION").unwrap_or(0.6); + let fallible = std::env::var("DF_PREFETCH_TRY").is_ok(); let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; + REGISTRY.lock().unwrap().push(Arc::downgrade(&shared)); let seq = SEQ.fetch_add(1, Ordering::Relaxed); PRIORITY.lock().unwrap().insert((partition, seq), false); let ranges = Arc::new(coalesce(plan, chunk)); @@ -110,6 +160,7 @@ impl PrefetchReader { Arc::clone(&live), Arc::clone(&pool), fraction, + fallible, partition, seq, )); @@ -130,10 +181,12 @@ impl Drop for PrefetchReader { self.shared.notify.notify_waiters(); if std::env::var("DF_PREFETCH_STATS").is_ok() { eprintln!( - "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={}", + "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={} evict_mib={} denied={}", HIT_BYTES.load(Ordering::Relaxed) >> 20, MISS_BYTES.load(Ordering::Relaxed) >> 20, - WAIT_US.load(Ordering::Relaxed) / 1000 + WAIT_US.load(Ordering::Relaxed) / 1000, + EVICT_BYTES.load(Ordering::Relaxed) >> 20, + DENIED.load(Ordering::Relaxed) ); } } @@ -179,6 +232,7 @@ async fn prefetch_worker( live: Arc, pool: Arc, fraction: f64, + fallible: bool, partition: usize, seq: u64, ) { @@ -200,16 +254,27 @@ async fn prefetch_worker( tokio::time::timeout(Duration::from_millis(2), s.notify.notified()).await; continue; } + if fallible { + // Optional bytes: take only what the pool grants now. + if s.reservation.try_grow(len).is_err() { + DENIED.fetch_add(1, Ordering::Relaxed); + let _ = tokio::time::timeout(Duration::from_millis(5), s.notify.notified()) + .await; + continue; + } + } else { + s.reservation.grow(len); + } if next .compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed) .is_err() { + s.reservation.shrink(len); continue; } if i + 1 == ranges.len() { set_done(partition, seq); } - s.reservation.lock().unwrap().grow(len); s.entries .lock() .unwrap() @@ -235,7 +300,7 @@ async fn prefetch_worker( Err(_) => { // Let the decoder fetch this range itself. entries.remove(&r.start); - s.reservation.lock().unwrap().shrink(len); + s.reservation.shrink(len); } } } @@ -300,7 +365,7 @@ impl Shared { } } if freed > 0 { - self.reservation.lock().unwrap().shrink(freed); + self.reservation.shrink(freed); self.notify.notify_waiters(); } Lookup::Hit(single.unwrap_or_else(|| out.freeze())) diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs index f84899eba37b7..23caa014a7a26 100644 --- a/datafusion/datasource-parquet/src/push_decoder.rs +++ b/datafusion/datasource-parquet/src/push_decoder.rs @@ -795,8 +795,24 @@ impl ReadAheadMemory { /// Reserve bytes the decoder needs. Never fails and can exceed the pool /// limit: the decoder cannot continue without these bytes. - fn reserve_required(&self, bytes: u64) { - self.reservation.grow(to_usize(bytes)); + fn reserve_required(&self, bytes: u64) -> Result<()> { + let bytes = to_usize(bytes); + // PROTOTYPE: DF_REQUIRED_STRICT orders required reservations: + // try the pool, reclaim optional prefetch bytes, retry, then fail. + if std::env::var("DF_REQUIRED_STRICT").is_err() { + self.reservation.grow(bytes); + return Ok(()); + } + if self.reservation.try_grow(bytes).is_ok() { + return Ok(()); + } + crate::prefetch::reclaim_prefetch(bytes); + self.reservation.try_grow(bytes).map_err(|e| { + e.context(format!( + "required bytes for the next batch of {}", + self.reservation.consumer().name() + )) + }) } /// Bytes the pool can grant now. Unlimited if the pool has no limit or @@ -1164,7 +1180,7 @@ impl PushDecoderStreamState { .buffered_bytes(); let read_ahead = self.read_ahead.as_mut().expect("streaming policy"); let needed_bytes: u64 = needed.iter().map(|r| r.end - r.start).sum(); - read_ahead.memory.reserve_required(needed_bytes); + read_ahead.memory.reserve_required(needed_bytes)?; for range in &needed { read_ahead.fetched.insert((range.start, range.end)); } @@ -1487,7 +1503,7 @@ mod tests { let pool: Arc = Arc::new(GreedyMemoryPool::new(100)); let memory = ReadAheadMemory::new(Some(Arc::clone(&pool)), 0); // Required bytes can exceed the limit; speculative bytes cannot. - memory.reserve_required(150); + memory.reserve_required(150).unwrap(); assert_eq!(pool.reserved(), 150); assert_eq!(memory.available(), 0); assert!(!memory.try_reserve_speculative(1)); diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index c8e643af5a882..2758cc8d4906f 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -726,6 +726,7 @@ impl FileSource for ParquetSource { .table_parquet_options .global .read_ahead_bytes + .or(std::env::var("DF_PREFETCH").ok().map(|_| 0)) .map(|_| Arc::clone(context.memory_pool())); Ok(Box::new(self.build_morselizer( object_store, From 446700d0039e86b5ccfbd98e5264104350b531cb Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:31:56 -0500 Subject: [PATCH 4/9] prefetch headroom --- datafusion/datasource-parquet/src/prefetch.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index 5f6d555a8f5c9..d17537eb9a83b 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -214,7 +214,8 @@ fn coalesce(plan: ScanPlan, chunk: u64) -> Vec> { fn pool_has_room(pool: &dyn MemoryPool, fraction: f64, add: usize) -> bool { match pool.memory_limit() { MemoryLimit::Finite(limit) => { - (pool.reserved() + add) as f64 <= limit as f64 * fraction + let headroom = env_usize("DF_PREFETCH_HEADROOM").unwrap_or(0); + (pool.reserved() + add + headroom) as f64 <= limit as f64 * fraction } _ => true, } From 875d27354538255de28884d1bf193275c5e5a5e3 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:59:36 -0500 Subject: [PATCH 5/9] promote missed ranges to required fetches --- datafusion/datasource-parquet/src/prefetch.rs | 86 ++++++++++++++++--- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index d17537eb9a83b..c1f34db98316f 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use std::ops::Range; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, Weak}; use std::time::Duration; @@ -95,6 +95,10 @@ struct Shared { entries: Mutex>, reservation: MemoryReservation, notify: Notify, + /// The coalesced fetch ranges in decode order. A range is fetched once, + /// by whichever side claims it first. + ranges: Vec>, + claimed: Vec, } pub(crate) struct PrefetchReader { @@ -137,25 +141,27 @@ impl PrefetchReader { ) -> Self { let reservation = MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")).register(&pool); + let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; + let ranges = coalesce(plan, chunk); + let claimed = ranges.iter().map(|_| AtomicBool::new(false)).collect(); let shared = Arc::new(Shared { entries: Mutex::new(BTreeMap::new()), reservation, notify: Notify::new(), + ranges, + claimed, }); let fraction = env_f64("DF_PREFETCH_FRACTION").unwrap_or(0.6); let fallible = std::env::var("DF_PREFETCH_TRY").is_ok(); - let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; REGISTRY.lock().unwrap().push(Arc::downgrade(&shared)); let seq = SEQ.fetch_add(1, Ordering::Relaxed); PRIORITY.lock().unwrap().insert((partition, seq), false); - let ranges = Arc::new(coalesce(plan, chunk)); let next = Arc::new(AtomicUsize::new(0)); let live = Arc::new(AtomicUsize::new(prefetch_readers.len())); for reader in prefetch_readers { tokio::spawn(prefetch_worker( Arc::downgrade(&shared), reader, - Arc::clone(&ranges), Arc::clone(&next), Arc::clone(&live), Arc::clone(&pool), @@ -181,12 +187,13 @@ impl Drop for PrefetchReader { self.shared.notify.notify_waiters(); if std::env::var("DF_PREFETCH_STATS").is_ok() { eprintln!( - "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={} evict_mib={} denied={}", + "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={} evict_mib={} denied={} promoted_mib={}", HIT_BYTES.load(Ordering::Relaxed) >> 20, MISS_BYTES.load(Ordering::Relaxed) >> 20, WAIT_US.load(Ordering::Relaxed) / 1000, EVICT_BYTES.load(Ordering::Relaxed) >> 20, - DENIED.load(Ordering::Relaxed) + DENIED.load(Ordering::Relaxed), + PROMOTED.load(Ordering::Relaxed) >> 20 ); } } @@ -228,7 +235,6 @@ fn pool_has_room(pool: &dyn MemoryPool, fraction: f64, add: usize) -> bool { async fn prefetch_worker( shared: Weak, mut reader: Box, - ranges: Arc>>, next: Arc, live: Arc, pool: Arc, @@ -240,13 +246,18 @@ async fn prefetch_worker( loop { let Some(s) = shared.upgrade() else { return }; let i = next.load(Ordering::Relaxed); - if i >= ranges.len() { + if i >= s.ranges.len() { if live.fetch_sub(1, Ordering::Relaxed) == 1 { set_done(partition, seq); } return; } - let r = ranges[i].clone(); + if s.claimed[i].load(Ordering::Relaxed) { + // The decoder promoted this range to a required fetch. + let _ = next.compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed); + continue; + } + let r = s.ranges[i].clone(); let len = (r.end - r.start) as usize; if !may_fetch(partition, seq) || !pool_has_room(pool.as_ref(), fraction, len) { // Wait for consumption, or poll again soon, because other @@ -269,11 +280,12 @@ async fn prefetch_worker( if next .compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed) .is_err() + || s.claimed[i].swap(true, Ordering::Relaxed) { s.reservation.shrink(len); continue; } - if i + 1 == ranges.len() { + if i + 1 == s.ranges.len() { set_done(partition, seq); } s.entries @@ -375,14 +387,19 @@ impl Shared { impl PrefetchReader { async fn read(&mut self, range: Range) -> ParquetResult { + let shared = Arc::clone(&self.shared); loop { - let notified = self.shared.notify.notified(); - match self.shared.lookup(&range) { + let notified = shared.notify.notified(); + match shared.lookup(&range) { Lookup::Hit(b) => { HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); return Ok(b); } Lookup::Miss => { + if std::env::var("DF_PREFETCH_PROMOTE").is_ok() && self.promote(&range).await + { + continue; + } MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); return self.inner.get_bytes(range).await; } @@ -396,6 +413,51 @@ impl PrefetchReader { } } +impl PrefetchReader { + /// The decoder needs `range` now. If an unclaimed fetch range covers it, + /// fetch that whole range as required bytes, so the prefetch worker does + /// not fetch it again later. Return false if no such range exists. + async fn promote(&mut self, range: &Range) -> bool { + let s = Arc::clone(&self.shared); + let Some(i) = s + .ranges + .iter() + .position(|r| r.start <= range.start && range.end <= r.end) + else { + return false; + }; + if s.claimed[i].swap(true, Ordering::Relaxed) { + return false; + } + let r = s.ranges[i].clone(); + let len = (r.end - r.start) as usize; + PROMOTED.fetch_add(len as u64, Ordering::Relaxed); + s.reservation.grow(len); + s.entries + .lock() + .unwrap() + .insert(r.start, (r.end, Entry::InFlight)); + let result = self.inner.get_bytes(r.clone()).await; + let mut entries = s.entries.lock().unwrap(); + let ok = match result { + Ok(b) => { + entries.insert(r.start, (r.end, Entry::Ready { bytes: b, consumed: 0 })); + true + } + Err(_) => { + entries.remove(&r.start); + s.reservation.shrink(len); + false + } + }; + drop(entries); + s.notify.notify_waiters(); + ok + } +} + +static PROMOTED: AtomicU64 = AtomicU64::new(0); + impl AsyncFileReader for PrefetchReader { fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, ParquetResult> { self.read(range).boxed() From 33f38f403573f69b0dca60c4ac23d7174e20539f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:16:50 -0500 Subject: [PATCH 6/9] PROTOTYPE: per-file fetch window with read cursor; drop reclaim, gates, priority, promote --- .../datasource-parquet/src/opener/mod.rs | 38 +- datafusion/datasource-parquet/src/prefetch.rs | 575 ++++++++---------- .../datasource-parquet/src/push_decoder.rs | 18 +- 3 files changed, 260 insertions(+), 371 deletions(-) diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index 11ed27c8130cd..aa26a2f9527c0 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1944,23 +1944,27 @@ impl RowGroupsPrunedParquetOpen { std::env::var("DF_PREFETCH").is_ok(), prepared_memory_pool.clone(), ) { - (true, Some(pool)) => Box::new(crate::prefetch::PrefetchReader::spawn( - prepared.async_file_reader, - (0..crate::prefetch::env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4)) - .map(|_| { - prepared.parquet_file_reader_factory.create_reader( - prepared.partition_index, - prepared.partitioned_file.clone(), - prepared.metadata_size_hint, - &prepared.metrics, - ) - }) - .map(|r| r.map(|r| r as Box)) - .collect::>>()?, - decoder.scan_plan(), - pool, - prepared.partition_index, - )), + (true, Some(pool)) => { + let config = crate::prefetch::PrefetchConfig::from_env(); + Box::new(crate::prefetch::PrefetchReader::spawn( + prepared.async_file_reader, + (0..config.concurrency) + .map(|_| { + prepared.parquet_file_reader_factory.create_reader( + prepared.partition_index, + prepared.partitioned_file.clone(), + prepared.metadata_size_hint, + &prepared.metrics, + ) + }) + .map(|r| r.map(|r| r as Box)) + .collect::>>()?, + decoder.scan_plan(), + pool, + prepared.partition_index, + config, + )) + } _ => prepared.async_file_reader, }; let stream = PushDecoderStreamState { diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index c1f34db98316f..23e45a0e0da4e 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -1,74 +1,30 @@ -// PROTOTYPE: background prefetch of a file's scan plan into the memory pool. +// PROTOTYPE: a fetch window over one file's scan plan. // -// A spawned task fetches the decoder's `scan_plan()` ranges in decode order, -// independent of whether the stream is being polled. The decoder's reader -// serves requests from the prefetched bytes when they cover the request. -// The task reserves every byte it holds from the memory pool, and stops -// fetching while the pool is above `DF_PREFETCH_FRACTION` of its limit, so -// other operators keep headroom. +// The decoder's `scan_plan()` lists the byte ranges of the file in decode +// order. This module coalesces them into chunks and keeps a window of chunks +// in memory, starting at the read cursor. The cursor is the chunk that holds +// the decoder's current read. +// +// Each chunk has one owner at a time, so no chunk is fetched twice: +// +// - The decoder fetches a chunk itself when it reads from a chunk that no +// worker has claimed. These are required bytes, so the reservation grows +// without a check. +// - Workers fetch chunks ahead of the cursor, up to `budget` bytes from the +// cursor. These are optional bytes, so the reservation uses `try_grow` and +// a worker waits when the pool refuses. +// +// When the cursor moves past a chunk, the window frees the chunk. A chunk +// that is in flight at that time is freed when its fetch completes. Dropping +// the reader aborts the workers and frees the reservation. -use std::collections::BTreeMap; use std::ops::Range; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::Duration; -static HIT_BYTES: AtomicU64 = AtomicU64::new(0); -static MISS_BYTES: AtomicU64 = AtomicU64::new(0); -static WAIT_US: AtomicU64 = AtomicU64::new(0); -static EVICT_BYTES: AtomicU64 = AtomicU64::new(0); -static DENIED: AtomicU64 = AtomicU64::new(0); - -/// Every live prefetch buffer, so a memory pool under pressure can reclaim -/// optional bytes. -static REGISTRY: Mutex>> = Mutex::new(Vec::new()); - -/// Drop up to `bytes` of prefetched data that the decoder has not started to -/// read, furthest from the decoder first. Returns the bytes freed. The -/// decoder fetches a dropped range itself when it reaches it. -pub fn reclaim_prefetch(bytes: usize) -> usize { - let shareds: Vec> = { - let Ok(mut reg) = REGISTRY.try_lock() else { return 0 }; - reg.retain(|w| w.strong_count() > 0); - reg.iter().filter_map(Weak::upgrade).collect() - }; - // Gather candidates across all buffers, then evict the furthest ahead of - // each buffer's read position first, round robin by distance. - let mut cands: Vec<(u64, usize, u64)> = Vec::new(); // (distance, shared idx, start) - for (i, s) in shareds.iter().enumerate() { - let Ok(entries) = s.entries.try_lock() else { continue }; - let pos = entries.keys().next().copied().unwrap_or(0); - for (start, (_, e)) in entries.iter() { - if let Entry::Ready { consumed: 0, .. } = e { - cands.push((start - pos, i, *start)); - } - } - } - cands.sort_by(|a, b| b.0.cmp(&a.0)); - let mut freed = 0; - for (_, i, start) in cands { - if freed >= bytes { - break; - } - let s = &shareds[i]; - let Ok(mut entries) = s.entries.try_lock() else { continue }; - if let Some((end, Entry::Ready { consumed: 0, .. })) = entries.get(&start) { - let len = (*end - start) as usize; - entries.remove(&start); - drop(entries); - s.reservation.shrink(len); - freed += len; - } - } - EVICT_BYTES.fetch_add(freed as u64, Ordering::Relaxed); - freed -} - - use bytes::Bytes; -use datafusion_execution::memory_pool::{ - MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation, -}; +use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; use futures::FutureExt; use futures::future::BoxFuture; use parquet::arrow::async_reader::AsyncFileReader; @@ -76,140 +32,140 @@ use parquet::arrow::push_decoder::ScanPlan; use parquet::errors::Result as ParquetResult; use parquet::file::metadata::ParquetMetaData; use tokio::sync::Notify; +use tokio::task::JoinSet; -pub(crate) fn env_f64(name: &str) -> Option { - std::env::var(name).ok().and_then(|v| v.parse().ok()) -} +static HIT_BYTES: AtomicU64 = AtomicU64::new(0); +static MISS_BYTES: AtomicU64 = AtomicU64::new(0); +static REQUIRED_BYTES: AtomicU64 = AtomicU64::new(0); +static WAIT_US: AtomicU64 = AtomicU64::new(0); +static DENIED: AtomicU64 = AtomicU64::new(0); pub(crate) fn env_usize(name: &str) -> Option { std::env::var(name).ok().and_then(|v| v.parse().ok()) } -enum Entry { +/// The settings of a prefetch window. The opener reads them once per file. +#[derive(Clone, Copy)] +pub(crate) struct PrefetchConfig { + /// The largest chunk, in bytes. + pub chunk: u64, + /// The most bytes that workers hold or fetch, counted from the cursor. + pub budget: u64, + /// The number of workers. Each worker has one request in flight. + pub concurrency: usize, +} + +impl PrefetchConfig { + pub(crate) fn from_env() -> Self { + Self { + chunk: env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64, + budget: env_usize("DF_PREFETCH_BUDGET").unwrap_or(32 << 20) as u64, + concurrency: env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4), + } + } +} + +enum State { + Pending, InFlight, - Ready { bytes: Bytes, consumed: u64 }, + Ready(Bytes), + Freed, +} + +struct Chunk { + range: Range, + state: State, +} + +struct Window { + chunks: Vec, + /// The index of the chunk that holds the decoder's current read. Each + /// chunk before it is `Freed`, or `InFlight` and freed on completion. + cursor: usize, } struct Shared { - /// start -> (end, entry) - entries: Mutex>, + window: Mutex, reservation: MemoryReservation, notify: Notify, - /// The coalesced fetch ranges in decode order. A range is fetched once, - /// by whichever side claims it first. - ranges: Vec>, - claimed: Vec, + budget: u64, } pub(crate) struct PrefetchReader { inner: Box, shared: Arc, - partition: usize, - seq: u64, -} - -/// Per partition: file sequence number -> whether that file has issued all -/// its fetches. A file fetches only after every earlier open file of the -/// same partition has issued all its fetches, so the file being decoded -/// gets I/O and memory before look-ahead files. -static PRIORITY: Mutex> = Mutex::new(BTreeMap::new()); -static SEQ: AtomicU64 = AtomicU64::new(0); - -fn may_fetch(partition: usize, seq: u64) -> bool { - PRIORITY - .lock() - .unwrap() - .range((partition, 0)..(partition, seq)) - .all(|(_, done)| *done) -} - -fn set_done(partition: usize, seq: u64) { - if let Some(d) = PRIORITY.lock().unwrap().get_mut(&(partition, seq)) { - *d = true; - } + /// Dropping the set aborts the workers. + _workers: JoinSet<()>, } impl PrefetchReader { - /// Spawn one prefetch worker per reader in `prefetch_readers` for - /// `plan`, and wrap `inner` so it reads from the prefetched bytes. + /// Wrap `inner` in a window over `plan`, with one worker for each reader + /// in `prefetch_readers`. pub(crate) fn spawn( inner: Box, prefetch_readers: Vec>, plan: ScanPlan, pool: Arc, partition: usize, + config: PrefetchConfig, ) -> Self { - let reservation = - MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")).register(&pool); - let chunk = env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64; - let ranges = coalesce(plan, chunk); - let claimed = ranges.iter().map(|_| AtomicBool::new(false)).collect(); + let chunks = coalesce(plan, config.chunk) + .into_iter() + .map(|range| Chunk { + range, + state: State::Pending, + }) + .collect(); let shared = Arc::new(Shared { - entries: Mutex::new(BTreeMap::new()), - reservation, + window: Mutex::new(Window { chunks, cursor: 0 }), + reservation: MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")) + .register(&pool), notify: Notify::new(), - ranges, - claimed, + budget: config.budget, }); - let fraction = env_f64("DF_PREFETCH_FRACTION").unwrap_or(0.6); - let fallible = std::env::var("DF_PREFETCH_TRY").is_ok(); - REGISTRY.lock().unwrap().push(Arc::downgrade(&shared)); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - PRIORITY.lock().unwrap().insert((partition, seq), false); - let next = Arc::new(AtomicUsize::new(0)); - let live = Arc::new(AtomicUsize::new(prefetch_readers.len())); + let mut workers = JoinSet::new(); for reader in prefetch_readers { - tokio::spawn(prefetch_worker( - Arc::downgrade(&shared), - reader, - Arc::clone(&next), - Arc::clone(&live), - Arc::clone(&pool), - fraction, - fallible, - partition, - seq, - )); + workers.spawn(worker(Arc::clone(&shared), reader)); } Self { inner, shared, - partition, - seq, + _workers: workers, } } } impl Drop for PrefetchReader { fn drop(&mut self) { - PRIORITY.lock().unwrap().remove(&(self.partition, self.seq)); - // The workers hold only a `Weak`. Waking them lets them observe the drop. - self.shared.notify.notify_waiters(); if std::env::var("DF_PREFETCH_STATS").is_ok() { eprintln!( - "PREFETCH_STATS hit_mib={} miss_mib={} wait_ms={} evict_mib={} denied={} promoted_mib={}", + "PREFETCH_STATS hit_mib={} miss_mib={} required_mib={} wait_ms={} denied={}", HIT_BYTES.load(Ordering::Relaxed) >> 20, MISS_BYTES.load(Ordering::Relaxed) >> 20, + REQUIRED_BYTES.load(Ordering::Relaxed) >> 20, WAIT_US.load(Ordering::Relaxed) / 1000, - EVICT_BYTES.load(Ordering::Relaxed) >> 20, - DENIED.load(Ordering::Relaxed), - PROMOTED.load(Ordering::Relaxed) >> 20 + DENIED.load(Ordering::Relaxed) ); } } } -/// Merge adjacent plan ranges into fetches of at most `chunk` bytes. +/// Merge adjacent plan ranges into chunks of at most `chunk` bytes. The +/// chunks do not overlap. fn coalesce(plan: ScanPlan, chunk: u64) -> Vec> { let mut out: Vec> = Vec::new(); for planned in plan { - let r = planned.range; + let mut r = planned.range; if let Some(last) = out.last_mut() { - if r.start == last.end && last.end - last.start + (r.end - r.start) <= chunk { - last.end = r.end; + if r.start >= last.start && r.start < last.end { + // Keep only the part after the previous chunk. + r.start = last.end; + } + if r.start >= r.end { continue; } - if r.start >= last.start && r.end <= last.end { + if r.start == last.end && r.end - last.start <= chunk { + last.end = r.end; continue; } } @@ -218,246 +174,189 @@ fn coalesce(plan: ScanPlan, chunk: u64) -> Vec> { out } -fn pool_has_room(pool: &dyn MemoryPool, fraction: f64, add: usize) -> bool { - match pool.memory_limit() { - MemoryLimit::Finite(limit) => { - let headroom = env_usize("DF_PREFETCH_HEADROOM").unwrap_or(0); - (pool.reserved() + add + headroom) as f64 <= limit as f64 * fraction +fn len(r: &Range) -> usize { + (r.end - r.start) as usize +} + +impl Shared { + /// Release the memory of `chunk`. If the chunk is in flight, `complete` + /// releases the memory when the fetch ends. + fn free(&self, chunk: &mut Chunk) { + if let State::Ready(_) = std::mem::replace(&mut chunk.state, State::Freed) { + self.reservation.shrink(len(&chunk.range)); } - _ => true, } -} -/// One worker keeps one request in flight. Each worker takes the next range -/// in decode order as soon as its previous request completes, so a slow -/// range does not hold back the others. -#[allow(clippy::too_many_arguments)] -async fn prefetch_worker( - shared: Weak, - mut reader: Box, - next: Arc, - live: Arc, - pool: Arc, - fraction: f64, - fallible: bool, - partition: usize, - seq: u64, -) { - loop { - let Some(s) = shared.upgrade() else { return }; - let i = next.load(Ordering::Relaxed); - if i >= s.ranges.len() { - if live.fetch_sub(1, Ordering::Relaxed) == 1 { - set_done(partition, seq); + /// Store the result of the fetch of chunk `i`. + fn complete(&self, i: usize, result: ParquetResult) { + let mut w = self.window.lock().unwrap(); + let chunk = &mut w.chunks[i]; + match (&chunk.state, result) { + (State::InFlight, Ok(bytes)) => chunk.state = State::Ready(bytes), + (State::InFlight, Err(_)) => { + // The decoder reads this range directly. + chunk.state = State::Freed; + self.reservation.shrink(len(&chunk.range)); } - return; - } - if s.claimed[i].load(Ordering::Relaxed) { - // The decoder promoted this range to a required fetch. - let _ = next.compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed); - continue; + // The cursor moved past the chunk while it was in flight. + _ => self.reservation.shrink(len(&chunk.range)), } - let r = s.ranges[i].clone(); - let len = (r.end - r.start) as usize; - if !may_fetch(partition, seq) || !pool_has_room(pool.as_ref(), fraction, len) { - // Wait for consumption, or poll again soon, because other - // partitions and earlier files also free memory and I/O. - let _ = - tokio::time::timeout(Duration::from_millis(2), s.notify.notified()).await; - continue; - } - if fallible { - // Optional bytes: take only what the pool grants now. - if s.reservation.try_grow(len).is_err() { - DENIED.fetch_add(1, Ordering::Relaxed); - let _ = tokio::time::timeout(Duration::from_millis(5), s.notify.notified()) - .await; + drop(w); + self.notify.notify_waiters(); + } + + /// Claim the first pending chunk ahead of the cursor for a worker. The + /// bytes from the cursor to the end of that chunk must fit in the budget, + /// and the pool must grant them. Return `Err` when no pending chunk is + /// left. + fn claim_optional(&self) -> Result)>, ()> { + let mut w = self.window.lock().unwrap(); + let cursor = w.cursor; + let mut ahead = 0; + for i in cursor..w.chunks.len() { + ahead += len(&w.chunks[i].range) as u64; + if !matches!(w.chunks[i].state, State::Pending) { continue; } - } else { - s.reservation.grow(len); - } - if next - .compare_exchange(i, i + 1, Ordering::Relaxed, Ordering::Relaxed) - .is_err() - || s.claimed[i].swap(true, Ordering::Relaxed) - { - s.reservation.shrink(len); - continue; - } - if i + 1 == s.ranges.len() { - set_done(partition, seq); - } - s.entries - .lock() - .unwrap() - .insert(r.start, (r.end, Entry::InFlight)); - drop(s); - let result = reader.get_bytes(r.clone()).await; - let Some(s) = shared.upgrade() else { return }; - { - let mut entries = s.entries.lock().unwrap(); - match result { - Ok(b) => { - entries.insert( - r.start, - ( - r.end, - Entry::Ready { - bytes: b, - consumed: 0, - }, - ), - ); - } - Err(_) => { - // Let the decoder fetch this range itself. - entries.remove(&r.start); - s.reservation.shrink(len); - } + if ahead > self.budget && i > cursor { + return Ok(None); + } + let r = w.chunks[i].range.clone(); + if self.reservation.try_grow(len(&r)).is_err() { + DENIED.fetch_add(1, Ordering::Relaxed); + return Ok(None); } + w.chunks[i].state = State::InFlight; + return Ok(Some((i, r))); } - s.notify.notify_waiters(); + Err(()) } -} -enum Lookup { - Miss, - Wait, - Hit(Bytes), -} - -impl Shared { - /// Serve `range` from one or more contiguous entries. - fn lookup(&self, range: &Range) -> Lookup { - let mut entries = self.entries.lock().unwrap(); - let Some((&first, _)) = entries.range(..=range.start).next_back() else { - return Lookup::Miss; + /// Move the cursor to the chunk that holds `range.start`, then serve + /// `range` or say what the decoder must do first. + fn step(&self, range: &Range) -> Step { + let mut guard = self.window.lock().unwrap(); + let w = &mut *guard; + let Some(first) = (w.cursor..w.chunks.len()) + .find(|&i| w.chunks[i].range.contains(&range.start)) + else { + return Step::Direct; }; - let mut parts = Vec::new(); - let mut pos = range.start; - for (&start, (end, entry)) in entries.range(first..) { - if start > pos { - return Lookup::Miss; - } - if *end <= pos { - continue; - } - match entry { - Entry::InFlight => return Lookup::Wait, - Entry::Ready { .. } => parts.push(start), + for i in w.cursor..first { + self.free(&mut w.chunks[i]); + } + w.cursor = first; + let mut last = first; + while w.chunks[last].range.end < range.end { + last += 1; + if last == w.chunks.len() + || w.chunks[last].range.start != w.chunks[last - 1].range.end + { + // The range leaves the plan. Read it directly. + return Step::Direct; } - pos = *end; - if pos >= range.end { - break; + } + let mut claims = Vec::new(); + let mut wait = false; + for i in first..=last { + match w.chunks[i].state { + State::Pending => { + // Required bytes: the decoder cannot continue without them. + let r = w.chunks[i].range.clone(); + self.reservation.grow(len(&r)); + REQUIRED_BYTES.fetch_add(r.end - r.start, Ordering::Relaxed); + w.chunks[i].state = State::InFlight; + claims.push((i, r)); + } + State::InFlight => wait = true, + State::Ready(_) => {} + State::Freed => return Step::Direct, } } - if pos < range.end { - return Lookup::Miss; + if !claims.is_empty() { + return Step::Fetch(claims); + } + if wait { + return Step::Wait; } - let mut out = bytes::BytesMut::new(); - let mut single = None; - let mut freed = 0; - for start in &parts { - let (end, entry) = entries.get_mut(start).unwrap(); - let Entry::Ready { bytes, consumed } = entry else { + let mut pieces = (first..=last).map(|i| { + let c = &w.chunks[i]; + let State::Ready(bytes) = &c.state else { unreachable!() }; - let lo = range.start.max(*start); - let hi = range.end.min(*end); - let piece = bytes.slice((lo - start) as usize..(hi - start) as usize); - if parts.len() == 1 { - single = Some(piece); - } else { - out.extend_from_slice(&piece); + let lo = range.start.max(c.range.start) - c.range.start; + let hi = range.end.min(c.range.end) - c.range.start; + bytes.slice(lo as usize..hi as usize) + }); + if first == last { + return Step::Hit(pieces.next().unwrap()); + } + let mut out = bytes::BytesMut::with_capacity(len(range)); + pieces.for_each(|p| out.extend_from_slice(&p)); + Step::Hit(out.freeze()) + } +} + +/// Fetch optional chunks until none is pending. +async fn worker(shared: Arc, mut reader: Box) { + loop { + let notified = shared.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + match shared.claim_optional() { + Err(()) => return, + Ok(Some((i, range))) => { + let result = reader.get_bytes(range).await; + shared.complete(i, result); } - *consumed += hi - lo; - if *consumed >= *end - start { - freed += (*end - start) as usize; - entries.remove(start); + // The cursor moves with a notification, but the pool frees memory + // without one. So poll the pool as well. + Ok(None) => { + let _ = tokio::time::timeout(Duration::from_millis(5), notified).await; } } - if freed > 0 { - self.reservation.shrink(freed); - self.notify.notify_waiters(); - } - Lookup::Hit(single.unwrap_or_else(|| out.freeze())) } } +enum Step { + Hit(Bytes), + Wait, + Fetch(Vec<(usize, Range)>), + Direct, +} + impl PrefetchReader { async fn read(&mut self, range: Range) -> ParquetResult { let shared = Arc::clone(&self.shared); loop { let notified = shared.notify.notified(); - match shared.lookup(&range) { - Lookup::Hit(b) => { + tokio::pin!(notified); + notified.as_mut().enable(); + match shared.step(&range) { + Step::Hit(b) => { HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); return Ok(b); } - Lookup::Miss => { - if std::env::var("DF_PREFETCH_PROMOTE").is_ok() && self.promote(&range).await - { - continue; - } + Step::Direct => { MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); return self.inner.get_bytes(range).await; } - Lookup::Wait => { + Step::Wait => { let t = std::time::Instant::now(); notified.await; WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); } + Step::Fetch(claims) => { + for (i, r) in claims { + let result = self.inner.get_bytes(r).await; + shared.complete(i, result); + } + } } } } } -impl PrefetchReader { - /// The decoder needs `range` now. If an unclaimed fetch range covers it, - /// fetch that whole range as required bytes, so the prefetch worker does - /// not fetch it again later. Return false if no such range exists. - async fn promote(&mut self, range: &Range) -> bool { - let s = Arc::clone(&self.shared); - let Some(i) = s - .ranges - .iter() - .position(|r| r.start <= range.start && range.end <= r.end) - else { - return false; - }; - if s.claimed[i].swap(true, Ordering::Relaxed) { - return false; - } - let r = s.ranges[i].clone(); - let len = (r.end - r.start) as usize; - PROMOTED.fetch_add(len as u64, Ordering::Relaxed); - s.reservation.grow(len); - s.entries - .lock() - .unwrap() - .insert(r.start, (r.end, Entry::InFlight)); - let result = self.inner.get_bytes(r.clone()).await; - let mut entries = s.entries.lock().unwrap(); - let ok = match result { - Ok(b) => { - entries.insert(r.start, (r.end, Entry::Ready { bytes: b, consumed: 0 })); - true - } - Err(_) => { - entries.remove(&r.start); - s.reservation.shrink(len); - false - } - }; - drop(entries); - s.notify.notify_waiters(); - ok - } -} - -static PROMOTED: AtomicU64 = AtomicU64::new(0); - impl AsyncFileReader for PrefetchReader { fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, ParquetResult> { self.read(range).boxed() diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs index 23caa014a7a26..14c99882a751f 100644 --- a/datafusion/datasource-parquet/src/push_decoder.rs +++ b/datafusion/datasource-parquet/src/push_decoder.rs @@ -797,22 +797,8 @@ impl ReadAheadMemory { /// limit: the decoder cannot continue without these bytes. fn reserve_required(&self, bytes: u64) -> Result<()> { let bytes = to_usize(bytes); - // PROTOTYPE: DF_REQUIRED_STRICT orders required reservations: - // try the pool, reclaim optional prefetch bytes, retry, then fail. - if std::env::var("DF_REQUIRED_STRICT").is_err() { - self.reservation.grow(bytes); - return Ok(()); - } - if self.reservation.try_grow(bytes).is_ok() { - return Ok(()); - } - crate::prefetch::reclaim_prefetch(bytes); - self.reservation.try_grow(bytes).map_err(|e| { - e.context(format!( - "required bytes for the next batch of {}", - self.reservation.consumer().name() - )) - }) + self.reservation.grow(bytes); + Ok(()) } /// Bytes the pool can grant now. Unlimited if the pool has no limit or From d7585092c4865efed75c1a9b7b52a1cdf6a9d47b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:18:26 -0500 Subject: [PATCH 7/9] PROTOTYPE: claim guard, cursor wakeups, forward-only plans, split large ranges --- .../datasource-parquet/src/opener/mod.rs | 7 +- datafusion/datasource-parquet/src/prefetch.rs | 124 +++++++++++++----- 2 files changed, 97 insertions(+), 34 deletions(-) diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index aa26a2f9527c0..b6dcf40eb3e5f 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1946,7 +1946,7 @@ impl RowGroupsPrunedParquetOpen { ) { (true, Some(pool)) => { let config = crate::prefetch::PrefetchConfig::from_env(); - Box::new(crate::prefetch::PrefetchReader::spawn( + match crate::prefetch::PrefetchReader::spawn( prepared.async_file_reader, (0..config.concurrency) .map(|_| { @@ -1963,7 +1963,10 @@ impl RowGroupsPrunedParquetOpen { pool, prepared.partition_index, config, - )) + ) { + Ok(r) => Box::new(r), + Err(inner) => inner, + } } _ => prepared.async_file_reader, }; diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index 23e45a0e0da4e..ea6b8c41ac20e 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -100,7 +100,8 @@ pub(crate) struct PrefetchReader { impl PrefetchReader { /// Wrap `inner` in a window over `plan`, with one worker for each reader - /// in `prefetch_readers`. + /// in `prefetch_readers`. Return `inner` unchanged if the plan does not + /// fit a window. pub(crate) fn spawn( inner: Box, prefetch_readers: Vec>, @@ -108,8 +109,12 @@ impl PrefetchReader { pool: Arc, partition: usize, config: PrefetchConfig, - ) -> Self { - let chunks = coalesce(plan, config.chunk) + ) -> Result> { + let Some(ranges) = coalesce(plan, config.chunk) else { + // The plan moves backward. Read without prefetch. + return Err(inner); + }; + let chunks = ranges .into_iter() .map(|range| Chunk { range, @@ -127,11 +132,11 @@ impl PrefetchReader { for reader in prefetch_readers { workers.spawn(worker(Arc::clone(&shared), reader)); } - Self { + Ok(Self { inner, shared, _workers: workers, - } + }) } } @@ -150,28 +155,60 @@ impl Drop for PrefetchReader { } } -/// Merge adjacent plan ranges into chunks of at most `chunk` bytes. The -/// chunks do not overlap. -fn coalesce(plan: ScanPlan, chunk: u64) -> Vec> { +/// Merge adjacent plan ranges into chunks of at most `chunk` bytes, and split +/// larger ranges. The window needs chunks in increasing byte order, so return +/// `None` if the plan moves backward. +fn coalesce(plan: ScanPlan, chunk: u64) -> Option>> { let mut out: Vec> = Vec::new(); for planned in plan { let mut r = planned.range; - if let Some(last) = out.last_mut() { - if r.start >= last.start && r.start < last.end { - // Keep only the part after the previous chunk. - r.start = last.end; + if let Some(last) = out.last() { + if r.start < last.start { + return None; } - if r.start >= r.end { - continue; - } - if r.start == last.end && r.end - last.start <= chunk { - last.end = r.end; - continue; + // Keep only the part after the previous chunk. + r.start = r.start.max(last.end); + } + while r.start < r.end { + match out.last_mut() { + Some(last) if last.end == r.start && last.end - last.start < chunk => { + let end = r.end.min(last.start + chunk); + last.end = end; + r.start = end; + } + _ => { + let end = r.end.min(r.start + chunk); + out.push(r.start..end); + r.start = end; + } } } - out.push(r); } - out + Some(out) +} + +/// A claim on an `InFlight` chunk. Dropping an unfinished claim returns the +/// chunk to `Pending` and releases its memory, so no reader waits forever. +struct Claim<'a> { + shared: &'a Shared, + i: usize, + done: bool, +} + +impl Drop for Claim<'_> { + fn drop(&mut self) { + if self.done { + return; + } + let mut w = self.shared.window.lock().unwrap(); + let chunk = &mut w.chunks[self.i]; + if matches!(chunk.state, State::InFlight) { + chunk.state = State::Pending; + } + self.shared.reservation.shrink(len(&chunk.range)); + drop(w); + self.shared.notify.notify_waiters(); + } } fn len(r: &Range) -> usize { @@ -187,6 +224,24 @@ impl Shared { } } + /// Fetch the claimed chunk `i` and store the result. If the future is + /// dropped first, the guard returns the chunk to `Pending`. + async fn fetch( + self: &Arc, + reader: &mut Box, + i: usize, + range: Range, + ) { + let mut guard = Claim { + shared: self, + i, + done: false, + }; + let result = reader.get_bytes(range).await; + guard.done = true; + self.complete(i, result); + } + /// Store the result of the fetch of chunk `i`. fn complete(&self, i: usize, result: ParquetResult) { let mut w = self.window.lock().unwrap(); @@ -206,7 +261,7 @@ impl Shared { } /// Claim the first pending chunk ahead of the cursor for a worker. The - /// bytes from the cursor to the end of that chunk must fit in the budget, + /// bytes after the cursor chunk up to the end of that chunk must fit in the budget, /// and the pool must grant them. Return `Err` when no pending chunk is /// left. fn claim_optional(&self) -> Result)>, ()> { @@ -214,11 +269,13 @@ impl Shared { let cursor = w.cursor; let mut ahead = 0; for i in cursor..w.chunks.len() { - ahead += len(&w.chunks[i].range) as u64; + if i > cursor { + ahead += len(&w.chunks[i].range) as u64; + } if !matches!(w.chunks[i].state, State::Pending) { continue; } - if ahead > self.budget && i > cursor { + if ahead > self.budget { return Ok(None); } let r = w.chunks[i].range.clone(); @@ -242,10 +299,14 @@ impl Shared { else { return Step::Direct; }; - for i in w.cursor..first { - self.free(&mut w.chunks[i]); + if first > w.cursor { + for i in w.cursor..first { + self.free(&mut w.chunks[i]); + } + w.cursor = first; + // The budget now reaches further, so wake the workers. + self.notify.notify_waiters(); } - w.cursor = first; let mut last = first; while w.chunks[last].range.end < range.end { last += 1; @@ -305,10 +366,7 @@ async fn worker(shared: Arc, mut reader: Box) { notified.as_mut().enable(); match shared.claim_optional() { Err(()) => return, - Ok(Some((i, range))) => { - let result = reader.get_bytes(range).await; - shared.complete(i, result); - } + Ok(Some((i, range))) => shared.fetch(&mut reader, i, range).await, // The cursor moves with a notification, but the pool frees memory // without one. So poll the pool as well. Ok(None) => { @@ -347,9 +405,11 @@ impl PrefetchReader { WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); } Step::Fetch(claims) => { + // The decoder has one reader, so it fetches the claims in + // turn. A read spans more than one chunk only at a chunk + // boundary, so a claim list is short. for (i, r) in claims { - let result = self.inner.get_bytes(r).await; - shared.complete(i, result); + shared.fetch(&mut self.inner, i, r).await; } } } From 740711eb5491677be3996a17fdf09bd53a3bbbce Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:31:58 -0500 Subject: [PATCH 8/9] PROTOTYPE: allow plans that move backward without overlap --- datafusion/datasource-parquet/src/prefetch.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index ea6b8c41ac20e..0d5e2794d4fdc 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -156,18 +156,22 @@ impl Drop for PrefetchReader { } /// Merge adjacent plan ranges into chunks of at most `chunk` bytes, and split -/// larger ranges. The window needs chunks in increasing byte order, so return -/// `None` if the plan moves backward. +/// larger ranges. The chunks stay in decode order, which can move backward in +/// the file, for example when row groups are read in reverse. The window needs +/// chunks that do not overlap, so return `None` if a range overlaps the start +/// of the previous chunk. fn coalesce(plan: ScanPlan, chunk: u64) -> Option>> { let mut out: Vec> = Vec::new(); for planned in plan { let mut r = planned.range; if let Some(last) = out.last() { - if r.start < last.start { + if r.start < last.start && r.end > last.start { return None; } - // Keep only the part after the previous chunk. - r.start = r.start.max(last.end); + if r.start >= last.start { + // Keep only the part after the previous chunk. + r.start = r.start.max(last.end); + } } while r.start < r.end { match out.last_mut() { From 5a5e2de1163642ecb045e03742f1dbb7a88c6ec8 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:38:21 -0500 Subject: [PATCH 9/9] PROTOTYPE: default 4 MiB chunks --- datafusion/datasource-parquet/src/prefetch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs index 0d5e2794d4fdc..059d6b7d1444f 100644 --- a/datafusion/datasource-parquet/src/prefetch.rs +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -58,7 +58,7 @@ pub(crate) struct PrefetchConfig { impl PrefetchConfig { pub(crate) fn from_env() -> Self { Self { - chunk: env_usize("DF_PREFETCH_CHUNK").unwrap_or(8 << 20) as u64, + chunk: env_usize("DF_PREFETCH_CHUNK").unwrap_or(4 << 20) as u64, budget: env_usize("DF_PREFETCH_BUDGET").unwrap_or(32 << 20) as u64, concurrency: env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4), }