diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs index 80d2205a1fab0..1b08314620f69 100644 --- a/datafusion/datasource-parquet/src/mod.rs +++ b/datafusion/datasource-parquet/src/mod.rs @@ -33,6 +33,7 @@ mod metrics; mod nested_schema_pruning; mod opener; mod page_filter; +pub mod prefetch; mod projection_read_plan; mod pruning; mod push_decoder; diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index 6cfafad459d31..b6dcf40eb3e5f 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1928,6 +1928,7 @@ impl RowGroupsPrunedParquetOpen { .file_metrics .row_groups_pruned_dynamic_filter .clone(); + let prepared_memory_pool = prepared.memory_pool.clone(); let read_ahead = read_ahead_bytes.map(|window| { let memory = ReadAheadMemory::new(prepared.memory_pool, prepared.partition_index); @@ -1938,11 +1939,42 @@ impl RowGroupsPrunedParquetOpen { ); ReadAhead::new(window, decoder.scan_plan(), memory, metrics) }); + // PROTOTYPE: prefetch this file's scan plan in the background. + let reader: Box = match ( + std::env::var("DF_PREFETCH").is_ok(), + prepared_memory_pool.clone(), + ) { + (true, Some(pool)) => { + let config = crate::prefetch::PrefetchConfig::from_env(); + match crate::prefetch::PrefetchReader::spawn( + prepared.async_file_reader, + (0..config.concurrency) + .map(|_| { + prepared.parquet_file_reader_factory.create_reader( + prepared.partition_index, + prepared.partitioned_file.clone(), + prepared.metadata_size_hint, + &prepared.metrics, + ) + }) + .map(|r| r.map(|r| r as Box)) + .collect::>>()?, + decoder.scan_plan(), + pool, + prepared.partition_index, + config, + ) { + Ok(r) => Box::new(r), + Err(inner) => inner, + } + } + _ => prepared.async_file_reader, + }; let stream = PushDecoderStreamState { decoder: Some(decoder), active_reader: None, rg_plan, - reader: Some(prepared.async_file_reader), + reader: Some(reader), decoder_projection, arrow_reader_metrics, predicate_cache_inner_records, diff --git a/datafusion/datasource-parquet/src/prefetch.rs b/datafusion/datasource-parquet/src/prefetch.rs new file mode 100644 index 0000000000000..059d6b7d1444f --- /dev/null +++ b/datafusion/datasource-parquet/src/prefetch.rs @@ -0,0 +1,449 @@ +// PROTOTYPE: a fetch window over one file's scan plan. +// +// The decoder's `scan_plan()` lists the byte ranges of the file in decode +// order. This module coalesces them into chunks and keeps a window of chunks +// in memory, starting at the read cursor. The cursor is the chunk that holds +// the decoder's current read. +// +// Each chunk has one owner at a time, so no chunk is fetched twice: +// +// - The decoder fetches a chunk itself when it reads from a chunk that no +// worker has claimed. These are required bytes, so the reservation grows +// without a check. +// - Workers fetch chunks ahead of the cursor, up to `budget` bytes from the +// cursor. These are optional bytes, so the reservation uses `try_grow` and +// a worker waits when the pool refuses. +// +// When the cursor moves past a chunk, the window frees the chunk. A chunk +// that is in flight at that time is freed when its fetch completes. Dropping +// the reader aborts the workers and frees the reservation. + +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; +use futures::FutureExt; +use futures::future::BoxFuture; +use parquet::arrow::async_reader::AsyncFileReader; +use parquet::arrow::push_decoder::ScanPlan; +use parquet::errors::Result as ParquetResult; +use parquet::file::metadata::ParquetMetaData; +use tokio::sync::Notify; +use tokio::task::JoinSet; + +static HIT_BYTES: AtomicU64 = AtomicU64::new(0); +static MISS_BYTES: AtomicU64 = AtomicU64::new(0); +static REQUIRED_BYTES: AtomicU64 = AtomicU64::new(0); +static WAIT_US: AtomicU64 = AtomicU64::new(0); +static DENIED: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn env_usize(name: &str) -> Option { + std::env::var(name).ok().and_then(|v| v.parse().ok()) +} + +/// The settings of a prefetch window. The opener reads them once per file. +#[derive(Clone, Copy)] +pub(crate) struct PrefetchConfig { + /// The largest chunk, in bytes. + pub chunk: u64, + /// The most bytes that workers hold or fetch, counted from the cursor. + pub budget: u64, + /// The number of workers. Each worker has one request in flight. + pub concurrency: usize, +} + +impl PrefetchConfig { + pub(crate) fn from_env() -> Self { + Self { + chunk: env_usize("DF_PREFETCH_CHUNK").unwrap_or(4 << 20) as u64, + budget: env_usize("DF_PREFETCH_BUDGET").unwrap_or(32 << 20) as u64, + concurrency: env_usize("DF_PREFETCH_CONCURRENCY").unwrap_or(4), + } + } +} + +enum State { + Pending, + InFlight, + Ready(Bytes), + Freed, +} + +struct Chunk { + range: Range, + state: State, +} + +struct Window { + chunks: Vec, + /// The index of the chunk that holds the decoder's current read. Each + /// chunk before it is `Freed`, or `InFlight` and freed on completion. + cursor: usize, +} + +struct Shared { + window: Mutex, + reservation: MemoryReservation, + notify: Notify, + budget: u64, +} + +pub(crate) struct PrefetchReader { + inner: Box, + shared: Arc, + /// Dropping the set aborts the workers. + _workers: JoinSet<()>, +} + +impl PrefetchReader { + /// Wrap `inner` in a window over `plan`, with one worker for each reader + /// in `prefetch_readers`. Return `inner` unchanged if the plan does not + /// fit a window. + pub(crate) fn spawn( + inner: Box, + prefetch_readers: Vec>, + plan: ScanPlan, + pool: Arc, + partition: usize, + config: PrefetchConfig, + ) -> Result> { + let Some(ranges) = coalesce(plan, config.chunk) else { + // The plan moves backward. Read without prefetch. + return Err(inner); + }; + let chunks = ranges + .into_iter() + .map(|range| Chunk { + range, + state: State::Pending, + }) + .collect(); + let shared = Arc::new(Shared { + window: Mutex::new(Window { chunks, cursor: 0 }), + reservation: MemoryConsumer::new(format!("ParquetPrefetch[{partition}]")) + .register(&pool), + notify: Notify::new(), + budget: config.budget, + }); + let mut workers = JoinSet::new(); + for reader in prefetch_readers { + workers.spawn(worker(Arc::clone(&shared), reader)); + } + Ok(Self { + inner, + shared, + _workers: workers, + }) + } +} + +impl Drop for PrefetchReader { + fn drop(&mut self) { + if std::env::var("DF_PREFETCH_STATS").is_ok() { + eprintln!( + "PREFETCH_STATS hit_mib={} miss_mib={} required_mib={} wait_ms={} denied={}", + HIT_BYTES.load(Ordering::Relaxed) >> 20, + MISS_BYTES.load(Ordering::Relaxed) >> 20, + REQUIRED_BYTES.load(Ordering::Relaxed) >> 20, + WAIT_US.load(Ordering::Relaxed) / 1000, + DENIED.load(Ordering::Relaxed) + ); + } + } +} + +/// Merge adjacent plan ranges into chunks of at most `chunk` bytes, and split +/// larger ranges. The chunks stay in decode order, which can move backward in +/// the file, for example when row groups are read in reverse. The window needs +/// chunks that do not overlap, so return `None` if a range overlaps the start +/// of the previous chunk. +fn coalesce(plan: ScanPlan, chunk: u64) -> Option>> { + let mut out: Vec> = Vec::new(); + for planned in plan { + let mut r = planned.range; + if let Some(last) = out.last() { + if r.start < last.start && r.end > last.start { + return None; + } + if r.start >= last.start { + // Keep only the part after the previous chunk. + r.start = r.start.max(last.end); + } + } + while r.start < r.end { + match out.last_mut() { + Some(last) if last.end == r.start && last.end - last.start < chunk => { + let end = r.end.min(last.start + chunk); + last.end = end; + r.start = end; + } + _ => { + let end = r.end.min(r.start + chunk); + out.push(r.start..end); + r.start = end; + } + } + } + } + Some(out) +} + +/// A claim on an `InFlight` chunk. Dropping an unfinished claim returns the +/// chunk to `Pending` and releases its memory, so no reader waits forever. +struct Claim<'a> { + shared: &'a Shared, + i: usize, + done: bool, +} + +impl Drop for Claim<'_> { + fn drop(&mut self) { + if self.done { + return; + } + let mut w = self.shared.window.lock().unwrap(); + let chunk = &mut w.chunks[self.i]; + if matches!(chunk.state, State::InFlight) { + chunk.state = State::Pending; + } + self.shared.reservation.shrink(len(&chunk.range)); + drop(w); + self.shared.notify.notify_waiters(); + } +} + +fn len(r: &Range) -> usize { + (r.end - r.start) as usize +} + +impl Shared { + /// Release the memory of `chunk`. If the chunk is in flight, `complete` + /// releases the memory when the fetch ends. + fn free(&self, chunk: &mut Chunk) { + if let State::Ready(_) = std::mem::replace(&mut chunk.state, State::Freed) { + self.reservation.shrink(len(&chunk.range)); + } + } + + /// Fetch the claimed chunk `i` and store the result. If the future is + /// dropped first, the guard returns the chunk to `Pending`. + async fn fetch( + self: &Arc, + reader: &mut Box, + i: usize, + range: Range, + ) { + let mut guard = Claim { + shared: self, + i, + done: false, + }; + let result = reader.get_bytes(range).await; + guard.done = true; + self.complete(i, result); + } + + /// Store the result of the fetch of chunk `i`. + fn complete(&self, i: usize, result: ParquetResult) { + let mut w = self.window.lock().unwrap(); + let chunk = &mut w.chunks[i]; + match (&chunk.state, result) { + (State::InFlight, Ok(bytes)) => chunk.state = State::Ready(bytes), + (State::InFlight, Err(_)) => { + // The decoder reads this range directly. + chunk.state = State::Freed; + self.reservation.shrink(len(&chunk.range)); + } + // The cursor moved past the chunk while it was in flight. + _ => self.reservation.shrink(len(&chunk.range)), + } + drop(w); + self.notify.notify_waiters(); + } + + /// Claim the first pending chunk ahead of the cursor for a worker. The + /// bytes after the cursor chunk up to the end of that chunk must fit in the budget, + /// and the pool must grant them. Return `Err` when no pending chunk is + /// left. + fn claim_optional(&self) -> Result)>, ()> { + let mut w = self.window.lock().unwrap(); + let cursor = w.cursor; + let mut ahead = 0; + for i in cursor..w.chunks.len() { + if i > cursor { + ahead += len(&w.chunks[i].range) as u64; + } + if !matches!(w.chunks[i].state, State::Pending) { + continue; + } + if ahead > self.budget { + return Ok(None); + } + let r = w.chunks[i].range.clone(); + if self.reservation.try_grow(len(&r)).is_err() { + DENIED.fetch_add(1, Ordering::Relaxed); + return Ok(None); + } + w.chunks[i].state = State::InFlight; + return Ok(Some((i, r))); + } + Err(()) + } + + /// Move the cursor to the chunk that holds `range.start`, then serve + /// `range` or say what the decoder must do first. + fn step(&self, range: &Range) -> Step { + let mut guard = self.window.lock().unwrap(); + let w = &mut *guard; + let Some(first) = (w.cursor..w.chunks.len()) + .find(|&i| w.chunks[i].range.contains(&range.start)) + else { + return Step::Direct; + }; + if first > w.cursor { + for i in w.cursor..first { + self.free(&mut w.chunks[i]); + } + w.cursor = first; + // The budget now reaches further, so wake the workers. + self.notify.notify_waiters(); + } + let mut last = first; + while w.chunks[last].range.end < range.end { + last += 1; + if last == w.chunks.len() + || w.chunks[last].range.start != w.chunks[last - 1].range.end + { + // The range leaves the plan. Read it directly. + return Step::Direct; + } + } + let mut claims = Vec::new(); + let mut wait = false; + for i in first..=last { + match w.chunks[i].state { + State::Pending => { + // Required bytes: the decoder cannot continue without them. + let r = w.chunks[i].range.clone(); + self.reservation.grow(len(&r)); + REQUIRED_BYTES.fetch_add(r.end - r.start, Ordering::Relaxed); + w.chunks[i].state = State::InFlight; + claims.push((i, r)); + } + State::InFlight => wait = true, + State::Ready(_) => {} + State::Freed => return Step::Direct, + } + } + if !claims.is_empty() { + return Step::Fetch(claims); + } + if wait { + return Step::Wait; + } + let mut pieces = (first..=last).map(|i| { + let c = &w.chunks[i]; + let State::Ready(bytes) = &c.state else { + unreachable!() + }; + let lo = range.start.max(c.range.start) - c.range.start; + let hi = range.end.min(c.range.end) - c.range.start; + bytes.slice(lo as usize..hi as usize) + }); + if first == last { + return Step::Hit(pieces.next().unwrap()); + } + let mut out = bytes::BytesMut::with_capacity(len(range)); + pieces.for_each(|p| out.extend_from_slice(&p)); + Step::Hit(out.freeze()) + } +} + +/// Fetch optional chunks until none is pending. +async fn worker(shared: Arc, mut reader: Box) { + loop { + let notified = shared.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + match shared.claim_optional() { + Err(()) => return, + Ok(Some((i, range))) => shared.fetch(&mut reader, i, range).await, + // The cursor moves with a notification, but the pool frees memory + // without one. So poll the pool as well. + Ok(None) => { + let _ = tokio::time::timeout(Duration::from_millis(5), notified).await; + } + } + } +} + +enum Step { + Hit(Bytes), + Wait, + Fetch(Vec<(usize, Range)>), + Direct, +} + +impl PrefetchReader { + async fn read(&mut self, range: Range) -> ParquetResult { + let shared = Arc::clone(&self.shared); + loop { + let notified = shared.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + match shared.step(&range) { + Step::Hit(b) => { + HIT_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + return Ok(b); + } + Step::Direct => { + MISS_BYTES.fetch_add(range.end - range.start, Ordering::Relaxed); + return self.inner.get_bytes(range).await; + } + Step::Wait => { + let t = std::time::Instant::now(); + notified.await; + WAIT_US.fetch_add(t.elapsed().as_micros() as u64, Ordering::Relaxed); + } + Step::Fetch(claims) => { + // The decoder has one reader, so it fetches the claims in + // turn. A read spans more than one chunk only at a chunk + // boundary, so a claim list is short. + for (i, r) in claims { + shared.fetch(&mut self.inner, i, r).await; + } + } + } + } + } +} + +impl AsyncFileReader for PrefetchReader { + fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, ParquetResult> { + self.read(range).boxed() + } + + fn get_byte_ranges( + &mut self, + ranges: Vec>, + ) -> BoxFuture<'_, ParquetResult>> { + async move { + let mut out = Vec::with_capacity(ranges.len()); + for r in ranges { + out.push(self.read(r).await?); + } + Ok(out) + } + .boxed() + } + + fn get_metadata<'a>( + &'a mut self, + options: Option<&'a parquet::arrow::arrow_reader::ArrowReaderOptions>, + ) -> BoxFuture<'a, ParquetResult>> { + self.inner.get_metadata(options) + } +} diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs index f84899eba37b7..14c99882a751f 100644 --- a/datafusion/datasource-parquet/src/push_decoder.rs +++ b/datafusion/datasource-parquet/src/push_decoder.rs @@ -795,8 +795,10 @@ impl ReadAheadMemory { /// Reserve bytes the decoder needs. Never fails and can exceed the pool /// limit: the decoder cannot continue without these bytes. - fn reserve_required(&self, bytes: u64) { - self.reservation.grow(to_usize(bytes)); + fn reserve_required(&self, bytes: u64) -> Result<()> { + let bytes = to_usize(bytes); + self.reservation.grow(bytes); + Ok(()) } /// Bytes the pool can grant now. Unlimited if the pool has no limit or @@ -1164,7 +1166,7 @@ impl PushDecoderStreamState { .buffered_bytes(); let read_ahead = self.read_ahead.as_mut().expect("streaming policy"); let needed_bytes: u64 = needed.iter().map(|r| r.end - r.start).sum(); - read_ahead.memory.reserve_required(needed_bytes); + read_ahead.memory.reserve_required(needed_bytes)?; for range in &needed { read_ahead.fetched.insert((range.start, range.end)); } @@ -1487,7 +1489,7 @@ mod tests { let pool: Arc = Arc::new(GreedyMemoryPool::new(100)); let memory = ReadAheadMemory::new(Some(Arc::clone(&pool)), 0); // Required bytes can exceed the limit; speculative bytes cannot. - memory.reserve_required(150); + memory.reserve_required(150).unwrap(); assert_eq!(pool.reserved(), 150); assert_eq!(memory.available(), 0); assert!(!memory.try_reserve_speculative(1)); diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index c8e643af5a882..2758cc8d4906f 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -726,6 +726,7 @@ impl FileSource for ParquetSource { .table_parquet_options .global .read_ahead_bytes + .or(std::env::var("DF_PREFETCH").ok().map(|_| 0)) .map(|_| Arc::clone(context.memory_pool())); Ok(Box::new(self.build_morselizer( object_store, diff --git a/datafusion/datasource/src/file_stream/scan_state.rs b/datafusion/datasource/src/file_stream/scan_state.rs index d1e57604f971c..1ef90bd74887c 100644 --- a/datafusion/datasource/src/file_stream/scan_state.rs +++ b/datafusion/datasource/src/file_stream/scan_state.rs @@ -81,6 +81,10 @@ pub(super) struct ScanState { /// Once the I/O completes, yields the next planner and is pushed back /// onto `ready_planners`. pending_planner: Option, + /// PROTOTYPE: extra planners blocked on I/O, for files opened ahead. + pending_ahead: Vec, + /// PROTOTYPE: files to open ahead of the active reader (`DF_FILE_LOOKAHEAD`). + lookahead: usize, /// Metrics for the active scan queues. metrics: FileStreamMetrics, } @@ -102,6 +106,11 @@ impl ScanState { ready_morsels: Default::default(), reader: None, pending_planner: None, + pending_ahead: Vec::new(), + lookahead: std::env::var("DF_FILE_LOOKAHEAD") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0), metrics, } } @@ -123,8 +132,8 @@ impl ScanState { /// The return [`ScanAndReturn`] tells `poll_inner` how to update the /// outer `FileStreamState`. pub(super) fn poll_scan(&mut self, cx: &mut Context<'_>) -> ScanAndReturn { - let _processing_timer: ScopedTimerGuard<'_> = - self.metrics.time_processing.timer(); + let time_processing = self.metrics.time_processing.clone(); + let _processing_timer: ScopedTimerGuard<'_> = time_processing.timer(); // Try and resolve outstanding IO first. If it is still pending, check // the current reader or ready morsels before yielding. New planning @@ -154,6 +163,14 @@ impl ScanState { } } + // PROTOTYPE: open files ahead while a reader is active, so their + // metadata I/O and prefetch overlap the current file's decode. + if self.lookahead > 0 && self.reader.is_some() { + if let Err(err) = self.plan_ahead(cx) { + return ScanAndReturn::Error(err); + } + } + // Next try and get the next batch from the active reader, if any. if let Some(reader) = self.reader.as_mut() { match reader.poll_next_unpin(cx) { @@ -227,6 +244,15 @@ impl ScanState { if self.pending_planner.is_some() { return ScanAndReturn::Return(Poll::Pending); } + if !self.pending_ahead.is_empty() && self.ready_planners.is_empty() { + if let Err(err) = self.plan_ahead(cx) { + return ScanAndReturn::Error(err); + } + if self.ready_planners.is_empty() && self.ready_morsels.is_empty() { + return ScanAndReturn::Return(Poll::Pending); + } + return ScanAndReturn::Continue; + } // No reader or morsel, so try to produce more work via CPU planning. if let Some(planner) = self.ready_planners.pop_front() { @@ -290,6 +316,55 @@ impl ScanState { } } +impl ScanState { + /// PROTOTYPE: poll the look-ahead I/O, plan ready files, and open new + /// files until `lookahead` files are queued behind the active reader. + /// Errors are returned regardless of `on_error`. + fn plan_ahead(&mut self, cx: &mut Context<'_>) -> Result<()> { + let mut i = 0; + while i < self.pending_ahead.len() { + match self.pending_ahead[i].poll_unpin(cx) { + Poll::Pending => i += 1, + Poll::Ready(r) => { + self.pending_ahead.swap_remove(i); + self.ready_planners.push_back(r?); + } + } + } + while let Some(planner) = self.ready_planners.pop_front() { + if let Some(mut plan) = planner.plan()? { + self.ready_morsels.extend(plan.take_morsels()); + self.ready_planners.extend(plan.take_ready_planners()); + if let Some(mut p) = plan.take_pending_planner() { + // Poll once so its waker is registered. + match p.poll_unpin(cx) { + Poll::Pending => self.pending_ahead.push(p), + Poll::Ready(r) => self.ready_planners.push_back(r?), + } + } + } else { + self.metrics.files_processed.add(1); + } + } + while self.ready_morsels.len() + self.pending_ahead.len() < self.lookahead { + let Some(file) = self.work_source.pop_front() else { break }; + self.metrics.files_opened.add(1); + let planner = self.morselizer.plan_file(file)?; + if let Some(mut plan) = planner.plan()? { + self.ready_morsels.extend(plan.take_morsels()); + self.ready_planners.extend(plan.take_ready_planners()); + if let Some(mut p) = plan.take_pending_planner() { + match p.poll_unpin(cx) { + Poll::Pending => self.pending_ahead.push(p), + Poll::Ready(r) => self.ready_planners.push_back(r?), + } + } + } + } + Ok(()) + } +} + /// What should be done on the next iteration of [`ScanState::poll_scan`]? pub(super) enum ScanAndReturn { /// Poll again.