From d0e87e83a025aa006f9d61335418eb5936a3ea42 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 7 Oct 2026 14:38:25 -0700 Subject: [PATCH 01/10] Add dedicated page index decoding to push decoder --- parquet/src/file/metadata/parser.rs | 21 +-- parquet/src/file/metadata/push_decoder.rs | 184 +++++++++++++++++++- parquet/src/file/metadata/reader.rs | 201 +++++++++++++++++++++- 3 files changed, 387 insertions(+), 19 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22f0564a9c4f..4599452187bb 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -20,12 +20,12 @@ //! These functions parse thrift-encoded metadata from a byte slice //! into the corresponding Rust structures -use std::{ops::Range, sync::Arc}; +use std::ops::Range; use bytes::Bytes; use crate::errors::{ParquetError, Result}; -use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; +use crate::file::metadata::page_index::{PageIndex, PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, @@ -235,10 +235,10 @@ pub(crate) fn decode_metadata( parquet_metadata_from_bytes(buf, options) } -/// Parses page indexes from the provided bytes and replaces those in the metadata. +/// Parses page indexes from the provided bytes. /// /// Arguments -/// * `metadata` - The ParquetMetaData whose page index will be replaced. +/// * `metadata` - The footer metadata describing the page index locations. /// * `column_index_policy` - The policy for handling column index parsing (e.g., /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., @@ -250,14 +250,14 @@ pub(crate) fn decode_metadata( /// * `bytes` - The byte slice containing the page index data. /// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( - metadata: &mut ParquetMetaData, + metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, column_index_mask: &ColumnChunkMask, offset_index_mask: &ColumnChunkMask, bytes: &Bytes, start_offset: u64, -) -> Result<()> { +) -> Result> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); let mut builder = PageIndexBuilder::default(); @@ -285,17 +285,12 @@ pub(crate) fn parse_page_index( )?; } - // Always replace the page index, even if both policies are Skip. - // This ensures that repeated calls to read_page_indexes replace the existing index - // rather than preserving it. let page_index = builder.build(); if page_index.has_column_indexes() || page_index.has_offset_indexes() { - metadata.set_page_index(Some(Arc::new(page_index))); + Ok(Some(page_index)) } else { - // If no indexes were read, clear the page index - metadata.set_page_index(None); + Ok(None) } - Ok(()) } fn parse_column_index( diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 90fd3ee7fa0e..efaad69a9019 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -20,6 +20,7 @@ use crate::DecodeResult; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, @@ -466,8 +467,8 @@ impl ParquetMetaDataPushDecoder { let buffer = self.get_bytes(&page_index_range)?; let offset = page_index_range.start; - parse_page_index( - &mut metadata, + let page_index = parse_page_index( + &metadata, self.column_index_policy, self.offset_index_policy, &self.column_index_mask, @@ -475,6 +476,13 @@ impl ParquetMetaDataPushDecoder { &buffer, offset, )?; + // install the new page index or clear the old one + if let Some(page_index) = page_index { + metadata.set_page_index(Some(Arc::new(page_index))); + } else { + metadata.set_page_index(None); + } + self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); } @@ -489,6 +497,176 @@ impl ParquetMetaDataPushDecoder { } } + /// Try to decode the page indexes from the pushed data, returning the + /// decoded [`PageIndex`] or an error if not enough data is available. + /// + /// This method can only be called after [`Self::try_new_with_metadata`], + /// which initializes the decoder with previously decoded metadata. + /// + /// This method is useful when you want to: + /// 1. First decode the footer metadata without reading the page indexes + /// (using [`PageIndexPolicy::Skip`]) + /// 2. Later selectively decode a subset of the page indexes using + /// [`Self::with_column_index_mask`] or [`Self::with_offset_index_mask`] + /// + /// When the data can be accessed through a [`ChunkReader`], prefer + /// [`ParquetMetaDataReader::read_page_index`] (or `read_page_index_async` for asynchronous + /// I/O), which handles the decoder state machine and I/O requests. + /// + /// # Example + /// + /// This example shows how to first read the footer metadata without the + /// page index, and then later selectively decode only a subset of the + /// page indexes (column index for column 0, offset index for columns 0, 1, + /// and 4). + /// + #[cfg_attr( + feature = "arrow", + doc = r#" +```rust +# use std::ops::Range; +# use bytes::Bytes; +# use arrow_array::record_batch; +# use parquet::DecodeResult; +# use parquet::arrow::ArrowWriter; +# use parquet::errors::ParquetError; +# use parquet::file::metadata::{ +# ParquetMetaData, ParquetMetaDataPushDecoder, PageIndexPolicy, ColumnChunkMask +# }; +# +# fn selective_page_index_decode() -> Result<(), ParquetError> { +# let file_bytes = { +# let mut buffer = vec![0]; +# let batch = record_batch!( +# ("col0", Int32, [1, 2, 3]), +# ("col1", Int32, [4, 5, 6]), +# ("col2", Int32, [7, 8, 9]), +# ("col3", Int32, [10, 11, 12]), +# ("col4", Int32, [13, 14, 15]) +# ).unwrap(); +# let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap(); +# writer.write(&batch).unwrap(); +# writer.close().unwrap(); +# Bytes::from(buffer) +# }; +# // mimic IO by returning a function that returns the bytes for a given range +# let get_range = |range: &Range| -> Bytes { +# let start = range.start as usize; +# let end = range.end as usize; +# file_bytes.slice(start..end) +# }; +# let file_len = file_bytes.len() as u64; +# +// Step 1: Read footer metadata without page indexes +let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Skip); + +// Decode the metadata (without page indexes) +let metadata = loop { + match decoder.try_decode() { + Ok(DecodeResult::Data(metadata)) => break metadata, + Ok(DecodeResult::NeedsData(ranges)) => { + let data = ranges.iter().map(|range| get_range(range)).collect(); + decoder.push_ranges(ranges, data).unwrap(); + } + Ok(DecodeResult::Finished) => unreachable!(), + Err(e) => return Err(e), + } +}; + +// At this point, we have the metadata but no page indexes +assert!(metadata.page_index().is_none()); + +// Step 2: Later, selectively decode a subset of the page indexes +// Create a new decoder initialized with the previously decoded metadata +let mut decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_len, metadata) + .unwrap() + // Request column index for column 0 only + .with_column_index_mask(ColumnChunkMask::columns([0])) + // Request offset index for columns 0, 1, and 4 + .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); + +// Decode the selected page indexes +let page_index = loop { + match decoder.try_decode_page_index() { + Ok(DecodeResult::Data(page_index)) => break page_index, + Ok(DecodeResult::NeedsData(ranges)) => { + let data = ranges.iter().map(|range| get_range(range)).collect(); + decoder.push_ranges(ranges, data).unwrap(); + } + Ok(DecodeResult::Finished) => unreachable!(), + Err(e) => return Err(e), + } +}; + +// The returned PageIndex contains only the requested subset +assert!(page_index.is_some()); +# Ok(()) +# } +``` +"# + )] + /// + /// # Returns + /// + /// * `Ok(DecodeResult::Data(Some(page_index)))` - Successfully decoded the + /// requested page indexes based on the configured policies and masks + /// * `Ok(DecodeResult::Data(None))` - No page indexes were requested + /// (both policies set to [`PageIndexPolicy::Skip`] or masks select nothing) + /// * `Ok(DecodeResult::NeedsData(ranges))` - More data is needed to decode + /// the page indexes. Push the requested ranges and call this method again. + /// * `Ok(DecodeResult::Finished)` - The decoder has finished and no more + /// data can be decoded + /// * `Err(_)` - An error occurred during decoding + pub fn try_decode_page_index(&mut self) -> Result>> { + // stripped down state machine copied from try_decode. need to be in the + // `ReadingPageIndex` state initially or this will error. + match std::mem::replace(&mut self.state, DecodeState::Intermediate) { + DecodeState::ReadingPageIndex(metadata) => { + // First determine if any page indexes are needed based on + // the specified policies + let range = range_for_page_index( + &metadata, + self.column_index_policy, + self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, + ); + + let Some(page_index_range) = range else { + self.state = DecodeState::Finished; + return Ok(DecodeResult::Data(None)); + }; + + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(needs_range(page_index_range)); + } + + let buffer = self.get_bytes(&page_index_range)?; + let offset = page_index_range.start; + let page_index = parse_page_index( + &metadata, + self.column_index_policy, + self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, + &buffer, + offset, + )?; + + self.state = DecodeState::Finished; + Ok(DecodeResult::Data(page_index)) + } + + DecodeState::Finished => Ok(DecodeResult::Finished), + _ => Err(general_err!( + "ParquetMetaDataPushDecoder: internal error, invalid state" + )), + } + } + /// Returns the bytes for the given range from the internal buffer fn get_bytes(&self, range: &Range) -> Result { let start = range.start; @@ -503,7 +681,7 @@ impl ParquetMetaDataPushDecoder { } /// returns a DecodeResults that describes needing the given range -fn needs_range(range: Range) -> DecodeResult { +fn needs_range(range: Range) -> DecodeResult { DecodeResult::NeedsData(vec![range]) } diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 78710600bfe5..4f5e3dd6e7c9 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -23,6 +23,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; #[cfg(feature = "arrow")] use crate::file::metadata::dictionary::decode_dictionary_page; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::parser::decode_metadata; use crate::file::metadata::thrift::parquet_schema_from_bytes; use crate::file::metadata::{ @@ -573,6 +574,78 @@ impl ParquetMetaDataReader { self.read_page_indexes_sized(reader, reader.len()) } + /// Reads and returns the page index structures described by the stored + /// [`ParquetMetaData`]. + /// + /// Unlike [`Self::read_page_indexes`], this returns the decoded [`PageIndex`] separately + /// instead of attaching it to the metadata. This is useful for populating an external page + /// index cache. The configured page index policies and masks determine which indexes are + /// returned. + /// + /// This operation consumes `self` and the stored [`ParquetMetaData`]. Use + /// [`Self::new_with_metadata`] to construct the reader. + pub fn read_page_index(self, reader: &R) -> Result> { + self.read_page_index_sized(reader, reader.len()) + } + + /// Reads and returns the page index structures described by the stored + /// [`ParquetMetaData`], using `file_size` as the size of the original Parquet file. + /// + /// This variant supports a [`Bytes`] reader containing only a suffix of the file. It returns + /// [`ParquetError::NeedMoreData`] if that suffix does not contain the requested page indexes. + /// See [`Self::read_page_index`] for ownership and configuration details. + pub fn read_page_index_sized( + mut self, + reader: &R, + file_size: u64, + ) -> Result> { + let Some(metadata) = self.metadata.take() else { + return Err(general_err!( + "Tried to read page indexes without ParquetMetaData metadata" + )); + }; + + let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? + .with_offset_index_policy(self.offset_index) + .with_column_index_policy(self.column_index) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) + .with_metadata_options(self.metadata_options.clone()); + let mut push_decoder = self.prepare_push_decoder(push_decoder); + + let range = match needs_page_index_data(&mut push_decoder)? { + NeedsIndexData::No(page_index) => return Ok(page_index), + NeedsIndexData::Yes(range) => range, + }; + + let file_range = file_size.saturating_sub(reader.len())..file_size; + if !(file_range.contains(&range.start) && file_range.contains(&range.end)) { + return if range.end > file_size { + Err(eof_err!( + "Parquet file too small. Range {range:?} is beyond file bounds {file_size}", + )) + } else { + Err(ParquetError::NeedMoreData( + (file_size - range.start).try_into()?, + )) + }; + } + + if let Some(metadata_size) = self.metadata_size { + let metadata_range = file_size.saturating_sub(metadata_size as u64)..file_size; + if range.end > metadata_range.start { + return Err(eof_err!( + "Parquet file too small. Page index range {range:?} overlaps with file metadata {metadata_range:?}", + )); + } + } + + let bytes_needed = usize::try_from(range.end - range.start)?; + let bytes = reader.get_bytes(range.start - file_range.start, bytes_needed)?; + push_decoder.push_range(range, bytes)?; + parse_page_index_data(&mut push_decoder) + } + /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// This variant is used when `reader` cannot access the entire Parquet file (e.g. it is /// a [`Bytes`] struct containing the tail of the file). @@ -731,6 +804,47 @@ impl ParquetMetaDataReader { self.load_page_index_with_remainder(fetch, None).await } + /// Asynchronously reads and returns the page index structures described by the stored + /// [`ParquetMetaData`]. + /// + /// Unlike [`Self::load_page_index`], this returns the decoded [`PageIndex`] separately instead + /// of attaching it to the metadata. This is useful for populating an external page index + /// cache. This operation consumes `self` and the stored metadata. + #[cfg(all(feature = "async", feature = "arrow"))] + pub async fn read_page_index_async( + mut self, + mut fetch: F, + ) -> Result> { + let Some(metadata) = self.metadata.take() else { + return Err(general_err!("Footer metadata is not present")); + }; + + // MetadataFetch uses absolute file offsets, so the actual file size is not needed here. + let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(u64::MAX, metadata)? + .with_offset_index_policy(self.offset_index) + .with_column_index_policy(self.column_index) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) + .with_metadata_options(self.metadata_options.clone()); + let mut push_decoder = self.prepare_push_decoder(push_decoder); + + let range = match needs_page_index_data(&mut push_decoder)? { + NeedsIndexData::No(page_index) => return Ok(page_index), + NeedsIndexData::Yes(range) => range, + }; + let bytes = fetch.fetch(range.clone()).await?; + if bytes.len() as u64 != range.end - range.start { + return Err(general_err!( + "Corrupted parquet file: index data length mismatch, expected {}, got {}", + range.end - range.start, + bytes.len() + )); + } + + push_decoder.push_range(range, bytes)?; + parse_page_index_data(&mut push_decoder) + } + /// Reads and decodes the dictionary page of a column chunk into an Arrow array. /// /// Returns `Ok(None)` if the column chunk has no dictionary page, or if @@ -1114,16 +1228,18 @@ impl ParquetMetaDataReader { /// The bounds needed to read page indexes // this is an internal enum, so it is ok to allow differences in enum size -enum NeedsIndexData { +enum NeedsIndexData { /// no additional data is needed (e.g. the indexes weren't requested) - No(ParquetMetaData), + No(T), /// Additional data is needed, with the range that are required Yes(Range), } /// Determines a single combined range of bytes needed to read the page indexes, /// or returns the metadata if no additional data is needed (e.g. if no page indexes are requested) -fn needs_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result { +fn needs_index_data( + push_decoder: &mut ParquetMetaDataPushDecoder, +) -> Result> { match push_decoder.try_decode()? { DecodeResult::NeedsData(ranges) => { let range = ranges @@ -1137,6 +1253,24 @@ fn needs_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result Result>> { + match push_decoder.try_decode_page_index()? { + DecodeResult::NeedsData(ranges) => { + let range = ranges + .into_iter() + .reduce(|a, b| a.start.min(b.start)..a.end.max(b.end)) + .ok_or_else(|| general_err!("Internal error: no ranges provided"))?; + Ok(NeedsIndexData::Yes(range)) + } + DecodeResult::Data(page_index) => Ok(NeedsIndexData::No(page_index)), + DecodeResult::Finished => Err(general_err!("Internal error: decoder was finished")), + } +} + /// Given a push decoder that has had the needed ranges pushed to it, /// attempt to decode indexes and return the updated metadata. fn parse_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result { @@ -1149,6 +1283,20 @@ fn parse_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result Result> { + match push_decoder.try_decode_page_index()? { + DecodeResult::NeedsData(_) => Err(general_err!( + "Internal error: decoder still needs data after reading required range" + )), + DecodeResult::Data(page_index) => Ok(page_index), + DecodeResult::Finished => Err(general_err!("Internal error: decoder was finished")), + } +} + /// Returns the `[start, end)` byte range of a column chunk's dictionary /// page, or `None` if the column chunk has no dictionary page or is not a /// `BYTE_ARRAY` column (the only physical type [`decode_dictionary_page`] @@ -1331,6 +1479,31 @@ mod tests { ); } + #[test] + #[cfg_attr(miri, ignore)] // Takes too long + fn test_read_page_index() { + use crate::file::metadata::page_index::PageIndexProvider; + + let file = get_test_file("alltypes_tiny_pages.parquet"); + let metadata = ParquetMetaDataReader::new() + .parse_and_finish(&file) + .unwrap(); + + let reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()); + assert!(reader.read_page_index(&file).unwrap().is_none()); + + let reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0])); + let page_index = reader.read_page_index(&file).unwrap().unwrap(); + + assert!(page_index.column_index(0, 0).is_some()); + assert!(page_index.offset_index(0, 0).is_some()); + assert!(page_index.column_index(0, 1).is_none()); + assert!(page_index.offset_index(0, 1).is_none()); + } + #[test] fn test_chunk_mask() { let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); @@ -1437,6 +1610,28 @@ mod async_tests { Ok(buf.into()) } + #[tokio::test] + async fn test_read_page_index_async() { + use crate::file::metadata::page_index::PageIndexProvider; + + let mut file = get_test_file("alltypes_tiny_pages.parquet"); + let metadata = ParquetMetaDataReader::new() + .parse_and_finish(&file) + .unwrap(); + let reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0])); + let fetch = MetadataFetchFn(|range| futures::future::ready(read_range(&mut file, range))); + + let page_index = reader.read_page_index_async(fetch).await.unwrap().unwrap(); + + assert!(page_index.column_index(0, 0).is_some()); + assert!(page_index.offset_index(0, 0).is_some()); + assert!(page_index.column_index(0, 1).is_none()); + assert!(page_index.offset_index(0, 1).is_none()); + } + #[tokio::test] async fn test_simple() { let mut file = get_test_file("nulls.snappy.parquet"); From 2da8253c93bae5db788411f07ec2b503d9f3588c Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 7 Oct 2026 16:23:14 -0700 Subject: [PATCH 02/10] add more tests --- parquet/src/file/metadata/push_decoder.rs | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index efaad69a9019..cf8386cefc03 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -746,6 +746,7 @@ pub(crate) fn range_for_page_index( mod tests { use super::*; use crate::arrow::ArrowWriter; + use crate::file::metadata::page_index::PageIndexProvider; use crate::file::properties::WriterProperties; use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringViewArray}; use bytes::Bytes; @@ -797,6 +798,66 @@ mod tests { assert!(masked.end - masked.start < all.end - all.start); } + #[test] + fn test_page_index_decoder_incremental() { + let metadata = test_metadata_without_page_index(); + let mut decoder = + ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata) + .unwrap() + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1])); + + let ranges = expect_needs_data(decoder.try_decode_page_index()); + assert_eq!(ranges.len(), 1); + push_ranges_to_metadata_decoder(&mut decoder, ranges); + + let page_index = expect_data(decoder.try_decode_page_index()).unwrap(); + expect_finished(decoder.try_decode_page_index()); + + for row_group_idx in 0..2 { + for column_idx in 0..3 { + assert_eq!( + page_index.column_index(row_group_idx, column_idx).is_some(), + row_group_idx == 1 && matches!(column_idx, 0 | 2) + ); + assert_eq!( + page_index.offset_index(row_group_idx, column_idx).is_some(), + row_group_idx == 0 && column_idx == 1 + ); + } + } + } + + #[test] + fn test_page_index_decoder_no_indexes_requested() { + let metadata = test_metadata_without_page_index(); + let mut decoder = + ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Skip); + + assert!(expect_data(decoder.try_decode_page_index()).is_none()); + expect_finished(decoder.try_decode_page_index()); + + let metadata = test_metadata_without_page_index(); + let mut decoder = + ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata) + .unwrap() + .with_page_index_mask(ColumnChunkMask::none()); + + assert!(expect_data(decoder.try_decode_page_index()).is_none()); + expect_finished(decoder.try_decode_page_index()); + } + + #[test] + fn test_page_index_decoder_requires_metadata() { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); + let err = decoder.try_decode_page_index().unwrap_err(); + assert!(err.to_string().contains("invalid state")); + } + /// It is possible to feed some, but not all, of the footer into the metadata decoder /// before asked. This avoids multiple IO requests #[test] @@ -988,6 +1049,16 @@ mod tests { metadata_decoder.push_ranges(ranges, data).unwrap(); } + fn test_metadata_without_page_index() -> ParquetMetaData { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Skip); + push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]); + let metadata = expect_data(decoder.try_decode()); + assert!(metadata.page_index().is_none()); + metadata + } + /// Expect that the [`DecodeResult`] is a [`DecodeResult::Data`] and return the corresponding element fn expect_data(result: Result>) -> T { match result.expect("Expected Ok(DecodeResult::Data(T))") { From 7e6518aa5a0c5e58d61d182f01bd085e883b5c00 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 7 Oct 2026 16:34:55 -0700 Subject: [PATCH 03/10] clean up docs --- parquet/src/file/metadata/push_decoder.rs | 101 ++-------------------- 1 file changed, 5 insertions(+), 96 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index cf8386cefc03..a69fac47a6d7 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -510,104 +510,10 @@ impl ParquetMetaDataPushDecoder { /// [`Self::with_column_index_mask`] or [`Self::with_offset_index_mask`] /// /// When the data can be accessed through a [`ChunkReader`], prefer - /// [`ParquetMetaDataReader::read_page_index`] (or `read_page_index_async` for asynchronous + /// [`ParquetMetaDataReader::read_page_index`] (or + /// [`ParquetMetaDataReader::read_page_index_async`] for asynchronous /// I/O), which handles the decoder state machine and I/O requests. /// - /// # Example - /// - /// This example shows how to first read the footer metadata without the - /// page index, and then later selectively decode only a subset of the - /// page indexes (column index for column 0, offset index for columns 0, 1, - /// and 4). - /// - #[cfg_attr( - feature = "arrow", - doc = r#" -```rust -# use std::ops::Range; -# use bytes::Bytes; -# use arrow_array::record_batch; -# use parquet::DecodeResult; -# use parquet::arrow::ArrowWriter; -# use parquet::errors::ParquetError; -# use parquet::file::metadata::{ -# ParquetMetaData, ParquetMetaDataPushDecoder, PageIndexPolicy, ColumnChunkMask -# }; -# -# fn selective_page_index_decode() -> Result<(), ParquetError> { -# let file_bytes = { -# let mut buffer = vec![0]; -# let batch = record_batch!( -# ("col0", Int32, [1, 2, 3]), -# ("col1", Int32, [4, 5, 6]), -# ("col2", Int32, [7, 8, 9]), -# ("col3", Int32, [10, 11, 12]), -# ("col4", Int32, [13, 14, 15]) -# ).unwrap(); -# let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap(); -# writer.write(&batch).unwrap(); -# writer.close().unwrap(); -# Bytes::from(buffer) -# }; -# // mimic IO by returning a function that returns the bytes for a given range -# let get_range = |range: &Range| -> Bytes { -# let start = range.start as usize; -# let end = range.end as usize; -# file_bytes.slice(start..end) -# }; -# let file_len = file_bytes.len() as u64; -# -// Step 1: Read footer metadata without page indexes -let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Skip); - -// Decode the metadata (without page indexes) -let metadata = loop { - match decoder.try_decode() { - Ok(DecodeResult::Data(metadata)) => break metadata, - Ok(DecodeResult::NeedsData(ranges)) => { - let data = ranges.iter().map(|range| get_range(range)).collect(); - decoder.push_ranges(ranges, data).unwrap(); - } - Ok(DecodeResult::Finished) => unreachable!(), - Err(e) => return Err(e), - } -}; - -// At this point, we have the metadata but no page indexes -assert!(metadata.page_index().is_none()); - -// Step 2: Later, selectively decode a subset of the page indexes -// Create a new decoder initialized with the previously decoded metadata -let mut decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_len, metadata) - .unwrap() - // Request column index for column 0 only - .with_column_index_mask(ColumnChunkMask::columns([0])) - // Request offset index for columns 0, 1, and 4 - .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); - -// Decode the selected page indexes -let page_index = loop { - match decoder.try_decode_page_index() { - Ok(DecodeResult::Data(page_index)) => break page_index, - Ok(DecodeResult::NeedsData(ranges)) => { - let data = ranges.iter().map(|range| get_range(range)).collect(); - decoder.push_ranges(ranges, data).unwrap(); - } - Ok(DecodeResult::Finished) => unreachable!(), - Err(e) => return Err(e), - } -}; - -// The returned PageIndex contains only the requested subset -assert!(page_index.is_some()); -# Ok(()) -# } -``` -"# - )] - /// /// # Returns /// /// * `Ok(DecodeResult::Data(Some(page_index)))` - Successfully decoded the @@ -619,6 +525,9 @@ assert!(page_index.is_some()); /// * `Ok(DecodeResult::Finished)` - The decoder has finished and no more /// data can be decoded /// * `Err(_)` - An error occurred during decoding + /// + /// [`ParquetMetaDataReader::read_page_index`]: crate::file::metadata::ParquetMetaDataReader::read_page_index + /// [`ParquetMetaDataReader::read_page_index_async`]: crate::file::metadata::ParquetMetaDataReader::read_page_index_async pub fn try_decode_page_index(&mut self) -> Result>> { // stripped down state machine copied from try_decode. need to be in the // `ReadingPageIndex` state initially or this will error. From 5d1163fde8e32e5ac2e75df2a48a8d2eba641d4d Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 7 Oct 2026 21:18:18 -0700 Subject: [PATCH 04/10] refactor to reuse page index decoding logic --- parquet/src/file/metadata/push_decoder.rs | 157 +++++++++++----------- 1 file changed, 75 insertions(+), 82 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index a69fac47a6d7..7feb4a5ae5cf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -439,52 +439,25 @@ impl ParquetMetaDataPushDecoder { self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); } - DecodeState::ReadingPageIndex(mut metadata) => { - // First determine if any page indexes are needed based on - // the specified policies - let range = range_for_page_index( - &metadata, - self.column_index_policy, - self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, - ); - - let Some(page_index_range) = range else { - // There is nothing to decode because both policies are Skip, the masks - // select nothing, or the file has no index ranges. Clear any previously - // loaded indexes while preserving the compatibility behavior that - // PageIndexPolicy::Required accepts files with no page indexes at all. - metadata.set_page_index(None); - self.state = DecodeState::Finished; - return Ok(DecodeResult::Data(*metadata)); - }; - - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + DecodeState::ReadingPageIndex(metadata) => { + match self.try_decode_page_index_for_metadata(metadata)? { + PageIndexDecodeResult::NeedsData(ranges) => { + return Ok(DecodeResult::NeedsData(ranges)); + } + PageIndexDecodeResult::Data { + mut metadata, + page_index, + } => { + // Install the newly decoded indexes, or clear any existing indexes if + // none were requested or present in the file. + if let Some(page_index) = page_index { + metadata.set_page_index(Some(Arc::new(page_index))); + } else { + metadata.set_page_index(None); + } + return Ok(DecodeResult::Data(*metadata)); + } } - - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; - let page_index = parse_page_index( - &metadata, - self.column_index_policy, - self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, - &buffer, - offset, - )?; - // install the new page index or clear the old one - if let Some(page_index) = page_index { - metadata.set_page_index(Some(Arc::new(page_index))); - } else { - metadata.set_page_index(None); - } - - self.state = DecodeState::Finished; - return Ok(DecodeResult::Data(*metadata)); } DecodeState::Finished => return Ok(DecodeResult::Finished), @@ -529,46 +502,15 @@ impl ParquetMetaDataPushDecoder { /// [`ParquetMetaDataReader::read_page_index`]: crate::file::metadata::ParquetMetaDataReader::read_page_index /// [`ParquetMetaDataReader::read_page_index_async`]: crate::file::metadata::ParquetMetaDataReader::read_page_index_async pub fn try_decode_page_index(&mut self) -> Result>> { - // stripped down state machine copied from try_decode. need to be in the - // `ReadingPageIndex` state initially or this will error. match std::mem::replace(&mut self.state, DecodeState::Intermediate) { DecodeState::ReadingPageIndex(metadata) => { - // First determine if any page indexes are needed based on - // the specified policies - let range = range_for_page_index( - &metadata, - self.column_index_policy, - self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, - ); - - let Some(page_index_range) = range else { - self.state = DecodeState::Finished; - return Ok(DecodeResult::Data(None)); - }; - - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); - } - - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; - let page_index = parse_page_index( - &metadata, - self.column_index_policy, - self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, - &buffer, - offset, - )?; - - self.state = DecodeState::Finished; - Ok(DecodeResult::Data(page_index)) + Ok(match self.try_decode_page_index_for_metadata(metadata)? { + PageIndexDecodeResult::NeedsData(ranges) => DecodeResult::NeedsData(ranges), + PageIndexDecodeResult::Data { page_index, .. } => { + DecodeResult::Data(page_index) + } + }) } - DecodeState::Finished => Ok(DecodeResult::Finished), _ => Err(general_err!( "ParquetMetaDataPushDecoder: internal error, invalid state" @@ -576,6 +518,49 @@ impl ParquetMetaDataPushDecoder { } } + fn try_decode_page_index_for_metadata( + &mut self, + metadata: Box, + ) -> Result { + let range = range_for_page_index( + &metadata, + self.column_index_policy, + self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, + ); + + let Some(page_index_range) = range else { + self.state = DecodeState::Finished; + return Ok(PageIndexDecodeResult::Data { + metadata, + page_index: None, + }); + }; + + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(PageIndexDecodeResult::NeedsData(vec![page_index_range])); + } + + let buffer = self.get_bytes(&page_index_range)?; + let page_index = parse_page_index( + &metadata, + self.column_index_policy, + self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, + &buffer, + page_index_range.start, + )?; + + self.state = DecodeState::Finished; + Ok(PageIndexDecodeResult::Data { + metadata, + page_index, + }) + } + /// Returns the bytes for the given range from the internal buffer fn get_bytes(&self, range: &Range) -> Result { let start = range.start; @@ -610,6 +595,14 @@ enum DecodeState { Intermediate, } +enum PageIndexDecodeResult { + NeedsData(Vec>), + Data { + metadata: Box, + page_index: Option, + }, +} + /// Returns the byte range needed to read the offset/page indexes, based on the /// specified policies and masks /// From a273efe19537eb98ef0916ef1f3d9b50585ed505 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 9 Oct 2026 21:57:46 -0700 Subject: [PATCH 05/10] implement some suggestions from review --- parquet/src/file/metadata/push_decoder.rs | 68 ++++++++----------- parquet/src/file/metadata/reader.rs | 82 ++++++++++++++--------- 2 files changed, 78 insertions(+), 72 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 7feb4a5ae5cf..065221c82ba9 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -441,13 +441,10 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(metadata) => { match self.try_decode_page_index_for_metadata(metadata)? { - PageIndexDecodeResult::NeedsData(ranges) => { + DecodeResult::NeedsData(ranges) => { return Ok(DecodeResult::NeedsData(ranges)); } - PageIndexDecodeResult::Data { - mut metadata, - page_index, - } => { + DecodeResult::Data((mut metadata, page_index)) => { // Install the newly decoded indexes, or clear any existing indexes if // none were requested or present in the file. if let Some(page_index) = page_index { @@ -457,6 +454,7 @@ impl ParquetMetaDataPushDecoder { } return Ok(DecodeResult::Data(*metadata)); } + DecodeResult::Finished => return Ok(DecodeResult::Finished), } } @@ -470,8 +468,8 @@ impl ParquetMetaDataPushDecoder { } } - /// Try to decode the page indexes from the pushed data, returning the - /// decoded [`PageIndex`] or an error if not enough data is available. + /// Try to decode the page indexes from the pushed data, returning the stored + /// [`ParquetMetaData`] together with the decoded [`PageIndex`]. /// /// This method can only be called after [`Self::try_new_with_metadata`], /// which initializes the decoder with previously decoded metadata. @@ -483,32 +481,36 @@ impl ParquetMetaDataPushDecoder { /// [`Self::with_column_index_mask`] or [`Self::with_offset_index_mask`] /// /// When the data can be accessed through a [`ChunkReader`], prefer - /// [`ParquetMetaDataReader::read_page_index`] (or - /// [`ParquetMetaDataReader::read_page_index_async`] for asynchronous + /// [`ParquetMetaDataReader::read_page_index_and_finish`] (or + /// [`ParquetMetaDataReader::load_page_index_and_finish`] for asynchronous /// I/O), which handles the decoder state machine and I/O requests. /// /// # Returns /// - /// * `Ok(DecodeResult::Data(Some(page_index)))` - Successfully decoded the - /// requested page indexes based on the configured policies and masks - /// * `Ok(DecodeResult::Data(None))` - No page indexes were requested - /// (both policies set to [`PageIndexPolicy::Skip`] or masks select nothing) + /// * `Ok(DecodeResult::Data((metadata, Some(page_index))))` - Successfully decoded the + /// requested page indexes based on the configured policies and masks. The newly decoded page + /// index is returned separately; any page index already attached to `metadata` is unchanged. + /// * `Ok(DecodeResult::Data((metadata, None)))` - No page indexes were requested or found + /// according to the configured policies and masks. The stored metadata is still returned. /// * `Ok(DecodeResult::NeedsData(ranges))` - More data is needed to decode /// the page indexes. Push the requested ranges and call this method again. /// * `Ok(DecodeResult::Finished)` - The decoder has finished and no more /// data can be decoded /// * `Err(_)` - An error occurred during decoding /// - /// [`ParquetMetaDataReader::read_page_index`]: crate::file::metadata::ParquetMetaDataReader::read_page_index - /// [`ParquetMetaDataReader::read_page_index_async`]: crate::file::metadata::ParquetMetaDataReader::read_page_index_async - pub fn try_decode_page_index(&mut self) -> Result>> { + /// [`ParquetMetaDataReader::read_page_index_and_finish`]: crate::file::metadata::ParquetMetaDataReader::read_page_index_and_finish + /// [`ParquetMetaDataReader::load_page_index_and_finish`]: crate::file::metadata::ParquetMetaDataReader::load_page_index_and_finish + pub fn try_decode_page_index( + &mut self, + ) -> Result)>> { match std::mem::replace(&mut self.state, DecodeState::Intermediate) { DecodeState::ReadingPageIndex(metadata) => { Ok(match self.try_decode_page_index_for_metadata(metadata)? { - PageIndexDecodeResult::NeedsData(ranges) => DecodeResult::NeedsData(ranges), - PageIndexDecodeResult::Data { page_index, .. } => { - DecodeResult::Data(page_index) + DecodeResult::NeedsData(ranges) => DecodeResult::NeedsData(ranges), + DecodeResult::Data((metadata, page_index)) => { + DecodeResult::Data((*metadata, page_index)) } + DecodeResult::Finished => DecodeResult::Finished, }) } DecodeState::Finished => Ok(DecodeResult::Finished), @@ -521,7 +523,7 @@ impl ParquetMetaDataPushDecoder { fn try_decode_page_index_for_metadata( &mut self, metadata: Box, - ) -> Result { + ) -> Result, Option)>> { let range = range_for_page_index( &metadata, self.column_index_policy, @@ -532,15 +534,12 @@ impl ParquetMetaDataPushDecoder { let Some(page_index_range) = range else { self.state = DecodeState::Finished; - return Ok(PageIndexDecodeResult::Data { - metadata, - page_index: None, - }); + return Ok(DecodeResult::Data((metadata, None))); }; if !self.buffers.has_range(&page_index_range) { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(PageIndexDecodeResult::NeedsData(vec![page_index_range])); + return Ok(DecodeResult::NeedsData(vec![page_index_range])); } let buffer = self.get_bytes(&page_index_range)?; @@ -555,10 +554,7 @@ impl ParquetMetaDataPushDecoder { )?; self.state = DecodeState::Finished; - Ok(PageIndexDecodeResult::Data { - metadata, - page_index, - }) + Ok(DecodeResult::Data((metadata, page_index))) } /// Returns the bytes for the given range from the internal buffer @@ -595,14 +591,6 @@ enum DecodeState { Intermediate, } -enum PageIndexDecodeResult { - NeedsData(Vec>), - Data { - metadata: Box, - page_index: Option, - }, -} - /// Returns the byte range needed to read the offset/page indexes, based on the /// specified policies and masks /// @@ -715,7 +703,7 @@ mod tests { assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut decoder, ranges); - let page_index = expect_data(decoder.try_decode_page_index()).unwrap(); + let page_index = expect_data(decoder.try_decode_page_index()).1.unwrap(); expect_finished(decoder.try_decode_page_index()); for row_group_idx in 0..2 { @@ -740,7 +728,7 @@ mod tests { .unwrap() .with_page_index_policy(PageIndexPolicy::Skip); - assert!(expect_data(decoder.try_decode_page_index()).is_none()); + assert!(expect_data(decoder.try_decode_page_index()).1.is_none()); expect_finished(decoder.try_decode_page_index()); let metadata = test_metadata_without_page_index(); @@ -749,7 +737,7 @@ mod tests { .unwrap() .with_page_index_mask(ColumnChunkMask::none()); - assert!(expect_data(decoder.try_decode_page_index()).is_none()); + assert!(expect_data(decoder.try_decode_page_index()).1.is_none()); expect_finished(decoder.try_decode_page_index()); } diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 4f5e3dd6e7c9..bbb1e8c6847c 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -574,31 +574,36 @@ impl ParquetMetaDataReader { self.read_page_indexes_sized(reader, reader.len()) } - /// Reads and returns the page index structures described by the stored - /// [`ParquetMetaData`]. + /// Reads the page index structures described by the stored [`ParquetMetaData`], returning the + /// metadata together with the decoded page index. /// - /// Unlike [`Self::read_page_indexes`], this returns the decoded [`PageIndex`] separately - /// instead of attaching it to the metadata. This is useful for populating an external page - /// index cache. The configured page index policies and masks determine which indexes are - /// returned. + /// Unlike [`Self::read_page_indexes`], this returns the newly decoded [`PageIndex`] separately; + /// any page index already attached to the returned metadata is unchanged. This is useful for + /// populating an external page index cache. The configured page index policies and masks + /// determine which indexes are returned. The page index is `None` when no indexes were + /// requested or found. /// - /// This operation consumes `self` and the stored [`ParquetMetaData`]. Use - /// [`Self::new_with_metadata`] to construct the reader. - pub fn read_page_index(self, reader: &R) -> Result> { - self.read_page_index_sized(reader, reader.len()) + /// This operation consumes `self`, but returns the stored [`ParquetMetaData`] as the first + /// tuple element. Use [`Self::new_with_metadata`] to construct the reader. + pub fn read_page_index_and_finish( + self, + reader: &R, + ) -> Result<(ParquetMetaData, Option)> { + self.read_page_index_sized_and_finish(reader, reader.len()) } - /// Reads and returns the page index structures described by the stored - /// [`ParquetMetaData`], using `file_size` as the size of the original Parquet file. + /// Reads the page index structures described by the stored [`ParquetMetaData`], returning the + /// metadata and decoded page index, and using `file_size` as the size of the original Parquet + /// file. /// /// This variant supports a [`Bytes`] reader containing only a suffix of the file. It returns /// [`ParquetError::NeedMoreData`] if that suffix does not contain the requested page indexes. - /// See [`Self::read_page_index`] for ownership and configuration details. - pub fn read_page_index_sized( + /// See [`Self::read_page_index_and_finish`] for ownership and configuration details. + pub fn read_page_index_sized_and_finish( mut self, reader: &R, file_size: u64, - ) -> Result> { + ) -> Result<(ParquetMetaData, Option)> { let Some(metadata) = self.metadata.take() else { return Err(general_err!( "Tried to read page indexes without ParquetMetaData metadata" @@ -804,17 +809,19 @@ impl ParquetMetaDataReader { self.load_page_index_with_remainder(fetch, None).await } - /// Asynchronously reads and returns the page index structures described by the stored - /// [`ParquetMetaData`]. + /// Asynchronously loads the page index structures described by the stored [`ParquetMetaData`], + /// returning the metadata together with the decoded page index. /// - /// Unlike [`Self::load_page_index`], this returns the decoded [`PageIndex`] separately instead - /// of attaching it to the metadata. This is useful for populating an external page index - /// cache. This operation consumes `self` and the stored metadata. + /// Unlike [`Self::load_page_index`], this returns the newly decoded [`PageIndex`] separately; + /// any page index already attached to the returned metadata is unchanged. This is useful for + /// populating an external page index cache. The page index is `None` when no indexes were + /// requested or found. This operation consumes `self`, but returns the stored metadata as the + /// first tuple element. #[cfg(all(feature = "async", feature = "arrow"))] - pub async fn read_page_index_async( + pub async fn load_page_index_and_finish( mut self, mut fetch: F, - ) -> Result> { + ) -> Result<(ParquetMetaData, Option)> { let Some(metadata) = self.metadata.take() else { return Err(general_err!("Footer metadata is not present")); }; @@ -1254,10 +1261,10 @@ fn needs_index_data( } /// Determines a single combined range of bytes needed to read the page indexes, or returns the -/// decoded page index if no additional data is needed. +/// metadata and decoded page index if no additional data is needed. fn needs_page_index_data( push_decoder: &mut ParquetMetaDataPushDecoder, -) -> Result>> { +) -> Result)>> { match push_decoder.try_decode_page_index()? { DecodeResult::NeedsData(ranges) => { let range = ranges @@ -1283,11 +1290,11 @@ fn parse_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result Result> { +) -> Result<(ParquetMetaData, Option)> { match push_decoder.try_decode_page_index()? { DecodeResult::NeedsData(_) => Err(general_err!( "Internal error: decoder still needs data after reading required range" @@ -1481,7 +1488,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] // Takes too long - fn test_read_page_index() { + fn test_read_page_index_and_finish() { use crate::file::metadata::page_index::PageIndexProvider; let file = get_test_file("alltypes_tiny_pages.parquet"); @@ -1490,13 +1497,19 @@ mod tests { .unwrap(); let reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()); - assert!(reader.read_page_index(&file).unwrap().is_none()); + assert!( + reader + .read_page_index_and_finish(&file) + .unwrap() + .1 + .is_none() + ); let reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) .with_column_index_mask(ColumnChunkMask::columns([0])) .with_offset_index_mask(ColumnChunkMask::columns([0])); - let page_index = reader.read_page_index(&file).unwrap().unwrap(); + let page_index = reader.read_page_index_and_finish(&file).unwrap().1.unwrap(); assert!(page_index.column_index(0, 0).is_some()); assert!(page_index.offset_index(0, 0).is_some()); @@ -1611,7 +1624,7 @@ mod async_tests { } #[tokio::test] - async fn test_read_page_index_async() { + async fn test_load_page_index_and_finish() { use crate::file::metadata::page_index::PageIndexProvider; let mut file = get_test_file("alltypes_tiny_pages.parquet"); @@ -1624,7 +1637,12 @@ mod async_tests { .with_offset_index_mask(ColumnChunkMask::columns([0])); let fetch = MetadataFetchFn(|range| futures::future::ready(read_range(&mut file, range))); - let page_index = reader.read_page_index_async(fetch).await.unwrap().unwrap(); + let page_index = reader + .load_page_index_and_finish(fetch) + .await + .unwrap() + .1 + .unwrap(); assert!(page_index.column_index(0, 0).is_some()); assert!(page_index.offset_index(0, 0).is_some()); From 69dec650d086386f114e07eb1dee5e76592fe4c3 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 9 Oct 2026 22:28:55 -0700 Subject: [PATCH 06/10] refactor to remove some duplicate code --- parquet/src/file/metadata/reader.rs | 186 ++++++++-------------------- 1 file changed, 49 insertions(+), 137 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index bbb1e8c6847c..1b46b543dfb3 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -23,7 +23,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; #[cfg(feature = "arrow")] use crate::file::metadata::dictionary::decode_dictionary_page; -use crate::file::metadata::page_index::PageIndex; +use crate::file::metadata::page_index::{PageIndex, PageIndexProvider}; use crate::file::metadata::parser::decode_metadata; use crate::file::metadata::thrift::parquet_schema_from_bytes; use crate::file::metadata::{ @@ -603,6 +603,14 @@ impl ParquetMetaDataReader { mut self, reader: &R, file_size: u64, + ) -> Result<(ParquetMetaData, Option)> { + self.read_page_index_sized_impl(reader, file_size) + } + + fn read_page_index_sized_impl( + &mut self, + reader: &R, + file_size: u64, ) -> Result<(ParquetMetaData, Option)> { let Some(metadata) = self.metadata.take() else { return Err(general_err!( @@ -664,65 +672,10 @@ impl ParquetMetaDataReader { reader: &R, file_size: u64, ) -> Result<()> { - let Some(metadata) = self.metadata.take() else { - return Err(general_err!( - "Tried to read page indexes without ParquetMetaData metadata" - )); - }; - - let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? - .with_offset_index_policy(self.offset_index) - .with_column_index_policy(self.column_index) - .with_offset_index_mask(self.offset_index_mask.clone()) - .with_column_index_mask(self.column_index_mask.clone()) - .with_metadata_options(self.metadata_options.clone()); - let mut push_decoder = self.prepare_push_decoder(push_decoder); - - // Get bounds needed for page indexes (if any are present in the file). - let range = match needs_index_data(&mut push_decoder)? { - NeedsIndexData::No(metadata) => { - self.metadata = Some(metadata); - return Ok(()); - } - NeedsIndexData::Yes(range) => range, - }; - - // Check to see if needed range is within `file_range`. Checking `range.end` seems - // redundant, but it guards against `range_for_page_index()` returning garbage. - let file_range = file_size.saturating_sub(reader.len())..file_size; - if !(file_range.contains(&range.start) && file_range.contains(&range.end)) { - // Requested range starts beyond EOF - return if range.end > file_size { - Err(eof_err!( - "Parquet file too small. Range {range:?} is beyond file bounds {file_size}", - )) - } else { - // Ask for a larger buffer - Err(ParquetError::NeedMoreData( - (file_size - range.start).try_into()?, - )) - }; - } - - // Perform extra sanity check to make sure `range` and the footer metadata don't - // overlap. - if let Some(metadata_size) = self.metadata_size { - let metadata_range = file_size.saturating_sub(metadata_size as u64)..file_size; - if range.end > metadata_range.start { - return Err(eof_err!( - "Parquet file too small. Page index range {range:?} overlaps with file metadata {metadata_range:?}", - )); - } - } - - // add the needed ranges to the decoder - let bytes_needed = usize::try_from(range.end - range.start)?; - let bytes = reader.get_bytes(range.start - file_range.start, bytes_needed)?; - - push_decoder.push_range(range, bytes)?; - let metadata = parse_index_data(&mut push_decoder)?; + let (mut metadata, page_index) = self.read_page_index_sized_impl(reader, file_size)?; + metadata + .set_page_index(page_index.map(|index| Arc::new(index) as Arc)); self.metadata = Some(metadata); - Ok(()) } @@ -820,36 +773,9 @@ impl ParquetMetaDataReader { #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index_and_finish( mut self, - mut fetch: F, + fetch: F, ) -> Result<(ParquetMetaData, Option)> { - let Some(metadata) = self.metadata.take() else { - return Err(general_err!("Footer metadata is not present")); - }; - - // MetadataFetch uses absolute file offsets, so the actual file size is not needed here. - let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(u64::MAX, metadata)? - .with_offset_index_policy(self.offset_index) - .with_column_index_policy(self.column_index) - .with_offset_index_mask(self.offset_index_mask.clone()) - .with_column_index_mask(self.column_index_mask.clone()) - .with_metadata_options(self.metadata_options.clone()); - let mut push_decoder = self.prepare_push_decoder(push_decoder); - - let range = match needs_page_index_data(&mut push_decoder)? { - NeedsIndexData::No(page_index) => return Ok(page_index), - NeedsIndexData::Yes(range) => range, - }; - let bytes = fetch.fetch(range.clone()).await?; - if bytes.len() as u64 != range.end - range.start { - return Err(general_err!( - "Corrupted parquet file: index data length mismatch, expected {}, got {}", - range.end - range.start, - bytes.len() - )); - } - - push_decoder.push_range(range, bytes)?; - parse_page_index_data(&mut push_decoder) + self.load_page_index_with_remainder_impl(fetch, None).await } /// Reads and decodes the dictionary page of a column chunk into an Arrow array. @@ -902,9 +828,24 @@ impl ParquetMetaDataReader { #[cfg(all(feature = "async", feature = "arrow"))] async fn load_page_index_with_remainder( &mut self, - mut fetch: F, + fetch: F, remainder: Option<(usize, Bytes)>, ) -> Result<()> { + let (mut metadata, page_index) = self + .load_page_index_with_remainder_impl(fetch, remainder) + .await?; + metadata + .set_page_index(page_index.map(|index| Arc::new(index) as Arc)); + self.metadata = Some(metadata); + Ok(()) + } + + #[cfg(all(feature = "async", feature = "arrow"))] + async fn load_page_index_with_remainder_impl( + &mut self, + mut fetch: F, + remainder: Option<(usize, Bytes)>, + ) -> Result<(ParquetMetaData, Option)> { let Some(metadata) = self.metadata.take() else { return Err(general_err!("Footer metadata is not present")); }; @@ -921,11 +862,8 @@ impl ParquetMetaDataReader { let mut push_decoder = self.prepare_push_decoder(push_decoder); // Get bounds needed for page indexes (if any are present in the file). - let range = match needs_index_data(&mut push_decoder)? { - NeedsIndexData::No(metadata) => { - self.metadata = Some(metadata); - return Ok(()); - } + let range = match needs_page_index_data(&mut push_decoder)? { + NeedsIndexData::No(page_index) => return Ok(page_index), NeedsIndexData::Yes(range) => range, }; @@ -956,9 +894,7 @@ impl ParquetMetaDataReader { )); } push_decoder.push_range(range.clone(), bytes)?; - let metadata = parse_index_data(&mut push_decoder)?; - self.metadata = Some(metadata); - Ok(()) + parse_page_index_data(&mut push_decoder) } // One-shot parse of footer. @@ -1242,24 +1178,6 @@ enum NeedsIndexData { Yes(Range), } -/// Determines a single combined range of bytes needed to read the page indexes, -/// or returns the metadata if no additional data is needed (e.g. if no page indexes are requested) -fn needs_index_data( - push_decoder: &mut ParquetMetaDataPushDecoder, -) -> Result> { - match push_decoder.try_decode()? { - DecodeResult::NeedsData(ranges) => { - let range = ranges - .into_iter() - .reduce(|a, b| a.start.min(b.start)..a.end.max(b.end)) - .ok_or_else(|| general_err!("Internal error: no ranges provided"))?; - Ok(NeedsIndexData::Yes(range)) - } - DecodeResult::Data(metadata) => Ok(NeedsIndexData::No(metadata)), - DecodeResult::Finished => Err(general_err!("Internal error: decoder was finished")), - } -} - /// Determines a single combined range of bytes needed to read the page indexes, or returns the /// metadata and decoded page index if no additional data is needed. fn needs_page_index_data( @@ -1278,18 +1196,6 @@ fn needs_page_index_data( } } -/// Given a push decoder that has had the needed ranges pushed to it, -/// attempt to decode indexes and return the updated metadata. -fn parse_index_data(push_decoder: &mut ParquetMetaDataPushDecoder) -> Result { - match push_decoder.try_decode()? { - DecodeResult::NeedsData(_) => Err(general_err!( - "Internal error: decoder still needs data after reading required range" - )), - DecodeResult::Data(metadata) => Ok(metadata), - DecodeResult::Finished => Err(general_err!("Internal error: decoder was finished")), - } -} - /// Given a push decoder that has had the needed ranges pushed to it, decode and return the metadata /// and page indexes. fn parse_page_index_data( @@ -1493,23 +1399,29 @@ mod tests { let file = get_test_file("alltypes_tiny_pages.parquet"); let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) .parse_and_finish(&file) .unwrap(); - - let reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()); - assert!( - reader - .read_page_index_and_finish(&file) - .unwrap() - .1 - .is_none() - ); + let original_page_index = Arc::clone(metadata.page_index().unwrap()); + + let reader = ParquetMetaDataReader::new_with_metadata(metadata); + let (metadata, page_index) = reader.read_page_index_and_finish(&file).unwrap(); + assert!(page_index.is_none()); + assert!(Arc::ptr_eq( + metadata.page_index().unwrap(), + &original_page_index + )); let reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) .with_column_index_mask(ColumnChunkMask::columns([0])) .with_offset_index_mask(ColumnChunkMask::columns([0])); - let page_index = reader.read_page_index_and_finish(&file).unwrap().1.unwrap(); + let (metadata, page_index) = reader.read_page_index_and_finish(&file).unwrap(); + let page_index = page_index.unwrap(); + assert!(Arc::ptr_eq( + metadata.page_index().unwrap(), + &original_page_index + )); assert!(page_index.column_index(0, 0).is_some()); assert!(page_index.offset_index(0, 0).is_some()); From 129ea6bb895884b90429a64fc48246946443da75 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 13 Oct 2026 22:45:44 -0700 Subject: [PATCH 07/10] clean up test of reading page index after skipping it --- parquet/src/file/metadata/push_decoder.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 065221c82ba9..941d448f80c3 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -689,8 +689,10 @@ mod tests { } #[test] - fn test_page_index_decoder_incremental() { + fn test_decode_page_index_after_metadata() { let metadata = test_metadata_without_page_index(); + assert!(metadata.page_index().is_none()); + let mut decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata) .unwrap() @@ -703,7 +705,9 @@ mod tests { assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut decoder, ranges); - let page_index = expect_data(decoder.try_decode_page_index()).1.unwrap(); + let (metadata, page_index) = expect_data(decoder.try_decode_page_index()); + assert!(metadata.page_index().is_none()); + let page_index = page_index.unwrap(); expect_finished(decoder.try_decode_page_index()); for row_group_idx in 0..2 { From be1ea0fa95e1e305a2e8caecda31531aa8d3e672 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 13 Oct 2026 23:06:37 -0700 Subject: [PATCH 08/10] add example to read_page_index_and_finish --- parquet/src/file/metadata/push_decoder.rs | 4 ++- parquet/src/file/metadata/reader.rs | 36 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 941d448f80c3..5a60c6299bdf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -483,7 +483,9 @@ impl ParquetMetaDataPushDecoder { /// When the data can be accessed through a [`ChunkReader`], prefer /// [`ParquetMetaDataReader::read_page_index_and_finish`] (or /// [`ParquetMetaDataReader::load_page_index_and_finish`] for asynchronous - /// I/O), which handles the decoder state machine and I/O requests. + /// I/O), which handles the decoder state machine and I/O requests. See the + /// [`read_page_index_and_finish` example][`ParquetMetaDataReader::read_page_index_and_finish`] + /// for the complete two-stage workflow. /// /// # Returns /// diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 1b46b543dfb3..7f1092a700ca 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -585,6 +585,42 @@ impl ParquetMetaDataReader { /// /// This operation consumes `self`, but returns the stored [`ParquetMetaData`] as the first /// tuple element. Use [`Self::new_with_metadata`] to construct the reader. + /// + /// # Example + /// + /// Read the footer metadata first, then load selected page indexes separately when they are + /// needed: + /// + /// ```no_run + /// # use parquet::errors::Result; + /// use parquet::file::metadata::page_index::PageIndexProvider; + /// use parquet::file::metadata::{ + /// ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader, + /// }; + /// # fn open_parquet_file() -> std::fs::File { unimplemented!() } + /// # fn example() -> Result<()> { + /// let file = open_parquet_file(); + /// let metadata = ParquetMetaDataReader::new() + /// .with_page_index_policy(PageIndexPolicy::Skip) + /// .parse_and_finish(&file)?; + /// assert!(metadata.page_index().is_none()); + /// + /// let mask = ColumnChunkMask::columns([0]); + /// let (metadata, page_index) = ParquetMetaDataReader::new_with_metadata(metadata) + /// .with_column_index_policy(PageIndexPolicy::Required) + /// .with_offset_index_policy(PageIndexPolicy::Required) + /// .with_column_index_mask(mask.clone()) + /// .with_offset_index_mask(mask) + /// .read_page_index_and_finish(&file)?; + /// + /// // The metadata is returned unchanged and the selected indexes can be cached separately. + /// assert!(metadata.page_index().is_none()); + /// let page_index = page_index.expect("page indexes are required"); + /// assert!(page_index.column_index(0, 0).is_some()); + /// assert!(page_index.offset_index(0, 0).is_some()); + /// # Ok(()) + /// # } + /// ``` pub fn read_page_index_and_finish( self, reader: &R, From fb321d5ea3e9bd3dd0fd986a16385439e017b3e3 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 14 Oct 2026 15:22:03 -0400 Subject: [PATCH 09/10] Fix small error case but, and update docs --- parquet/src/file/metadata/push_decoder.rs | 83 ++++++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 5a60c6299bdf..d3573c8376fe 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -275,10 +275,16 @@ impl ParquetMetaDataPushDecoder { /// This can be used to parse and populate the page index structures /// after the metadata has already been decoded. /// - /// When decoding page indexes, any existing page indexes in `metadata` will be - /// replaced based on the configured policies. See - /// [`crate::file::metadata::ParquetMetaDataReader::read_page_indexes`] for details - /// on the replacement behavior. + /// The page indexes can then be decoded in one of two ways: + /// + /// * [`Self::try_decode`] attaches the decoded page indexes to the returned + /// metadata, replacing (or clearing) any existing page indexes + /// See [`ParquetMetaDataReader::read_page_indexes`] for details. + /// * [`Self::try_decode_page_index`] returns the decoded page indexes + /// separately and leaves any page indexes already attached to `metadata` + /// unchanged. + /// + /// [`ParquetMetaDataReader::read_page_indexes`]: [`crate::file::metadata::ParquetMetaDataReader::read_page_indexes`] pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); @@ -471,8 +477,10 @@ impl ParquetMetaDataPushDecoder { /// Try to decode the page indexes from the pushed data, returning the stored /// [`ParquetMetaData`] together with the decoded [`PageIndex`]. /// - /// This method can only be called after [`Self::try_new_with_metadata`], - /// which initializes the decoder with previously decoded metadata. + /// This method can only be called once the footer metadata is known: either + /// after [`Self::try_new_with_metadata`], or after [`Self::try_decode`] has + /// decoded the footer metadata and requested page index data. Calling it + /// earlier returns an error and leaves the decoder unchanged. /// /// This method is useful when you want to: /// 1. First decode the footer metadata without reading the page indexes @@ -515,10 +523,17 @@ impl ParquetMetaDataPushDecoder { DecodeResult::Finished => DecodeResult::Finished, }) } - DecodeState::Finished => Ok(DecodeResult::Finished), - _ => Err(general_err!( - "ParquetMetaDataPushDecoder: internal error, invalid state" - )), + DecodeState::Finished => { + self.state = DecodeState::Finished; + Ok(DecodeResult::Finished) + } + other => { + // Restore the original state so the decoder remains usable + self.state = other; + Err(general_err!( + "ParquetMetaDataPushDecoder: try_decode_page_index requires a decoder created with try_new_with_metadata" + )) + } } } @@ -751,7 +766,53 @@ mod tests { fn test_page_index_decoder_requires_metadata() { let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); let err = decoder.try_decode_page_index().unwrap_err(); - assert!(err.to_string().contains("invalid state")); + assert!( + err.to_string() + .contains("requires a decoder created with try_new_with_metadata"), + "unexpected error: {err}" + ); + } + + /// A failed call to `try_decode_page_index` must not leave the decoder in + /// an unusable state: decoding the metadata via `try_decode` should still + /// work afterwards, at every stage of the footer decoding state machine. + #[test] + fn test_decode_after_failed_page_index_decode() { + let file_len = test_file_len(); + let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap(); + + // Before any data is pushed, the decoder is waiting for the footer + decoder.try_decode_page_index().unwrap_err(); + let footer_ranges = expect_needs_data(decoder.try_decode()); + assert_eq!(footer_ranges.len(), 1); + push_ranges_to_metadata_decoder(&mut decoder, footer_ranges); + + // The footer is available but not yet decoded + decoder.try_decode_page_index().unwrap_err(); + let metadata_ranges = expect_needs_data(decoder.try_decode()); + assert_eq!(metadata_ranges.len(), 1); + push_ranges_to_metadata_decoder(&mut decoder, metadata_ranges); + + // The metadata is available but not yet decoded + decoder.try_decode_page_index().unwrap_err(); + let index_ranges = expect_needs_data(decoder.try_decode()); + assert_eq!(index_ranges.len(), 1); + + // The metadata is decoded and the decoder is waiting for page index + // data, so try_decode_page_index is now valid and asks for the same range + let index_ranges_again = expect_needs_data(decoder.try_decode_page_index()); + assert_eq!(index_ranges, index_ranges_again); + push_ranges_to_metadata_decoder(&mut decoder, index_ranges); + + // The regular decode path still works and attaches the page index + let metadata = expect_data(decoder.try_decode()); + assert_eq!(metadata.num_row_groups(), 2); + assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); + + // Once finished, both entry points keep reporting Finished + expect_finished(decoder.try_decode_page_index()); + expect_finished(decoder.try_decode_page_index()); + expect_finished(decoder.try_decode()); } /// It is possible to feed some, but not all, of the footer into the metadata decoder From 7bb72d54fd31c248782f09ea9b6ef359dfff6b81 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 14 Oct 2026 13:20:31 -0700 Subject: [PATCH 10/10] restore push decoder state on decoding errors --- parquet/src/file/metadata/push_decoder.rs | 41 +++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index d3573c8376fe..202fa0d1633b 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -559,8 +559,14 @@ impl ParquetMetaDataPushDecoder { return Ok(DecodeResult::NeedsData(vec![page_index_range])); } - let buffer = self.get_bytes(&page_index_range)?; - let page_index = parse_page_index( + let buffer = match self.get_bytes(&page_index_range) { + Ok(buffer) => buffer, + Err(error) => { + self.state = DecodeState::ReadingPageIndex(metadata); + return Err(error); + } + }; + let page_index = match parse_page_index( &metadata, self.column_index_policy, self.offset_index_policy, @@ -568,7 +574,13 @@ impl ParquetMetaDataPushDecoder { &self.offset_index_mask, &buffer, page_index_range.start, - )?; + ) { + Ok(page_index) => page_index, + Err(error) => { + self.state = DecodeState::ReadingPageIndex(metadata); + return Err(error); + } + }; self.state = DecodeState::Finished; Ok(DecodeResult::Data((metadata, page_index))) @@ -773,6 +785,29 @@ mod tests { ); } + #[test] + fn test_page_index_decoder_restores_state_after_decode_error() { + let metadata = test_metadata_without_page_index(); + let mut decoder = + ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required); + + let ranges = expect_needs_data(decoder.try_decode_page_index()); + let invalid_data = ranges + .iter() + .map(|range| Bytes::from(vec![0; (range.end - range.start) as usize])) + .collect(); + decoder.push_ranges(ranges.clone(), invalid_data).unwrap(); + decoder.try_decode_page_index().unwrap_err(); + + decoder.clear_all_ranges(); + push_ranges_to_metadata_decoder(&mut decoder, ranges); + let (metadata, page_index) = expect_data(decoder.try_decode_page_index()); + assert!(metadata.page_index().is_none()); + assert!(page_index.is_some_and(|index| index.is_complete())); + } + /// A failed call to `try_decode_page_index` must not leave the decoder in /// an unusable state: decoding the metadata via `try_decode` should still /// work afterwards, at every stage of the footer decoding state machine.