From 1c7798468aff9b57a0171cec262e7518334ba9c9 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:00:56 -0700 Subject: [PATCH 01/62] allow for serialization of custom page index providers --- parquet/src/file/metadata/page_index.rs | 21 +--- parquet/src/file/metadata/writer.rs | 121 ++++++++++++------------ parquet/src/file/writer.rs | 73 +++++++++++++- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9ded57d0a28e..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -460,22 +460,6 @@ impl PageIndex { pub fn into_builder(self) -> PageIndexBuilder { self.into() } - - /// Returns a reference to the raw column indexes structure - /// - /// This method provides access to the underlying column index data for serialization - /// and other low-level operations. - pub(crate) fn column_indexes_raw(&self) -> Option<&Vec>>> { - self.column_indexes.as_ref() - } - - /// Returns a reference to the raw offset indexes structure - /// - /// This method provides access to the underlying offset index data for serialization - /// and other low-level operations. - pub(crate) fn offset_indexes_raw(&self) -> Option<&Vec>>> { - self.offset_indexes.as_ref() - } } impl PageIndexProvider for PageIndex { @@ -536,7 +520,10 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { + pub(crate) fn empty_index( + num_row_groups: usize, + num_columns: usize, + ) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a1708ccf2c3a..d6de7e45c8c5 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::FileMeta; use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; @@ -54,8 +55,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { buf: &'a mut TrackedWrite, schema_descr: &'a SchemaDescPtr, row_groups: Vec, - column_indexes: Option>>>, - offset_indexes: Option>>>, + page_index: Option>, key_value_metadata: Option>, created_by: Option, object_writer: MetadataObjectWriter, @@ -71,14 +71,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized offset indexes. fn write_offset_indexes( &mut self, - offset_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut offset_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let offidx_vec = offset_indexes.as_mut().unwrap(); + + // we've already checked before calling that the offset indexes are populated + assert!(page_index.has_offset_indexes()); + // iter row group // iter each column // write offset index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(offset_index) = &offset_indexes[row_group_idx][column_idx] { + if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); self.object_writer.write_offset_index( offset_index, @@ -91,10 +98,11 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); + offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); } } } - Ok(()) + Ok(offset_indexes) } /// Serialize all the column indexes to the `self.buf` @@ -104,14 +112,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized column indexes. fn write_column_indexes( &mut self, - column_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut column_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let colidx_vec = column_indexes.as_mut().unwrap(); + + // we've already checked before calling that the column indexes are populated + assert!(page_index.has_column_indexes()); + // iter row group // iter each column // write column index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(column_index) = &column_indexes[row_group_idx][column_idx] { + if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); // only update column_metadata if the write succeeds if self.object_writer.write_column_index( @@ -127,21 +142,28 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } + colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); } } } - Ok(()) + Ok(column_indexes) } - /// Serialize the column indexes and transform to `Option` - fn finalize_column_indexes(&mut self) -> Result>>>> { - let column_indexes = std::mem::take(&mut self.column_indexes); - - // Write column indexes to file - if let Some(column_indexes) = column_indexes.as_ref() { - self.write_column_indexes(column_indexes)?; + /// Serialize the column indexes and transform to `Option>>>` + fn finalize_column_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_column_indexes()) + { + return Ok(None); } + // Write column indexes to file while assembling a column index + let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; + // check to see if the index is `None` for every row group and column chunk let all_none = column_indexes .as_ref() @@ -155,13 +177,19 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes(&mut self) -> Result>>>> { - let offset_indexes = std::mem::take(&mut self.offset_indexes); + fn finalize_offset_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_offset_indexes()) + { + return Ok(None); + } // Write offset indexes to file - if let Some(offset_indexes) = offset_indexes.as_ref() { - self.write_offset_indexes(offset_indexes)?; - } + let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; // check to see if the index is `None` for every row group and column chunk let all_none = offset_indexes @@ -180,8 +208,9 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); // serialize page indexes and transform to the proper form for use in ParquetMetaData - let column_indexes = self.finalize_column_indexes()?; - let offset_indexes = self.finalize_offset_indexes()?; + let page_index = self.page_index.take(); + let column_indexes = self.finalize_column_indexes(&page_index)?; + let offset_indexes = self.finalize_offset_indexes(&page_index)?; // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -275,8 +304,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { buf, schema_descr, row_groups, - column_indexes: None, - offset_indexes: None, + page_index: None, key_value_metadata: None, created_by, object_writer: Default::default(), @@ -285,19 +313,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } } - pub fn with_column_indexes( - mut self, - column_indexes: Vec>>, - ) -> Self { - self.column_indexes = Some(column_indexes); - self - } - - pub fn with_offset_indexes( - mut self, - offset_indexes: Vec>>, - ) -> Self { - self.offset_indexes = Some(offset_indexes); + pub fn with_page_index(mut self, page_index: Arc) -> Self { + self.page_index = Some(page_index); self } @@ -333,17 +350,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -///
-/// -/// **NOTE:** -/// The serialization of custom [`PageIndexProvider`]s is not currently supported. -/// The only supported page index structure is [`PageIndex`]. If the metadata -/// contains any other [`PageIndexProvider`] implementation, the [`ColumnIndex`] -/// and [`OffsetIndex`] structures are silently omitted from the output. See -/// for more details. -/// -///
-/// /// # Output Format /// /// The format of the metadata is as follows: @@ -464,21 +470,10 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - // Downcast to PageIndex to access raw index structures for serialization. - // Page indexes from custom PageIndexProviders are not written. See - // if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && let Some(page_index) = page_index_arc - .as_any() - .downcast_ref::() + && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) { - if let Some(column_indexes) = page_index.column_indexes_raw() { - encoder = encoder.with_column_indexes(column_indexes.clone()); - } - - if let Some(offset_indexes) = page_index.offset_indexes_raw() { - encoder = encoder.with_offset_indexes(offset_indexes.clone()); - } + encoder = encoder.with_page_index(page_index_arc.clone()); } if let Some(key_value_metadata) = key_value_metadata { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 6c0f3e8dd283..18b7e6e9999e 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -18,6 +18,7 @@ //! [`SerializedFileWriter`]: Low level Parquet writer API use crate::bloom_filter::Sbbf; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::offset_index::OffsetIndexMetaData; @@ -377,10 +378,27 @@ impl SerializedFileWriter { encoder = encoder.with_key_value_metadata(key_value_metadata) } - encoder = encoder.with_column_indexes(column_indexes); - if !self.props.offset_index_disabled() { - encoder = encoder.with_offset_indexes(offset_indexes); + // check for empty column index + let column_indexes = if column_indexes.is_empty() + || column_indexes + .iter() + .all(|cis| cis.iter().all(|ci| ci.is_none())) + { + None + } else { + Some(column_indexes) + }; + // offset index will always be created unless explicitly disabled + let offset_indexes = if self.props.offset_index_disabled() { + None + } else { + Some(offset_indexes) + }; + if column_indexes.is_some() || offset_indexes.is_some() { + let page_index = PageIndex::new(column_indexes, offset_indexes); + encoder = encoder.with_page_index(Arc::new(page_index)); } + encoder.finish() } @@ -2201,6 +2219,55 @@ mod tests { assert!(b_idx.is_none(), "{b_idx:?}"); } + #[test] + fn test_offset_index_disabled() { + let message_type = " + message test_schema { + REQUIRED INT32 a; + REQUIRED INT32 b; + } + "; + // write file with indexes disabled (including offset indexes) + let schema = Arc::new(parse_message_type(message_type).unwrap()); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::None) + .set_offset_index_disabled(true) + .build(); + let mut file = Vec::with_capacity(1024); + let mut file_writer = + SerializedFileWriter::new(&mut file, schema, Arc::new(props)).unwrap(); + + let mut row_group_writer = file_writer.next_row_group().unwrap(); + let mut a_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = a_writer.typed::(); + col_writer.write_batch(&[1, 2, 3], None, None).unwrap(); + a_writer.close().unwrap(); + + let mut b_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = b_writer.typed::(); + col_writer.write_batch(&[4, 5, 6], None, None).unwrap(); + b_writer.close().unwrap(); + row_group_writer.close().unwrap(); + + let metadata = file_writer.finish().unwrap(); + assert_eq!(metadata.num_row_groups(), 1); + let row_group = metadata.row_group(0); + assert_eq!(row_group.num_columns(), 2); + // no page indexes should exist + assert!(row_group.column(0).offset_index_offset().is_none()); + assert!(row_group.column(0).column_index_offset().is_none()); + assert!(row_group.column(1).offset_index_offset().is_none()); + assert!(row_group.column(1).column_index_offset().is_none()); + + drop(file_writer); + + // read file and request page index...should be `None` + let options = ReadOptionsBuilder::new().with_page_index().build(); + let reader = SerializedFileReader::new_with_options(Bytes::from(file), options).unwrap(); + + assert!(reader.metadata().page_index().is_none()); + } + #[test] fn test_byte_array_size_statistics() { let message_type = " From b2d3a1a40b749a0fada964a49b52d9395913c73d Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:11:03 -0700 Subject: [PATCH 02/62] clippy --- parquet/src/file/metadata/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index d6de7e45c8c5..63cdf540d146 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -152,7 +152,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the column indexes and transform to `Option>>>` fn finalize_column_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -179,7 +179,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the offset indexes and transform to `Option` fn finalize_offset_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -209,8 +209,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // serialize page indexes and transform to the proper form for use in ParquetMetaData let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(&page_index)?; - let offset_indexes = self.finalize_offset_indexes(&page_index)?; + let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; + let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; // We only include ColumnOrder for leaf nodes. let column_orders = self From 0638b8715900d033907b5e745580d1afa4924069 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 9 Sep 2026 13:31:12 -0700 Subject: [PATCH 03/62] add test of using custom page index provider with row selection --- .../custom_page_index_provider.rs | 546 ++++++++++++++++++ parquet/tests/arrow_reader/mod.rs | 1 + 2 files changed, 547 insertions(+) create mode 100644 parquet/tests/arrow_reader/custom_page_index_provider.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs new file mode 100644 index 000000000000..00e008a13ac6 --- /dev/null +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for custom PageIndexProvider implementation. +//! +//! This test validates reading with a custom PageIndexProvider through the +//! actual read path, ensuring that: +//! - Columns with page indexes use offset-index-driven fetch +//! - Columns without page indexes fall back to whole-column-chunk fetching +//! - Correct results are returned in both cases + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, + RowSelectionPolicy, +}; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::index_reader::{decode_column_index, decode_offset_index}; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::sync::Arc; +use tempfile::NamedTempFile; + +/// A custom PageIndexProvider that only stores indexes for a subset of columns +/// +/// This provider mimics a real-world scenario where an application has loaded +/// page indexes selectively based on query predicates and projections. +#[derive(Debug, Clone)] +struct SelectivePageIndexProvider { + metadata: ParquetMetaData, + file_bytes: Bytes, + // indexes are accessed first by row_group index and then by column index + column_indexes: Option>>, + offset_indexes: Option>>, +} + +impl SelectivePageIndexProvider { + fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + Self { + metadata, + file_bytes, + column_indexes: None, + offset_indexes: None, + } + } + + /// Fetch and parse the column index for the given row group and column + fn fetch_column_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.column_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_column_index(&idx_bytes, column.column_type())?; + e.insert(idx); + } + } + Ok(()) + } + + /// Fetch and parse the offset index for the given row group and column + fn fetch_offset_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.offset_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_offset_index(&idx_bytes)?; + e.insert(idx); + } + } + Ok(()) + } +} + +impl PageIndexProvider for SelectivePageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.offset_indexes.is_some() + } + + fn has_column_indexes(&self) -> bool { + self.column_indexes.is_some() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Create a test parquet file with multiple row groups and multiple pages per column +fn create_test_file() -> NamedTempFile { + let temp_file = tempfile::Builder::new() + .prefix("custom_page_index_test") + .suffix(".parquet") + .tempfile() + .expect("tempfile creation"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + + // Create small pages and small row groups for testing + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_row_count_limit(10) // Small pages for multiple pages per column + .set_write_batch_size(10) + .set_max_row_group_row_count(Some(50)) // Small row groups + .build(); + + let file = temp_file.reopen().unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + + // Write 3 row groups with 50 rows each (150 rows total) + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = (row_group * 50) + (batch_num * 10); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (offset..offset + 10).collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + (offset..offset + 10) + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap(); + writer.write(&batch).unwrap(); + } + writer.flush().unwrap(); + } + + // check that file has properties we wanted (3 row groups, 5 pages per chunk) + let metadata = writer.close().unwrap(); + assert_eq!(metadata.num_row_groups(), 3); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for row_group_idx in 0..3 { + for column_idx in 0..4 { + assert!(page_index.column_index(row_group_idx, column_idx).is_some()); + assert!(page_index.offset_index(row_group_idx, column_idx).is_some()); + assert!( + page_index + .num_data_pages(row_group_idx, column_idx) + .is_some_and(|n| n == 5) + ); + } + } + temp_file +} + +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + provider.fetch_column_index(0, 1).unwrap(); + provider.fetch_offset_index(0, 1).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0).unwrap(); + provider.fetch_offset_index(2, 0).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); + + // Success! The test validates that: + // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping + // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data + // 3. For row group 1, no columns had indexes but all returned correct data + // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data + // 5. The reader correctly handled the mix of indexed and non-indexed columns +} + +#[test] +fn test_read_all_columns_with_partial_indexes() { + // This test focuses on verifying that columns without page indexes + // fall back to whole-column-chunk fetching and return correct results + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Load metadata without page indexes + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes ONLY for column 0 in all row groups + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + for row_group_idx in 0..metadata.num_row_groups() { + provider.fetch_column_index(row_group_idx, 0).unwrap(); + provider.fetch_offset_index(row_group_idx, 0).unwrap(); + } + + // Install provider + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read with row selection + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Select rows 10-20 from row group 0 + let selection = RowSelection::from(vec![ + parquet::arrow::arrow_reader::RowSelector::skip(10), + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let batches: Vec = reader.collect::, _>>().unwrap(); + + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + + // Verify all columns have correct data + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + let id = ids.value(i); + assert_eq!(values.value(i), id * 2); + assert_eq!(names.value(i), format!("name{id}")); + assert_eq!(scores.value(i), id * 3); + } + } +} + +#[test] +fn test_custom_provider_with_no_row_selection() { + // Test that reading without RowSelection also works correctly + // with a custom page index provider + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes for first column only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read all data + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + let reader = builder.build().unwrap(); + + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Should read all 150 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 150); + + // Verify data integrity + let mut row_count: i32 = 0; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + assert_eq!(ids.value(i), row_count); + assert_eq!(values.value(i), row_count * 2); + assert_eq!(names.value(i), format!("name{row_count}")); + assert_eq!(scores.value(i), row_count * 3); + row_count += 1; + } + } +} diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index c7c77e0a444c..6c47a00b9b3d 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -42,6 +42,7 @@ mod bad_data; mod bloom_filter; #[cfg(feature = "crc")] mod checksum; +mod custom_page_index_provider; mod int96_stats_roundtrip; mod invalid_utf8; mod io; From da1ff75a0409616fc2d443776176e1efd04cf22c Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 09:57:22 -0700 Subject: [PATCH 04/62] refactor and add usage stats --- .../custom_page_index_provider.rs | 582 ++++++++---------- 1 file changed, 248 insertions(+), 334 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 00e008a13ac6..2736b7bc35e2 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -41,41 +41,260 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fs::File; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use tempfile::NamedTempFile; +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().as_ref(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0, metadata).unwrap(); + provider.fetch_offset_index(0, 0, metadata).unwrap(); + provider.fetch_column_index(0, 1, metadata).unwrap(); + provider.fetch_offset_index(0, 1, metadata).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0, metadata).unwrap(); + provider.fetch_offset_index(2, 0, metadata).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Reset statistics so validation checks don't affect the final counts + provider.reset_stats(); + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = ArrowReaderMetadata::try_new( + metadata_with_custom_index.clone(), + ArrowReaderOptions::default(), + ) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Check statistics + // Note: We need to get the provider reference from the metadata to check stats + let provider_ref = metadata_with_custom_index + .page_index() + .unwrap() + .as_any() + .downcast_ref::() + .expect("Expected SelectivePageIndexProvider"); + + // Check that the custom provider was used + let (hits, misses) = provider_ref.stats(); + assert!(hits > 0, "provider should have hits"); + assert!(misses > 0, "provider should have misses"); + + // Since we didn't use predicates, the column index should be untouched + let (col_hits, col_misses) = provider_ref.column_index_stats(); + assert_eq!(col_hits, 0, "column index should not be used"); + assert_eq!(col_misses, 0, "column index should not be used"); + + let (off_hits, off_misses) = provider_ref.offset_index_stats(); + assert!(off_hits > 0, "offset index should have hits"); + assert!(off_misses > 0, "offset index should have misses"); + + // Step 8: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); +} + /// A custom PageIndexProvider that only stores indexes for a subset of columns /// /// This provider mimics a real-world scenario where an application has loaded /// page indexes selectively based on query predicates and projections. -#[derive(Debug, Clone)] +#[derive(Debug)] struct SelectivePageIndexProvider { - metadata: ParquetMetaData, file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index column_indexes: Option>>, offset_indexes: Option>>, + // Usage statistics + column_index_hits: Arc, + column_index_misses: Arc, + offset_index_hits: Arc, + offset_index_misses: Arc, } impl SelectivePageIndexProvider { - fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes) -> Self { Self { - metadata, file_bytes, column_indexes: None, offset_indexes: None, + column_index_hits: Arc::new(AtomicUsize::new(0)), + column_index_misses: Arc::new(AtomicUsize::new(0)), + offset_index_hits: Arc::new(AtomicUsize::new(0)), + offset_index_misses: Arc::new(AtomicUsize::new(0)), } } + /// Get statistics about column index usage + pub fn column_index_stats(&self) -> (usize, usize) { + ( + self.column_index_hits.load(Ordering::Relaxed), + self.column_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get statistics about offset index usage + pub fn offset_index_stats(&self) -> (usize, usize) { + ( + self.offset_index_hits.load(Ordering::Relaxed), + self.offset_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get combined statistics as (hits, misses) for both index types + pub fn stats(&self) -> (usize, usize) { + let (col_hits, col_misses) = self.column_index_stats(); + let (off_hits, off_misses) = self.offset_index_stats(); + (col_hits + off_hits, col_misses + off_misses) + } + + /// Reset all statistics counters to zero + pub fn reset_stats(&self) { + self.column_index_hits.store(0, Ordering::Relaxed); + self.column_index_misses.store(0, Ordering::Relaxed); + self.offset_index_hits.store(0, Ordering::Relaxed); + self.offset_index_misses.store(0, Ordering::Relaxed); + } + /// Fetch and parse the column index for the given row group and column fn fetch_column_index( &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.column_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.column_index_range(); if let Some(range) = range { let idx_bytes = self @@ -93,11 +312,12 @@ impl SelectivePageIndexProvider { &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.offset_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.offset_index_range(); if let Some(range) = range { let idx_bytes = self @@ -125,10 +345,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - self.column_indexes + let result = self + .column_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.column_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.column_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn offset_index( @@ -136,10 +365,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - self.offset_indexes + let result = self + .offset_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.offset_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.offset_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn as_any(&self) -> &dyn std::any::Any { @@ -220,327 +458,3 @@ fn create_test_file() -> NamedTempFile { } temp_file } - -#[test] -fn test_read_with_custom_page_index_provider() { - // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Verify we have multiple row groups - assert_eq!(metadata.num_row_groups(), 3); - - // Step 3: Create a custom provider and populate it selectively - // Simulate a scenario where: - // - We only populate indexes for row groups 0 and 2 (skipping row group 1) - // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - provider.fetch_column_index(0, 1).unwrap(); - provider.fetch_offset_index(0, 1).unwrap(); - - // Populate indexes for row group 2, column 0 only - provider.fetch_column_index(2, 0).unwrap(); - provider.fetch_offset_index(2, 0).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - - // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Step 5: Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Create a RowSelection that: - // - Selects rows 20-30 (in row group 0) - // - Selects rows 60-70 (in row group 1) - // - Selects rows 120-130 (in row group 2) - let selection = RowSelection::from(vec![ - // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), - // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), - // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), - // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Step 7: Verify correct results - // We expect 30 rows total: 10 from each selected range - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 30); - - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - - // Expected values for the selected rows: - // rows 20-30: id=20..30, value=40..60, score=60..90 - // rows 60-70: id=60..70, value=120..140, score=180..210 - // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); - assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" - ); - - // Success! The test validates that: - // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping - // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data - // 3. For row group 1, no columns had indexes but all returned correct data - // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data - // 5. The reader correctly handled the mix of indexed and non-indexed columns -} - -#[test] -fn test_read_all_columns_with_partial_indexes() { - // This test focuses on verifying that columns without page indexes - // fall back to whole-column-chunk fetching and return correct results - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Load metadata without page indexes - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes ONLY for column 0 in all row groups - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - for row_group_idx in 0..metadata.num_row_groups() { - provider.fetch_column_index(row_group_idx, 0).unwrap(); - provider.fetch_offset_index(row_group_idx, 0).unwrap(); - } - - // Install provider - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read with row selection - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Select rows 10-20 from row group 0 - let selection = RowSelection::from(vec![ - parquet::arrow::arrow_reader::RowSelector::skip(10), - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - let batches: Vec = reader.collect::, _>>().unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 10); - - // Verify all columns have correct data - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - let id = ids.value(i); - assert_eq!(values.value(i), id * 2); - assert_eq!(names.value(i), format!("name{id}")); - assert_eq!(scores.value(i), id * 3); - } - } -} - -#[test] -fn test_custom_provider_with_no_row_selection() { - // Test that reading without RowSelection also works correctly - // with a custom page index provider - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes for first column only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read all data - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - let reader = builder.build().unwrap(); - - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Should read all 150 rows - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 150); - - // Verify data integrity - let mut row_count: i32 = 0; - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - assert_eq!(ids.value(i), row_count); - assert_eq!(values.value(i), row_count * 2); - assert_eq!(names.value(i), format!("name{row_count}")); - assert_eq!(scores.value(i), row_count * 3); - row_count += 1; - } - } -} From a39dd8fdbf6e2b77001ece99e189cd5783b7b618 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 9 Sep 2026 13:31:12 -0700 Subject: [PATCH 05/62] add test of using custom page index provider with row selection --- .../custom_page_index_provider.rs | 546 ++++++++++++++++++ parquet/tests/arrow_reader/mod.rs | 1 + 2 files changed, 547 insertions(+) create mode 100644 parquet/tests/arrow_reader/custom_page_index_provider.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs new file mode 100644 index 000000000000..00e008a13ac6 --- /dev/null +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for custom PageIndexProvider implementation. +//! +//! This test validates reading with a custom PageIndexProvider through the +//! actual read path, ensuring that: +//! - Columns with page indexes use offset-index-driven fetch +//! - Columns without page indexes fall back to whole-column-chunk fetching +//! - Correct results are returned in both cases + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, + RowSelectionPolicy, +}; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::index_reader::{decode_column_index, decode_offset_index}; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::sync::Arc; +use tempfile::NamedTempFile; + +/// A custom PageIndexProvider that only stores indexes for a subset of columns +/// +/// This provider mimics a real-world scenario where an application has loaded +/// page indexes selectively based on query predicates and projections. +#[derive(Debug, Clone)] +struct SelectivePageIndexProvider { + metadata: ParquetMetaData, + file_bytes: Bytes, + // indexes are accessed first by row_group index and then by column index + column_indexes: Option>>, + offset_indexes: Option>>, +} + +impl SelectivePageIndexProvider { + fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + Self { + metadata, + file_bytes, + column_indexes: None, + offset_indexes: None, + } + } + + /// Fetch and parse the column index for the given row group and column + fn fetch_column_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.column_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_column_index(&idx_bytes, column.column_type())?; + e.insert(idx); + } + } + Ok(()) + } + + /// Fetch and parse the offset index for the given row group and column + fn fetch_offset_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.offset_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_offset_index(&idx_bytes)?; + e.insert(idx); + } + } + Ok(()) + } +} + +impl PageIndexProvider for SelectivePageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.offset_indexes.is_some() + } + + fn has_column_indexes(&self) -> bool { + self.column_indexes.is_some() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Create a test parquet file with multiple row groups and multiple pages per column +fn create_test_file() -> NamedTempFile { + let temp_file = tempfile::Builder::new() + .prefix("custom_page_index_test") + .suffix(".parquet") + .tempfile() + .expect("tempfile creation"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + + // Create small pages and small row groups for testing + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_row_count_limit(10) // Small pages for multiple pages per column + .set_write_batch_size(10) + .set_max_row_group_row_count(Some(50)) // Small row groups + .build(); + + let file = temp_file.reopen().unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + + // Write 3 row groups with 50 rows each (150 rows total) + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = (row_group * 50) + (batch_num * 10); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (offset..offset + 10).collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + (offset..offset + 10) + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap(); + writer.write(&batch).unwrap(); + } + writer.flush().unwrap(); + } + + // check that file has properties we wanted (3 row groups, 5 pages per chunk) + let metadata = writer.close().unwrap(); + assert_eq!(metadata.num_row_groups(), 3); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for row_group_idx in 0..3 { + for column_idx in 0..4 { + assert!(page_index.column_index(row_group_idx, column_idx).is_some()); + assert!(page_index.offset_index(row_group_idx, column_idx).is_some()); + assert!( + page_index + .num_data_pages(row_group_idx, column_idx) + .is_some_and(|n| n == 5) + ); + } + } + temp_file +} + +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + provider.fetch_column_index(0, 1).unwrap(); + provider.fetch_offset_index(0, 1).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0).unwrap(); + provider.fetch_offset_index(2, 0).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); + + // Success! The test validates that: + // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping + // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data + // 3. For row group 1, no columns had indexes but all returned correct data + // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data + // 5. The reader correctly handled the mix of indexed and non-indexed columns +} + +#[test] +fn test_read_all_columns_with_partial_indexes() { + // This test focuses on verifying that columns without page indexes + // fall back to whole-column-chunk fetching and return correct results + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Load metadata without page indexes + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes ONLY for column 0 in all row groups + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + for row_group_idx in 0..metadata.num_row_groups() { + provider.fetch_column_index(row_group_idx, 0).unwrap(); + provider.fetch_offset_index(row_group_idx, 0).unwrap(); + } + + // Install provider + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read with row selection + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Select rows 10-20 from row group 0 + let selection = RowSelection::from(vec![ + parquet::arrow::arrow_reader::RowSelector::skip(10), + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let batches: Vec = reader.collect::, _>>().unwrap(); + + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + + // Verify all columns have correct data + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + let id = ids.value(i); + assert_eq!(values.value(i), id * 2); + assert_eq!(names.value(i), format!("name{id}")); + assert_eq!(scores.value(i), id * 3); + } + } +} + +#[test] +fn test_custom_provider_with_no_row_selection() { + // Test that reading without RowSelection also works correctly + // with a custom page index provider + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes for first column only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read all data + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + let reader = builder.build().unwrap(); + + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Should read all 150 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 150); + + // Verify data integrity + let mut row_count: i32 = 0; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + assert_eq!(ids.value(i), row_count); + assert_eq!(values.value(i), row_count * 2); + assert_eq!(names.value(i), format!("name{row_count}")); + assert_eq!(scores.value(i), row_count * 3); + row_count += 1; + } + } +} diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index c7c77e0a444c..6c47a00b9b3d 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -42,6 +42,7 @@ mod bad_data; mod bloom_filter; #[cfg(feature = "crc")] mod checksum; +mod custom_page_index_provider; mod int96_stats_roundtrip; mod invalid_utf8; mod io; From 30631fa71dddaea0c7e90a40d851f229a97dd30e Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 09:57:22 -0700 Subject: [PATCH 06/62] refactor and add usage stats --- .../custom_page_index_provider.rs | 582 ++++++++---------- 1 file changed, 248 insertions(+), 334 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 00e008a13ac6..2736b7bc35e2 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -41,41 +41,260 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fs::File; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use tempfile::NamedTempFile; +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().as_ref(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0, metadata).unwrap(); + provider.fetch_offset_index(0, 0, metadata).unwrap(); + provider.fetch_column_index(0, 1, metadata).unwrap(); + provider.fetch_offset_index(0, 1, metadata).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0, metadata).unwrap(); + provider.fetch_offset_index(2, 0, metadata).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Reset statistics so validation checks don't affect the final counts + provider.reset_stats(); + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = ArrowReaderMetadata::try_new( + metadata_with_custom_index.clone(), + ArrowReaderOptions::default(), + ) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Check statistics + // Note: We need to get the provider reference from the metadata to check stats + let provider_ref = metadata_with_custom_index + .page_index() + .unwrap() + .as_any() + .downcast_ref::() + .expect("Expected SelectivePageIndexProvider"); + + // Check that the custom provider was used + let (hits, misses) = provider_ref.stats(); + assert!(hits > 0, "provider should have hits"); + assert!(misses > 0, "provider should have misses"); + + // Since we didn't use predicates, the column index should be untouched + let (col_hits, col_misses) = provider_ref.column_index_stats(); + assert_eq!(col_hits, 0, "column index should not be used"); + assert_eq!(col_misses, 0, "column index should not be used"); + + let (off_hits, off_misses) = provider_ref.offset_index_stats(); + assert!(off_hits > 0, "offset index should have hits"); + assert!(off_misses > 0, "offset index should have misses"); + + // Step 8: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); +} + /// A custom PageIndexProvider that only stores indexes for a subset of columns /// /// This provider mimics a real-world scenario where an application has loaded /// page indexes selectively based on query predicates and projections. -#[derive(Debug, Clone)] +#[derive(Debug)] struct SelectivePageIndexProvider { - metadata: ParquetMetaData, file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index column_indexes: Option>>, offset_indexes: Option>>, + // Usage statistics + column_index_hits: Arc, + column_index_misses: Arc, + offset_index_hits: Arc, + offset_index_misses: Arc, } impl SelectivePageIndexProvider { - fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes) -> Self { Self { - metadata, file_bytes, column_indexes: None, offset_indexes: None, + column_index_hits: Arc::new(AtomicUsize::new(0)), + column_index_misses: Arc::new(AtomicUsize::new(0)), + offset_index_hits: Arc::new(AtomicUsize::new(0)), + offset_index_misses: Arc::new(AtomicUsize::new(0)), } } + /// Get statistics about column index usage + pub fn column_index_stats(&self) -> (usize, usize) { + ( + self.column_index_hits.load(Ordering::Relaxed), + self.column_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get statistics about offset index usage + pub fn offset_index_stats(&self) -> (usize, usize) { + ( + self.offset_index_hits.load(Ordering::Relaxed), + self.offset_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get combined statistics as (hits, misses) for both index types + pub fn stats(&self) -> (usize, usize) { + let (col_hits, col_misses) = self.column_index_stats(); + let (off_hits, off_misses) = self.offset_index_stats(); + (col_hits + off_hits, col_misses + off_misses) + } + + /// Reset all statistics counters to zero + pub fn reset_stats(&self) { + self.column_index_hits.store(0, Ordering::Relaxed); + self.column_index_misses.store(0, Ordering::Relaxed); + self.offset_index_hits.store(0, Ordering::Relaxed); + self.offset_index_misses.store(0, Ordering::Relaxed); + } + /// Fetch and parse the column index for the given row group and column fn fetch_column_index( &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.column_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.column_index_range(); if let Some(range) = range { let idx_bytes = self @@ -93,11 +312,12 @@ impl SelectivePageIndexProvider { &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.offset_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.offset_index_range(); if let Some(range) = range { let idx_bytes = self @@ -125,10 +345,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - self.column_indexes + let result = self + .column_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.column_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.column_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn offset_index( @@ -136,10 +365,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - self.offset_indexes + let result = self + .offset_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.offset_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.offset_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn as_any(&self) -> &dyn std::any::Any { @@ -220,327 +458,3 @@ fn create_test_file() -> NamedTempFile { } temp_file } - -#[test] -fn test_read_with_custom_page_index_provider() { - // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Verify we have multiple row groups - assert_eq!(metadata.num_row_groups(), 3); - - // Step 3: Create a custom provider and populate it selectively - // Simulate a scenario where: - // - We only populate indexes for row groups 0 and 2 (skipping row group 1) - // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - provider.fetch_column_index(0, 1).unwrap(); - provider.fetch_offset_index(0, 1).unwrap(); - - // Populate indexes for row group 2, column 0 only - provider.fetch_column_index(2, 0).unwrap(); - provider.fetch_offset_index(2, 0).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - - // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Step 5: Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Create a RowSelection that: - // - Selects rows 20-30 (in row group 0) - // - Selects rows 60-70 (in row group 1) - // - Selects rows 120-130 (in row group 2) - let selection = RowSelection::from(vec![ - // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), - // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), - // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), - // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Step 7: Verify correct results - // We expect 30 rows total: 10 from each selected range - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 30); - - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - - // Expected values for the selected rows: - // rows 20-30: id=20..30, value=40..60, score=60..90 - // rows 60-70: id=60..70, value=120..140, score=180..210 - // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); - assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" - ); - - // Success! The test validates that: - // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping - // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data - // 3. For row group 1, no columns had indexes but all returned correct data - // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data - // 5. The reader correctly handled the mix of indexed and non-indexed columns -} - -#[test] -fn test_read_all_columns_with_partial_indexes() { - // This test focuses on verifying that columns without page indexes - // fall back to whole-column-chunk fetching and return correct results - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Load metadata without page indexes - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes ONLY for column 0 in all row groups - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - for row_group_idx in 0..metadata.num_row_groups() { - provider.fetch_column_index(row_group_idx, 0).unwrap(); - provider.fetch_offset_index(row_group_idx, 0).unwrap(); - } - - // Install provider - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read with row selection - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Select rows 10-20 from row group 0 - let selection = RowSelection::from(vec![ - parquet::arrow::arrow_reader::RowSelector::skip(10), - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - let batches: Vec = reader.collect::, _>>().unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 10); - - // Verify all columns have correct data - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - let id = ids.value(i); - assert_eq!(values.value(i), id * 2); - assert_eq!(names.value(i), format!("name{id}")); - assert_eq!(scores.value(i), id * 3); - } - } -} - -#[test] -fn test_custom_provider_with_no_row_selection() { - // Test that reading without RowSelection also works correctly - // with a custom page index provider - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes for first column only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read all data - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - let reader = builder.build().unwrap(); - - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Should read all 150 rows - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 150); - - // Verify data integrity - let mut row_count: i32 = 0; - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - assert_eq!(ids.value(i), row_count); - assert_eq!(values.value(i), row_count * 2); - assert_eq!(names.value(i), format!("name{row_count}")); - assert_eq!(scores.value(i), row_count * 3); - row_count += 1; - } - } -} From 5458721eebe9a5af19f7fdeb89de8726de7e9379 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:00:56 -0700 Subject: [PATCH 07/62] allow for serialization of custom page index providers --- parquet/src/file/metadata/page_index.rs | 21 +--- parquet/src/file/metadata/writer.rs | 121 ++++++++++++------------ parquet/src/file/writer.rs | 73 +++++++++++++- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9ded57d0a28e..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -460,22 +460,6 @@ impl PageIndex { pub fn into_builder(self) -> PageIndexBuilder { self.into() } - - /// Returns a reference to the raw column indexes structure - /// - /// This method provides access to the underlying column index data for serialization - /// and other low-level operations. - pub(crate) fn column_indexes_raw(&self) -> Option<&Vec>>> { - self.column_indexes.as_ref() - } - - /// Returns a reference to the raw offset indexes structure - /// - /// This method provides access to the underlying offset index data for serialization - /// and other low-level operations. - pub(crate) fn offset_indexes_raw(&self) -> Option<&Vec>>> { - self.offset_indexes.as_ref() - } } impl PageIndexProvider for PageIndex { @@ -536,7 +520,10 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { + pub(crate) fn empty_index( + num_row_groups: usize, + num_columns: usize, + ) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a1708ccf2c3a..d6de7e45c8c5 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::FileMeta; use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; @@ -54,8 +55,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { buf: &'a mut TrackedWrite, schema_descr: &'a SchemaDescPtr, row_groups: Vec, - column_indexes: Option>>>, - offset_indexes: Option>>>, + page_index: Option>, key_value_metadata: Option>, created_by: Option, object_writer: MetadataObjectWriter, @@ -71,14 +71,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized offset indexes. fn write_offset_indexes( &mut self, - offset_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut offset_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let offidx_vec = offset_indexes.as_mut().unwrap(); + + // we've already checked before calling that the offset indexes are populated + assert!(page_index.has_offset_indexes()); + // iter row group // iter each column // write offset index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(offset_index) = &offset_indexes[row_group_idx][column_idx] { + if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); self.object_writer.write_offset_index( offset_index, @@ -91,10 +98,11 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); + offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); } } } - Ok(()) + Ok(offset_indexes) } /// Serialize all the column indexes to the `self.buf` @@ -104,14 +112,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized column indexes. fn write_column_indexes( &mut self, - column_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut column_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let colidx_vec = column_indexes.as_mut().unwrap(); + + // we've already checked before calling that the column indexes are populated + assert!(page_index.has_column_indexes()); + // iter row group // iter each column // write column index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(column_index) = &column_indexes[row_group_idx][column_idx] { + if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); // only update column_metadata if the write succeeds if self.object_writer.write_column_index( @@ -127,21 +142,28 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } + colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); } } } - Ok(()) + Ok(column_indexes) } - /// Serialize the column indexes and transform to `Option` - fn finalize_column_indexes(&mut self) -> Result>>>> { - let column_indexes = std::mem::take(&mut self.column_indexes); - - // Write column indexes to file - if let Some(column_indexes) = column_indexes.as_ref() { - self.write_column_indexes(column_indexes)?; + /// Serialize the column indexes and transform to `Option>>>` + fn finalize_column_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_column_indexes()) + { + return Ok(None); } + // Write column indexes to file while assembling a column index + let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; + // check to see if the index is `None` for every row group and column chunk let all_none = column_indexes .as_ref() @@ -155,13 +177,19 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes(&mut self) -> Result>>>> { - let offset_indexes = std::mem::take(&mut self.offset_indexes); + fn finalize_offset_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_offset_indexes()) + { + return Ok(None); + } // Write offset indexes to file - if let Some(offset_indexes) = offset_indexes.as_ref() { - self.write_offset_indexes(offset_indexes)?; - } + let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; // check to see if the index is `None` for every row group and column chunk let all_none = offset_indexes @@ -180,8 +208,9 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); // serialize page indexes and transform to the proper form for use in ParquetMetaData - let column_indexes = self.finalize_column_indexes()?; - let offset_indexes = self.finalize_offset_indexes()?; + let page_index = self.page_index.take(); + let column_indexes = self.finalize_column_indexes(&page_index)?; + let offset_indexes = self.finalize_offset_indexes(&page_index)?; // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -275,8 +304,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { buf, schema_descr, row_groups, - column_indexes: None, - offset_indexes: None, + page_index: None, key_value_metadata: None, created_by, object_writer: Default::default(), @@ -285,19 +313,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } } - pub fn with_column_indexes( - mut self, - column_indexes: Vec>>, - ) -> Self { - self.column_indexes = Some(column_indexes); - self - } - - pub fn with_offset_indexes( - mut self, - offset_indexes: Vec>>, - ) -> Self { - self.offset_indexes = Some(offset_indexes); + pub fn with_page_index(mut self, page_index: Arc) -> Self { + self.page_index = Some(page_index); self } @@ -333,17 +350,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -///
-/// -/// **NOTE:** -/// The serialization of custom [`PageIndexProvider`]s is not currently supported. -/// The only supported page index structure is [`PageIndex`]. If the metadata -/// contains any other [`PageIndexProvider`] implementation, the [`ColumnIndex`] -/// and [`OffsetIndex`] structures are silently omitted from the output. See -/// for more details. -/// -///
-/// /// # Output Format /// /// The format of the metadata is as follows: @@ -464,21 +470,10 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - // Downcast to PageIndex to access raw index structures for serialization. - // Page indexes from custom PageIndexProviders are not written. See - // if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && let Some(page_index) = page_index_arc - .as_any() - .downcast_ref::() + && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) { - if let Some(column_indexes) = page_index.column_indexes_raw() { - encoder = encoder.with_column_indexes(column_indexes.clone()); - } - - if let Some(offset_indexes) = page_index.offset_indexes_raw() { - encoder = encoder.with_offset_indexes(offset_indexes.clone()); - } + encoder = encoder.with_page_index(page_index_arc.clone()); } if let Some(key_value_metadata) = key_value_metadata { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 6c0f3e8dd283..18b7e6e9999e 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -18,6 +18,7 @@ //! [`SerializedFileWriter`]: Low level Parquet writer API use crate::bloom_filter::Sbbf; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::offset_index::OffsetIndexMetaData; @@ -377,10 +378,27 @@ impl SerializedFileWriter { encoder = encoder.with_key_value_metadata(key_value_metadata) } - encoder = encoder.with_column_indexes(column_indexes); - if !self.props.offset_index_disabled() { - encoder = encoder.with_offset_indexes(offset_indexes); + // check for empty column index + let column_indexes = if column_indexes.is_empty() + || column_indexes + .iter() + .all(|cis| cis.iter().all(|ci| ci.is_none())) + { + None + } else { + Some(column_indexes) + }; + // offset index will always be created unless explicitly disabled + let offset_indexes = if self.props.offset_index_disabled() { + None + } else { + Some(offset_indexes) + }; + if column_indexes.is_some() || offset_indexes.is_some() { + let page_index = PageIndex::new(column_indexes, offset_indexes); + encoder = encoder.with_page_index(Arc::new(page_index)); } + encoder.finish() } @@ -2201,6 +2219,55 @@ mod tests { assert!(b_idx.is_none(), "{b_idx:?}"); } + #[test] + fn test_offset_index_disabled() { + let message_type = " + message test_schema { + REQUIRED INT32 a; + REQUIRED INT32 b; + } + "; + // write file with indexes disabled (including offset indexes) + let schema = Arc::new(parse_message_type(message_type).unwrap()); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::None) + .set_offset_index_disabled(true) + .build(); + let mut file = Vec::with_capacity(1024); + let mut file_writer = + SerializedFileWriter::new(&mut file, schema, Arc::new(props)).unwrap(); + + let mut row_group_writer = file_writer.next_row_group().unwrap(); + let mut a_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = a_writer.typed::(); + col_writer.write_batch(&[1, 2, 3], None, None).unwrap(); + a_writer.close().unwrap(); + + let mut b_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = b_writer.typed::(); + col_writer.write_batch(&[4, 5, 6], None, None).unwrap(); + b_writer.close().unwrap(); + row_group_writer.close().unwrap(); + + let metadata = file_writer.finish().unwrap(); + assert_eq!(metadata.num_row_groups(), 1); + let row_group = metadata.row_group(0); + assert_eq!(row_group.num_columns(), 2); + // no page indexes should exist + assert!(row_group.column(0).offset_index_offset().is_none()); + assert!(row_group.column(0).column_index_offset().is_none()); + assert!(row_group.column(1).offset_index_offset().is_none()); + assert!(row_group.column(1).column_index_offset().is_none()); + + drop(file_writer); + + // read file and request page index...should be `None` + let options = ReadOptionsBuilder::new().with_page_index().build(); + let reader = SerializedFileReader::new_with_options(Bytes::from(file), options).unwrap(); + + assert!(reader.metadata().page_index().is_none()); + } + #[test] fn test_byte_array_size_statistics() { let message_type = " From 979141a093023289d3041bc72a6dd4bbca23f5fd Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:11:03 -0700 Subject: [PATCH 08/62] clippy --- parquet/src/file/metadata/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index d6de7e45c8c5..63cdf540d146 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -152,7 +152,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the column indexes and transform to `Option>>>` fn finalize_column_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -179,7 +179,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the offset indexes and transform to `Option` fn finalize_offset_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -209,8 +209,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // serialize page indexes and transform to the proper form for use in ParquetMetaData let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(&page_index)?; - let offset_indexes = self.finalize_offset_indexes(&page_index)?; + let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; + let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; // We only include ColumnOrder for leaf nodes. let column_orders = self From 3a8ee6ae551004b2993de8d3483e4ee309454cae Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 17:43:51 -0700 Subject: [PATCH 09/62] add plumbing for page index selective population --- parquet/benches/metadata.rs | 28 ++- parquet/src/arrow/arrow_reader/mod.rs | 56 ++++- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 16 ++ parquet/src/file/metadata/parser.rs | 68 +++--- parquet/src/file/metadata/push_decoder.rs | 221 +++++++++++++++++--- parquet/src/file/metadata/reader.rs | 108 ++++++++++ parquet/src/file/page_index/index_reader.rs | 11 - parquet/tests/arrow_reader/io/mod.rs | 4 +- 10 files changed, 441 insertions(+), 77 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 0ec1b3cd1733..d9fef3d1a5d0 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, ParquetMetaData, - ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, - RowGroupMetaData, + ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, + PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; use parquet::file::writer::TrackedWrite; @@ -191,6 +191,28 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&data) + .unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Required); + c.bench_function("read page index", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + c.bench_function("read page index reduced columns", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + let meta_data = get_footer_bytes(data.clone()); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata", |b| { diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 73bcb2048ad5..8f8953e69933 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,8 +45,8 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, - ParquetStatisticsPolicy, RowGroupMetaData, + PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; use crate::schema::types::SchemaDescriptor; @@ -593,8 +593,10 @@ pub struct ArrowReaderOptions { /// [ARROW_SCHEMA_META_KEY]: crate::arrow::ARROW_SCHEMA_META_KEY supplied_schema: Option, - pub(crate) column_index: PageIndexPolicy, - pub(crate) offset_index: PageIndexPolicy, + column_index: PageIndexPolicy, + offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -774,6 +776,32 @@ impl ArrowReaderOptions { self } + /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// + /// The column index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when filtering by a predicate + /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// + /// The offset index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when projecting a small subset + /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet /// footer will be skipped. /// @@ -924,6 +952,20 @@ impl ArrowReaderOptions { self.column_index } + /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// + /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. + pub fn offset_index_selection(&self) -> &PageIndexSelection { + &self.offset_index_selection + } + + /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// + /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. + pub fn column_index_selection(&self) -> &PageIndexSelection { + &self.column_index_selection + } + /// Retrieve the currently set metadata decoding options. pub fn metadata_options(&self) -> &ParquetMetaDataOptions { &self.metadata_options @@ -970,7 +1012,9 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self @@ -1019,6 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index 4bd65cfead59..a73d2c0d3fbe 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,7 +242,9 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 166a7eb7ab6a..65c084108c7f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, ParquetMetaDataReader}; +pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..6a5755fec3a7 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,6 +17,7 @@ //! Page Index structures for efficient page-level skipping +use crate::file::metadata::PageIndexSelection; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -558,6 +559,21 @@ impl PageIndexBuilder { } } + /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy + /// + /// For sparse indexes, this can save a great deal of memory + pub fn new_with_selection( + num_row_groups: usize, + num_columns: usize, + _column_index_selection: PageIndexSelection, + _offset_index_selection: PageIndexSelection, + ) -> Self { + Self { + column_indexes: Self::empty_index(num_row_groups, num_columns), + offset_indexes: Self::empty_index(num_row_groups, num_columns), + } + } + /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 861bb2e53b08..b2a35172e8af 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,13 +26,15 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use bytes::Bytes; +use crate::file::reader::ChunkReader; +use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -250,8 +252,9 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - bytes: &Bytes, - start_offset: u64, + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip { @@ -259,25 +262,28 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = PageIndexBuilder::new_with_selection( + num_row_groups, + num_columns, + column_index_selection.clone(), + offset_index_selection.clone(), + ); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, + column_index_selection, &mut builder, bytes, - start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, + offset_index_selection, &mut builder, bytes, - start_offset, )?; } @@ -293,27 +299,27 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_column_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -325,27 +331,27 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_offset_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index c5e77cc22468..b55468fd2980 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -21,8 +21,10 @@ use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; -use crate::file::metadata::{FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions}; -use crate::file::page_index::index_reader::acc_range; +use crate::file::metadata::{ + ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, +}; use crate::file::reader::ChunkReader; use bytes::Bytes; use std::ops::Range; @@ -226,6 +228,10 @@ pub struct ParquetMetaDataPushDecoder { column_index_policy: PageIndexPolicy, /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, + /// which rows and columns of the column index should be read + column_index_selection: PageIndexSelection, + /// which rows and columns of the offset index should be read + offset_index_selection: PageIndexSelection, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -250,6 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, + column_index_selection: PageIndexSelection::all(), + offset_index_selection: PageIndexSelection::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -300,6 +308,25 @@ impl ParquetMetaDataPushDecoder { self } + /// Select the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection.clone(); + self.offset_index_selection = selection; + self + } + + /// Select the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Select the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Set the options to use when decoding the Parquet metadata. pub fn with_metadata_options(mut self, options: Option>) -> Self { self.metadata_parser = self.metadata_parser.with_metadata_options(options); @@ -408,30 +435,36 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let range = range_for_page_index( + let ranges = ranges_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_selection, + &self.offset_index_selection, ); - let Some(page_index_range) = range else { + if ranges.is_empty() { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - }; + } + + let needed_ranges = ranges + .into_iter() + .filter(|r| !self.buffers.has_range(r)) + .collect::>(); - if !self.buffers.has_range(&page_index_range) { + if !needed_ranges.is_empty() { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + return Ok(DecodeResult::NeedsData(needed_ranges)); } - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, - &buffer, - offset, + &self.column_index_selection, + &self.offset_index_selection, + &self.buffers, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); @@ -481,25 +514,81 @@ enum DecodeState { Intermediate, } -/// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// Returns the minimum set of non-overlapping ranges needed to cover the requested +/// offset and column indexes. +/// +/// If no page indexes are present in the file, or none are actually +/// requested by the policies passed in, this will return an empty +/// vector. /// -/// Returns None if no page indexes are needed -pub fn range_for_page_index( +/// This may result in more ranges to fetch but may also result in fewer bytes +/// being read (and stored). To get a single range for all indexes, sort +/// the resultant vector by the range starts, and then create a single range +/// using `start` from the head and `end` from the tail. +/// +/// ```rust +/// # use core::ops::Range; +/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { +/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// let range = (ranges.first()?.start..ranges.last()?.end); +/// Some(range) +/// # } +/// ``` +pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, -) -> Option> { - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, +) -> Vec> { + let mut result = Vec::new(); + + fn add_ranges( + metadata: &ParquetMetaData, + selection: &PageIndexSelection, + ranges: &mut Vec>, + f: T, + ) where + T: Fn(&ColumnChunkMetaData) -> Option>, + { + for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { + if selection.includes_row_group(rg_idx) { + for (col_idx, col) in rg.columns().iter().enumerate() { + if selection.includes_column(col_idx) { + if let Some(range) = f(col) { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); + } + } + } + } + } } } - range + + if column_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + column_index_selection, + &mut result, + ColumnChunkMetaData::column_index_range, + ); + } + if offset_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + offset_index_selection, + &mut result, + ColumnChunkMetaData::offset_index_range, + ); + } + result } // These tests use the arrow writer to create a parquet file in memory @@ -651,6 +740,90 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } + /// Decode the metadata incrementally, with a partial page index + #[test] + fn test_metadata_decoder_incremental_partial_page_index() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + + /// Decode the metadata incrementally, with a partial page index, but coalesced + /// range request. + #[test] + fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let mut ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + // collapse ranges into a single range + ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + let range = ranges.first().unwrap().start..ranges.last().unwrap().end; + push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e4cec0fd0df5..c3ce6ab6ce23 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -27,6 +27,7 @@ use crate::file::metadata::{ use crate::file::reader::ChunkReader; use crate::schema::types::SchemaDescriptor; use bytes::Bytes; +use std::collections::BTreeSet; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -71,6 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -104,6 +107,89 @@ impl From for PageIndexPolicy { } } +/// Selects the row groups and columns for which page indexes are read. +/// +/// This is independent of [`PageIndexPolicy`], which controls whether selected +/// indexes are optional or required. By default, all row groups and columns are selected. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct PageIndexSelection { + // using i32 because that's how thrift vectors are sized + row_groups: Option>>, + columns: Option>>, +} + +// TODO(ets): add unit tests for PageIndexSelection +impl PageIndexSelection { + /// Select all row groups and columns. + pub fn all() -> Self { + Self::default() + } + + /// Select page indexes for only the listed columns. + /// + /// Any indices in `columns` that are less than zero will be ignored. + pub fn columns(columns: impl IntoIterator) -> Self { + Self { + row_groups: None, + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + /// Select page indexes for only the listed row groups. + /// + /// Any indices in `row_groups` that are less than zero will be ignored. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: None, + } + } + + /// Select page indexes for only the listed row groups and columns. + /// + /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + pub fn row_groups_and_columns( + row_groups: impl IntoIterator, + columns: impl IntoIterator, + ) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + // test if `idx` is in the row group set. returns false if idx > i32::MAX + pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.row_groups + .as_ref() + .is_none_or(|keep| keep.contains(&idx)) + } + + // test if `idx` is in the column set. returns false if idx > i32::MAX + pub(crate) fn includes_column(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + } + + // FIXME(ets): these will be used later + /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { + self.row_groups.as_deref() + } + + pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { + self.columns.as_deref() + }*/ +} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -137,6 +223,24 @@ impl ParquetMetaDataReader { self } + /// Selects the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { + self.with_column_index_selection(selection.clone()) + .with_offset_index_selection(selection) + } + + /// Selects the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Selects the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Sets the [`ParquetMetaDataOptions`] to use when decoding pub fn with_metadata_options(mut self, options: Option) -> Self { self.metadata_options = options.map(Arc::new); @@ -341,6 +445,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -488,6 +594,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/src/file/page_index/index_reader.rs b/parquet/src/file/page_index/index_reader.rs index 963ee0ea9782..235f0b241887 100644 --- a/parquet/src/file/page_index/index_reader.rs +++ b/parquet/src/file/page_index/index_reader.rs @@ -30,17 +30,6 @@ use crate::parquet_thrift::{ }; use crate::thrift_struct; use std::io::Write; -use std::ops::Range; - -/// Computes the covering range of two optional ranges -/// -/// For example `acc_range(Some(7..9), Some(1..3)) = Some(1..9)` -pub(crate) fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } -} /// Decode a Thrift [`OffsetIndex`] from the provided bytes. /// diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 3fb63ffdb87b..2f0348b835f5 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -132,7 +132,9 @@ impl AsyncFileReader for TestReader { if let Some(options) = options { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self.metadata = Some(Arc::new( From 1df722eb61f2a7b92f6a7f1e9c083228a0f99ea1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 13:23:30 -0700 Subject: [PATCH 10/62] more tests --- .../custom_page_index_provider.rs | 2 +- parquet/tests/arrow_reader/mod.rs | 1 + parquet/tests/arrow_reader/page_index.rs | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 parquet/tests/arrow_reader/page_index.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 2736b7bc35e2..5846db69b7a1 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -386,7 +386,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> NamedTempFile { +pub(super) fn create_test_file() -> NamedTempFile { let temp_file = tempfile::Builder::new() .prefix("custom_page_index_test") .suffix(".parquet") diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index 6c47a00b9b3d..867b69d7bbf5 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -47,6 +47,7 @@ mod int96_stats_roundtrip; mod invalid_utf8; mod io; mod large_string_overflow; +mod page_index; mod parquet_testing; #[cfg(feature = "async")] mod predicate_cache; diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs new file mode 100644 index 000000000000..821d1d4cc55d --- /dev/null +++ b/parquet/tests/arrow_reader/page_index.rs @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests of selective page index population + +use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use std::fs::File; + +use crate::custom_page_index_provider::create_test_file; + +#[test] +fn test_parse_selected_columns() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_none()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_none()); + } +} + +#[test] +fn test_parse_selected_columns_mixed() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_some()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups() { + // test populating PageIndex with a subset of row groups + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate indexes for row groups 0 and 2 + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) + .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + + for col in 0..num_cols { + // row group 0 has both indexes + assert!(page_index.column_index(0, col).is_some()); + assert!(page_index.offset_index(0, col).is_some()); + // row group 1 has no indexes + assert!(page_index.column_index(1, col).is_none()); + assert!(page_index.offset_index(1, col).is_none()); + // row group 2 has both indexes + assert!(page_index.column_index(2, col).is_some()); + assert!(page_index.offset_index(2, col).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups_and_columns() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + + // row groups 0 and 2 should have no indexes + [0, 2].into_iter().for_each(|i| { + let rg_idx = metadata.page_index_for_row_group(i); + for col in 0..num_cols { + assert!(rg_idx.column_index(col).is_none()); + assert!(rg_idx.offset_index(col).is_none()); + } + }); + + // row group 1 should have column index for column 0 and offset index for columns 0 & 2 + let rg_idx = metadata.page_index_for_row_group(1); + assert!(rg_idx.column_index(0).is_some()); + assert!(rg_idx.offset_index(0).is_some()); + assert!(rg_idx.column_index(1).is_none()); + assert!(rg_idx.offset_index(1).is_none()); + assert!(rg_idx.column_index(2).is_none()); + assert!(rg_idx.offset_index(2).is_some()); + assert!(rg_idx.column_index(3).is_none()); + assert!(rg_idx.offset_index(3).is_none()); +} + +#[test] +fn test_page_index_sizes() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // no index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_none()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 7393); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 7817); + + // full index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 13897); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 14321); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12776); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 13200); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12056); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 12480); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 11326); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 11750); +} From 9eacc746f0aa604387a4c7aa0bb50a60fed76128 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:05:59 -0700 Subject: [PATCH 11/62] rename PageIndexSelection to ColumnChunkMask --- parquet/benches/metadata.rs | 8 +-- parquet/src/arrow/arrow_reader/mod.rs | 46 +++++++------- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 8 +-- parquet/src/file/metadata/parser.rs | 29 +++++---- parquet/src/file/metadata/push_decoder.rs | 76 +++++++++++------------ parquet/src/file/metadata/reader.rs | 55 +++++++++------- parquet/tests/arrow_reader/io/mod.rs | 4 +- parquet/tests/arrow_reader/page_index.rs | 30 ++++----- 10 files changed, 135 insertions(+), 127 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index d9fef3d1a5d0..b5ee07c49530 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, - PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ColumnChunkMask, ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, + PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; @@ -205,8 +205,8 @@ fn criterion_benchmark(c: &mut Criterion) { let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); c.bench_function("read page index reduced columns", |b| { b.iter(|| { reader.read_page_indexes(&data).unwrap(); diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 8f8953e69933..492ea6063462 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,7 +45,7 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; @@ -595,8 +595,8 @@ pub struct ArrowReaderOptions { column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -776,29 +776,29 @@ impl ArrowReaderOptions { self } - /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when filtering by a predicate - /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } - /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [OffsetIndex] structure. /// /// The offset index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when projecting a small subset - /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -952,18 +952,18 @@ impl ArrowReaderOptions { self.column_index } - /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// Retrieve the currently set [`ColumnChunkMask`] for the offset index. /// - /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. - pub fn offset_index_selection(&self) -> &PageIndexSelection { - &self.offset_index_selection + /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask]. + pub fn offset_index_mask(&self) -> &ColumnChunkMask { + &self.offset_index_mask } - /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// Retrieve the currently set [`ColumnChunkMask`] for the column index. /// - /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. - pub fn column_index_selection(&self) -> &PageIndexSelection { - &self.column_index_selection + /// This can be set via [`with_column_index_mask`][Self::with_column_index_mask]. + pub fn column_index_mask(&self) -> &ColumnChunkMask { + &self.column_index_mask } /// Retrieve the currently set metadata decoding options. @@ -1013,8 +1013,8 @@ impl ParquetMetaDataReader { self = self .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self @@ -1063,8 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()) + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index a73d2c0d3fbe..c7db5d0abbc1 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -243,8 +243,8 @@ impl AsyncFileReader for ParquetObjectReader { metadata = metadata .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 65c084108c7f..12162a7f8f85 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 6a5755fec3a7..ea1b17f3af38 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,7 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::PageIndexSelection; +use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -562,11 +562,11 @@ impl PageIndexBuilder { /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy /// /// For sparse indexes, this can save a great deal of memory - pub fn new_with_selection( + pub fn new_with_mask( num_row_groups: usize, num_columns: usize, - _column_index_selection: PageIndexSelection, - _offset_index_selection: PageIndexSelection, + _column_index_mask: ColumnChunkMask, + _offset_index_mask: ColumnChunkMask, ) -> Self { Self { column_indexes: Self::empty_index(num_row_groups, num_columns), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index b2a35172e8af..47b93463147c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,8 +26,7 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; @@ -252,8 +251,8 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip @@ -262,17 +261,17 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_selection( + let mut builder = PageIndexBuilder::new_with_mask( num_row_groups, num_columns, - column_index_selection.clone(), - offset_index_selection.clone(), + column_index_mask.clone(), + offset_index_mask.clone(), ); if column_index_policy != PageIndexPolicy::Skip { parse_column_index( metadata, column_index_policy, - column_index_selection, + column_index_mask, &mut builder, bytes, )?; @@ -281,7 +280,7 @@ pub(crate) fn parse_page_index( parse_offset_index( metadata, offset_index_policy, - offset_index_selection, + offset_index_mask, &mut builder, bytes, )?; @@ -299,7 +298,7 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -307,12 +306,12 @@ fn parse_column_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); @@ -331,7 +330,7 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -339,12 +338,12 @@ fn parse_offset_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index b55468fd2980..3999fdf9a9fb 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,7 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; @@ -229,9 +229,9 @@ pub struct ParquetMetaDataPushDecoder { /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, /// which rows and columns of the column index should be read - column_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, /// which rows and columns of the offset index should be read - offset_index_selection: PageIndexSelection, + offset_index_mask: ColumnChunkMask, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -256,8 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, - column_index_selection: PageIndexSelection::all(), - offset_index_selection: PageIndexSelection::all(), + column_index_mask: ColumnChunkMask::all(), + offset_index_mask: ColumnChunkMask::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -309,21 +309,21 @@ impl ParquetMetaDataPushDecoder { } /// Select the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection.clone(); - self.offset_index_selection = selection; + pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask.clone(); + self.offset_index_mask = mask; self } /// Select the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Select the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -439,8 +439,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, ); if ranges.is_empty() { @@ -462,8 +462,8 @@ impl ParquetMetaDataPushDecoder { &mut metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, &self.buffers, )?; self.state = DecodeState::Finished; @@ -526,10 +526,10 @@ enum DecodeState { /// the resultant vector by the range starts, and then create a single range /// using `start` from the head and `end` from the tail. /// -/// ```rust +/// ```ignore /// # use core::ops::Range; /// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// ranges.sort_by_key(|r| r.start); /// let range = (ranges.first()?.start..ranges.last()?.end); /// Some(range) /// # } @@ -538,33 +538,33 @@ pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Vec> { let mut result = Vec::new(); fn add_ranges( metadata: &ParquetMetaData, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, ranges: &mut Vec>, f: T, ) where T: Fn(&ColumnChunkMetaData) -> Option>, { for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if selection.includes_row_group(rg_idx) { + if mask.includes_row_group(rg_idx) { for (col_idx, col) in rg.columns().iter().enumerate() { - if selection.includes_column(col_idx) { - if let Some(range) = f(col) { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } + if mask.includes_column(col_idx) + && let Some(range) = f(col) + { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); } } } @@ -575,7 +575,7 @@ pub fn ranges_for_page_index( if column_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - column_index_selection, + column_index_mask, &mut result, ColumnChunkMetaData::column_index_range, ); @@ -583,7 +583,7 @@ pub fn ranges_for_page_index( if offset_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - offset_index_selection, + offset_index_mask, &mut result, ColumnChunkMetaData::offset_index_range, ); @@ -747,7 +747,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -788,7 +788,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -803,7 +803,7 @@ mod tests { // one for column index and one for offset index for each row group assert_eq!(ranges.len(), 4); // collapse ranges into a single range - ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + ranges.sort_by_key(|r| r.start); let range = ranges.first().unwrap().start..ranges.last().unwrap().end; push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index c3ce6ab6ce23..cdaeb1855552 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -72,8 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -107,25 +107,34 @@ impl From for PageIndexPolicy { } } -/// Selects the row groups and columns for which page indexes are read. +/// Struct to specify column chunks for which metadata is required. /// -/// This is independent of [`PageIndexPolicy`], which controls whether selected -/// indexes are optional or required. By default, all row groups and columns are selected. +/// Column chunks are identified by row group index and column index. This struct +/// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), +/// horizontal slices (via [`Self::row_groups`]), or the intersection of the two +/// (via [`Self::row_groups_and_columns`]). +/// +/// # Example +/// TODO(ets): example +/// +/// At present this is only used to select elements of the [Page Index] for decoding. +/// +/// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct PageIndexSelection { +pub struct ColumnChunkMask { // using i32 because that's how thrift vectors are sized row_groups: Option>>, columns: Option>>, } -// TODO(ets): add unit tests for PageIndexSelection -impl PageIndexSelection { +// TODO(ets): add unit tests for ColumnChunkMask +impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { Self::default() } - /// Select page indexes for only the listed columns. + /// Select only the listed columns. /// /// Any indices in `columns` that are less than zero will be ignored. pub fn columns(columns: impl IntoIterator) -> Self { @@ -135,7 +144,7 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups. + /// Select only the listed row groups. /// /// Any indices in `row_groups` that are less than zero will be ignored. pub fn row_groups(row_groups: impl IntoIterator) -> Self { @@ -147,9 +156,9 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups and columns. + /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, @@ -224,20 +233,20 @@ impl ParquetMetaDataReader { } /// Selects the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { - self.with_column_index_selection(selection.clone()) - .with_offset_index_selection(selection) + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) } /// Selects the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Selects the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -445,8 +454,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -594,8 +603,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 2f0348b835f5..bb9b5aeba446 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -133,8 +133,8 @@ impl AsyncFileReader for TestReader { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self.metadata = Some(Arc::new( diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 821d1d4cc55d..e594e78e900d 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -17,7 +17,7 @@ //! Tests of selective page index population -use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -31,8 +31,8 @@ fn test_parse_selected_columns() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -66,8 +66,8 @@ fn test_parse_selected_columns_mixed() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -101,8 +101,8 @@ fn test_parse_selected_row_groups() { // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) - .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups([0, 2])) + .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -134,8 +134,8 @@ fn test_parse_selected_row_groups_and_columns() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -197,8 +197,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -212,8 +212,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -228,8 +228,8 @@ fn test_page_index_sizes() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); From 69ab61adc0adbeb6e3ac72c0d3db2c504cec1dca Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:17:29 -0700 Subject: [PATCH 12/62] docs and cleanup --- parquet/src/file/metadata/page_index.rs | 16 ---- parquet/src/file/metadata/parser.rs | 12 ++- parquet/src/file/metadata/push_decoder.rs | 2 + parquet/src/file/metadata/reader.rs | 94 +++++++++++++++++------ 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index ea1b17f3af38..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,6 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -559,21 +558,6 @@ impl PageIndexBuilder { } } - /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy - /// - /// For sparse indexes, this can save a great deal of memory - pub fn new_with_mask( - num_row_groups: usize, - num_columns: usize, - _column_index_mask: ColumnChunkMask, - _offset_index_mask: ColumnChunkMask, - ) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } - } - /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 47b93463147c..a76eea9066c0 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,7 +245,8 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). -/// * `bytes` - The byte slice containing the page index data. +/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing +/// the page indexes. /// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, @@ -261,13 +262,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_mask( - num_row_groups, - num_columns, - column_index_mask.clone(), - offset_index_mask.clone(), - ); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -277,6 +274,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 3999fdf9a9fb..6b747b34aabf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -701,6 +701,8 @@ mod tests { // expect the second request to read the offset indexes let ranges = expect_needs_data(metadata_decoder.try_decode()); + // since we're reading all indexes, there should be a single range needed + assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); // expect the third request to read the actual data diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index cdaeb1855552..186ccfcce330 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -114,11 +114,28 @@ impl From for PageIndexPolicy { /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). /// -/// # Example -/// TODO(ets): example -/// /// At present this is only used to select elements of the [Page Index] for decoding. /// +/// # Examples +/// +/// To select columns 0 and 1 from all row groups: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::columns([0, 1]); +/// ``` +/// +/// To select all columns from row group 2: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups([2]); +/// ``` +/// +/// To select columns 1 and 3 from row group 0: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups_and_columns([0], [1, 3]); +/// ``` +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ColumnChunkMask { @@ -127,7 +144,6 @@ pub struct ColumnChunkMask { columns: Option>>, } -// TODO(ets): add unit tests for ColumnChunkMask impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { @@ -136,22 +152,22 @@ impl ColumnChunkMask { /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. + /// Any indices in `columns` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all columns. pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + columns: Self::iter_to_set(columns), } } /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. + /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all row groups. pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), + row_groups: Self::iter_to_set(row_groups), columns: None, } } @@ -159,20 +175,22 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. + /// Passing an empty set for `row_groups` is treated as selecting all row groups, and + /// an empty set for `columns` as selectiong all columns. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, ) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + row_groups: Self::iter_to_set(row_groups), + columns: Self::iter_to_set(columns), } } - // test if `idx` is in the row group set. returns false if idx > i32::MAX - pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + /// Test if `idx` is in the row group set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_row_group(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; @@ -181,22 +199,20 @@ impl ColumnChunkMask { .is_none_or(|keep| keep.contains(&idx)) } - // test if `idx` is in the column set. returns false if idx > i32::MAX - pub(crate) fn includes_column(&self, idx: usize) -> bool { + /// Test if `idx` is in the column set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_column(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) } - // FIXME(ets): these will be used later - /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { - self.row_groups.as_deref() + fn iter_to_set(indices: impl IntoIterator) -> Option>> { + let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); + (!set.is_empty()).then_some(Arc::new(set)) } - - pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { - self.columns.as_deref() - }*/ } impl ParquetMetaDataReader { @@ -1509,4 +1525,32 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } + + #[test] + fn test_chunk_mask() { + // basic test + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + // test that negative numbers are ignored + let mask = ColumnChunkMask::columns([-1, 0, 1]); + assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(!mask.includes_column(2)); + + // empty set should behave as "all" + let mask = ColumnChunkMask::columns([]); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(mask.includes_column(2)); + + // indexing beyond i32::MAX should return false + let mask = ColumnChunkMask::all(); + assert!(mask.includes_column(i32::MAX as usize)); + assert!(!mask.includes_column(u32::MAX as usize)); + } } From b7b529de8a2f95b66110092eb567cbc3930a7fd2 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 09:42:51 -0700 Subject: [PATCH 13/62] more miri friendly --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 2736b7bc35e2..09b41df1155f 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -408,7 +408,7 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = temp_file.reopen().unwrap(); + let file = File::create(temp_file.path()).unwrap(); let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); // Write 3 row groups with 50 rows each (150 rows total) From b682ebecff792b5d2efdd455633a4b7492335010 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 10:13:48 -0700 Subject: [PATCH 14/62] rework tests to do no file i/o --- .../custom_page_index_provider.rs | 28 ++++++------------- parquet/tests/arrow_reader/page_index.rs | 24 ++++++---------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 5846db69b7a1..b864a466001c 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -39,21 +39,17 @@ use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::fs::File; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::NamedTempFile; #[test] fn test_read_with_custom_page_index_provider() { // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + let file_bytes = create_test_file(); // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, + file_bytes.clone(), ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), ) .unwrap(); @@ -68,7 +64,7 @@ fn test_read_with_custom_page_index_provider() { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(file_bytes); + let mut provider = SelectivePageIndexProvider::new(file_bytes.clone()); // Populate indexes for row group 0, columns 0 and 1 provider.fetch_column_index(0, 0, metadata).unwrap(); @@ -105,8 +101,8 @@ fn test_read_with_custom_page_index_provider() { .unwrap(); // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + let builder = + ParquetRecordBatchReaderBuilder::new_with_metadata(file_bytes.clone(), arrow_metadata); // Create a RowSelection that: // - Selects rows 20-30 (in row group 0) @@ -386,13 +382,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -pub(super) fn create_test_file() -> NamedTempFile { - let temp_file = tempfile::Builder::new() - .prefix("custom_page_index_test") - .suffix(".parquet") - .tempfile() - .expect("tempfile creation"); - +pub(super) fn create_test_file() -> Bytes { let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("value", DataType::Int32, false), @@ -408,8 +398,8 @@ pub(super) fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = temp_file.reopen().unwrap(); - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + let mut buffer = Vec::with_capacity(1024); + let mut writer = ArrowWriter::try_new(&mut buffer, schema.clone(), Some(props)).unwrap(); // Write 3 row groups with 50 rows each (150 rows total) for row_group in 0..3 { @@ -456,5 +446,5 @@ pub(super) fn create_test_file() -> NamedTempFile { ); } } - temp_file + Bytes::from(buffer) } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e594e78e900d..c605eac59834 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -18,7 +18,6 @@ //! Tests of selective page index population use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; -use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -26,7 +25,6 @@ use crate::custom_page_index_provider::create_test_file; fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -35,7 +33,7 @@ fn test_parse_selected_columns() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -61,7 +59,6 @@ fn test_parse_selected_columns() { fn test_parse_selected_columns_mixed() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -70,7 +67,7 @@ fn test_parse_selected_columns_mixed() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -96,7 +93,6 @@ fn test_parse_selected_columns_mixed() { fn test_parse_selected_row_groups() { // test populating PageIndex with a subset of row groups let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() @@ -105,7 +101,7 @@ fn test_parse_selected_row_groups() { .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_cols = metadata.file_metadata().schema_descr().num_columns(); assert!(metadata.page_index().is_some()); @@ -128,7 +124,6 @@ fn test_parse_selected_row_groups() { fn test_parse_selected_row_groups_and_columns() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -138,7 +133,7 @@ fn test_parse_selected_row_groups_and_columns() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); let num_cols = metadata.file_metadata().schema_descr().num_columns(); @@ -168,13 +163,12 @@ fn test_parse_selected_row_groups_and_columns() { fn test_page_index_sizes() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // no index let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_none()); #[cfg(not(feature = "encryption"))] @@ -186,7 +180,7 @@ fn test_page_index_sizes() { let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -201,7 +195,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -216,7 +210,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -232,7 +226,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] From bd587b28de5b5577006f264cff8e7313570c795e Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 08:24:34 -0700 Subject: [PATCH 15/62] apply suggestions from review --- parquet/src/arrow/mod.rs | 107 ++++++++++++++++++++ parquet/src/file/metadata/page_index.rs | 5 +- parquet/src/file/metadata/writer.rs | 128 ++++++++---------------- 3 files changed, 149 insertions(+), 91 deletions(-) diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs index 8309a35e9b1d..b60f211a19e0 100644 --- a/parquet/src/arrow/mod.rs +++ b/parquet/src/arrow/mod.rs @@ -595,6 +595,113 @@ mod test { assert_eq!(original_metadata, roundtrip_metadata); } + #[test] + #[cfg_attr(miri, ignore)] // Takes too long + fn test_metadata_read_write_roundtrip_missing_page_index() { + let parquet_bytes = create_parquet_file(); + + // read the metadata from the file but skip the page indexes + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + let original_metadata = ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&parquet_bytes) + .unwrap(); + + // metadata_to_bytes should zero out the page index locations. if they aren't + // then reading metadata_bytes will fail with EOF + let metadata_bytes = metadata_to_bytes(&original_metadata); + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + let roundtrip_metadata = ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Optional) + .parse_and_finish(&metadata_bytes) + .expect("page index locations should have been cleared"); + + assert!(roundtrip_metadata.page_index().is_none()); + } + + #[test] + fn test_metadata_read_write_roundtrip_offset_index_only() { + // `Chunk` statistics: the file has an offset index, but no column index + let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let batch = RecordBatch::try_from_iter(vec![("id", array)]).unwrap(); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Chunk) + .build(); + let mut buf = vec![]; + let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let read = |bytes: &Bytes| { + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Optional) + .parse_and_finish(bytes) + .unwrap() + }; + let original = read(&Bytes::from(buf)); + let page_index = original.page_index().unwrap(); + assert!(!page_index.has_column_indexes() && page_index.has_offset_indexes()); + let roundtrip = read(&metadata_to_bytes(&original)); + assert_eq!( + normalize_locations(original), + normalize_locations(roundtrip) + ); + } + + #[test] + fn test_metadata_read_write_roundtrip_custom_page_index() { + use crate::file::metadata::page_index::PageIndexProvider; + use crate::file::page_index::{ + column_index::ColumnIndexMetaData, offset_index::OffsetIndexMetaData, + }; + + /// A custom provider that forwards to another provider + #[derive(Debug)] + struct Forward(Arc); + impl PageIndexProvider for Forward { + fn has_offset_indexes(&self) -> bool { + self.0.has_offset_indexes() + } + fn has_column_indexes(&self) -> bool { + self.0.has_column_indexes() + } + fn column_index(&self, rg: usize, col: usize) -> Option<&ColumnIndexMetaData> { + self.0.column_index(rg, col) + } + fn offset_index(&self, rg: usize, col: usize) -> Option<&OffsetIndexMetaData> { + self.0.offset_index(rg, col) + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + let read = |bytes: &Bytes| { + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(bytes) + .unwrap() + }; + let original = read(&create_parquet_file()); + let provider = Forward(original.page_index().unwrap().clone()); + let custom = original + .clone() + .into_builder() + .set_page_index(Some(Arc::new(provider))) + .build(); + let roundtrip = read(&metadata_to_bytes(&custom)); + assert_eq!( + normalize_locations(original), + normalize_locations(roundtrip) + ); + } + /// Sets the page index offset locations in the metadata to `None` /// /// This is because the offsets are used to find the relative location of the index diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -520,10 +520,7 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - pub(crate) fn empty_index( - num_row_groups: usize, - num_columns: usize, - ) -> Option>>> { + fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 63cdf540d146..596907d726fe 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; +use crate::file::metadata::page_index::PageIndexProvider; use crate::file::metadata::thrift::FileMeta; -use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; +use crate::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; use crate::{ basic::ColumnOrder, @@ -69,14 +69,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Note: also updates the `ColumnChunk::offset_index_offset` and /// `ColumnChunk::offset_index_length` to reflect the position and length /// of the serialized offset indexes. - fn write_offset_indexes( - &mut self, - page_index: &Arc, - ) -> Result>>>> { - let mut offset_indexes = - PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); - let offidx_vec = offset_indexes.as_mut().unwrap(); - + fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { // we've already checked before calling that the offset indexes are populated assert!(page_index.has_offset_indexes()); @@ -98,11 +91,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); - offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); + } else { + column_metadata.offset_index_offset = None; + column_metadata.offset_index_length = None; } } } - Ok(offset_indexes) + Ok(()) } /// Serialize all the column indexes to the `self.buf` @@ -110,17 +105,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Note: also updates the `ColumnChunk::column_index_offset` and /// `ColumnChunk::column_index_length` to reflect the position and length /// of the serialized column indexes. - fn write_column_indexes( - &mut self, - page_index: &Arc, - ) -> Result>>>> { - let mut column_indexes = - PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); - let colidx_vec = column_indexes.as_mut().unwrap(); - - // we've already checked before calling that the column indexes are populated - assert!(page_index.has_column_indexes()); - + fn write_column_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { // iter row group // iter each column // write column index to the file @@ -142,64 +127,23 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } - colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); + } else { + column_metadata.column_index_offset = None; + column_metadata.column_index_length = None; } } } - Ok(column_indexes) - } - - /// Serialize the column indexes and transform to `Option>>>` - fn finalize_column_indexes( - &mut self, - page_index: Option<&Arc>, - ) -> Result>>>> { - if page_index - .as_ref() - .is_none_or(|pi| !pi.has_column_indexes()) - { - return Ok(None); - } - - // Write column indexes to file while assembling a column index - let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; - - // check to see if the index is `None` for every row group and column chunk - let all_none = column_indexes - .as_ref() - .is_some_and(|ci| ci.iter().all(|cii| cii.iter().all(|idx| idx.is_none()))); - - if all_none { - Ok(None) - } else { - Ok(column_indexes) - } + Ok(()) } - /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes( - &mut self, - page_index: Option<&Arc>, - ) -> Result>>>> { - if page_index - .as_ref() - .is_none_or(|pi| !pi.has_offset_indexes()) - { - return Ok(None); - } - - // Write offset indexes to file - let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; - - // check to see if the index is `None` for every row group and column chunk - let all_none = offset_indexes - .as_ref() - .is_some_and(|oi| oi.iter().all(|oii| oii.iter().all(|idx| idx.is_none()))); - - if all_none { - Ok(None) - } else { - Ok(offset_indexes) + fn clear_page_indexes(&mut self) { + for row_group in self.row_groups.iter_mut() { + for column_metadata in row_group.columns.iter_mut() { + column_metadata.column_index_offset = None; + column_metadata.column_index_length = None; + column_metadata.offset_index_offset = None; + column_metadata.offset_index_length = None; + } } } @@ -207,10 +151,15 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { pub fn finish(mut self) -> Result { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); - // serialize page indexes and transform to the proper form for use in ParquetMetaData + // serialize page indexes and update index locations let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; - let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; + if let Some(pi) = page_index.as_deref() { + self.write_column_indexes(pi)?; + self.write_offset_indexes(pi)?; + } else { + // no page indexes at all, then clear index locations + self.clear_page_indexes(); + } // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -278,14 +227,17 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { self.buf.write_all(&metadata_len.to_le_bytes())?; self.buf.write_all(self.object_writer.get_file_magic())?; + // Note: we simply pass the original reference counted page index provider to the + // builder. Either it came from ParquetMetaDataWriter where it was cloned, and this + // reference will go away when ParquetMetaDataWriter::finish drops the newly created + // metadata, or it was passed an owned version from SerializedFileWriter which then + // takes it back via the returned metadata. + let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(page_index); + // If row group metadata was encrypted, we replace the encrypted row groups with // unencrypted metadata before it is returned to users. This allows the metadata // to be usable for retrieving the row group statistics for example, without users // needing to decrypt the metadata. - let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(Some(Arc::new( - PageIndex::new(column_indexes, offset_indexes), - ))); - Ok(match unencrypted_row_groups { Some(rg) => builder.set_row_groups(rg).build(), None => builder.set_row_groups(row_groups).build(), @@ -350,6 +302,10 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// +/// The writer serializes the page index of any [`PageIndexProvider`]. If the +/// provider has no index for a column chunk, the writer clears the offset and +/// length of that index in the [`ColumnChunkMetaData`]. +/// /// # Output Format /// /// The format of the metadata is as follows: @@ -470,10 +426,8 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) - { - encoder = encoder.with_page_index(page_index_arc.clone()); + if let Some(page_index_arc) = self.metadata.page_index.clone() { + encoder = encoder.with_page_index(page_index_arc); } if let Some(key_value_metadata) = key_value_metadata { From 997f184ac3a6269a02513dad3aba713cecad4de1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 08:27:40 -0700 Subject: [PATCH 16/62] remove assert --- parquet/src/file/metadata/writer.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 596907d726fe..cbba43261ec3 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -70,9 +70,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// `ColumnChunk::offset_index_length` to reflect the position and length /// of the serialized offset indexes. fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { - // we've already checked before calling that the offset indexes are populated - assert!(page_index.has_offset_indexes()); - // iter row group // iter each column // write offset index to the file From 0969c2329209a7bab57a51cabb93415d7968ff92 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 09:00:52 -0700 Subject: [PATCH 17/62] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 09b41df1155f..f5b5bd32cd2a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -67,7 +67,7 @@ fn test_read_with_custom_page_index_provider() { // Simulate a scenario where: // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only + // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap let mut provider = SelectivePageIndexProvider::new(file_bytes); // Populate indexes for row group 0, columns 0 and 1 @@ -76,9 +76,11 @@ fn test_read_with_custom_page_index_provider() { provider.fetch_column_index(0, 1, metadata).unwrap(); provider.fetch_offset_index(0, 1, metadata).unwrap(); - // Populate indexes for row group 2, column 0 only + // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) provider.fetch_column_index(2, 0, metadata).unwrap(); provider.fetch_offset_index(2, 0, metadata).unwrap(); + provider.fetch_column_index(2, 2, metadata).unwrap(); + provider.fetch_offset_index(2, 2, metadata).unwrap(); // Verify the provider has the expected indexes assert!(provider.column_index(0, 0).is_some()); From d93796e9b51004d54055af58717f7415ab717989 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 09:06:13 -0700 Subject: [PATCH 18/62] add another assert --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index f5b5bd32cd2a..ca5359f08e5b 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -90,6 +90,7 @@ fn test_read_with_custom_page_index_provider() { assert!(provider.column_index(0, 2).is_none()); // Not populated assert!(provider.column_index(2, 0).is_some()); assert!(provider.column_index(2, 1).is_none()); // Not populated + assert!(provider.column_index(2, 2).is_some()); // Reset statistics so validation checks don't affect the final counts provider.reset_stats(); From 2a2fd09e34ae4b1521a87baa268830649408a070 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 09:35:38 -0700 Subject: [PATCH 19/62] clippy --- parquet/src/file/metadata/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index cbba43261ec3..766c3f1e787c 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -134,8 +134,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } fn clear_page_indexes(&mut self) { - for row_group in self.row_groups.iter_mut() { - for column_metadata in row_group.columns.iter_mut() { + for row_group in &mut self.row_groups { + for column_metadata in &mut row_group.columns { column_metadata.column_index_offset = None; column_metadata.column_index_length = None; column_metadata.offset_index_offset = None; From c687e6ed9e1669c63fc884f7e02ed06a58017f92 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 11:24:27 -0700 Subject: [PATCH 20/62] restore old page index location behavior add flag to zero out page locations if desired --- parquet/src/arrow/mod.rs | 15 ++++++++++++--- parquet/src/file/metadata/writer.rs | 25 ++++++++++++++++++++++++- parquet/src/file/writer.rs | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs index 4b9a694e7304..e5cbafea5e64 100644 --- a/parquet/src/arrow/mod.rs +++ b/parquet/src/arrow/mod.rs @@ -596,7 +596,6 @@ mod test { } #[test] - #[cfg_attr(miri, ignore)] // Takes too long fn test_metadata_read_write_roundtrip_missing_page_index() { let parquet_bytes = create_parquet_file(); @@ -608,9 +607,9 @@ mod test { .parse_and_finish(&parquet_bytes) .unwrap(); - // metadata_to_bytes should zero out the page index locations. if they aren't + // metadata_to_bytes_no_page_idx should zero out the page index locations. if they aren't // then reading metadata_bytes will fail with EOF - let metadata_bytes = metadata_to_bytes(&original_metadata); + let metadata_bytes = metadata_to_bytes_no_page_idx(&original_metadata); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); let roundtrip_metadata = ParquetMetaDataReader::new() .with_metadata_options(Some(options)) @@ -754,6 +753,16 @@ mod test { Bytes::from(buf) } + // like metadata_to_bytes, but do not preserve page index location info + fn metadata_to_bytes_no_page_idx(metadata: &ParquetMetaData) -> Bytes { + let mut buf = vec![]; + ParquetMetaDataWriter::new(&mut buf, metadata) + .with_preserve_page_index_locations(false) + .finish() + .unwrap(); + Bytes::from(buf) + } + #[test] fn test_mask_from_column_names() { let schema = parse_schema( diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 766c3f1e787c..a439efc5e6da 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -61,6 +61,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { object_writer: MetadataObjectWriter, writer_version: i32, write_path_in_schema: bool, + preserve_page_index_locations: bool, } impl<'a, W: Write> ThriftMetadataWriter<'a, W> { @@ -153,7 +154,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { if let Some(pi) = page_index.as_deref() { self.write_column_indexes(pi)?; self.write_offset_indexes(pi)?; - } else { + } else if !self.preserve_page_index_locations { // no page indexes at all, then clear index locations self.clear_page_indexes(); } @@ -248,6 +249,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { created_by: Option, writer_version: i32, write_path_in_schema: bool, + preserve_page_index_locations: bool, ) -> Self { Self { buf, @@ -259,6 +261,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { object_writer: Default::default(), writer_version, write_path_in_schema, + preserve_page_index_locations, } } @@ -365,6 +368,7 @@ pub struct ParquetMetaDataWriter<'a, W: Write> { buf: TrackedWrite, metadata: &'a ParquetMetaData, write_path_in_schema: bool, + preserve_page_index_locations: bool, } impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { @@ -390,6 +394,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { buf, metadata, write_path_in_schema: true, + preserve_page_index_locations: true, } } @@ -402,6 +407,23 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { } } + /// Set whether or not to preserve the page index location metadata in the Thrift + /// `ColumnMetaData`. + /// + /// Because this struct is often used to externalize the footer metadata, it is + /// usually desirable to preserve this location information, even when the + /// page indexes are not duplicated (for instance if the provided `ParquetMetaData` + /// returns no `PageIndexProvider`). As such, this defaults to `true`. + /// + /// Set this to `false` to reset the location metadata if no page indexes are + /// present. + pub fn with_preserve_page_index_locations(self, val: bool) -> Self { + Self { + preserve_page_index_locations: val, + ..self + } + } + /// Write the metadata to the buffer pub fn finish(mut self) -> Result<()> { let file_metadata = self.metadata.file_metadata(); @@ -421,6 +443,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { created_by, file_metadata.version(), self.write_path_in_schema, + self.preserve_page_index_locations, ); if let Some(page_index_arc) = self.metadata.page_index.clone() { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 18b7e6e9999e..af60326ca387 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -367,6 +367,7 @@ impl SerializedFileWriter { Some(self.props.created_by().to_string()), self.props.writer_version().as_num(), write_path_in_schema, + false, ); #[cfg(feature = "encryption")] From 55eb9675b2b89d682023daa328a0f46091ac20c7 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 11:37:35 -0700 Subject: [PATCH 21/62] fix docs --- parquet/src/file/metadata/writer.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a439efc5e6da..2ce8396f8c09 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -302,9 +302,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -/// The writer serializes the page index of any [`PageIndexProvider`]. If the -/// provider has no index for a column chunk, the writer clears the offset and -/// length of that index in the [`ColumnChunkMetaData`]. +/// The writer serializes the page index of any [`PageIndexProvider`] attached +/// to the input [`ParquetMetaData`]. By default, if the provider has no index +/// for a column chunk, the offset and length of that index in the [`ColumnChunkMetaData`] +/// are preserved in the output metadata. This is done so one can store the footer +/// metadata externally but keep the page indexes in the original file. If this +/// behavior is not desired, then [`Self::with_preserve_page_index_locations`] should +/// be set to `false`. /// /// # Output Format /// From 92fb20c5bf9f4238c15359f9de9763d7a5fe69ef Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 12:01:15 -0700 Subject: [PATCH 22/62] honor preserve_page_index_locations in write_x_indexes --- parquet/src/file/metadata/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 2ce8396f8c09..65f8e50a7270 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -89,7 +89,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); - } else { + } else if !self.preserve_page_index_locations { column_metadata.offset_index_offset = None; column_metadata.offset_index_length = None; } @@ -125,7 +125,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } - } else { + } else if !self.preserve_page_index_locations { column_metadata.column_index_offset = None; column_metadata.column_index_length = None; } From 2489741dacebede25e2344b60b27a18b48820b67 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:45:29 -0700 Subject: [PATCH 23/62] implement review suggestions --- .../custom_page_index_provider.rs | 414 +++++++++--------- 1 file changed, 214 insertions(+), 200 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index ca5359f08e5b..6c93193f91c9 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -23,14 +23,17 @@ //! - Columns without page indexes fall back to whole-column-chunk fetching //! - Correct results are returned in both cases -use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow::compute::concat_batches; +use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use bytes::Bytes; +use parquet::DecodeResult; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelectionPolicy, }; +use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::file::metadata::page_index::PageIndexProvider; use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use parquet::file::page_index::column_index::ColumnIndexMetaData; @@ -38,22 +41,104 @@ use parquet::file::page_index::index_reader::{decode_column_index, decode_offset use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::fs::File; +use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::NamedTempFile; #[test] -fn test_read_with_custom_page_index_provider() { +fn test_read_with_custom_page_index_provider_sync() { + run_test(Reader::Sync); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_push() { + run_test(Reader::Push); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[cfg(feature = "async")] +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_async() { + run_test(Reader::Async); +} + +#[derive(Debug, Clone, Copy)] +enum Reader { + Sync, + Push, + #[cfg(feature = "async")] + Async, +} + +/// Read `file` with `reader`, using `metadata` and `selection` +fn read( + reader: Reader, + file: Bytes, + metadata: ArrowReaderMetadata, + selection: RowSelection, + requested: &mut Vec>, +) -> Vec { + match reader { + Reader::Sync => ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .collect::>() + .unwrap(), + Reader::Push => { + let mut decoder = ParquetPushDecoderBuilder::new_with_metadata(metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let mut batches = vec![]; + loop { + match decoder.try_decode().unwrap() { + DecodeResult::NeedsData(ranges) => { + requested.extend(ranges.iter().cloned()); + let data = ranges + .iter() + .map(|r| file.slice(r.start as usize..r.end as usize)) + .collect(); + decoder.push_ranges(ranges, data).unwrap(); + } + DecodeResult::Data(batch) => batches.push(batch), + DecodeResult::Finished => return batches, + } + } + } + #[cfg(feature = "async")] + Reader::Async => { + use futures::TryStreamExt; + let input = std::io::Cursor::new(file.to_vec()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + parquet::arrow::ParquetRecordBatchStreamBuilder::new_with_metadata(input, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .try_collect() + .await + .unwrap() + }) + } + } +} + +fn run_test(reader: Reader) { // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + let file_bytes = create_test_file(); // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, + file_bytes.clone(), ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), ) .unwrap(); @@ -68,32 +153,11 @@ fn test_read_with_custom_page_index_provider() { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap - let mut provider = SelectivePageIndexProvider::new(file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0, metadata).unwrap(); - provider.fetch_offset_index(0, 0, metadata).unwrap(); - provider.fetch_column_index(0, 1, metadata).unwrap(); - provider.fetch_offset_index(0, 1, metadata).unwrap(); - - // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) - provider.fetch_column_index(2, 0, metadata).unwrap(); - provider.fetch_offset_index(2, 0, metadata).unwrap(); - provider.fetch_column_index(2, 2, metadata).unwrap(); - provider.fetch_offset_index(2, 2, metadata).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - assert!(provider.column_index(2, 2).is_some()); - - // Reset statistics so validation checks don't affect the final counts - provider.reset_stats(); + let provider = SelectivePageIndexProvider::new( + file_bytes.clone(), + metadata, + &[(0, 0), (0, 1), (2, 0), (2, 2)], + ); // Step 4: Install the custom provider into metadata let mut metadata_builder = metadata.clone().into_builder(); @@ -108,8 +172,6 @@ fn test_read_with_custom_page_index_provider() { .unwrap(); // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); // Create a RowSelection that: // - Selects rows 20-30 (in row group 0) @@ -130,17 +192,33 @@ fn test_read_with_custom_page_index_provider() { parquet::arrow::arrow_reader::RowSelector::select(10), ]); - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); + let mut requested = Vec::>::new(); + let batches = read( + reader, + file_bytes.clone(), + arrow_metadata, + selection, + &mut requested, + ); // Step 7: Check statistics + + // only the push test will populate requested + if matches!(reader, Reader::Push) { + let fetched = |rg: usize, col: usize| -> u64 { + let (start, len) = metadata.row_group(rg).column(col).byte_range(); + requested + .iter() + .filter(|r| r.start >= start && r.end <= start + len) + .map(|r| r.end - r.start) + .sum() + }; + let chunk_len = |rg: usize, col: usize| metadata.row_group(rg).column(col).byte_range().1; + assert!(fetched(0, 0) < chunk_len(0, 0)); // offset index: selected pages only + assert_eq!(fetched(0, 2), chunk_len(0, 2)); // no offset index: whole chunk + } + // Note: We need to get the provider reference from the metadata to check stats let provider_ref = metadata_with_custom_index .page_index() @@ -168,61 +246,15 @@ fn test_read_with_custom_page_index_provider() { let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 30); - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - // Expected values for the selected rows: // rows 20-30: id=20..30, value=40..60, score=60..90 // rows 60-70: id=60..70, value=120..140, score=180..210 // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); + let ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected = make_batch(&ids); assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" + concat_batches(&expected.schema(), &batches).unwrap(), + expected ); } @@ -232,32 +264,46 @@ fn test_read_with_custom_page_index_provider() { /// page indexes selectively based on query predicates and projections. #[derive(Debug)] struct SelectivePageIndexProvider { - file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index - column_indexes: Option>>, - offset_indexes: Option>>, + column_indexes: Option>, + offset_indexes: Option>, // Usage statistics - column_index_hits: Arc, - column_index_misses: Arc, - offset_index_hits: Arc, - offset_index_misses: Arc, + column_index_hits: AtomicUsize, + column_index_misses: AtomicUsize, + offset_index_hits: AtomicUsize, + offset_index_misses: AtomicUsize, } impl SelectivePageIndexProvider { - fn new(file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes, metadata: &ParquetMetaData, chunks: &[(usize, usize)]) -> Self { + let mut column_indexes = HashMap::new(); + let mut offset_indexes = HashMap::new(); + for &(rg, col) in chunks { + column_indexes.insert( + (rg, col), + Self::fetch_column_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + offset_indexes.insert( + (rg, col), + Self::fetch_offset_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + } Self { - file_bytes, - column_indexes: None, - offset_indexes: None, - column_index_hits: Arc::new(AtomicUsize::new(0)), - column_index_misses: Arc::new(AtomicUsize::new(0)), - offset_index_hits: Arc::new(AtomicUsize::new(0)), - offset_index_misses: Arc::new(AtomicUsize::new(0)), + column_indexes: Some(column_indexes), + offset_indexes: Some(offset_indexes), + column_index_hits: AtomicUsize::new(0), + column_index_misses: AtomicUsize::new(0), + offset_index_hits: AtomicUsize::new(0), + offset_index_misses: AtomicUsize::new(0), } } /// Get statistics about column index usage - pub fn column_index_stats(&self) -> (usize, usize) { + fn column_index_stats(&self) -> (usize, usize) { ( self.column_index_hits.load(Ordering::Relaxed), self.column_index_misses.load(Ordering::Relaxed), @@ -265,7 +311,7 @@ impl SelectivePageIndexProvider { } /// Get statistics about offset index usage - pub fn offset_index_stats(&self) -> (usize, usize) { + fn offset_index_stats(&self) -> (usize, usize) { ( self.offset_index_hits.load(Ordering::Relaxed), self.offset_index_misses.load(Ordering::Relaxed), @@ -273,64 +319,44 @@ impl SelectivePageIndexProvider { } /// Get combined statistics as (hits, misses) for both index types - pub fn stats(&self) -> (usize, usize) { + fn stats(&self) -> (usize, usize) { let (col_hits, col_misses) = self.column_index_stats(); let (off_hits, off_misses) = self.offset_index_stats(); (col_hits + off_hits, col_misses + off_misses) } - /// Reset all statistics counters to zero - pub fn reset_stats(&self) { - self.column_index_hits.store(0, Ordering::Relaxed); - self.column_index_misses.store(0, Ordering::Relaxed); - self.offset_index_hits.store(0, Ordering::Relaxed); - self.offset_index_misses.store(0, Ordering::Relaxed); - } - /// Fetch and parse the column index for the given row group and column fn fetch_column_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.column_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.column_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_column_index(&idx_bytes, column.column_type())?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_column_index(&idx_bytes, column.column_type())?)) + } else { + Ok(None) } - Ok(()) } /// Fetch and parse the offset index for the given row group and column fn fetch_offset_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.offset_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.offset_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_offset_index(&idx_bytes)?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_offset_index(&idx_bytes)?)) + } else { + Ok(None) } - Ok(()) } } @@ -351,8 +377,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .column_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.column_index_hits.fetch_add(1, Ordering::Relaxed); @@ -371,8 +396,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .offset_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.offset_index_hits.fetch_add(1, Ordering::Relaxed); @@ -389,20 +413,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> NamedTempFile { - let temp_file = tempfile::Builder::new() - .prefix("custom_page_index_test") - .suffix(".parquet") - .tempfile() - .expect("tempfile creation"); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("value", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - Field::new("score", DataType::Int32, false), - ])); - +fn create_test_file() -> Bytes { // Create small pages and small row groups for testing let props = WriterProperties::builder() .set_statistics_enabled(EnabledStatistics::Page) @@ -411,37 +422,12 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = File::create(temp_file.path()).unwrap(); - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); - // Write 3 row groups with 50 rows each (150 rows total) - for row_group in 0..3 { - for batch_num in 0..5 { - let offset = (row_group * 50) + (batch_num * 10); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from( - (offset..offset + 10).collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 2).collect::>(), - )), - Arc::new(StringArray::from( - (offset..offset + 10) - .map(|x| format!("name{x}")) - .collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 3).collect::>(), - )), - ], - ) - .unwrap(); - writer.write(&batch).unwrap(); - } - writer.flush().unwrap(); - } + let batch = make_batch(&(0..150).collect::>().as_slice()); + let mut buffer = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); // check that file has properties we wanted (3 row groups, 5 pages per chunk) let metadata = writer.close().unwrap(); @@ -459,5 +445,33 @@ fn create_test_file() -> NamedTempFile { ); } } - temp_file + Bytes::from(buffer) +} + +fn make_batch(ids: &[i32]) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + // Write 3 row groups with 50 rows each (150 rows total) + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + ids.iter() + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap() } From 835170329bae2e3948ac900d676888e8484752af Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:56:14 -0700 Subject: [PATCH 24/62] clippy --- .../tests/arrow_reader/custom_page_index_provider.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 6c93193f91c9..8447146418f1 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -52,7 +52,9 @@ fn test_read_with_custom_page_index_provider_sync() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } @@ -60,7 +62,9 @@ fn test_read_with_custom_page_index_provider_push() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } @@ -423,7 +427,7 @@ fn create_test_file() -> Bytes { .build(); // Write 3 row groups with 50 rows each (150 rows total) - let batch = make_batch(&(0..150).collect::>().as_slice()); + let batch = make_batch((0..150).collect::>().as_slice()); let mut buffer = Vec::with_capacity(1024); let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); From 57d70307f880f932cdae1f8f6ddafd43e09a2fa8 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 15:23:34 -0700 Subject: [PATCH 25/62] tests now pass --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 8447146418f1..b6348ecd221a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -50,21 +50,13 @@ fn test_read_with_custom_page_index_provider_sync() { run_test(Reader::Sync); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } From 82087c5e8da815ded0e22aeb5f87962428d7d2b2 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 16:11:00 -0700 Subject: [PATCH 26/62] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/src/file/metadata/reader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 186ccfcce330..d8e3decef59e 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -109,7 +109,8 @@ impl From for PageIndexPolicy { /// Struct to specify column chunks for which metadata is required. /// -/// Column chunks are identified by row group index and column index. This struct +/// Column chunks are identified by row group index and leaf column index (the index of the +/// column in [`SchemaDescriptor::columns`], not the index of a root or Arrow field). This struct /// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). From 7b30fa0cf545e0c562b88c025bcb8e37dbefed28 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 07:53:10 -0700 Subject: [PATCH 27/62] revert most changes to push decoder --- parquet/src/file/metadata/push_decoder.rs | 189 +++------------------- 1 file changed, 24 insertions(+), 165 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 6b747b34aabf..4c8b514251ca 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,8 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; use bytes::Bytes; @@ -435,27 +434,20 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let ranges = ranges_for_page_index( + let range = range_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, ); - if ranges.is_empty() { + let Some(page_index_range) = range else { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - } - - let needed_ranges = ranges - .into_iter() - .filter(|r| !self.buffers.has_range(r)) - .collect::>(); + }; - if !needed_ranges.is_empty() { + if !self.buffers.has_range(&page_index_range) { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(DecodeResult::NeedsData(needed_ranges)); + return Ok(needs_range(page_index_range)); } parse_page_index( @@ -514,81 +506,32 @@ enum DecodeState { Intermediate, } -/// Returns the minimum set of non-overlapping ranges needed to cover the requested -/// offset and column indexes. +/// Returns the byte range needed to read the offset/page indexes, based on the +/// specified policies /// -/// If no page indexes are present in the file, or none are actually -/// requested by the policies passed in, this will return an empty -/// vector. -/// -/// This may result in more ranges to fetch but may also result in fewer bytes -/// being read (and stored). To get a single range for all indexes, sort -/// the resultant vector by the range starts, and then create a single range -/// using `start` from the head and `end` from the tail. -/// -/// ```ignore -/// # use core::ops::Range; -/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by_key(|r| r.start); -/// let range = (ranges.first()?.start..ranges.last()?.end); -/// Some(range) -/// # } -/// ``` -pub fn ranges_for_page_index( +/// Returns None if no page indexes are needed +pub fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_mask: &ColumnChunkMask, - offset_index_mask: &ColumnChunkMask, -) -> Vec> { - let mut result = Vec::new(); - - fn add_ranges( - metadata: &ParquetMetaData, - mask: &ColumnChunkMask, - ranges: &mut Vec>, - f: T, - ) where - T: Fn(&ColumnChunkMetaData) -> Option>, - { - for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if mask.includes_row_group(rg_idx) { - for (col_idx, col) in rg.columns().iter().enumerate() { - if mask.includes_column(col_idx) - && let Some(range) = f(col) - { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } - } - } - } +) -> Option> { + fn acc_range(a: Option>, b: Option>) -> Option> { + match (a, b) { + (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), + (None, x) | (x, None) => x, } } - if column_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - column_index_mask, - &mut result, - ColumnChunkMetaData::column_index_range, - ); - } - if offset_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - offset_index_mask, - &mut result, - ColumnChunkMetaData::offset_index_range, - ); + let mut range = None; + for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { + if column_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.column_index_range()); + } + if offset_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.offset_index_range()); + } } - result + range } // These tests use the arrow writer to create a parquet file in memory @@ -742,90 +685,6 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } - /// Decode the metadata incrementally, with a partial page index - #[test] - fn test_metadata_decoder_incremental_partial_page_index() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - - /// Decode the metadata incrementally, with a partial page index, but coalesced - /// range request. - #[test] - fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let mut ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - // collapse ranges into a single range - ranges.sort_by_key(|r| r.start); - let range = ranges.first().unwrap().start..ranges.last().unwrap().end; - push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) From 3cefc59282e1fdd31eeeb0082f9db9732ab715b4 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 09:33:16 -0700 Subject: [PATCH 28/62] address a number of issues found in review, primarily documentation changes, and changes to chunk mask semantics and storage. Also fixes the preloading reader issue. --- parquet/src/arrow/arrow_reader/mod.rs | 28 ++- parquet/src/arrow/async_reader/store.rs | 12 +- parquet/src/file/metadata/reader.rs | 183 ++++++++++++------ parquet/tests/arrow_reader/page_index.rs | 123 ++++++++++++ .../arrow_reader/partial_offset_index.rs | 67 ++++++- 5 files changed, 333 insertions(+), 80 deletions(-) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 492ea6063462..9788f597d95b 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -776,6 +776,12 @@ impl ArrowReaderOptions { self } + /// Sets the same [`ColumnChunkMask`] for both page-index structures. + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) + } + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed @@ -783,6 +789,11 @@ impl ArrowReaderOptions { /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// The mask applies only if the column-index policy is not [`PageIndexPolicy::Skip`] + /// (the default), or an underlying reader is configured to preload the index. It is + /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; + /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. + /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -796,6 +807,10 @@ impl ArrowReaderOptions { /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// Page pruning also needs the offset index for predicate columns, so callers should + /// include those columns in addition to projected columns. The same loading and policy + /// qualifications as [`Self::with_column_index_mask`] apply. + /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; @@ -1012,9 +1027,16 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + + // Preload settings on the underlying reader may enable an index even when the + // corresponding options policy is `Skip`, so apply non-default masks independently. + if !options.column_index_mask().is_all() { + self = self.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + self = self.with_offset_index_mask(options.offset_index_mask().clone()); } self diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index c7db5d0abbc1..5528cc05b2bd 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,9 +242,15 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + if let Some(options) = options { + if !options.column_index_mask().is_all() { + metadata = metadata.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + metadata = metadata.with_offset_index_mask(options.offset_index_mask().clone()); + } } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index d8e3decef59e..8987912499fb 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -91,10 +91,12 @@ pub enum PageIndexPolicy { Skip, /// Read the page index if it exists, otherwise do not error. Optional, - /// Require the page index to exist, and error if it does not. + /// Require offset indexes for every selected column chunk when any selected + /// page index ranges exist. /// - /// Only enforced for the offset index; this is the same as [`Self::Optional`] - /// for the column index. + /// For compatibility, a file with no page index ranges is accepted. This is only + /// enforced for the offset index; this is the same as [`Self::Optional`] for the + /// column index. Required, } @@ -138,11 +140,12 @@ impl From for PageIndexPolicy { /// ``` /// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct ColumnChunkMask { - // using i32 because that's how thrift vectors are sized - row_groups: Option>>, - columns: Option>>, + // `None` means all, while `Some(empty)` means none. Store u32 because + // Parquet/Thrift collections cannot contain more than i32::MAX entries. + row_groups: Option>, + columns: Option>, } impl ColumnChunkMask { @@ -151,11 +154,18 @@ impl ColumnChunkMask { Self::default() } + /// Select no row groups or columns. + pub fn none() -> Self { + Self { + row_groups: Some(Arc::from([])), + columns: Some(Arc::from([])), + } + } + /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all columns. - pub fn columns(columns: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no columns. + pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, columns: Self::iter_to_set(columns), @@ -164,9 +174,8 @@ impl ColumnChunkMask { /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all row groups. - pub fn row_groups(row_groups: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no row groups. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { row_groups: Self::iter_to_set(row_groups), columns: None, @@ -175,12 +184,10 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. - /// Passing an empty set for `row_groups` is treated as selecting all row groups, and - /// an empty set for `columns` as selectiong all columns. + /// An empty iterator for either dimension selects no column chunks. pub fn row_groups_and_columns( - row_groups: impl IntoIterator, - columns: impl IntoIterator, + row_groups: impl IntoIterator, + columns: impl IntoIterator, ) -> Self { Self { row_groups: Self::iter_to_set(row_groups), @@ -189,30 +196,79 @@ impl ColumnChunkMask { } /// Test if `idx` is in the row group set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_row_group(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { - return false; - }; - self.row_groups - .as_ref() - .is_none_or(|keep| keep.contains(&idx)) + Self::includes_index(self.row_groups.as_ref(), idx) } /// Test if `idx` is in the column set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_column(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { + Self::includes_index(self.columns.as_ref(), idx) + } + + fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { + // return false for out-of-bounds index + let Ok(idx) = u32::try_from(idx) else { return false; }; - self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + keep.is_none_or(|keep| keep.binary_search(&idx).is_ok()) + } + + /// Returns `true` when this mask selects every column chunk. + pub fn is_all(&self) -> bool { + self.row_groups.is_none() && self.columns.is_none() + } + + /// Returns selected row groups, or `None` when all row groups are selected. + pub fn selected_row_groups(&self) -> Option<&[u32]> { + self.row_groups.as_deref() + } + + /// Returns selected leaf columns, or `None` when all columns are selected. + pub fn selected_columns(&self) -> Option<&[u32]> { + self.columns.as_deref() + } + + /// Creates a mask selecting the leaf columns in an Arrow projection. + #[cfg(feature = "arrow")] + pub fn from_projection( + projection: &crate::arrow::ProjectionMask, + schema: &SchemaDescriptor, + ) -> Self { + Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } - fn iter_to_set(indices: impl IntoIterator) -> Option>> { - let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); - (!set.is_empty()).then_some(Arc::new(set)) + /// Returns an iterator over the row group indices selected by this mask + pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + Self::axis_indices(self.row_groups.as_deref(), num_row_groups) + } + + /// Returns an iterator over the column indices selected by this mask + pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + Self::axis_indices(self.columns.as_deref(), num_columns) + } + + fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + match axis { + None => Box::new(0..len), + Some(indices) => Box::new( + indices + .iter() + .map(|&i| i as usize) + .take_while(move |&i| i < len), + ), + } + } + + fn iter_to_set(indices: impl IntoIterator) -> Option> { + Some( + indices + .into_iter() + .filter_map(|i| u32::try_from(i).ok()) + .collect::>() + .into_iter() + .collect::>() + .into(), + ) } } @@ -249,19 +305,29 @@ impl ParquetMetaDataReader { self } - /// Selects the row groups and columns for which both page index structures are read. + /// Selects the row groups and leaf columns for which both page index structures are read. + /// + /// The mask applies only to index structures whose policy is not [`PageIndexPolicy::Skip`]. + /// This reader fetches one byte range covering the selected indexes, so a sparse mask + /// primarily reduces decoding and memory rather than necessarily reducing I/O. pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { self.with_column_index_mask(mask.clone()) .with_offset_index_mask(mask) } - /// Selects the row groups and columns for which column indexes are read. + /// Selects the row groups and leaf columns for which column indexes are read. + /// + /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self } - /// Selects the row groups and columns for which offset indexes are read. + /// Selects the row groups and leaf columns for which offset indexes are read. + /// + /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self @@ -1121,6 +1187,23 @@ mod tests { "EOF: Parquet file too small. Size is 1728 but need 1729" ); } + + #[test] + fn test_chunk_mask() { + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + let mask = ColumnChunkMask::columns([]); + assert!(!mask.includes_column(0)); + assert_eq!(mask.selected_columns(), Some([].as_slice())); + + let all = ColumnChunkMask::all(); + assert!(all.is_all()); + assert!(all.includes_column(u32::MAX as usize)); + } } #[cfg(all(feature = "async", feature = "arrow", test))] @@ -1526,32 +1609,4 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } - - #[test] - fn test_chunk_mask() { - // basic test - let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); - assert!(mask.includes_row_group(0)); - assert!(!mask.includes_row_group(1)); - assert!(!mask.includes_column(0)); - assert!(mask.includes_column(1)); - - // test that negative numbers are ignored - let mask = ColumnChunkMask::columns([-1, 0, 1]); - assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(!mask.includes_column(2)); - - // empty set should behave as "all" - let mask = ColumnChunkMask::columns([]); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(mask.includes_column(2)); - - // indexing beyond i32::MAX should return false - let mask = ColumnChunkMask::all(); - assert!(mask.includes_column(i32::MAX as usize)); - assert!(!mask.includes_column(u32::MAX as usize)); - } } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c605eac59834..c84e1c7b09c9 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -21,6 +21,129 @@ use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataR use crate::custom_page_index_provider::create_test_file; +fn assert_page_index_cells( + metadata: &parquet::file::metadata::ParquetMetaData, + expect_ci: impl Fn(usize, usize) -> bool, + expect_oi: impl Fn(usize, usize) -> bool, +) { + let page_index = metadata.page_index().expect("page index should be loaded"); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + for rg in 0..metadata.num_row_groups() { + for col in 0..num_cols { + assert_eq!( + page_index.column_index(rg, col).is_some(), + expect_ci(rg, col), + "column index rg={rg} col={col}" + ); + assert_eq!( + page_index.offset_index(rg, col).is_some(), + expect_oi(rg, col), + "offset index rg={rg} col={col}" + ); + } + } +} + +#[test] +fn test_arrow_reader_options_page_index_masks() { + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let column_mask = ColumnChunkMask::row_groups_and_columns([1], [0]); + let offset_mask = ColumnChunkMask::columns([2]); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(column_mask.clone()) + .with_offset_index_mask(offset_mask.clone()); + assert_eq!(options.column_index_mask(), &column_mask); + assert_eq!(options.offset_index_mask(), &offset_mask); + + let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap(); + assert_page_index_cells( + metadata.metadata(), + |rg, col| rg == 1 && col == 0, + |_, col| col == 2, + ); + + let metadata = ParquetMetaDataReader::new() + .with_arrow_reader_options(Some(&options)) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells(&metadata, |rg, col| rg == 1 && col == 0, |_, col| col == 2); +} + +#[test] +fn test_parse_with_page_index_mask() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([2], [1, 3])) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| rg == 2 && (col == 1 || col == 3), + |rg, col| rg == 2 && (col == 1 || col == 3), + ); +} + +/*#[test] +// C11 coverage +fn test_repeated_page_index_reads_merge_cells() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + ); +} + +#[test] +// C3 coverage +fn test_partial_page_statistics_remain_aligned() { + use arrow_array::Array; + use parquet::arrow::arrow_reader::statistics::StatisticsConverter; + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([1])); + let metadata = ArrowReaderMetadata::load(&file, options).unwrap(); + let parquet_metadata = metadata.metadata(); + let page_index = parquet_metadata.page_index().unwrap().as_ref(); + let row_groups = parquet_metadata.row_groups(); + let row_group_indices = [0, 1, 2]; + let converter = StatisticsConverter::try_new( + "id", + metadata.schema(), + parquet_metadata.file_metadata().schema_descr(), + ) + .unwrap(); + + let mins = converter + .data_page_mins(page_index, row_group_indices.iter()) + .unwrap(); + let row_counts = converter + .data_page_row_counts(page_index, row_groups, row_group_indices.iter()) + .unwrap() + .unwrap(); + assert_eq!(mins.len(), row_counts.len()); + assert_eq!(row_counts.null_count(), row_counts.len()); +}*/ + #[test] fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns diff --git a/parquet/tests/arrow_reader/partial_offset_index.rs b/parquet/tests/arrow_reader/partial_offset_index.rs index 72489da5c64a..28aa4bb18568 100644 --- a/parquet/tests/arrow_reader/partial_offset_index.rs +++ b/parquet/tests/arrow_reader/partial_offset_index.rs @@ -30,7 +30,7 @@ use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::arrow::{ArrowWriter, ProjectionMask}; use parquet::column::writer::ColumnCloseResult; use parquet::errors::Result; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use parquet::file::properties::WriterProperties; use parquet::file::writer::SerializedFileWriter; use std::sync::Arc; @@ -144,15 +144,23 @@ fn configure(builder: ArrowReaderBuilder, trigger: Trigger) -> ArrowReader } } -fn read_sync(file: &Bytes, trigger: Trigger) -> Result> { - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options())?; +fn read_sync( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options)?; Ok(configure(builder, trigger) .build()? .collect::>()?) } #[cfg(feature = "async")] -fn read_async(file: &Bytes, trigger: Trigger) -> Result> { +fn read_async( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { use futures::TryStreamExt; use parquet::arrow::ParquetRecordBatchStreamBuilder; @@ -161,13 +169,17 @@ fn read_async(file: &Bytes, trigger: Trigger) -> Result> { .unwrap(); runtime.block_on(async { let input = std::io::Cursor::new(file.clone()); - let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options()).await?; + let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options).await?; configure(builder, trigger).build()?.try_collect().await }) } -fn read_push_decoder(file: &Bytes, trigger: Trigger) -> Result> { - let metadata = ArrowReaderMetadata::load(file, options())?; +fn read_push_decoder( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let metadata = ArrowReaderMetadata::load(file, options)?; let builder = ParquetPushDecoderBuilder::new_with_metadata(metadata); let mut decoder = configure(builder, trigger).build()?; let mut batches = vec![]; @@ -202,10 +214,45 @@ fn check(has_index: impl Fn(usize, usize) -> bool) { let expected = test_batch().slice(150, 100); for trigger in [Trigger::RowSelection, Trigger::RowFilter] { let results = [ - ("sync", read_sync(&file, trigger)), - ("push decoder", read_push_decoder(&file, trigger)), + ("sync", read_sync(&file, trigger, options())), + ("push decoder", read_push_decoder(&file, trigger, options())), #[cfg(feature = "async")] - ("async", read_async(&file, trigger)), + ("async", read_async(&file, trigger, options())), + ]; + for (reader, result) in results { + let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); + let actual = concat_batches(&expected.schema(), &batches).unwrap(); + assert_eq!(actual, expected, "{reader} reader, {trigger:?}"); + } + } +} + +#[test] +fn test_read_with_offset_index_mask() { + let file = test_file(|_, _| true); + let masked_options = + || options().with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])); + let metadata = ArrowReaderMetadata::load(&file, masked_options()).unwrap(); + let page_index = metadata.metadata().page_index().unwrap(); + for row_group in 0..2 { + for column in 0..3 { + assert_eq!( + page_index.offset_index(row_group, column).is_some(), + row_group == 1 && column == 0 + ); + } + } + + let expected = test_batch().slice(150, 100); + for trigger in [Trigger::RowSelection, Trigger::RowFilter] { + let results = [ + ("sync", read_sync(&file, trigger, masked_options())), + ( + "push decoder", + read_push_decoder(&file, trigger, masked_options()), + ), + #[cfg(feature = "async")] + ("async", read_async(&file, trigger, masked_options())), ]; for (reader, result) in results { let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); From 4ee4125c5da460172d858cdd34039044dd131484 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:06:06 -0700 Subject: [PATCH 29/62] modify range_for_page_index to take masks --- parquet/src/file/metadata/push_decoder.rs | 71 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 4c8b514251ca..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -438,6 +438,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, ); let Some(page_index_range) = range else { @@ -507,28 +509,38 @@ enum DecodeState { } /// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// specified policies and masks /// /// Returns None if no page indexes are needed -pub fn range_for_page_index( +pub(crate) fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Option> { - fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } - } - - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + let mut range: Option> = None; + for (policy, mask, column_index) in [ + (column_index_policy, column_index_mask, true), + (offset_index_policy, offset_index_mask, false), + ] { + if policy != PageIndexPolicy::Skip { + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); + } + } + } } } range @@ -564,6 +576,33 @@ mod tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[test] + fn test_range_for_page_index_respects_column_mask() { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); + push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]); + let metadata = expect_data(decoder.try_decode()); + + let all = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::all(), + &ColumnChunkMask::all(), + ) + .unwrap(); + let masked = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::columns([0]), + &ColumnChunkMask::all(), + ) + .unwrap(); + + assert!(all.start <= masked.start && masked.end <= all.end); + assert!(masked.end - masked.start < all.end - all.start); + } + /// It is possible to feed some, but not all, of the footer into the metadata decoder /// before asked. This avoids multiple IO requests #[test] From f6a8fde4e2ab2a2b8ff63a15b0b89b2486a04a06 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:30:47 -0700 Subject: [PATCH 30/62] more efficient page index parser traversal --- parquet/src/file/metadata/parser.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index a76eea9066c0..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,7 +247,6 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. -/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -303,15 +302,9 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -335,15 +328,9 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; From 682edbfc0a7ccf4321bf6bcfd8ad2384a822f181 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:36:06 -0700 Subject: [PATCH 31/62] address StatisticsConverter misalignment --- parquet/src/arrow/arrow_reader/statistics.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/statistics.rs b/parquet/src/arrow/arrow_reader/statistics.rs index 486e937a79de..32684b2b1210 100644 --- a/parquet/src/arrow/arrow_reader/statistics.rs +++ b/parquet/src/arrow/arrow_reader/statistics.rs @@ -2042,6 +2042,15 @@ impl<'a> StatisticsConverter<'a> { let mut nulls = NullBufferBuilder::new(0); for rg_idx in row_group_indices { let Some(offset_index) = page_index.offset_index(*rg_idx, parquet_index) else { + // A partial page index may contain the ColumnIndex but not the OffsetIndex. + // Preserve page-wise alignment with data_page_{mins,maxes,null_counts} by + // emitting one null row count for every page known from the ColumnIndex. + let num_pages = page_index + .column_index(*rg_idx, parquet_index) + .map(|index| index.num_pages() as usize) + .unwrap_or(0); + row_counts.resize(row_counts.len() + num_pages, 0); + nulls.append_n_nulls(num_pages); continue; }; let page_locations = offset_index.page_locations(); From 18f34a8d36b46ba0ae7c732bc50f5f22583cd227 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:41:44 -0700 Subject: [PATCH 32/62] enable C3 test --- parquet/tests/arrow_reader/page_index.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c84e1c7b09c9..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -107,10 +107,9 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} +}*/ #[test] -// C3 coverage fn test_partial_page_statistics_remain_aligned() { use arrow_array::Array; use parquet::arrow::arrow_reader::statistics::StatisticsConverter; @@ -142,7 +141,7 @@ fn test_partial_page_statistics_remain_aligned() { .unwrap(); assert_eq!(mins.len(), row_counts.len()); assert_eq!(row_counts.null_count(), row_counts.len()); -}*/ +} #[test] fn test_parse_selected_columns() { From 81693113b564e9997c4518e0098bec0e4f5c7757 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:17:05 -0700 Subject: [PATCH 33/62] fix C11: preserve existing indexes across multiple parse calls --- parquet/src/file/metadata/page_index.rs | 42 +++++++++++++++++++++++ parquet/src/file/metadata/parser.rs | 11 ++++-- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 ++++- parquet/tests/arrow_reader/page_index.rs | 22 ++++++++++-- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..18e3d4c85851 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,6 +513,36 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { + /// Creates a new [`PageIndexBuilder`] populated from an existing provider. + pub(crate) fn new_from_provider( + page_index: &dyn PageIndexProvider, + num_row_groups: usize, + num_columns: usize, + ) -> Self { + let mut builder = Self::default(); + if page_index.has_column_indexes() { + builder.allocate_column_indexes(num_row_groups, num_columns); + } + if page_index.has_offset_indexes() { + builder.allocate_offset_indexes(num_row_groups, num_columns); + } + for row_group in 0..num_row_groups { + for column in 0..num_columns { + if builder.column_indexes.is_some() + && let Some(index) = page_index.column_index(row_group, column) + { + builder.put_column_index(index.clone(), row_group, column); + } + if builder.offset_indexes.is_some() + && let Some(index) = page_index.offset_index(row_group, column) + { + builder.put_offset_index(index.clone(), row_group, column); + } + } + } + builder + } + /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -570,6 +600,12 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.column_indexes.is_none() { + self.allocate_column_indexes(num_row_groups, num_columns); + } + } + /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -582,6 +618,12 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.offset_indexes.is_none() { + self.allocate_offset_indexes(num_row_groups, num_columns); + } + } + /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..d146c89434a7 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,9 +261,14 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = metadata + .page_index() + .map(|index| { + PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) + }) + .unwrap_or_default(); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); + builder.ensure_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -273,7 +278,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); + builder.ensure_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..8e396fc8e441 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,7 +272,8 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. + /// after the metadata has already been decoded. Newly decoded page indexes + /// are merged with any page indexes already present in `metadata`. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 8987912499fb..4a4a3b9a894d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -514,6 +514,10 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// Newly read indexes are merged with any existing page index. Existing entries outside + /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are + /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -523,6 +527,7 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -664,7 +669,8 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. + /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as + /// described by [`Self::read_page_indexes()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..4c2353fb49c5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,8 +87,7 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage +#[test] fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -107,7 +106,24 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -}*/ +} + +#[test] +fn test_repeated_page_index_reads_preserve_skipped_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells(&metadata, |_, _| true, |_, _| true); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 09337c842f721327d973a5fc4255f0197021d1ab Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:28:41 -0700 Subject: [PATCH 34/62] Fix page index loading with suffix prefetch --- parquet/src/file/metadata/reader.rs | 37 +++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e4cec0fd0df5..088f3281db11 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -688,10 +688,9 @@ impl ParquetMetaDataReader { } else { let metadata_start = suffix_len - metadata_offset; let slice = suffix.slice(metadata_start..suffix_len - FOOTER_SIZE); - Ok(( - self.decode_footer_metadata(slice, file_size, footer)?, - Some((0, suffix.slice(..metadata_start))), - )) + // `MetadataSuffixFetch` does not expose the file size, so the file offset of + // the bytes preceding the metadata is unknown and they cannot be reused safely. + Ok((self.decode_footer_metadata(slice, file_size, footer)?, None)) } } @@ -1347,6 +1346,36 @@ mod async_tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[tokio::test] + async fn test_page_index_via_suffix_with_prefetch() { + let mut file = get_test_file("alltypes_tiny_pages.parquet"); + let mut suffix_file = file.try_clone().unwrap(); + let len = file.len(); + let fetch_count = AtomicUsize::new(0); + let suffix_fetch_count = AtomicUsize::new(0); + + let mut fetch = |range| { + fetch_count.fetch_add(1, Ordering::SeqCst); + futures::future::ready(read_range(&mut file, range)) + }; + let mut suffix_fetch = |suffix| { + suffix_fetch_count.fetch_add(1, Ordering::SeqCst); + futures::future::ready(read_suffix(&mut suffix_file, suffix)) + }; + + let input = MetadataSuffixFetchFn(&mut fetch, &mut suffix_fetch); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_prefetch_hint(Some((len - 1000) as usize)) + .load_via_suffix_and_finish(input) + .await + .unwrap(); + + assert_eq!(suffix_fetch_count.load(Ordering::SeqCst), 1); + assert_eq!(fetch_count.load(Ordering::SeqCst), 1); + assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); + } + fn write_parquet_file(offset_index_disabled: bool) -> Result { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); let batch = RecordBatch::try_new( From 5b9c07d9ccbb0ff3d6c69554a5c406945f4215ef Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 17:43:51 -0700 Subject: [PATCH 35/62] add plumbing for page index selective population --- parquet/benches/metadata.rs | 28 ++- parquet/src/arrow/arrow_reader/mod.rs | 56 ++++- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 16 ++ parquet/src/file/metadata/parser.rs | 68 +++--- parquet/src/file/metadata/push_decoder.rs | 221 +++++++++++++++++--- parquet/src/file/metadata/reader.rs | 108 ++++++++++ parquet/src/file/page_index/index_reader.rs | 11 - parquet/tests/arrow_reader/io/mod.rs | 4 +- 10 files changed, 441 insertions(+), 77 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 0ec1b3cd1733..d9fef3d1a5d0 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, ParquetMetaData, - ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, - RowGroupMetaData, + ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, + PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; use parquet::file::writer::TrackedWrite; @@ -191,6 +191,28 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&data) + .unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Required); + c.bench_function("read page index", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + c.bench_function("read page index reduced columns", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + let meta_data = get_footer_bytes(data.clone()); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata", |b| { diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 73bcb2048ad5..8f8953e69933 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,8 +45,8 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, - ParquetStatisticsPolicy, RowGroupMetaData, + PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; use crate::schema::types::SchemaDescriptor; @@ -593,8 +593,10 @@ pub struct ArrowReaderOptions { /// [ARROW_SCHEMA_META_KEY]: crate::arrow::ARROW_SCHEMA_META_KEY supplied_schema: Option, - pub(crate) column_index: PageIndexPolicy, - pub(crate) offset_index: PageIndexPolicy, + column_index: PageIndexPolicy, + offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -774,6 +776,32 @@ impl ArrowReaderOptions { self } + /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// + /// The column index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when filtering by a predicate + /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// + /// The offset index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when projecting a small subset + /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet /// footer will be skipped. /// @@ -924,6 +952,20 @@ impl ArrowReaderOptions { self.column_index } + /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// + /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. + pub fn offset_index_selection(&self) -> &PageIndexSelection { + &self.offset_index_selection + } + + /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// + /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. + pub fn column_index_selection(&self) -> &PageIndexSelection { + &self.column_index_selection + } + /// Retrieve the currently set metadata decoding options. pub fn metadata_options(&self) -> &ParquetMetaDataOptions { &self.metadata_options @@ -970,7 +1012,9 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self @@ -1019,6 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index 4bd65cfead59..a73d2c0d3fbe 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,7 +242,9 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 166a7eb7ab6a..65c084108c7f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, ParquetMetaDataReader}; +pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..46254777e668 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,6 +17,7 @@ //! Page Index structures for efficient page-level skipping +use crate::file::metadata::PageIndexSelection; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -555,6 +556,21 @@ impl PageIndexBuilder { } } + /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy + /// + /// For sparse indexes, this can save a great deal of memory + pub fn new_with_selection( + num_row_groups: usize, + num_columns: usize, + _column_index_selection: PageIndexSelection, + _offset_index_selection: PageIndexSelection, + ) -> Self { + Self { + column_indexes: Self::empty_index(num_row_groups, num_columns), + offset_indexes: Self::empty_index(num_row_groups, num_columns), + } + } + /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 861bb2e53b08..b2a35172e8af 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,13 +26,15 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use bytes::Bytes; +use crate::file::reader::ChunkReader; +use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -250,8 +252,9 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - bytes: &Bytes, - start_offset: u64, + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip { @@ -259,25 +262,28 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = PageIndexBuilder::new_with_selection( + num_row_groups, + num_columns, + column_index_selection.clone(), + offset_index_selection.clone(), + ); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, + column_index_selection, &mut builder, bytes, - start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, + offset_index_selection, &mut builder, bytes, - start_offset, )?; } @@ -293,27 +299,27 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_column_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -325,27 +331,27 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_offset_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index c5e77cc22468..b55468fd2980 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -21,8 +21,10 @@ use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; -use crate::file::metadata::{FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions}; -use crate::file::page_index::index_reader::acc_range; +use crate::file::metadata::{ + ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, +}; use crate::file::reader::ChunkReader; use bytes::Bytes; use std::ops::Range; @@ -226,6 +228,10 @@ pub struct ParquetMetaDataPushDecoder { column_index_policy: PageIndexPolicy, /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, + /// which rows and columns of the column index should be read + column_index_selection: PageIndexSelection, + /// which rows and columns of the offset index should be read + offset_index_selection: PageIndexSelection, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -250,6 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, + column_index_selection: PageIndexSelection::all(), + offset_index_selection: PageIndexSelection::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -300,6 +308,25 @@ impl ParquetMetaDataPushDecoder { self } + /// Select the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection.clone(); + self.offset_index_selection = selection; + self + } + + /// Select the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Select the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Set the options to use when decoding the Parquet metadata. pub fn with_metadata_options(mut self, options: Option>) -> Self { self.metadata_parser = self.metadata_parser.with_metadata_options(options); @@ -408,30 +435,36 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let range = range_for_page_index( + let ranges = ranges_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_selection, + &self.offset_index_selection, ); - let Some(page_index_range) = range else { + if ranges.is_empty() { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - }; + } + + let needed_ranges = ranges + .into_iter() + .filter(|r| !self.buffers.has_range(r)) + .collect::>(); - if !self.buffers.has_range(&page_index_range) { + if !needed_ranges.is_empty() { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + return Ok(DecodeResult::NeedsData(needed_ranges)); } - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, - &buffer, - offset, + &self.column_index_selection, + &self.offset_index_selection, + &self.buffers, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); @@ -481,25 +514,81 @@ enum DecodeState { Intermediate, } -/// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// Returns the minimum set of non-overlapping ranges needed to cover the requested +/// offset and column indexes. +/// +/// If no page indexes are present in the file, or none are actually +/// requested by the policies passed in, this will return an empty +/// vector. /// -/// Returns None if no page indexes are needed -pub fn range_for_page_index( +/// This may result in more ranges to fetch but may also result in fewer bytes +/// being read (and stored). To get a single range for all indexes, sort +/// the resultant vector by the range starts, and then create a single range +/// using `start` from the head and `end` from the tail. +/// +/// ```rust +/// # use core::ops::Range; +/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { +/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// let range = (ranges.first()?.start..ranges.last()?.end); +/// Some(range) +/// # } +/// ``` +pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, -) -> Option> { - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, +) -> Vec> { + let mut result = Vec::new(); + + fn add_ranges( + metadata: &ParquetMetaData, + selection: &PageIndexSelection, + ranges: &mut Vec>, + f: T, + ) where + T: Fn(&ColumnChunkMetaData) -> Option>, + { + for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { + if selection.includes_row_group(rg_idx) { + for (col_idx, col) in rg.columns().iter().enumerate() { + if selection.includes_column(col_idx) { + if let Some(range) = f(col) { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); + } + } + } + } + } } } - range + + if column_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + column_index_selection, + &mut result, + ColumnChunkMetaData::column_index_range, + ); + } + if offset_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + offset_index_selection, + &mut result, + ColumnChunkMetaData::offset_index_range, + ); + } + result } // These tests use the arrow writer to create a parquet file in memory @@ -651,6 +740,90 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } + /// Decode the metadata incrementally, with a partial page index + #[test] + fn test_metadata_decoder_incremental_partial_page_index() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + + /// Decode the metadata incrementally, with a partial page index, but coalesced + /// range request. + #[test] + fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let mut ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + // collapse ranges into a single range + ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + let range = ranges.first().unwrap().start..ranges.last().unwrap().end; + push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 088f3281db11..9f31c250c33b 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -27,6 +27,7 @@ use crate::file::metadata::{ use crate::file::reader::ChunkReader; use crate::schema::types::SchemaDescriptor; use bytes::Bytes; +use std::collections::BTreeSet; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -71,6 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -104,6 +107,89 @@ impl From for PageIndexPolicy { } } +/// Selects the row groups and columns for which page indexes are read. +/// +/// This is independent of [`PageIndexPolicy`], which controls whether selected +/// indexes are optional or required. By default, all row groups and columns are selected. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct PageIndexSelection { + // using i32 because that's how thrift vectors are sized + row_groups: Option>>, + columns: Option>>, +} + +// TODO(ets): add unit tests for PageIndexSelection +impl PageIndexSelection { + /// Select all row groups and columns. + pub fn all() -> Self { + Self::default() + } + + /// Select page indexes for only the listed columns. + /// + /// Any indices in `columns` that are less than zero will be ignored. + pub fn columns(columns: impl IntoIterator) -> Self { + Self { + row_groups: None, + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + /// Select page indexes for only the listed row groups. + /// + /// Any indices in `row_groups` that are less than zero will be ignored. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: None, + } + } + + /// Select page indexes for only the listed row groups and columns. + /// + /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + pub fn row_groups_and_columns( + row_groups: impl IntoIterator, + columns: impl IntoIterator, + ) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + // test if `idx` is in the row group set. returns false if idx > i32::MAX + pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.row_groups + .as_ref() + .is_none_or(|keep| keep.contains(&idx)) + } + + // test if `idx` is in the column set. returns false if idx > i32::MAX + pub(crate) fn includes_column(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + } + + // FIXME(ets): these will be used later + /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { + self.row_groups.as_deref() + } + + pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { + self.columns.as_deref() + }*/ +} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -137,6 +223,24 @@ impl ParquetMetaDataReader { self } + /// Selects the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { + self.with_column_index_selection(selection.clone()) + .with_offset_index_selection(selection) + } + + /// Selects the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Selects the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Sets the [`ParquetMetaDataOptions`] to use when decoding pub fn with_metadata_options(mut self, options: Option) -> Self { self.metadata_options = options.map(Arc::new); @@ -341,6 +445,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -488,6 +594,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/src/file/page_index/index_reader.rs b/parquet/src/file/page_index/index_reader.rs index 963ee0ea9782..235f0b241887 100644 --- a/parquet/src/file/page_index/index_reader.rs +++ b/parquet/src/file/page_index/index_reader.rs @@ -30,17 +30,6 @@ use crate::parquet_thrift::{ }; use crate::thrift_struct; use std::io::Write; -use std::ops::Range; - -/// Computes the covering range of two optional ranges -/// -/// For example `acc_range(Some(7..9), Some(1..3)) = Some(1..9)` -pub(crate) fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } -} /// Decode a Thrift [`OffsetIndex`] from the provided bytes. /// diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 3fb63ffdb87b..2f0348b835f5 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -132,7 +132,9 @@ impl AsyncFileReader for TestReader { if let Some(options) = options { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self.metadata = Some(Arc::new( From 1c30c6a2f590cfee466a0fb81aa179c673230a24 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 13:23:30 -0700 Subject: [PATCH 36/62] more tests --- .../custom_page_index_provider.rs | 2 +- parquet/tests/arrow_reader/mod.rs | 1 + parquet/tests/arrow_reader/page_index.rs | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 parquet/tests/arrow_reader/page_index.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 147c9a7336c0..026637742798 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -404,7 +404,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> Bytes { +pub(super) fn create_test_file() -> Bytes { // Create small pages and small row groups for testing let props = WriterProperties::builder() .set_statistics_enabled(EnabledStatistics::Page) diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index 1dfcfb690160..680524f64e35 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -47,6 +47,7 @@ mod int96_stats_roundtrip; mod invalid_utf8; mod io; mod large_string_overflow; +mod page_index; mod parquet_testing; mod partial_offset_index; #[cfg(feature = "async")] diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs new file mode 100644 index 000000000000..821d1d4cc55d --- /dev/null +++ b/parquet/tests/arrow_reader/page_index.rs @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests of selective page index population + +use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use std::fs::File; + +use crate::custom_page_index_provider::create_test_file; + +#[test] +fn test_parse_selected_columns() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_none()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_none()); + } +} + +#[test] +fn test_parse_selected_columns_mixed() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_some()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups() { + // test populating PageIndex with a subset of row groups + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate indexes for row groups 0 and 2 + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) + .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + + for col in 0..num_cols { + // row group 0 has both indexes + assert!(page_index.column_index(0, col).is_some()); + assert!(page_index.offset_index(0, col).is_some()); + // row group 1 has no indexes + assert!(page_index.column_index(1, col).is_none()); + assert!(page_index.offset_index(1, col).is_none()); + // row group 2 has both indexes + assert!(page_index.column_index(2, col).is_some()); + assert!(page_index.offset_index(2, col).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups_and_columns() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + + // row groups 0 and 2 should have no indexes + [0, 2].into_iter().for_each(|i| { + let rg_idx = metadata.page_index_for_row_group(i); + for col in 0..num_cols { + assert!(rg_idx.column_index(col).is_none()); + assert!(rg_idx.offset_index(col).is_none()); + } + }); + + // row group 1 should have column index for column 0 and offset index for columns 0 & 2 + let rg_idx = metadata.page_index_for_row_group(1); + assert!(rg_idx.column_index(0).is_some()); + assert!(rg_idx.offset_index(0).is_some()); + assert!(rg_idx.column_index(1).is_none()); + assert!(rg_idx.offset_index(1).is_none()); + assert!(rg_idx.column_index(2).is_none()); + assert!(rg_idx.offset_index(2).is_some()); + assert!(rg_idx.column_index(3).is_none()); + assert!(rg_idx.offset_index(3).is_none()); +} + +#[test] +fn test_page_index_sizes() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // no index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_none()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 7393); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 7817); + + // full index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 13897); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 14321); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12776); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 13200); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12056); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 12480); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 11326); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 11750); +} From 3177084ca648afcfc553afd7ac15ebf9eab82878 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:05:59 -0700 Subject: [PATCH 37/62] rename PageIndexSelection to ColumnChunkMask --- parquet/benches/metadata.rs | 8 +-- parquet/src/arrow/arrow_reader/mod.rs | 46 +++++++------- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 8 +-- parquet/src/file/metadata/parser.rs | 29 +++++---- parquet/src/file/metadata/push_decoder.rs | 76 +++++++++++------------ parquet/src/file/metadata/reader.rs | 55 +++++++++------- parquet/tests/arrow_reader/io/mod.rs | 4 +- parquet/tests/arrow_reader/page_index.rs | 30 ++++----- 10 files changed, 135 insertions(+), 127 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index d9fef3d1a5d0..b5ee07c49530 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, - PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ColumnChunkMask, ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, + PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; @@ -205,8 +205,8 @@ fn criterion_benchmark(c: &mut Criterion) { let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); c.bench_function("read page index reduced columns", |b| { b.iter(|| { reader.read_page_indexes(&data).unwrap(); diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 8f8953e69933..492ea6063462 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,7 +45,7 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; @@ -595,8 +595,8 @@ pub struct ArrowReaderOptions { column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -776,29 +776,29 @@ impl ArrowReaderOptions { self } - /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when filtering by a predicate - /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } - /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [OffsetIndex] structure. /// /// The offset index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when projecting a small subset - /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -952,18 +952,18 @@ impl ArrowReaderOptions { self.column_index } - /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// Retrieve the currently set [`ColumnChunkMask`] for the offset index. /// - /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. - pub fn offset_index_selection(&self) -> &PageIndexSelection { - &self.offset_index_selection + /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask]. + pub fn offset_index_mask(&self) -> &ColumnChunkMask { + &self.offset_index_mask } - /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// Retrieve the currently set [`ColumnChunkMask`] for the column index. /// - /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. - pub fn column_index_selection(&self) -> &PageIndexSelection { - &self.column_index_selection + /// This can be set via [`with_column_index_mask`][Self::with_column_index_mask]. + pub fn column_index_mask(&self) -> &ColumnChunkMask { + &self.column_index_mask } /// Retrieve the currently set metadata decoding options. @@ -1013,8 +1013,8 @@ impl ParquetMetaDataReader { self = self .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self @@ -1063,8 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()) + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index a73d2c0d3fbe..c7db5d0abbc1 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -243,8 +243,8 @@ impl AsyncFileReader for ParquetObjectReader { metadata = metadata .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 65c084108c7f..12162a7f8f85 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 46254777e668..2e70b6ab7650 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,7 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::PageIndexSelection; +use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -559,11 +559,11 @@ impl PageIndexBuilder { /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy /// /// For sparse indexes, this can save a great deal of memory - pub fn new_with_selection( + pub fn new_with_mask( num_row_groups: usize, num_columns: usize, - _column_index_selection: PageIndexSelection, - _offset_index_selection: PageIndexSelection, + _column_index_mask: ColumnChunkMask, + _offset_index_mask: ColumnChunkMask, ) -> Self { Self { column_indexes: Self::empty_index(num_row_groups, num_columns), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index b2a35172e8af..47b93463147c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,8 +26,7 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; @@ -252,8 +251,8 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip @@ -262,17 +261,17 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_selection( + let mut builder = PageIndexBuilder::new_with_mask( num_row_groups, num_columns, - column_index_selection.clone(), - offset_index_selection.clone(), + column_index_mask.clone(), + offset_index_mask.clone(), ); if column_index_policy != PageIndexPolicy::Skip { parse_column_index( metadata, column_index_policy, - column_index_selection, + column_index_mask, &mut builder, bytes, )?; @@ -281,7 +280,7 @@ pub(crate) fn parse_page_index( parse_offset_index( metadata, offset_index_policy, - offset_index_selection, + offset_index_mask, &mut builder, bytes, )?; @@ -299,7 +298,7 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -307,12 +306,12 @@ fn parse_column_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); @@ -331,7 +330,7 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -339,12 +338,12 @@ fn parse_offset_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index b55468fd2980..3999fdf9a9fb 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,7 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; @@ -229,9 +229,9 @@ pub struct ParquetMetaDataPushDecoder { /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, /// which rows and columns of the column index should be read - column_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, /// which rows and columns of the offset index should be read - offset_index_selection: PageIndexSelection, + offset_index_mask: ColumnChunkMask, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -256,8 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, - column_index_selection: PageIndexSelection::all(), - offset_index_selection: PageIndexSelection::all(), + column_index_mask: ColumnChunkMask::all(), + offset_index_mask: ColumnChunkMask::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -309,21 +309,21 @@ impl ParquetMetaDataPushDecoder { } /// Select the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection.clone(); - self.offset_index_selection = selection; + pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask.clone(); + self.offset_index_mask = mask; self } /// Select the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Select the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -439,8 +439,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, ); if ranges.is_empty() { @@ -462,8 +462,8 @@ impl ParquetMetaDataPushDecoder { &mut metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, &self.buffers, )?; self.state = DecodeState::Finished; @@ -526,10 +526,10 @@ enum DecodeState { /// the resultant vector by the range starts, and then create a single range /// using `start` from the head and `end` from the tail. /// -/// ```rust +/// ```ignore /// # use core::ops::Range; /// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// ranges.sort_by_key(|r| r.start); /// let range = (ranges.first()?.start..ranges.last()?.end); /// Some(range) /// # } @@ -538,33 +538,33 @@ pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Vec> { let mut result = Vec::new(); fn add_ranges( metadata: &ParquetMetaData, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, ranges: &mut Vec>, f: T, ) where T: Fn(&ColumnChunkMetaData) -> Option>, { for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if selection.includes_row_group(rg_idx) { + if mask.includes_row_group(rg_idx) { for (col_idx, col) in rg.columns().iter().enumerate() { - if selection.includes_column(col_idx) { - if let Some(range) = f(col) { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } + if mask.includes_column(col_idx) + && let Some(range) = f(col) + { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); } } } @@ -575,7 +575,7 @@ pub fn ranges_for_page_index( if column_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - column_index_selection, + column_index_mask, &mut result, ColumnChunkMetaData::column_index_range, ); @@ -583,7 +583,7 @@ pub fn ranges_for_page_index( if offset_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - offset_index_selection, + offset_index_mask, &mut result, ColumnChunkMetaData::offset_index_range, ); @@ -747,7 +747,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -788,7 +788,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -803,7 +803,7 @@ mod tests { // one for column index and one for offset index for each row group assert_eq!(ranges.len(), 4); // collapse ranges into a single range - ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + ranges.sort_by_key(|r| r.start); let range = ranges.first().unwrap().start..ranges.last().unwrap().end; push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 9f31c250c33b..6e0b5f011ee0 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -72,8 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -107,25 +107,34 @@ impl From for PageIndexPolicy { } } -/// Selects the row groups and columns for which page indexes are read. +/// Struct to specify column chunks for which metadata is required. /// -/// This is independent of [`PageIndexPolicy`], which controls whether selected -/// indexes are optional or required. By default, all row groups and columns are selected. +/// Column chunks are identified by row group index and column index. This struct +/// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), +/// horizontal slices (via [`Self::row_groups`]), or the intersection of the two +/// (via [`Self::row_groups_and_columns`]). +/// +/// # Example +/// TODO(ets): example +/// +/// At present this is only used to select elements of the [Page Index] for decoding. +/// +/// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct PageIndexSelection { +pub struct ColumnChunkMask { // using i32 because that's how thrift vectors are sized row_groups: Option>>, columns: Option>>, } -// TODO(ets): add unit tests for PageIndexSelection -impl PageIndexSelection { +// TODO(ets): add unit tests for ColumnChunkMask +impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { Self::default() } - /// Select page indexes for only the listed columns. + /// Select only the listed columns. /// /// Any indices in `columns` that are less than zero will be ignored. pub fn columns(columns: impl IntoIterator) -> Self { @@ -135,7 +144,7 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups. + /// Select only the listed row groups. /// /// Any indices in `row_groups` that are less than zero will be ignored. pub fn row_groups(row_groups: impl IntoIterator) -> Self { @@ -147,9 +156,9 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups and columns. + /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, @@ -224,20 +233,20 @@ impl ParquetMetaDataReader { } /// Selects the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { - self.with_column_index_selection(selection.clone()) - .with_offset_index_selection(selection) + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) } /// Selects the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Selects the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -445,8 +454,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -594,8 +603,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 2f0348b835f5..bb9b5aeba446 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -133,8 +133,8 @@ impl AsyncFileReader for TestReader { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self.metadata = Some(Arc::new( diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 821d1d4cc55d..e594e78e900d 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -17,7 +17,7 @@ //! Tests of selective page index population -use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -31,8 +31,8 @@ fn test_parse_selected_columns() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -66,8 +66,8 @@ fn test_parse_selected_columns_mixed() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -101,8 +101,8 @@ fn test_parse_selected_row_groups() { // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) - .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups([0, 2])) + .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -134,8 +134,8 @@ fn test_parse_selected_row_groups_and_columns() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -197,8 +197,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -212,8 +212,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -228,8 +228,8 @@ fn test_page_index_sizes() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); From 942f2bfe0ece4a0c1bf6f2c88eaea10fd8735266 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:17:29 -0700 Subject: [PATCH 38/62] docs and cleanup --- parquet/src/file/metadata/page_index.rs | 16 ---- parquet/src/file/metadata/parser.rs | 12 ++- parquet/src/file/metadata/push_decoder.rs | 2 + parquet/src/file/metadata/reader.rs | 94 +++++++++++++++++------ 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 2e70b6ab7650..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,6 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -556,21 +555,6 @@ impl PageIndexBuilder { } } - /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy - /// - /// For sparse indexes, this can save a great deal of memory - pub fn new_with_mask( - num_row_groups: usize, - num_columns: usize, - _column_index_mask: ColumnChunkMask, - _offset_index_mask: ColumnChunkMask, - ) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } - } - /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 47b93463147c..a76eea9066c0 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,7 +245,8 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). -/// * `bytes` - The byte slice containing the page index data. +/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing +/// the page indexes. /// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, @@ -261,13 +262,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_mask( - num_row_groups, - num_columns, - column_index_mask.clone(), - offset_index_mask.clone(), - ); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -277,6 +274,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 3999fdf9a9fb..6b747b34aabf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -701,6 +701,8 @@ mod tests { // expect the second request to read the offset indexes let ranges = expect_needs_data(metadata_decoder.try_decode()); + // since we're reading all indexes, there should be a single range needed + assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); // expect the third request to read the actual data diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 6e0b5f011ee0..993554f1087d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -114,11 +114,28 @@ impl From for PageIndexPolicy { /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). /// -/// # Example -/// TODO(ets): example -/// /// At present this is only used to select elements of the [Page Index] for decoding. /// +/// # Examples +/// +/// To select columns 0 and 1 from all row groups: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::columns([0, 1]); +/// ``` +/// +/// To select all columns from row group 2: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups([2]); +/// ``` +/// +/// To select columns 1 and 3 from row group 0: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups_and_columns([0], [1, 3]); +/// ``` +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ColumnChunkMask { @@ -127,7 +144,6 @@ pub struct ColumnChunkMask { columns: Option>>, } -// TODO(ets): add unit tests for ColumnChunkMask impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { @@ -136,22 +152,22 @@ impl ColumnChunkMask { /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. + /// Any indices in `columns` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all columns. pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + columns: Self::iter_to_set(columns), } } /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. + /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all row groups. pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), + row_groups: Self::iter_to_set(row_groups), columns: None, } } @@ -159,20 +175,22 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. + /// Passing an empty set for `row_groups` is treated as selecting all row groups, and + /// an empty set for `columns` as selectiong all columns. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, ) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + row_groups: Self::iter_to_set(row_groups), + columns: Self::iter_to_set(columns), } } - // test if `idx` is in the row group set. returns false if idx > i32::MAX - pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + /// Test if `idx` is in the row group set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_row_group(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; @@ -181,22 +199,20 @@ impl ColumnChunkMask { .is_none_or(|keep| keep.contains(&idx)) } - // test if `idx` is in the column set. returns false if idx > i32::MAX - pub(crate) fn includes_column(&self, idx: usize) -> bool { + /// Test if `idx` is in the column set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_column(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) } - // FIXME(ets): these will be used later - /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { - self.row_groups.as_deref() + fn iter_to_set(indices: impl IntoIterator) -> Option>> { + let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); + (!set.is_empty()).then_some(Arc::new(set)) } - - pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { - self.columns.as_deref() - }*/ } impl ParquetMetaDataReader { @@ -1538,4 +1554,32 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } + + #[test] + fn test_chunk_mask() { + // basic test + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + // test that negative numbers are ignored + let mask = ColumnChunkMask::columns([-1, 0, 1]); + assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(!mask.includes_column(2)); + + // empty set should behave as "all" + let mask = ColumnChunkMask::columns([]); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(mask.includes_column(2)); + + // indexing beyond i32::MAX should return false + let mask = ColumnChunkMask::all(); + assert!(mask.includes_column(i32::MAX as usize)); + assert!(!mask.includes_column(u32::MAX as usize)); + } } From 199a41d2690668ce67f1b236bcdc0c6d8d4f0c08 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 10:13:48 -0700 Subject: [PATCH 39/62] rework tests to do no file i/o --- parquet/tests/arrow_reader/page_index.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e594e78e900d..c605eac59834 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -18,7 +18,6 @@ //! Tests of selective page index population use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; -use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -26,7 +25,6 @@ use crate::custom_page_index_provider::create_test_file; fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -35,7 +33,7 @@ fn test_parse_selected_columns() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -61,7 +59,6 @@ fn test_parse_selected_columns() { fn test_parse_selected_columns_mixed() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -70,7 +67,7 @@ fn test_parse_selected_columns_mixed() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -96,7 +93,6 @@ fn test_parse_selected_columns_mixed() { fn test_parse_selected_row_groups() { // test populating PageIndex with a subset of row groups let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() @@ -105,7 +101,7 @@ fn test_parse_selected_row_groups() { .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_cols = metadata.file_metadata().schema_descr().num_columns(); assert!(metadata.page_index().is_some()); @@ -128,7 +124,6 @@ fn test_parse_selected_row_groups() { fn test_parse_selected_row_groups_and_columns() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -138,7 +133,7 @@ fn test_parse_selected_row_groups_and_columns() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); let num_cols = metadata.file_metadata().schema_descr().num_columns(); @@ -168,13 +163,12 @@ fn test_parse_selected_row_groups_and_columns() { fn test_page_index_sizes() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // no index let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_none()); #[cfg(not(feature = "encryption"))] @@ -186,7 +180,7 @@ fn test_page_index_sizes() { let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -201,7 +195,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -216,7 +210,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -232,7 +226,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] From 3dfeeca12defd2b6ea40de025580b1e917a32711 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 16:11:00 -0700 Subject: [PATCH 40/62] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/src/file/metadata/reader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 993554f1087d..86f13e5fe8d6 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -109,7 +109,8 @@ impl From for PageIndexPolicy { /// Struct to specify column chunks for which metadata is required. /// -/// Column chunks are identified by row group index and column index. This struct +/// Column chunks are identified by row group index and leaf column index (the index of the +/// column in [`SchemaDescriptor::columns`], not the index of a root or Arrow field). This struct /// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). From 26962603f9a90979952052c5887dec37616bcb51 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 07:53:10 -0700 Subject: [PATCH 41/62] revert most changes to push decoder --- parquet/src/file/metadata/push_decoder.rs | 189 +++------------------- 1 file changed, 24 insertions(+), 165 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 6b747b34aabf..4c8b514251ca 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,8 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; use bytes::Bytes; @@ -435,27 +434,20 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let ranges = ranges_for_page_index( + let range = range_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, ); - if ranges.is_empty() { + let Some(page_index_range) = range else { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - } - - let needed_ranges = ranges - .into_iter() - .filter(|r| !self.buffers.has_range(r)) - .collect::>(); + }; - if !needed_ranges.is_empty() { + if !self.buffers.has_range(&page_index_range) { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(DecodeResult::NeedsData(needed_ranges)); + return Ok(needs_range(page_index_range)); } parse_page_index( @@ -514,81 +506,32 @@ enum DecodeState { Intermediate, } -/// Returns the minimum set of non-overlapping ranges needed to cover the requested -/// offset and column indexes. +/// Returns the byte range needed to read the offset/page indexes, based on the +/// specified policies /// -/// If no page indexes are present in the file, or none are actually -/// requested by the policies passed in, this will return an empty -/// vector. -/// -/// This may result in more ranges to fetch but may also result in fewer bytes -/// being read (and stored). To get a single range for all indexes, sort -/// the resultant vector by the range starts, and then create a single range -/// using `start` from the head and `end` from the tail. -/// -/// ```ignore -/// # use core::ops::Range; -/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by_key(|r| r.start); -/// let range = (ranges.first()?.start..ranges.last()?.end); -/// Some(range) -/// # } -/// ``` -pub fn ranges_for_page_index( +/// Returns None if no page indexes are needed +pub fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_mask: &ColumnChunkMask, - offset_index_mask: &ColumnChunkMask, -) -> Vec> { - let mut result = Vec::new(); - - fn add_ranges( - metadata: &ParquetMetaData, - mask: &ColumnChunkMask, - ranges: &mut Vec>, - f: T, - ) where - T: Fn(&ColumnChunkMetaData) -> Option>, - { - for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if mask.includes_row_group(rg_idx) { - for (col_idx, col) in rg.columns().iter().enumerate() { - if mask.includes_column(col_idx) - && let Some(range) = f(col) - { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } - } - } - } +) -> Option> { + fn acc_range(a: Option>, b: Option>) -> Option> { + match (a, b) { + (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), + (None, x) | (x, None) => x, } } - if column_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - column_index_mask, - &mut result, - ColumnChunkMetaData::column_index_range, - ); - } - if offset_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - offset_index_mask, - &mut result, - ColumnChunkMetaData::offset_index_range, - ); + let mut range = None; + for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { + if column_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.column_index_range()); + } + if offset_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.offset_index_range()); + } } - result + range } // These tests use the arrow writer to create a parquet file in memory @@ -742,90 +685,6 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } - /// Decode the metadata incrementally, with a partial page index - #[test] - fn test_metadata_decoder_incremental_partial_page_index() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - - /// Decode the metadata incrementally, with a partial page index, but coalesced - /// range request. - #[test] - fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let mut ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - // collapse ranges into a single range - ranges.sort_by_key(|r| r.start); - let range = ranges.first().unwrap().start..ranges.last().unwrap().end; - push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) From e303c539ecb6c304c2970c2ea0c65e1de43343c6 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 09:33:16 -0700 Subject: [PATCH 42/62] address a number of issues found in review, primarily documentation changes, and changes to chunk mask semantics and storage. Also fixes the preloading reader issue. --- parquet/src/arrow/arrow_reader/mod.rs | 28 ++- parquet/src/arrow/async_reader/store.rs | 12 +- parquet/src/file/metadata/reader.rs | 183 ++++++++++++------ parquet/tests/arrow_reader/page_index.rs | 123 ++++++++++++ .../arrow_reader/partial_offset_index.rs | 67 ++++++- 5 files changed, 333 insertions(+), 80 deletions(-) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 492ea6063462..9788f597d95b 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -776,6 +776,12 @@ impl ArrowReaderOptions { self } + /// Sets the same [`ColumnChunkMask`] for both page-index structures. + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) + } + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed @@ -783,6 +789,11 @@ impl ArrowReaderOptions { /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// The mask applies only if the column-index policy is not [`PageIndexPolicy::Skip`] + /// (the default), or an underlying reader is configured to preload the index. It is + /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; + /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. + /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -796,6 +807,10 @@ impl ArrowReaderOptions { /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// Page pruning also needs the offset index for predicate columns, so callers should + /// include those columns in addition to projected columns. The same loading and policy + /// qualifications as [`Self::with_column_index_mask`] apply. + /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; @@ -1012,9 +1027,16 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + + // Preload settings on the underlying reader may enable an index even when the + // corresponding options policy is `Skip`, so apply non-default masks independently. + if !options.column_index_mask().is_all() { + self = self.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + self = self.with_offset_index_mask(options.offset_index_mask().clone()); } self diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index c7db5d0abbc1..5528cc05b2bd 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,9 +242,15 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + if let Some(options) = options { + if !options.column_index_mask().is_all() { + metadata = metadata.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + metadata = metadata.with_offset_index_mask(options.offset_index_mask().clone()); + } } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 86f13e5fe8d6..af345d7193fd 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -91,10 +91,12 @@ pub enum PageIndexPolicy { Skip, /// Read the page index if it exists, otherwise do not error. Optional, - /// Require the page index to exist, and error if it does not. + /// Require offset indexes for every selected column chunk when any selected + /// page index ranges exist. /// - /// Only enforced for the offset index; this is the same as [`Self::Optional`] - /// for the column index. + /// For compatibility, a file with no page index ranges is accepted. This is only + /// enforced for the offset index; this is the same as [`Self::Optional`] for the + /// column index. Required, } @@ -138,11 +140,12 @@ impl From for PageIndexPolicy { /// ``` /// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct ColumnChunkMask { - // using i32 because that's how thrift vectors are sized - row_groups: Option>>, - columns: Option>>, + // `None` means all, while `Some(empty)` means none. Store u32 because + // Parquet/Thrift collections cannot contain more than i32::MAX entries. + row_groups: Option>, + columns: Option>, } impl ColumnChunkMask { @@ -151,11 +154,18 @@ impl ColumnChunkMask { Self::default() } + /// Select no row groups or columns. + pub fn none() -> Self { + Self { + row_groups: Some(Arc::from([])), + columns: Some(Arc::from([])), + } + } + /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all columns. - pub fn columns(columns: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no columns. + pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, columns: Self::iter_to_set(columns), @@ -164,9 +174,8 @@ impl ColumnChunkMask { /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all row groups. - pub fn row_groups(row_groups: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no row groups. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { row_groups: Self::iter_to_set(row_groups), columns: None, @@ -175,12 +184,10 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. - /// Passing an empty set for `row_groups` is treated as selecting all row groups, and - /// an empty set for `columns` as selectiong all columns. + /// An empty iterator for either dimension selects no column chunks. pub fn row_groups_and_columns( - row_groups: impl IntoIterator, - columns: impl IntoIterator, + row_groups: impl IntoIterator, + columns: impl IntoIterator, ) -> Self { Self { row_groups: Self::iter_to_set(row_groups), @@ -189,30 +196,79 @@ impl ColumnChunkMask { } /// Test if `idx` is in the row group set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_row_group(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { - return false; - }; - self.row_groups - .as_ref() - .is_none_or(|keep| keep.contains(&idx)) + Self::includes_index(self.row_groups.as_ref(), idx) } /// Test if `idx` is in the column set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_column(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { + Self::includes_index(self.columns.as_ref(), idx) + } + + fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { + // return false for out-of-bounds index + let Ok(idx) = u32::try_from(idx) else { return false; }; - self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + keep.is_none_or(|keep| keep.binary_search(&idx).is_ok()) + } + + /// Returns `true` when this mask selects every column chunk. + pub fn is_all(&self) -> bool { + self.row_groups.is_none() && self.columns.is_none() + } + + /// Returns selected row groups, or `None` when all row groups are selected. + pub fn selected_row_groups(&self) -> Option<&[u32]> { + self.row_groups.as_deref() + } + + /// Returns selected leaf columns, or `None` when all columns are selected. + pub fn selected_columns(&self) -> Option<&[u32]> { + self.columns.as_deref() + } + + /// Creates a mask selecting the leaf columns in an Arrow projection. + #[cfg(feature = "arrow")] + pub fn from_projection( + projection: &crate::arrow::ProjectionMask, + schema: &SchemaDescriptor, + ) -> Self { + Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } - fn iter_to_set(indices: impl IntoIterator) -> Option>> { - let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); - (!set.is_empty()).then_some(Arc::new(set)) + /// Returns an iterator over the row group indices selected by this mask + pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + Self::axis_indices(self.row_groups.as_deref(), num_row_groups) + } + + /// Returns an iterator over the column indices selected by this mask + pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + Self::axis_indices(self.columns.as_deref(), num_columns) + } + + fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + match axis { + None => Box::new(0..len), + Some(indices) => Box::new( + indices + .iter() + .map(|&i| i as usize) + .take_while(move |&i| i < len), + ), + } + } + + fn iter_to_set(indices: impl IntoIterator) -> Option> { + Some( + indices + .into_iter() + .filter_map(|i| u32::try_from(i).ok()) + .collect::>() + .into_iter() + .collect::>() + .into(), + ) } } @@ -249,19 +305,29 @@ impl ParquetMetaDataReader { self } - /// Selects the row groups and columns for which both page index structures are read. + /// Selects the row groups and leaf columns for which both page index structures are read. + /// + /// The mask applies only to index structures whose policy is not [`PageIndexPolicy::Skip`]. + /// This reader fetches one byte range covering the selected indexes, so a sparse mask + /// primarily reduces decoding and memory rather than necessarily reducing I/O. pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { self.with_column_index_mask(mask.clone()) .with_offset_index_mask(mask) } - /// Selects the row groups and columns for which column indexes are read. + /// Selects the row groups and leaf columns for which column indexes are read. + /// + /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self } - /// Selects the row groups and columns for which offset indexes are read. + /// Selects the row groups and leaf columns for which offset indexes are read. + /// + /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self @@ -1120,6 +1186,23 @@ mod tests { "EOF: Parquet file too small. Size is 1728 but need 1729" ); } + + #[test] + fn test_chunk_mask() { + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + let mask = ColumnChunkMask::columns([]); + assert!(!mask.includes_column(0)); + assert_eq!(mask.selected_columns(), Some([].as_slice())); + + let all = ColumnChunkMask::all(); + assert!(all.is_all()); + assert!(all.includes_column(u32::MAX as usize)); + } } #[cfg(all(feature = "async", feature = "arrow", test))] @@ -1555,32 +1638,4 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } - - #[test] - fn test_chunk_mask() { - // basic test - let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); - assert!(mask.includes_row_group(0)); - assert!(!mask.includes_row_group(1)); - assert!(!mask.includes_column(0)); - assert!(mask.includes_column(1)); - - // test that negative numbers are ignored - let mask = ColumnChunkMask::columns([-1, 0, 1]); - assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(!mask.includes_column(2)); - - // empty set should behave as "all" - let mask = ColumnChunkMask::columns([]); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(mask.includes_column(2)); - - // indexing beyond i32::MAX should return false - let mask = ColumnChunkMask::all(); - assert!(mask.includes_column(i32::MAX as usize)); - assert!(!mask.includes_column(u32::MAX as usize)); - } } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c605eac59834..c84e1c7b09c9 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -21,6 +21,129 @@ use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataR use crate::custom_page_index_provider::create_test_file; +fn assert_page_index_cells( + metadata: &parquet::file::metadata::ParquetMetaData, + expect_ci: impl Fn(usize, usize) -> bool, + expect_oi: impl Fn(usize, usize) -> bool, +) { + let page_index = metadata.page_index().expect("page index should be loaded"); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + for rg in 0..metadata.num_row_groups() { + for col in 0..num_cols { + assert_eq!( + page_index.column_index(rg, col).is_some(), + expect_ci(rg, col), + "column index rg={rg} col={col}" + ); + assert_eq!( + page_index.offset_index(rg, col).is_some(), + expect_oi(rg, col), + "offset index rg={rg} col={col}" + ); + } + } +} + +#[test] +fn test_arrow_reader_options_page_index_masks() { + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let column_mask = ColumnChunkMask::row_groups_and_columns([1], [0]); + let offset_mask = ColumnChunkMask::columns([2]); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(column_mask.clone()) + .with_offset_index_mask(offset_mask.clone()); + assert_eq!(options.column_index_mask(), &column_mask); + assert_eq!(options.offset_index_mask(), &offset_mask); + + let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap(); + assert_page_index_cells( + metadata.metadata(), + |rg, col| rg == 1 && col == 0, + |_, col| col == 2, + ); + + let metadata = ParquetMetaDataReader::new() + .with_arrow_reader_options(Some(&options)) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells(&metadata, |rg, col| rg == 1 && col == 0, |_, col| col == 2); +} + +#[test] +fn test_parse_with_page_index_mask() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([2], [1, 3])) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| rg == 2 && (col == 1 || col == 3), + |rg, col| rg == 2 && (col == 1 || col == 3), + ); +} + +/*#[test] +// C11 coverage +fn test_repeated_page_index_reads_merge_cells() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + ); +} + +#[test] +// C3 coverage +fn test_partial_page_statistics_remain_aligned() { + use arrow_array::Array; + use parquet::arrow::arrow_reader::statistics::StatisticsConverter; + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([1])); + let metadata = ArrowReaderMetadata::load(&file, options).unwrap(); + let parquet_metadata = metadata.metadata(); + let page_index = parquet_metadata.page_index().unwrap().as_ref(); + let row_groups = parquet_metadata.row_groups(); + let row_group_indices = [0, 1, 2]; + let converter = StatisticsConverter::try_new( + "id", + metadata.schema(), + parquet_metadata.file_metadata().schema_descr(), + ) + .unwrap(); + + let mins = converter + .data_page_mins(page_index, row_group_indices.iter()) + .unwrap(); + let row_counts = converter + .data_page_row_counts(page_index, row_groups, row_group_indices.iter()) + .unwrap() + .unwrap(); + assert_eq!(mins.len(), row_counts.len()); + assert_eq!(row_counts.null_count(), row_counts.len()); +}*/ + #[test] fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns diff --git a/parquet/tests/arrow_reader/partial_offset_index.rs b/parquet/tests/arrow_reader/partial_offset_index.rs index 72489da5c64a..28aa4bb18568 100644 --- a/parquet/tests/arrow_reader/partial_offset_index.rs +++ b/parquet/tests/arrow_reader/partial_offset_index.rs @@ -30,7 +30,7 @@ use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::arrow::{ArrowWriter, ProjectionMask}; use parquet::column::writer::ColumnCloseResult; use parquet::errors::Result; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use parquet::file::properties::WriterProperties; use parquet::file::writer::SerializedFileWriter; use std::sync::Arc; @@ -144,15 +144,23 @@ fn configure(builder: ArrowReaderBuilder, trigger: Trigger) -> ArrowReader } } -fn read_sync(file: &Bytes, trigger: Trigger) -> Result> { - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options())?; +fn read_sync( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options)?; Ok(configure(builder, trigger) .build()? .collect::>()?) } #[cfg(feature = "async")] -fn read_async(file: &Bytes, trigger: Trigger) -> Result> { +fn read_async( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { use futures::TryStreamExt; use parquet::arrow::ParquetRecordBatchStreamBuilder; @@ -161,13 +169,17 @@ fn read_async(file: &Bytes, trigger: Trigger) -> Result> { .unwrap(); runtime.block_on(async { let input = std::io::Cursor::new(file.clone()); - let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options()).await?; + let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options).await?; configure(builder, trigger).build()?.try_collect().await }) } -fn read_push_decoder(file: &Bytes, trigger: Trigger) -> Result> { - let metadata = ArrowReaderMetadata::load(file, options())?; +fn read_push_decoder( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let metadata = ArrowReaderMetadata::load(file, options)?; let builder = ParquetPushDecoderBuilder::new_with_metadata(metadata); let mut decoder = configure(builder, trigger).build()?; let mut batches = vec![]; @@ -202,10 +214,45 @@ fn check(has_index: impl Fn(usize, usize) -> bool) { let expected = test_batch().slice(150, 100); for trigger in [Trigger::RowSelection, Trigger::RowFilter] { let results = [ - ("sync", read_sync(&file, trigger)), - ("push decoder", read_push_decoder(&file, trigger)), + ("sync", read_sync(&file, trigger, options())), + ("push decoder", read_push_decoder(&file, trigger, options())), #[cfg(feature = "async")] - ("async", read_async(&file, trigger)), + ("async", read_async(&file, trigger, options())), + ]; + for (reader, result) in results { + let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); + let actual = concat_batches(&expected.schema(), &batches).unwrap(); + assert_eq!(actual, expected, "{reader} reader, {trigger:?}"); + } + } +} + +#[test] +fn test_read_with_offset_index_mask() { + let file = test_file(|_, _| true); + let masked_options = + || options().with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])); + let metadata = ArrowReaderMetadata::load(&file, masked_options()).unwrap(); + let page_index = metadata.metadata().page_index().unwrap(); + for row_group in 0..2 { + for column in 0..3 { + assert_eq!( + page_index.offset_index(row_group, column).is_some(), + row_group == 1 && column == 0 + ); + } + } + + let expected = test_batch().slice(150, 100); + for trigger in [Trigger::RowSelection, Trigger::RowFilter] { + let results = [ + ("sync", read_sync(&file, trigger, masked_options())), + ( + "push decoder", + read_push_decoder(&file, trigger, masked_options()), + ), + #[cfg(feature = "async")] + ("async", read_async(&file, trigger, masked_options())), ]; for (reader, result) in results { let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); From 9672e90836025fd1fc1f96cc2419a1a46a687852 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:06:06 -0700 Subject: [PATCH 43/62] modify range_for_page_index to take masks --- parquet/src/file/metadata/push_decoder.rs | 71 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 4c8b514251ca..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -438,6 +438,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, ); let Some(page_index_range) = range else { @@ -507,28 +509,38 @@ enum DecodeState { } /// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// specified policies and masks /// /// Returns None if no page indexes are needed -pub fn range_for_page_index( +pub(crate) fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Option> { - fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } - } - - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + let mut range: Option> = None; + for (policy, mask, column_index) in [ + (column_index_policy, column_index_mask, true), + (offset_index_policy, offset_index_mask, false), + ] { + if policy != PageIndexPolicy::Skip { + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); + } + } + } } } range @@ -564,6 +576,33 @@ mod tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[test] + fn test_range_for_page_index_respects_column_mask() { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); + push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]); + let metadata = expect_data(decoder.try_decode()); + + let all = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::all(), + &ColumnChunkMask::all(), + ) + .unwrap(); + let masked = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::columns([0]), + &ColumnChunkMask::all(), + ) + .unwrap(); + + assert!(all.start <= masked.start && masked.end <= all.end); + assert!(masked.end - masked.start < all.end - all.start); + } + /// It is possible to feed some, but not all, of the footer into the metadata decoder /// before asked. This avoids multiple IO requests #[test] From 1c5a1351bd07c6f3f6a36b6f7c2812a90c7400ca Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:30:47 -0700 Subject: [PATCH 44/62] more efficient page index parser traversal --- parquet/src/file/metadata/parser.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index a76eea9066c0..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,7 +247,6 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. -/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -303,15 +302,9 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -335,15 +328,9 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; From 2c597fb80308fdf1db21fc86d128b79ed01b83e1 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:36:06 -0700 Subject: [PATCH 45/62] address StatisticsConverter misalignment --- parquet/src/arrow/arrow_reader/statistics.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/statistics.rs b/parquet/src/arrow/arrow_reader/statistics.rs index 486e937a79de..32684b2b1210 100644 --- a/parquet/src/arrow/arrow_reader/statistics.rs +++ b/parquet/src/arrow/arrow_reader/statistics.rs @@ -2042,6 +2042,15 @@ impl<'a> StatisticsConverter<'a> { let mut nulls = NullBufferBuilder::new(0); for rg_idx in row_group_indices { let Some(offset_index) = page_index.offset_index(*rg_idx, parquet_index) else { + // A partial page index may contain the ColumnIndex but not the OffsetIndex. + // Preserve page-wise alignment with data_page_{mins,maxes,null_counts} by + // emitting one null row count for every page known from the ColumnIndex. + let num_pages = page_index + .column_index(*rg_idx, parquet_index) + .map(|index| index.num_pages() as usize) + .unwrap_or(0); + row_counts.resize(row_counts.len() + num_pages, 0); + nulls.append_n_nulls(num_pages); continue; }; let page_locations = offset_index.page_locations(); From 134a3b91c95aee64baa6212535664f6151d4ad4e Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:41:44 -0700 Subject: [PATCH 46/62] enable C3 test --- parquet/tests/arrow_reader/page_index.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c84e1c7b09c9..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -107,10 +107,9 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} +}*/ #[test] -// C3 coverage fn test_partial_page_statistics_remain_aligned() { use arrow_array::Array; use parquet::arrow::arrow_reader::statistics::StatisticsConverter; @@ -142,7 +141,7 @@ fn test_partial_page_statistics_remain_aligned() { .unwrap(); assert_eq!(mins.len(), row_counts.len()); assert_eq!(row_counts.null_count(), row_counts.len()); -}*/ +} #[test] fn test_parse_selected_columns() { From dc23d994fab26476c3d7b177cad68d8a150b2dd1 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:17:05 -0700 Subject: [PATCH 47/62] fix C11: preserve existing indexes across multiple parse calls --- parquet/src/file/metadata/page_index.rs | 42 +++++++++++++++++++++++ parquet/src/file/metadata/parser.rs | 11 ++++-- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 ++++- parquet/tests/arrow_reader/page_index.rs | 22 ++++++++++-- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..5f94fe3c9d88 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,6 +513,36 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { + /// Creates a new [`PageIndexBuilder`] populated from an existing provider. + pub(crate) fn new_from_provider( + page_index: &dyn PageIndexProvider, + num_row_groups: usize, + num_columns: usize, + ) -> Self { + let mut builder = Self::default(); + if page_index.has_column_indexes() { + builder.allocate_column_indexes(num_row_groups, num_columns); + } + if page_index.has_offset_indexes() { + builder.allocate_offset_indexes(num_row_groups, num_columns); + } + for row_group in 0..num_row_groups { + for column in 0..num_columns { + if builder.column_indexes.is_some() + && let Some(index) = page_index.column_index(row_group, column) + { + builder.put_column_index(index.clone(), row_group, column); + } + if builder.offset_indexes.is_some() + && let Some(index) = page_index.offset_index(row_group, column) + { + builder.put_offset_index(index.clone(), row_group, column); + } + } + } + builder + } + /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -567,6 +597,12 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.column_indexes.is_none() { + self.allocate_column_indexes(num_row_groups, num_columns); + } + } + /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -579,6 +615,12 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.offset_indexes.is_none() { + self.allocate_offset_indexes(num_row_groups, num_columns); + } + } + /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..d146c89434a7 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,9 +261,14 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = metadata + .page_index() + .map(|index| { + PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) + }) + .unwrap_or_default(); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); + builder.ensure_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -273,7 +278,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); + builder.ensure_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..8e396fc8e441 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,7 +272,8 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. + /// after the metadata has already been decoded. Newly decoded page indexes + /// are merged with any page indexes already present in `metadata`. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index af345d7193fd..cb8b9457d91f 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -514,6 +514,10 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// Newly read indexes are merged with any existing page index. Existing entries outside + /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are + /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -523,6 +527,7 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -664,7 +669,8 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. + /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as + /// described by [`Self::read_page_indexes()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..4c2353fb49c5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,8 +87,7 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage +#[test] fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -107,7 +106,24 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -}*/ +} + +#[test] +fn test_repeated_page_index_reads_preserve_skipped_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells(&metadata, |_, _| true, |_, _| true); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 78917edcc93308cc4020de6c3d56177c0fce9aea Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 24 Sep 2026 21:10:38 -0700 Subject: [PATCH 48/62] empty_index no longer needs pub(crate) --- parquet/src/file/metadata/page_index.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 18e3d4c85851..5f94fe3c9d88 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -550,10 +550,7 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - pub(crate) fn empty_index( - num_row_groups: usize, - num_columns: usize, - ) -> Option>>> { + fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { Some( (0..num_row_groups) .map(|_| { From 9fd6d8a68fe22dad63f29f759b9ad6c1fccf62f0 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 12:32:45 -0700 Subject: [PATCH 49/62] Revert "fix C11: preserve existing indexes across multiple parse calls" This reverts commit dc23d994fab26476c3d7b177cad68d8a150b2dd1. --- parquet/src/file/metadata/page_index.rs | 42 ----------------------- parquet/src/file/metadata/parser.rs | 11 ++---- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 +---- parquet/tests/arrow_reader/page_index.rs | 22 ++---------- 5 files changed, 8 insertions(+), 78 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 5f94fe3c9d88..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,36 +513,6 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { - /// Creates a new [`PageIndexBuilder`] populated from an existing provider. - pub(crate) fn new_from_provider( - page_index: &dyn PageIndexProvider, - num_row_groups: usize, - num_columns: usize, - ) -> Self { - let mut builder = Self::default(); - if page_index.has_column_indexes() { - builder.allocate_column_indexes(num_row_groups, num_columns); - } - if page_index.has_offset_indexes() { - builder.allocate_offset_indexes(num_row_groups, num_columns); - } - for row_group in 0..num_row_groups { - for column in 0..num_columns { - if builder.column_indexes.is_some() - && let Some(index) = page_index.column_index(row_group, column) - { - builder.put_column_index(index.clone(), row_group, column); - } - if builder.offset_indexes.is_some() - && let Some(index) = page_index.offset_index(row_group, column) - { - builder.put_offset_index(index.clone(), row_group, column); - } - } - } - builder - } - /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -597,12 +567,6 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } - pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - if self.column_indexes.is_none() { - self.allocate_column_indexes(num_row_groups, num_columns); - } - } - /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -615,12 +579,6 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } - pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - if self.offset_indexes.is_none() { - self.allocate_offset_indexes(num_row_groups, num_columns); - } - } - /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index d146c89434a7..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,14 +261,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = metadata - .page_index() - .map(|index| { - PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) - }) - .unwrap_or_default(); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { - builder.ensure_column_indexes(num_row_groups, num_columns); + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -278,7 +273,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.ensure_offset_indexes(num_row_groups, num_columns); + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 8e396fc8e441..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,8 +272,7 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. Newly decoded page indexes - /// are merged with any page indexes already present in `metadata`. + /// after the metadata has already been decoded. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 8c7d07840ec0..cee752f4dd79 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -518,10 +518,6 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. - /// - /// Newly read indexes are merged with any existing page index. Existing entries outside - /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are - /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -531,7 +527,6 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. - /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -673,8 +668,7 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as - /// described by [`Self::read_page_indexes()`]. + /// been obtained. See [`Self::new_with_metadata()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 4c2353fb49c5..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,7 +87,8 @@ fn test_parse_with_page_index_mask() { ); } -#[test] +/*#[test] +// C11 coverage fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -106,24 +107,7 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} - -#[test] -fn test_repeated_page_index_reads_preserve_skipped_index() { - let file = create_test_file(); - let metadata = ParquetMetaDataReader::new() - .with_column_index_policy(PageIndexPolicy::Required) - .with_offset_index_policy(PageIndexPolicy::Skip) - .parse_and_finish(&file) - .unwrap(); - - let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) - .with_column_index_policy(PageIndexPolicy::Skip) - .with_offset_index_policy(PageIndexPolicy::Required); - reader.read_page_indexes(&file).unwrap(); - let metadata = reader.finish().unwrap(); - assert_page_index_cells(&metadata, |_, _| true, |_, _| true); -} +}*/ #[test] fn test_partial_page_statistics_remain_aligned() { From bc975c40ee82dd29c37a74ddf6cc45ec72f50cb9 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 15:01:35 -0700 Subject: [PATCH 50/62] replace path for C11 --- parquet/src/file/metadata/parser.rs | 21 +++--- parquet/src/file/metadata/push_decoder.rs | 21 +++--- parquet/src/file/metadata/reader.rs | 13 ++++ parquet/tests/arrow_reader/page_index.rs | 88 +++++++++++++++++++++-- 4 files changed, 119 insertions(+), 24 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..84645687540d 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -237,10 +237,10 @@ pub(crate) fn decode_metadata( parquet_metadata_from_bytes(buf, options) } -/// Parses page index from the provided bytes and adds it to the metadata. +/// Parses page indexes from the provided bytes and replaces those in the metadata. /// /// Arguments -/// * `metadata` - The ParquetMetaData to which the parsed column index will be added. +/// * `metadata` - The ParquetMetaData whose page index will be replaced. /// * `column_index_policy` - The policy for handling column index parsing (e.g., /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., @@ -255,13 +255,10 @@ pub(crate) fn parse_page_index( offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { - if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip - { - return Ok(()); - } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); let mut builder = PageIndexBuilder::default(); + if column_index_policy != PageIndexPolicy::Skip { builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( @@ -283,12 +280,16 @@ pub(crate) fn parse_page_index( )?; } + // Always replace the page index, even if both policies are Skip. + // This ensures that repeated calls to read_page_indexes replace the existing index + // rather than preserving it. let page_index = builder.build(); - // if both indexes are missing from the file, return without modifying `metadata` - if !page_index.has_column_indexes() && !page_index.has_offset_indexes() { - return Ok(()); + if page_index.has_column_indexes() || page_index.has_offset_indexes() { + metadata.set_page_index(Some(Arc::new(page_index))); + } else { + // If no indexes were read, clear the page index + metadata.set_page_index(None); } - metadata.set_page_index(Some(Arc::new(page_index))); Ok(()) } diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..af106dd1f5e0 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -273,6 +273,11 @@ impl ParquetMetaDataPushDecoder { /// /// This can be used to parse and populate the page index structures /// after the metadata has already been decoded. + /// + /// When decoding page indexes, any existing page indexes in `metadata` will be + /// replaced based on the configured policies. See + /// [`crate::file::metadata::ParquetMetaDataReader::read_page_indexes`] for details + /// on the replacement behavior. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); @@ -442,14 +447,14 @@ impl ParquetMetaDataPushDecoder { &self.offset_index_mask, ); - let Some(page_index_range) = range else { - self.state = DecodeState::Finished; - return Ok(DecodeResult::Data(*metadata)); - }; - - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + // Even if there is no range to read (for example, both policies are Skip, + // the masks select nothing, or the file has no indexes), parse the result to + // replace any existing indexes. + if let Some(page_index_range) = range { + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(needs_range(page_index_range)); + } } parse_page_index( diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index cee752f4dd79..efd73a8b38fd 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -518,6 +518,13 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// # Behavior on multiple calls + /// + /// On success, this method replaces any existing page index in the metadata. Index types + /// configured as [`PageIndexPolicy::Skip`] are cleared; other index types contain only the + /// indexes selected by their configured masks and available according to their policies. If + /// neither index type is populated, the page index is cleared. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -527,6 +534,9 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// + /// See [`Self::read_page_indexes()`] for a description of the replacement behavior on + /// multiple calls. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -669,6 +679,9 @@ impl ParquetMetaDataReader { /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already /// been obtained. See [`Self::new_with_metadata()`]. + /// + /// See [`Self::read_page_indexes()`] for a description of the replacement behavior on + /// multiple calls. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..471f69f6201c 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,27 +87,103 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage -fn test_repeated_page_index_reads_merge_cells() { +#[test] +// C11 coverage: verify that repeated calls to read_page_indexes replace existing indexes +fn test_repeated_page_index_reads_replace() { let file = create_test_file(); + // First read: load indexes for row group 0, column 0 let metadata = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Required) .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) .parse_and_finish(&file) .unwrap(); + // Second read: load indexes for row group 1, column 1 + // This should REPLACE the previous indexes, not merge with them let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); reader.read_page_indexes(&file).unwrap(); let metadata = reader.finish().unwrap(); + + // After the second read, only row group 1, column 1 should have indexes + // The indexes from the first read (row group 0, column 0) should be gone assert_page_index_cells( &metadata, - |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), - |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| rg == 1 && col == 1, + |rg, col| rg == 1 && col == 1, + ); +} + +#[test] +// C11 coverage: verify that Skip policy clears the corresponding index type +fn test_page_index_skip_policy_clears_index() { + let file = create_test_file(); + // First read: load both column and offset indexes + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + + // Verify both indexes are present + let page_index = metadata.page_index().expect("page index should be loaded"); + assert!( + page_index.has_column_indexes(), + "column indexes should be present" + ); + assert!( + page_index.has_offset_indexes(), + "offset indexes should be present" + ); + + // Second read: Skip column index but load offset index + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + + // After the second read, only offset indexes should be present + let page_index = metadata.page_index().expect("page index should be loaded"); + assert!( + !page_index.has_column_indexes(), + "column indexes should be cleared" + ); + assert!( + page_index.has_offset_indexes(), + "offset indexes should be present" ); -}*/ + + // Third read: Skip both indexes + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Skip); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + + // After the third read, no page index should be present + assert!( + metadata.page_index().is_none(), + "page index should be cleared when both policies are Skip" + ); +} + +#[test] +fn test_empty_page_index_mask_clears_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + assert!(metadata.page_index().is_some()); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::none()); + reader.read_page_indexes(&file).unwrap(); + + assert!(reader.finish().unwrap().page_index().is_none()); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 0a9dab932a54dfc9ef3f055fdb658456bfb13652 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 15:52:49 -0700 Subject: [PATCH 51/62] restore previous compatibility exception for PageIndexPolicy::Required --- parquet/src/file/metadata/push_decoder.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index af106dd1f5e0..a631e153f992 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -447,14 +447,19 @@ impl ParquetMetaDataPushDecoder { &self.offset_index_mask, ); - // Even if there is no range to read (for example, both policies are Skip, - // the masks select nothing, or the file has no indexes), parse the result to - // replace any existing indexes. - if let Some(page_index_range) = range { - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); - } + let Some(page_index_range) = range else { + // There is nothing to decode because both policies are Skip, the masks + // select nothing, or the file has no index ranges. Clear any previously + // loaded indexes while preserving the compatibility behavior that + // PageIndexPolicy::Required accepts files with no page indexes at all. + metadata.set_page_index(None); + self.state = DecodeState::Finished; + return Ok(DecodeResult::Data(*metadata)); + }; + + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(needs_range(page_index_range)); } parse_page_index( From 372d06515e53e87bad0864e1002eddd712b6ff4d Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 16:10:25 -0700 Subject: [PATCH 52/62] remove mention of the C11 tag --- parquet/tests/arrow_reader/page_index.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 471f69f6201c..e1a830028647 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -88,8 +88,8 @@ fn test_parse_with_page_index_mask() { } #[test] -// C11 coverage: verify that repeated calls to read_page_indexes replace existing indexes fn test_repeated_page_index_reads_replace() { + // verify that repeated calls to read_page_indexes replace existing indexes let file = create_test_file(); // First read: load indexes for row group 0, column 0 let metadata = ParquetMetaDataReader::new() @@ -116,8 +116,8 @@ fn test_repeated_page_index_reads_replace() { } #[test] -// C11 coverage: verify that Skip policy clears the corresponding index type fn test_page_index_skip_policy_clears_index() { + // verify that Skip policy clears the corresponding index type let file = create_test_file(); // First read: load both column and offset indexes let metadata = ParquetMetaDataReader::new() From 3ab2fd4dd9e1f16c15f623088c03c3596f115bd5 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:39:46 -0700 Subject: [PATCH 53/62] finish docs for parse_page_index --- parquet/src/file/metadata/parser.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 84645687540d..3f2c0649089c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,6 +245,10 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). +/// * `column_index_mask` - The row groups and leaf columns whose column indexes are parsed. +/// Ignored when `column_index_policy` is [`PageIndexPolicy::Skip`]. +/// * `offset_index_mask` - The row groups and leaf columns whose offset indexes are parsed. +/// Ignored when `offset_index_policy` is [`PageIndexPolicy::Skip`]. /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. pub(crate) fn parse_page_index( From c47bb75b7e9c95fdf04c418bfb6bd57b7d1dfa17 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:53:00 -0700 Subject: [PATCH 54/62] import ProjectionMask --- parquet/src/file/metadata/reader.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index efd73a8b38fd..e144f37fdb2d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "arrow")] +use crate::arrow::ProjectionMask; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; @@ -234,10 +236,7 @@ impl ColumnChunkMask { /// Creates a mask selecting the leaf columns in an Arrow projection. #[cfg(feature = "arrow")] - pub fn from_projection( - projection: &crate::arrow::ProjectionMask, - schema: &SchemaDescriptor, - ) -> Self { + pub fn from_projection(projection: &ProjectionMask, schema: &SchemaDescriptor) -> Self { Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } From b9fd8847c44454e1fa3ec570e02774b8fd3962d0 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:04:40 -0700 Subject: [PATCH 55/62] address nit --- parquet/src/file/metadata/push_decoder.rs | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index a631e153f992..1019e808c3c1 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -534,21 +534,22 @@ pub(crate) fn range_for_page_index( (column_index_policy, column_index_mask, true), (offset_index_policy, offset_index_mask, false), ] { - if policy != PageIndexPolicy::Skip { - for row_group in mask.row_group_indices(metadata.num_row_groups()) { - for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { - let column = metadata.row_group(row_group).column(column); - let index = if column_index { - column.column_index_range() - } else { - column.offset_index_range() - }; - if let Some(index) = index { - range = Some(match range { - Some(range) => range.start.min(index.start)..range.end.max(index.end), - None => index, - }); - } + if policy == PageIndexPolicy::Skip { + continue; + } + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); } } } From ad865184537abeab2463330e0cc26bd39905072d Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:09:45 -0700 Subject: [PATCH 56/62] replace boxed iterator with more targeted concrete iterator impl --- parquet/src/file/metadata/reader.rs | 58 ++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e144f37fdb2d..7f7eb81c3090 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -34,6 +34,8 @@ use crate::schema::types::SchemaDescriptor; use arrow_array::ArrayRef; use bytes::Bytes; use std::collections::BTreeSet; +use std::iter::FusedIterator; +use std::slice::Iter; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -241,24 +243,22 @@ impl ColumnChunkMask { } /// Returns an iterator over the row group indices selected by this mask - pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + pub fn row_group_indices(&self, num_row_groups: usize) -> impl Iterator + '_ { Self::axis_indices(self.row_groups.as_deref(), num_row_groups) } /// Returns an iterator over the column indices selected by this mask - pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + pub fn column_indices(&self, num_columns: usize) -> impl Iterator + '_ { Self::axis_indices(self.columns.as_deref(), num_columns) } - fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + fn axis_indices(axis: Option<&[u32]>, len: usize) -> AxisIndices<'_> { match axis { - None => Box::new(0..len), - Some(indices) => Box::new( - indices - .iter() - .map(|&i| i as usize) - .take_while(move |&i| i < len), - ), + None => AxisIndices::All(0..len), + Some(indices) => { + let end = indices.partition_point(|&i| (i as usize) < len); + AxisIndices::Selected(indices[..end].iter()) + } } } @@ -275,6 +275,33 @@ impl ColumnChunkMask { } } +// helper for ColumnChunkMask selected index traversal +enum AxisIndices<'a> { + All(Range), + Selected(Iter<'a, u32>), +} + +impl Iterator for AxisIndices<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + match self { + Self::All(indices) => indices.next(), + Self::Selected(indices) => indices.next().map(|&i| i as usize), + } + } + + fn size_hint(&self) -> (usize, Option) { + match self { + Self::All(indices) => indices.size_hint(), + Self::Selected(indices) => indices.size_hint(), + } + } +} + +impl ExactSizeIterator for AxisIndices<'_> {} +impl FusedIterator for AxisIndices<'_> {} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -1301,6 +1328,17 @@ mod tests { let all = ColumnChunkMask::all(); assert!(all.is_all()); assert!(all.includes_column(u32::MAX as usize)); + + assert_eq!(all.row_group_indices(3).collect::>(), [0, 1, 2]); + assert_eq!(all.column_indices(2).collect::>(), [0, 1]); + + let none = ColumnChunkMask::none(); + assert_eq!(none.row_group_indices(3).count(), 0); + assert_eq!(none.column_indices(3).count(), 0); + + let sparse = ColumnChunkMask::row_groups_and_columns([0, 2, 4], [1, 3, 5]); + assert_eq!(sparse.row_group_indices(3).collect::>(), [0, 2]); + assert_eq!(sparse.column_indices(4).collect::>(), [1, 3]); } } From 4823984ecc27cb9291ae96a85e331a40334bc8c9 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:43:51 -0700 Subject: [PATCH 57/62] revert to using single passed in Bytes --- parquet/src/file/metadata/parser.rs | 55 +++++++++++++---------- parquet/src/file/metadata/push_decoder.rs | 5 ++- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 3f2c0649089c..22f0564a9c4f 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -20,9 +20,11 @@ //! These functions parse thrift-encoded metadata from a byte slice //! into the corresponding Rust structures -use std::sync::Arc; +use std::{ops::Range, sync::Arc}; -use crate::errors::ParquetError; +use bytes::Bytes; + +use crate::errors::{ParquetError, Result}; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ @@ -32,8 +34,6 @@ use crate::file::metadata::{ use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use crate::file::reader::ChunkReader; -use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -50,7 +50,6 @@ mod inner { use super::*; use crate::encryption::decrypt::FileDecryptionProperties; - use crate::errors::Result; /// API for decoding metadata that may be encrypted #[derive(Debug, Default)] @@ -108,7 +107,7 @@ mod inner { column: &ColumnChunkMetaData, row_group_index: usize, col_index: usize, - ) -> crate::errors::Result { + ) -> Result { use crate::encryption::decrypt::CryptoContext; match &column.column_crypto_metadata { Some(crypto_metadata) => { @@ -136,7 +135,7 @@ mod inner { column: &ColumnChunkMetaData, row_group_index: usize, col_index: usize, - ) -> crate::errors::Result { + ) -> Result { use crate::encryption::decrypt::CryptoContext; match &column.column_crypto_metadata { Some(crypto_metadata) => { @@ -162,7 +161,6 @@ mod inner { #[cfg(not(feature = "encryption"))] mod inner { use super::*; - use crate::errors::Result; use std::sync::Arc; /// parallel implementation when encryption feature is not enabled /// @@ -208,7 +206,7 @@ mod inner { column: &ColumnChunkMetaData, _row_group_index: usize, _col_index: usize, - ) -> crate::errors::Result { + ) -> Result { decode_column_index(bytes, column.column_type()) } @@ -218,7 +216,7 @@ mod inner { _column: &ColumnChunkMetaData, _row_group_index: usize, _col_index: usize, - ) -> crate::errors::Result { + ) -> Result { decode_offset_index(bytes) } } @@ -233,7 +231,7 @@ mod inner { pub(crate) fn decode_metadata( buf: &[u8], options: Option<&ParquetMetaDataOptions>, -) -> crate::errors::Result { +) -> Result { parquet_metadata_from_bytes(buf, options) } @@ -249,16 +247,17 @@ pub(crate) fn decode_metadata( /// Ignored when `column_index_policy` is [`PageIndexPolicy::Skip`]. /// * `offset_index_mask` - The row groups and leaf columns whose offset indexes are parsed. /// Ignored when `offset_index_policy` is [`PageIndexPolicy::Skip`]. -/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing -/// the page indexes. +/// * `bytes` - The byte slice containing the page index data. +/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, column_index_mask: &ColumnChunkMask, offset_index_mask: &ColumnChunkMask, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); let mut builder = PageIndexBuilder::default(); @@ -271,6 +270,7 @@ pub(crate) fn parse_page_index( column_index_mask, &mut builder, bytes, + start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { @@ -281,6 +281,7 @@ pub(crate) fn parse_page_index( offset_index_mask, &mut builder, bytes, + start_offset, )?; } @@ -302,8 +303,9 @@ fn parse_column_index( column_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } @@ -312,9 +314,9 @@ fn parse_column_index( for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = - inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; + inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -328,8 +330,9 @@ fn parse_offset_index( offset_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } @@ -338,9 +341,9 @@ fn parse_offset_index( for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = - inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; + inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); @@ -350,3 +353,9 @@ fn parse_offset_index( Ok(()) } + +fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range) -> Result<&[u8]> { + let start = usize::try_from(range.start - start_offset)?; + let end = usize::try_from(range.end - start_offset)?; + Ok(&bytes[start..end]) +} diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 1019e808c3c1..78179528047e 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -462,13 +462,16 @@ impl ParquetMetaDataPushDecoder { return Ok(needs_range(page_index_range)); } + let buffer = self.get_bytes(&page_index_range)?; + let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, &self.column_index_mask, &self.offset_index_mask, - &self.buffers, + &buffer, + offset, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); From b9206675e0ddb170f66bf8d0e53b6e7b7d3a22b5 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 30 Sep 2026 21:47:18 -0700 Subject: [PATCH 58/62] clean up documentation for ColumnChunkMask add is_none function enforce indices <= i32::MAX, not u32::MAX --- parquet/src/file/metadata/reader.rs | 38 ++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 7f7eb81c3090..dedff337e7b0 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -125,6 +125,9 @@ impl From for PageIndexPolicy { /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). /// +/// Because Parquet collection cannot contain more than [`i32::MAX`] values, user provided indices +/// outside that range will be discarded. +/// /// At present this is only used to select elements of the [Page Index] for decoding. /// /// # Examples @@ -150,8 +153,11 @@ impl From for PageIndexPolicy { /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct ColumnChunkMask { - // `None` means all, while `Some(empty)` means none. Store u32 because - // Parquet/Thrift collections cannot contain more than i32::MAX entries. + // `row_groups` and `columns` are optional `u32` slices, with elements + // sorted in ascending order (to permit binary searching for member indices). + // `None` means all indices are included, while an empty slice means none are. + // Indices are stored as `u32` because Parquet/Thrift collections cannot contain + // more than i32::MAX entries. row_groups: Option>, columns: Option>, } @@ -204,20 +210,25 @@ impl ColumnChunkMask { } /// Test if `idx` is in the row group set. + /// + /// Returns `false` if idx > [i32::MAX]. pub fn includes_row_group(&self, idx: usize) -> bool { Self::includes_index(self.row_groups.as_ref(), idx) } /// Test if `idx` is in the column set. + /// + /// Returns `false` if idx > [i32::MAX]. pub fn includes_column(&self, idx: usize) -> bool { Self::includes_index(self.columns.as_ref(), idx) } fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { // return false for out-of-bounds index - let Ok(idx) = u32::try_from(idx) else { + let Ok(idx) = i32::try_from(idx) else { return false; }; + let idx = idx as u32; keep.is_none_or(|keep| keep.binary_search(&idx).is_ok()) } @@ -226,12 +237,22 @@ impl ColumnChunkMask { self.row_groups.is_none() && self.columns.is_none() } + /// Returns `true` when this mask selects no column chunks. + pub fn is_none(&self) -> bool { + self.row_groups.as_ref().is_some_and(|i| i.is_empty()) + || self.columns.as_ref().is_some_and(|i| i.is_empty()) + } + /// Returns selected row groups, or `None` when all row groups are selected. + /// + /// This returns indices as `u32` because that is how they are stored internally. pub fn selected_row_groups(&self) -> Option<&[u32]> { self.row_groups.as_deref() } /// Returns selected leaf columns, or `None` when all columns are selected. + /// + /// This returns indices as `u32` because that is how they are stored internally. pub fn selected_columns(&self) -> Option<&[u32]> { self.columns.as_deref() } @@ -242,12 +263,14 @@ impl ColumnChunkMask { Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } - /// Returns an iterator over the row group indices selected by this mask + /// Returns an iterator over the row group indices selected by this mask. If the mask is `all` + /// then all indices in the range `[0, num_row_groups)` are returned. pub fn row_group_indices(&self, num_row_groups: usize) -> impl Iterator + '_ { Self::axis_indices(self.row_groups.as_deref(), num_row_groups) } - /// Returns an iterator over the column indices selected by this mask + /// Returns an iterator over the column indices selected by this mask. If the mask is `all` + /// then all indices in the range `[0, num_columns)` are returned. pub fn column_indices(&self, num_columns: usize) -> impl Iterator + '_ { Self::axis_indices(self.columns.as_deref(), num_columns) } @@ -262,11 +285,14 @@ impl ColumnChunkMask { } } + // Used during initialization, this helper takes an iterator over `usize` elements, + // discards out-of-range values and then sorts. fn iter_to_set(indices: impl IntoIterator) -> Option> { Some( indices .into_iter() - .filter_map(|i| u32::try_from(i).ok()) + .filter_map(|i| i32::try_from(i).ok()) // remove out-of-bounds values + .map(|i| i as u32) // store as u32. safe because 0 <= i <= i32::MAX .collect::>() .into_iter() .collect::>() From 58ef396e5631dadda8172f468046a753ece2a082 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 30 Sep 2026 21:59:33 -0700 Subject: [PATCH 59/62] fix test --- parquet/src/file/metadata/reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index dedff337e7b0..366fafe59f30 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -1353,7 +1353,7 @@ mod tests { let all = ColumnChunkMask::all(); assert!(all.is_all()); - assert!(all.includes_column(u32::MAX as usize)); + assert!(all.includes_column(i32::MAX as usize)); assert_eq!(all.row_group_indices(3).collect::>(), [0, 1, 2]); assert_eq!(all.column_indices(2).collect::>(), [0, 1]); From 117f3ad7dd5f50a84793b813c0394d3855f47ff1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 30 Sep 2026 22:16:58 -0700 Subject: [PATCH 60/62] mention cheap cloneability for ColumnChunkMask --- parquet/src/file/metadata/reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 366fafe59f30..798839ea8e95 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -117,7 +117,7 @@ impl From for PageIndexPolicy { } } -/// Struct to specify column chunks for which metadata is required. +/// A cheaply cloneable struct to specify column chunks for which metadata is required. /// /// Column chunks are identified by row group index and leaf column index (the index of the /// column in [`SchemaDescriptor::columns`], not the index of a root or Arrow field). This struct From e17b71f77a518dd6dacdbd6b97b00cd02a17f85e Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 1 Oct 2026 10:55:49 -0700 Subject: [PATCH 61/62] clean up the chunk api and docs --- parquet/src/file/metadata/reader.rs | 51 +++++++++++------------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 798839ea8e95..3bca5f05b52a 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -209,18 +209,14 @@ impl ColumnChunkMask { } } - /// Test if `idx` is in the row group set. - /// - /// Returns `false` if idx > [i32::MAX]. - pub fn includes_row_group(&self, idx: usize) -> bool { - Self::includes_index(self.row_groups.as_ref(), idx) + /// Returns whether row group `idx` is selected and within the bounds `[0, num_row_groups)`. + pub fn includes_row_group(&self, idx: usize, num_row_groups: usize) -> bool { + idx < num_row_groups && Self::includes_index(self.row_groups.as_ref(), idx) } - /// Test if `idx` is in the column set. - /// - /// Returns `false` if idx > [i32::MAX]. - pub fn includes_column(&self, idx: usize) -> bool { - Self::includes_index(self.columns.as_ref(), idx) + /// Returns whether column `idx` is selected and within the bounds `[0, num_columns)`. + pub fn includes_column(&self, idx: usize, num_columns: usize) -> bool { + idx < num_columns && Self::includes_index(self.columns.as_ref(), idx) } fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { @@ -237,26 +233,16 @@ impl ColumnChunkMask { self.row_groups.is_none() && self.columns.is_none() } - /// Returns `true` when this mask selects no column chunks. + /// Returns `true` when either configured axis is empty (and thus would select no column chunks). + /// + /// This does not consider the dimensions of a particular Parquet file, so may return `false` + /// when [`Self::column_indices`] or [`Self::row_group_indices`] returns an empty iterator due + /// to all elements exceeding the provided upper bound. pub fn is_none(&self) -> bool { self.row_groups.as_ref().is_some_and(|i| i.is_empty()) || self.columns.as_ref().is_some_and(|i| i.is_empty()) } - /// Returns selected row groups, or `None` when all row groups are selected. - /// - /// This returns indices as `u32` because that is how they are stored internally. - pub fn selected_row_groups(&self) -> Option<&[u32]> { - self.row_groups.as_deref() - } - - /// Returns selected leaf columns, or `None` when all columns are selected. - /// - /// This returns indices as `u32` because that is how they are stored internally. - pub fn selected_columns(&self) -> Option<&[u32]> { - self.columns.as_deref() - } - /// Creates a mask selecting the leaf columns in an Arrow projection. #[cfg(feature = "arrow")] pub fn from_projection(projection: &ProjectionMask, schema: &SchemaDescriptor) -> Self { @@ -1342,18 +1328,19 @@ mod tests { #[test] fn test_chunk_mask() { let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); - assert!(mask.includes_row_group(0)); - assert!(!mask.includes_row_group(1)); - assert!(!mask.includes_column(0)); - assert!(mask.includes_column(1)); + assert!(mask.includes_row_group(0, 1)); + assert!(!mask.includes_row_group(1, 1)); + assert!(!mask.includes_column(0, 2)); + assert!(mask.includes_column(1, 2)); let mask = ColumnChunkMask::columns([]); - assert!(!mask.includes_column(0)); - assert_eq!(mask.selected_columns(), Some([].as_slice())); + assert!(!mask.includes_column(0, 3)); + assert_eq!(mask.column_indices(3).count(), 0); let all = ColumnChunkMask::all(); assert!(all.is_all()); - assert!(all.includes_column(i32::MAX as usize)); + assert!(all.includes_column(2, 3)); + assert!(!all.includes_column(3, 3)); assert_eq!(all.row_group_indices(3).collect::>(), [0, 1, 2]); assert_eq!(all.column_indices(2).collect::>(), [0, 1]); From b7033a61754335fcbbba732db3f9a039d23f79a2 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Mon, 5 Oct 2026 21:26:11 -0700 Subject: [PATCH 62/62] add TODOs for followup --- parquet/src/arrow/arrow_reader/mod.rs | 7 +++++++ parquet/src/arrow/async_reader/store.rs | 2 ++ parquet/src/file/metadata/push_decoder.rs | 2 ++ parquet/src/file/metadata/reader.rs | 6 ++++++ 4 files changed, 17 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 9788f597d95b..132236c7cdf9 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -776,6 +776,8 @@ impl ArrowReaderOptions { self } + // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395 + /// Sets the same [`ColumnChunkMask`] for both page-index structures. pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { self.with_column_index_mask(mask.clone()) @@ -967,6 +969,8 @@ impl ArrowReaderOptions { self.column_index } + // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395 + /// Retrieve the currently set [`ColumnChunkMask`] for the offset index. /// /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask]. @@ -1032,6 +1036,9 @@ impl ParquetMetaDataReader { // Preload settings on the underlying reader may enable an index even when the // corresponding options policy is `Skip`, so apply non-default masks independently. + // + // TODO: https://github.com/apache/arrow-rs/issues/11395 may deprecate this API. Decide + // at that time if masks should be added to ParquetObjectReader. if !options.column_index_mask().is_all() { self = self.with_column_index_mask(options.column_index_mask().clone()); } diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index 5528cc05b2bd..a38d0c355811 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -244,6 +244,8 @@ impl AsyncFileReader for ParquetObjectReader { .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()); } + // TODO: https://github.com/apache/arrow-rs/issues/11395 may deprecate this API. Decide + // at that time if masks should be added to ParquetObjectReader. if let Some(options) = options { if !options.column_index_mask().is_all() { metadata = metadata.with_column_index_mask(options.column_index_mask().clone()); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 78179528047e..90fd3ee7fa0e 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -312,6 +312,8 @@ impl ParquetMetaDataPushDecoder { self } + // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395 + /// Select the row groups and columns for which both page index structures are read. pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask.clone(); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 3bca5f05b52a..78710600bfe5 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -91,6 +91,10 @@ pub struct ParquetMetaDataReader { file_decryption_properties: Option>, } +// TODO: https://github.com/apache/arrow-rs/issues/11395 is tracking unifying PageIndexPolicy +// with the ColumnChunkMask to reduce the dizzying number of APIs that exist to control reading +// the page indexes. + /// Describes the policy for reading page indexes #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum PageIndexPolicy { @@ -347,6 +351,8 @@ impl ParquetMetaDataReader { self } + // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395 + /// Selects the row groups and leaf columns for which both page index structures are read. /// /// The mask applies only to index structures whose policy is not [`PageIndexPolicy::Skip`].