From 1c7798468aff9b57a0171cec262e7518334ba9c9 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:00:56 -0700 Subject: [PATCH 01/78] allow for serialization of custom page index providers --- parquet/src/file/metadata/page_index.rs | 21 +--- parquet/src/file/metadata/writer.rs | 121 ++++++++++++------------ parquet/src/file/writer.rs | 73 +++++++++++++- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9ded57d0a28e..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -460,22 +460,6 @@ impl PageIndex { pub fn into_builder(self) -> PageIndexBuilder { self.into() } - - /// Returns a reference to the raw column indexes structure - /// - /// This method provides access to the underlying column index data for serialization - /// and other low-level operations. - pub(crate) fn column_indexes_raw(&self) -> Option<&Vec>>> { - self.column_indexes.as_ref() - } - - /// Returns a reference to the raw offset indexes structure - /// - /// This method provides access to the underlying offset index data for serialization - /// and other low-level operations. - pub(crate) fn offset_indexes_raw(&self) -> Option<&Vec>>> { - self.offset_indexes.as_ref() - } } impl PageIndexProvider for PageIndex { @@ -536,7 +520,10 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { + pub(crate) fn empty_index( + num_row_groups: usize, + num_columns: usize, + ) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a1708ccf2c3a..d6de7e45c8c5 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::FileMeta; use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; @@ -54,8 +55,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { buf: &'a mut TrackedWrite, schema_descr: &'a SchemaDescPtr, row_groups: Vec, - column_indexes: Option>>>, - offset_indexes: Option>>>, + page_index: Option>, key_value_metadata: Option>, created_by: Option, object_writer: MetadataObjectWriter, @@ -71,14 +71,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized offset indexes. fn write_offset_indexes( &mut self, - offset_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut offset_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let offidx_vec = offset_indexes.as_mut().unwrap(); + + // we've already checked before calling that the offset indexes are populated + assert!(page_index.has_offset_indexes()); + // iter row group // iter each column // write offset index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(offset_index) = &offset_indexes[row_group_idx][column_idx] { + if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); self.object_writer.write_offset_index( offset_index, @@ -91,10 +98,11 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); + offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); } } } - Ok(()) + Ok(offset_indexes) } /// Serialize all the column indexes to the `self.buf` @@ -104,14 +112,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized column indexes. fn write_column_indexes( &mut self, - column_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut column_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let colidx_vec = column_indexes.as_mut().unwrap(); + + // we've already checked before calling that the column indexes are populated + assert!(page_index.has_column_indexes()); + // iter row group // iter each column // write column index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(column_index) = &column_indexes[row_group_idx][column_idx] { + if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); // only update column_metadata if the write succeeds if self.object_writer.write_column_index( @@ -127,21 +142,28 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } + colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); } } } - Ok(()) + Ok(column_indexes) } - /// Serialize the column indexes and transform to `Option` - fn finalize_column_indexes(&mut self) -> Result>>>> { - let column_indexes = std::mem::take(&mut self.column_indexes); - - // Write column indexes to file - if let Some(column_indexes) = column_indexes.as_ref() { - self.write_column_indexes(column_indexes)?; + /// Serialize the column indexes and transform to `Option>>>` + fn finalize_column_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_column_indexes()) + { + return Ok(None); } + // Write column indexes to file while assembling a column index + let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; + // check to see if the index is `None` for every row group and column chunk let all_none = column_indexes .as_ref() @@ -155,13 +177,19 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes(&mut self) -> Result>>>> { - let offset_indexes = std::mem::take(&mut self.offset_indexes); + fn finalize_offset_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_offset_indexes()) + { + return Ok(None); + } // Write offset indexes to file - if let Some(offset_indexes) = offset_indexes.as_ref() { - self.write_offset_indexes(offset_indexes)?; - } + let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; // check to see if the index is `None` for every row group and column chunk let all_none = offset_indexes @@ -180,8 +208,9 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); // serialize page indexes and transform to the proper form for use in ParquetMetaData - let column_indexes = self.finalize_column_indexes()?; - let offset_indexes = self.finalize_offset_indexes()?; + let page_index = self.page_index.take(); + let column_indexes = self.finalize_column_indexes(&page_index)?; + let offset_indexes = self.finalize_offset_indexes(&page_index)?; // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -275,8 +304,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { buf, schema_descr, row_groups, - column_indexes: None, - offset_indexes: None, + page_index: None, key_value_metadata: None, created_by, object_writer: Default::default(), @@ -285,19 +313,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } } - pub fn with_column_indexes( - mut self, - column_indexes: Vec>>, - ) -> Self { - self.column_indexes = Some(column_indexes); - self - } - - pub fn with_offset_indexes( - mut self, - offset_indexes: Vec>>, - ) -> Self { - self.offset_indexes = Some(offset_indexes); + pub fn with_page_index(mut self, page_index: Arc) -> Self { + self.page_index = Some(page_index); self } @@ -333,17 +350,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -///
-/// -/// **NOTE:** -/// The serialization of custom [`PageIndexProvider`]s is not currently supported. -/// The only supported page index structure is [`PageIndex`]. If the metadata -/// contains any other [`PageIndexProvider`] implementation, the [`ColumnIndex`] -/// and [`OffsetIndex`] structures are silently omitted from the output. See -/// for more details. -/// -///
-/// /// # Output Format /// /// The format of the metadata is as follows: @@ -464,21 +470,10 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - // Downcast to PageIndex to access raw index structures for serialization. - // Page indexes from custom PageIndexProviders are not written. See - // if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && let Some(page_index) = page_index_arc - .as_any() - .downcast_ref::() + && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) { - if let Some(column_indexes) = page_index.column_indexes_raw() { - encoder = encoder.with_column_indexes(column_indexes.clone()); - } - - if let Some(offset_indexes) = page_index.offset_indexes_raw() { - encoder = encoder.with_offset_indexes(offset_indexes.clone()); - } + encoder = encoder.with_page_index(page_index_arc.clone()); } if let Some(key_value_metadata) = key_value_metadata { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 6c0f3e8dd283..18b7e6e9999e 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -18,6 +18,7 @@ //! [`SerializedFileWriter`]: Low level Parquet writer API use crate::bloom_filter::Sbbf; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::offset_index::OffsetIndexMetaData; @@ -377,10 +378,27 @@ impl SerializedFileWriter { encoder = encoder.with_key_value_metadata(key_value_metadata) } - encoder = encoder.with_column_indexes(column_indexes); - if !self.props.offset_index_disabled() { - encoder = encoder.with_offset_indexes(offset_indexes); + // check for empty column index + let column_indexes = if column_indexes.is_empty() + || column_indexes + .iter() + .all(|cis| cis.iter().all(|ci| ci.is_none())) + { + None + } else { + Some(column_indexes) + }; + // offset index will always be created unless explicitly disabled + let offset_indexes = if self.props.offset_index_disabled() { + None + } else { + Some(offset_indexes) + }; + if column_indexes.is_some() || offset_indexes.is_some() { + let page_index = PageIndex::new(column_indexes, offset_indexes); + encoder = encoder.with_page_index(Arc::new(page_index)); } + encoder.finish() } @@ -2201,6 +2219,55 @@ mod tests { assert!(b_idx.is_none(), "{b_idx:?}"); } + #[test] + fn test_offset_index_disabled() { + let message_type = " + message test_schema { + REQUIRED INT32 a; + REQUIRED INT32 b; + } + "; + // write file with indexes disabled (including offset indexes) + let schema = Arc::new(parse_message_type(message_type).unwrap()); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::None) + .set_offset_index_disabled(true) + .build(); + let mut file = Vec::with_capacity(1024); + let mut file_writer = + SerializedFileWriter::new(&mut file, schema, Arc::new(props)).unwrap(); + + let mut row_group_writer = file_writer.next_row_group().unwrap(); + let mut a_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = a_writer.typed::(); + col_writer.write_batch(&[1, 2, 3], None, None).unwrap(); + a_writer.close().unwrap(); + + let mut b_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = b_writer.typed::(); + col_writer.write_batch(&[4, 5, 6], None, None).unwrap(); + b_writer.close().unwrap(); + row_group_writer.close().unwrap(); + + let metadata = file_writer.finish().unwrap(); + assert_eq!(metadata.num_row_groups(), 1); + let row_group = metadata.row_group(0); + assert_eq!(row_group.num_columns(), 2); + // no page indexes should exist + assert!(row_group.column(0).offset_index_offset().is_none()); + assert!(row_group.column(0).column_index_offset().is_none()); + assert!(row_group.column(1).offset_index_offset().is_none()); + assert!(row_group.column(1).column_index_offset().is_none()); + + drop(file_writer); + + // read file and request page index...should be `None` + let options = ReadOptionsBuilder::new().with_page_index().build(); + let reader = SerializedFileReader::new_with_options(Bytes::from(file), options).unwrap(); + + assert!(reader.metadata().page_index().is_none()); + } + #[test] fn test_byte_array_size_statistics() { let message_type = " From b2d3a1a40b749a0fada964a49b52d9395913c73d Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:11:03 -0700 Subject: [PATCH 02/78] clippy --- parquet/src/file/metadata/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index d6de7e45c8c5..63cdf540d146 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -152,7 +152,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the column indexes and transform to `Option>>>` fn finalize_column_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -179,7 +179,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the offset indexes and transform to `Option` fn finalize_offset_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -209,8 +209,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // serialize page indexes and transform to the proper form for use in ParquetMetaData let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(&page_index)?; - let offset_indexes = self.finalize_offset_indexes(&page_index)?; + let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; + let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; // We only include ColumnOrder for leaf nodes. let column_orders = self From 0638b8715900d033907b5e745580d1afa4924069 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 9 Sep 2026 13:31:12 -0700 Subject: [PATCH 03/78] add test of using custom page index provider with row selection --- .../custom_page_index_provider.rs | 546 ++++++++++++++++++ parquet/tests/arrow_reader/mod.rs | 1 + 2 files changed, 547 insertions(+) create mode 100644 parquet/tests/arrow_reader/custom_page_index_provider.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs new file mode 100644 index 000000000000..00e008a13ac6 --- /dev/null +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for custom PageIndexProvider implementation. +//! +//! This test validates reading with a custom PageIndexProvider through the +//! actual read path, ensuring that: +//! - Columns with page indexes use offset-index-driven fetch +//! - Columns without page indexes fall back to whole-column-chunk fetching +//! - Correct results are returned in both cases + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, + RowSelectionPolicy, +}; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::index_reader::{decode_column_index, decode_offset_index}; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::sync::Arc; +use tempfile::NamedTempFile; + +/// A custom PageIndexProvider that only stores indexes for a subset of columns +/// +/// This provider mimics a real-world scenario where an application has loaded +/// page indexes selectively based on query predicates and projections. +#[derive(Debug, Clone)] +struct SelectivePageIndexProvider { + metadata: ParquetMetaData, + file_bytes: Bytes, + // indexes are accessed first by row_group index and then by column index + column_indexes: Option>>, + offset_indexes: Option>>, +} + +impl SelectivePageIndexProvider { + fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + Self { + metadata, + file_bytes, + column_indexes: None, + offset_indexes: None, + } + } + + /// Fetch and parse the column index for the given row group and column + fn fetch_column_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.column_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_column_index(&idx_bytes, column.column_type())?; + e.insert(idx); + } + } + Ok(()) + } + + /// Fetch and parse the offset index for the given row group and column + fn fetch_offset_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.offset_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_offset_index(&idx_bytes)?; + e.insert(idx); + } + } + Ok(()) + } +} + +impl PageIndexProvider for SelectivePageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.offset_indexes.is_some() + } + + fn has_column_indexes(&self) -> bool { + self.column_indexes.is_some() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Create a test parquet file with multiple row groups and multiple pages per column +fn create_test_file() -> NamedTempFile { + let temp_file = tempfile::Builder::new() + .prefix("custom_page_index_test") + .suffix(".parquet") + .tempfile() + .expect("tempfile creation"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + + // Create small pages and small row groups for testing + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_row_count_limit(10) // Small pages for multiple pages per column + .set_write_batch_size(10) + .set_max_row_group_row_count(Some(50)) // Small row groups + .build(); + + let file = temp_file.reopen().unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + + // Write 3 row groups with 50 rows each (150 rows total) + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = (row_group * 50) + (batch_num * 10); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (offset..offset + 10).collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + (offset..offset + 10) + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap(); + writer.write(&batch).unwrap(); + } + writer.flush().unwrap(); + } + + // check that file has properties we wanted (3 row groups, 5 pages per chunk) + let metadata = writer.close().unwrap(); + assert_eq!(metadata.num_row_groups(), 3); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for row_group_idx in 0..3 { + for column_idx in 0..4 { + assert!(page_index.column_index(row_group_idx, column_idx).is_some()); + assert!(page_index.offset_index(row_group_idx, column_idx).is_some()); + assert!( + page_index + .num_data_pages(row_group_idx, column_idx) + .is_some_and(|n| n == 5) + ); + } + } + temp_file +} + +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + provider.fetch_column_index(0, 1).unwrap(); + provider.fetch_offset_index(0, 1).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0).unwrap(); + provider.fetch_offset_index(2, 0).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); + + // Success! The test validates that: + // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping + // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data + // 3. For row group 1, no columns had indexes but all returned correct data + // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data + // 5. The reader correctly handled the mix of indexed and non-indexed columns +} + +#[test] +fn test_read_all_columns_with_partial_indexes() { + // This test focuses on verifying that columns without page indexes + // fall back to whole-column-chunk fetching and return correct results + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Load metadata without page indexes + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes ONLY for column 0 in all row groups + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + for row_group_idx in 0..metadata.num_row_groups() { + provider.fetch_column_index(row_group_idx, 0).unwrap(); + provider.fetch_offset_index(row_group_idx, 0).unwrap(); + } + + // Install provider + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read with row selection + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Select rows 10-20 from row group 0 + let selection = RowSelection::from(vec![ + parquet::arrow::arrow_reader::RowSelector::skip(10), + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let batches: Vec = reader.collect::, _>>().unwrap(); + + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + + // Verify all columns have correct data + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + let id = ids.value(i); + assert_eq!(values.value(i), id * 2); + assert_eq!(names.value(i), format!("name{id}")); + assert_eq!(scores.value(i), id * 3); + } + } +} + +#[test] +fn test_custom_provider_with_no_row_selection() { + // Test that reading without RowSelection also works correctly + // with a custom page index provider + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes for first column only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read all data + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + let reader = builder.build().unwrap(); + + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Should read all 150 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 150); + + // Verify data integrity + let mut row_count: i32 = 0; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + assert_eq!(ids.value(i), row_count); + assert_eq!(values.value(i), row_count * 2); + assert_eq!(names.value(i), format!("name{row_count}")); + assert_eq!(scores.value(i), row_count * 3); + row_count += 1; + } + } +} diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index c7c77e0a444c..6c47a00b9b3d 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -42,6 +42,7 @@ mod bad_data; mod bloom_filter; #[cfg(feature = "crc")] mod checksum; +mod custom_page_index_provider; mod int96_stats_roundtrip; mod invalid_utf8; mod io; From da1ff75a0409616fc2d443776176e1efd04cf22c Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 09:57:22 -0700 Subject: [PATCH 04/78] refactor and add usage stats --- .../custom_page_index_provider.rs | 582 ++++++++---------- 1 file changed, 248 insertions(+), 334 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 00e008a13ac6..2736b7bc35e2 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -41,41 +41,260 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fs::File; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use tempfile::NamedTempFile; +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().as_ref(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0, metadata).unwrap(); + provider.fetch_offset_index(0, 0, metadata).unwrap(); + provider.fetch_column_index(0, 1, metadata).unwrap(); + provider.fetch_offset_index(0, 1, metadata).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0, metadata).unwrap(); + provider.fetch_offset_index(2, 0, metadata).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Reset statistics so validation checks don't affect the final counts + provider.reset_stats(); + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = ArrowReaderMetadata::try_new( + metadata_with_custom_index.clone(), + ArrowReaderOptions::default(), + ) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Check statistics + // Note: We need to get the provider reference from the metadata to check stats + let provider_ref = metadata_with_custom_index + .page_index() + .unwrap() + .as_any() + .downcast_ref::() + .expect("Expected SelectivePageIndexProvider"); + + // Check that the custom provider was used + let (hits, misses) = provider_ref.stats(); + assert!(hits > 0, "provider should have hits"); + assert!(misses > 0, "provider should have misses"); + + // Since we didn't use predicates, the column index should be untouched + let (col_hits, col_misses) = provider_ref.column_index_stats(); + assert_eq!(col_hits, 0, "column index should not be used"); + assert_eq!(col_misses, 0, "column index should not be used"); + + let (off_hits, off_misses) = provider_ref.offset_index_stats(); + assert!(off_hits > 0, "offset index should have hits"); + assert!(off_misses > 0, "offset index should have misses"); + + // Step 8: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); +} + /// A custom PageIndexProvider that only stores indexes for a subset of columns /// /// This provider mimics a real-world scenario where an application has loaded /// page indexes selectively based on query predicates and projections. -#[derive(Debug, Clone)] +#[derive(Debug)] struct SelectivePageIndexProvider { - metadata: ParquetMetaData, file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index column_indexes: Option>>, offset_indexes: Option>>, + // Usage statistics + column_index_hits: Arc, + column_index_misses: Arc, + offset_index_hits: Arc, + offset_index_misses: Arc, } impl SelectivePageIndexProvider { - fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes) -> Self { Self { - metadata, file_bytes, column_indexes: None, offset_indexes: None, + column_index_hits: Arc::new(AtomicUsize::new(0)), + column_index_misses: Arc::new(AtomicUsize::new(0)), + offset_index_hits: Arc::new(AtomicUsize::new(0)), + offset_index_misses: Arc::new(AtomicUsize::new(0)), } } + /// Get statistics about column index usage + pub fn column_index_stats(&self) -> (usize, usize) { + ( + self.column_index_hits.load(Ordering::Relaxed), + self.column_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get statistics about offset index usage + pub fn offset_index_stats(&self) -> (usize, usize) { + ( + self.offset_index_hits.load(Ordering::Relaxed), + self.offset_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get combined statistics as (hits, misses) for both index types + pub fn stats(&self) -> (usize, usize) { + let (col_hits, col_misses) = self.column_index_stats(); + let (off_hits, off_misses) = self.offset_index_stats(); + (col_hits + off_hits, col_misses + off_misses) + } + + /// Reset all statistics counters to zero + pub fn reset_stats(&self) { + self.column_index_hits.store(0, Ordering::Relaxed); + self.column_index_misses.store(0, Ordering::Relaxed); + self.offset_index_hits.store(0, Ordering::Relaxed); + self.offset_index_misses.store(0, Ordering::Relaxed); + } + /// Fetch and parse the column index for the given row group and column fn fetch_column_index( &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.column_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.column_index_range(); if let Some(range) = range { let idx_bytes = self @@ -93,11 +312,12 @@ impl SelectivePageIndexProvider { &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.offset_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.offset_index_range(); if let Some(range) = range { let idx_bytes = self @@ -125,10 +345,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - self.column_indexes + let result = self + .column_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.column_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.column_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn offset_index( @@ -136,10 +365,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - self.offset_indexes + let result = self + .offset_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.offset_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.offset_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn as_any(&self) -> &dyn std::any::Any { @@ -220,327 +458,3 @@ fn create_test_file() -> NamedTempFile { } temp_file } - -#[test] -fn test_read_with_custom_page_index_provider() { - // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Verify we have multiple row groups - assert_eq!(metadata.num_row_groups(), 3); - - // Step 3: Create a custom provider and populate it selectively - // Simulate a scenario where: - // - We only populate indexes for row groups 0 and 2 (skipping row group 1) - // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - provider.fetch_column_index(0, 1).unwrap(); - provider.fetch_offset_index(0, 1).unwrap(); - - // Populate indexes for row group 2, column 0 only - provider.fetch_column_index(2, 0).unwrap(); - provider.fetch_offset_index(2, 0).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - - // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Step 5: Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Create a RowSelection that: - // - Selects rows 20-30 (in row group 0) - // - Selects rows 60-70 (in row group 1) - // - Selects rows 120-130 (in row group 2) - let selection = RowSelection::from(vec![ - // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), - // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), - // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), - // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Step 7: Verify correct results - // We expect 30 rows total: 10 from each selected range - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 30); - - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - - // Expected values for the selected rows: - // rows 20-30: id=20..30, value=40..60, score=60..90 - // rows 60-70: id=60..70, value=120..140, score=180..210 - // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); - assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" - ); - - // Success! The test validates that: - // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping - // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data - // 3. For row group 1, no columns had indexes but all returned correct data - // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data - // 5. The reader correctly handled the mix of indexed and non-indexed columns -} - -#[test] -fn test_read_all_columns_with_partial_indexes() { - // This test focuses on verifying that columns without page indexes - // fall back to whole-column-chunk fetching and return correct results - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Load metadata without page indexes - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes ONLY for column 0 in all row groups - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - for row_group_idx in 0..metadata.num_row_groups() { - provider.fetch_column_index(row_group_idx, 0).unwrap(); - provider.fetch_offset_index(row_group_idx, 0).unwrap(); - } - - // Install provider - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read with row selection - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Select rows 10-20 from row group 0 - let selection = RowSelection::from(vec![ - parquet::arrow::arrow_reader::RowSelector::skip(10), - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - let batches: Vec = reader.collect::, _>>().unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 10); - - // Verify all columns have correct data - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - let id = ids.value(i); - assert_eq!(values.value(i), id * 2); - assert_eq!(names.value(i), format!("name{id}")); - assert_eq!(scores.value(i), id * 3); - } - } -} - -#[test] -fn test_custom_provider_with_no_row_selection() { - // Test that reading without RowSelection also works correctly - // with a custom page index provider - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes for first column only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read all data - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - let reader = builder.build().unwrap(); - - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Should read all 150 rows - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 150); - - // Verify data integrity - let mut row_count: i32 = 0; - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - assert_eq!(ids.value(i), row_count); - assert_eq!(values.value(i), row_count * 2); - assert_eq!(names.value(i), format!("name{row_count}")); - assert_eq!(scores.value(i), row_count * 3); - row_count += 1; - } - } -} From a39dd8fdbf6e2b77001ece99e189cd5783b7b618 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 9 Sep 2026 13:31:12 -0700 Subject: [PATCH 05/78] add test of using custom page index provider with row selection --- .../custom_page_index_provider.rs | 546 ++++++++++++++++++ parquet/tests/arrow_reader/mod.rs | 1 + 2 files changed, 547 insertions(+) create mode 100644 parquet/tests/arrow_reader/custom_page_index_provider.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs new file mode 100644 index 000000000000..00e008a13ac6 --- /dev/null +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for custom PageIndexProvider implementation. +//! +//! This test validates reading with a custom PageIndexProvider through the +//! actual read path, ensuring that: +//! - Columns with page indexes use offset-index-driven fetch +//! - Columns without page indexes fall back to whole-column-chunk fetching +//! - Correct results are returned in both cases + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, + RowSelectionPolicy, +}; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::index_reader::{decode_column_index, decode_offset_index}; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::sync::Arc; +use tempfile::NamedTempFile; + +/// A custom PageIndexProvider that only stores indexes for a subset of columns +/// +/// This provider mimics a real-world scenario where an application has loaded +/// page indexes selectively based on query predicates and projections. +#[derive(Debug, Clone)] +struct SelectivePageIndexProvider { + metadata: ParquetMetaData, + file_bytes: Bytes, + // indexes are accessed first by row_group index and then by column index + column_indexes: Option>>, + offset_indexes: Option>>, +} + +impl SelectivePageIndexProvider { + fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + Self { + metadata, + file_bytes, + column_indexes: None, + offset_indexes: None, + } + } + + /// Fetch and parse the column index for the given row group and column + fn fetch_column_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.column_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_column_index(&idx_bytes, column.column_type())?; + e.insert(idx); + } + } + Ok(()) + } + + /// Fetch and parse the offset index for the given row group and column + fn fetch_offset_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.offset_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_offset_index(&idx_bytes)?; + e.insert(idx); + } + } + Ok(()) + } +} + +impl PageIndexProvider for SelectivePageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.offset_indexes.is_some() + } + + fn has_column_indexes(&self) -> bool { + self.column_indexes.is_some() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Create a test parquet file with multiple row groups and multiple pages per column +fn create_test_file() -> NamedTempFile { + let temp_file = tempfile::Builder::new() + .prefix("custom_page_index_test") + .suffix(".parquet") + .tempfile() + .expect("tempfile creation"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + + // Create small pages and small row groups for testing + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_row_count_limit(10) // Small pages for multiple pages per column + .set_write_batch_size(10) + .set_max_row_group_row_count(Some(50)) // Small row groups + .build(); + + let file = temp_file.reopen().unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + + // Write 3 row groups with 50 rows each (150 rows total) + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = (row_group * 50) + (batch_num * 10); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (offset..offset + 10).collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + (offset..offset + 10) + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap(); + writer.write(&batch).unwrap(); + } + writer.flush().unwrap(); + } + + // check that file has properties we wanted (3 row groups, 5 pages per chunk) + let metadata = writer.close().unwrap(); + assert_eq!(metadata.num_row_groups(), 3); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for row_group_idx in 0..3 { + for column_idx in 0..4 { + assert!(page_index.column_index(row_group_idx, column_idx).is_some()); + assert!(page_index.offset_index(row_group_idx, column_idx).is_some()); + assert!( + page_index + .num_data_pages(row_group_idx, column_idx) + .is_some_and(|n| n == 5) + ); + } + } + temp_file +} + +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + provider.fetch_column_index(0, 1).unwrap(); + provider.fetch_offset_index(0, 1).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0).unwrap(); + provider.fetch_offset_index(2, 0).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); + + // Success! The test validates that: + // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping + // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data + // 3. For row group 1, no columns had indexes but all returned correct data + // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data + // 5. The reader correctly handled the mix of indexed and non-indexed columns +} + +#[test] +fn test_read_all_columns_with_partial_indexes() { + // This test focuses on verifying that columns without page indexes + // fall back to whole-column-chunk fetching and return correct results + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Load metadata without page indexes + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes ONLY for column 0 in all row groups + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + for row_group_idx in 0..metadata.num_row_groups() { + provider.fetch_column_index(row_group_idx, 0).unwrap(); + provider.fetch_offset_index(row_group_idx, 0).unwrap(); + } + + // Install provider + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read with row selection + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Select rows 10-20 from row group 0 + let selection = RowSelection::from(vec![ + parquet::arrow::arrow_reader::RowSelector::skip(10), + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let batches: Vec = reader.collect::, _>>().unwrap(); + + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + + // Verify all columns have correct data + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + let id = ids.value(i); + assert_eq!(values.value(i), id * 2); + assert_eq!(names.value(i), format!("name{id}")); + assert_eq!(scores.value(i), id * 3); + } + } +} + +#[test] +fn test_custom_provider_with_no_row_selection() { + // Test that reading without RowSelection also works correctly + // with a custom page index provider + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes for first column only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read all data + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + let reader = builder.build().unwrap(); + + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Should read all 150 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 150); + + // Verify data integrity + let mut row_count: i32 = 0; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + assert_eq!(ids.value(i), row_count); + assert_eq!(values.value(i), row_count * 2); + assert_eq!(names.value(i), format!("name{row_count}")); + assert_eq!(scores.value(i), row_count * 3); + row_count += 1; + } + } +} diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index c7c77e0a444c..6c47a00b9b3d 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -42,6 +42,7 @@ mod bad_data; mod bloom_filter; #[cfg(feature = "crc")] mod checksum; +mod custom_page_index_provider; mod int96_stats_roundtrip; mod invalid_utf8; mod io; From 30631fa71dddaea0c7e90a40d851f229a97dd30e Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 09:57:22 -0700 Subject: [PATCH 06/78] refactor and add usage stats --- .../custom_page_index_provider.rs | 582 ++++++++---------- 1 file changed, 248 insertions(+), 334 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 00e008a13ac6..2736b7bc35e2 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -41,41 +41,260 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fs::File; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use tempfile::NamedTempFile; +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().as_ref(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0, metadata).unwrap(); + provider.fetch_offset_index(0, 0, metadata).unwrap(); + provider.fetch_column_index(0, 1, metadata).unwrap(); + provider.fetch_offset_index(0, 1, metadata).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0, metadata).unwrap(); + provider.fetch_offset_index(2, 0, metadata).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Reset statistics so validation checks don't affect the final counts + provider.reset_stats(); + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = ArrowReaderMetadata::try_new( + metadata_with_custom_index.clone(), + ArrowReaderOptions::default(), + ) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Check statistics + // Note: We need to get the provider reference from the metadata to check stats + let provider_ref = metadata_with_custom_index + .page_index() + .unwrap() + .as_any() + .downcast_ref::() + .expect("Expected SelectivePageIndexProvider"); + + // Check that the custom provider was used + let (hits, misses) = provider_ref.stats(); + assert!(hits > 0, "provider should have hits"); + assert!(misses > 0, "provider should have misses"); + + // Since we didn't use predicates, the column index should be untouched + let (col_hits, col_misses) = provider_ref.column_index_stats(); + assert_eq!(col_hits, 0, "column index should not be used"); + assert_eq!(col_misses, 0, "column index should not be used"); + + let (off_hits, off_misses) = provider_ref.offset_index_stats(); + assert!(off_hits > 0, "offset index should have hits"); + assert!(off_misses > 0, "offset index should have misses"); + + // Step 8: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); +} + /// A custom PageIndexProvider that only stores indexes for a subset of columns /// /// This provider mimics a real-world scenario where an application has loaded /// page indexes selectively based on query predicates and projections. -#[derive(Debug, Clone)] +#[derive(Debug)] struct SelectivePageIndexProvider { - metadata: ParquetMetaData, file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index column_indexes: Option>>, offset_indexes: Option>>, + // Usage statistics + column_index_hits: Arc, + column_index_misses: Arc, + offset_index_hits: Arc, + offset_index_misses: Arc, } impl SelectivePageIndexProvider { - fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes) -> Self { Self { - metadata, file_bytes, column_indexes: None, offset_indexes: None, + column_index_hits: Arc::new(AtomicUsize::new(0)), + column_index_misses: Arc::new(AtomicUsize::new(0)), + offset_index_hits: Arc::new(AtomicUsize::new(0)), + offset_index_misses: Arc::new(AtomicUsize::new(0)), } } + /// Get statistics about column index usage + pub fn column_index_stats(&self) -> (usize, usize) { + ( + self.column_index_hits.load(Ordering::Relaxed), + self.column_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get statistics about offset index usage + pub fn offset_index_stats(&self) -> (usize, usize) { + ( + self.offset_index_hits.load(Ordering::Relaxed), + self.offset_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get combined statistics as (hits, misses) for both index types + pub fn stats(&self) -> (usize, usize) { + let (col_hits, col_misses) = self.column_index_stats(); + let (off_hits, off_misses) = self.offset_index_stats(); + (col_hits + off_hits, col_misses + off_misses) + } + + /// Reset all statistics counters to zero + pub fn reset_stats(&self) { + self.column_index_hits.store(0, Ordering::Relaxed); + self.column_index_misses.store(0, Ordering::Relaxed); + self.offset_index_hits.store(0, Ordering::Relaxed); + self.offset_index_misses.store(0, Ordering::Relaxed); + } + /// Fetch and parse the column index for the given row group and column fn fetch_column_index( &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.column_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.column_index_range(); if let Some(range) = range { let idx_bytes = self @@ -93,11 +312,12 @@ impl SelectivePageIndexProvider { &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.offset_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.offset_index_range(); if let Some(range) = range { let idx_bytes = self @@ -125,10 +345,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - self.column_indexes + let result = self + .column_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.column_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.column_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn offset_index( @@ -136,10 +365,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - self.offset_indexes + let result = self + .offset_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.offset_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.offset_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn as_any(&self) -> &dyn std::any::Any { @@ -220,327 +458,3 @@ fn create_test_file() -> NamedTempFile { } temp_file } - -#[test] -fn test_read_with_custom_page_index_provider() { - // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Verify we have multiple row groups - assert_eq!(metadata.num_row_groups(), 3); - - // Step 3: Create a custom provider and populate it selectively - // Simulate a scenario where: - // - We only populate indexes for row groups 0 and 2 (skipping row group 1) - // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - provider.fetch_column_index(0, 1).unwrap(); - provider.fetch_offset_index(0, 1).unwrap(); - - // Populate indexes for row group 2, column 0 only - provider.fetch_column_index(2, 0).unwrap(); - provider.fetch_offset_index(2, 0).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - - // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Step 5: Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Create a RowSelection that: - // - Selects rows 20-30 (in row group 0) - // - Selects rows 60-70 (in row group 1) - // - Selects rows 120-130 (in row group 2) - let selection = RowSelection::from(vec![ - // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), - // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), - // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), - // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Step 7: Verify correct results - // We expect 30 rows total: 10 from each selected range - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 30); - - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - - // Expected values for the selected rows: - // rows 20-30: id=20..30, value=40..60, score=60..90 - // rows 60-70: id=60..70, value=120..140, score=180..210 - // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); - assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" - ); - - // Success! The test validates that: - // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping - // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data - // 3. For row group 1, no columns had indexes but all returned correct data - // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data - // 5. The reader correctly handled the mix of indexed and non-indexed columns -} - -#[test] -fn test_read_all_columns_with_partial_indexes() { - // This test focuses on verifying that columns without page indexes - // fall back to whole-column-chunk fetching and return correct results - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Load metadata without page indexes - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes ONLY for column 0 in all row groups - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - for row_group_idx in 0..metadata.num_row_groups() { - provider.fetch_column_index(row_group_idx, 0).unwrap(); - provider.fetch_offset_index(row_group_idx, 0).unwrap(); - } - - // Install provider - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read with row selection - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Select rows 10-20 from row group 0 - let selection = RowSelection::from(vec![ - parquet::arrow::arrow_reader::RowSelector::skip(10), - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - let batches: Vec = reader.collect::, _>>().unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 10); - - // Verify all columns have correct data - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - let id = ids.value(i); - assert_eq!(values.value(i), id * 2); - assert_eq!(names.value(i), format!("name{id}")); - assert_eq!(scores.value(i), id * 3); - } - } -} - -#[test] -fn test_custom_provider_with_no_row_selection() { - // Test that reading without RowSelection also works correctly - // with a custom page index provider - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes for first column only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read all data - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - let reader = builder.build().unwrap(); - - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Should read all 150 rows - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 150); - - // Verify data integrity - let mut row_count: i32 = 0; - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - assert_eq!(ids.value(i), row_count); - assert_eq!(values.value(i), row_count * 2); - assert_eq!(names.value(i), format!("name{row_count}")); - assert_eq!(scores.value(i), row_count * 3); - row_count += 1; - } - } -} From 5458721eebe9a5af19f7fdeb89de8726de7e9379 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:00:56 -0700 Subject: [PATCH 07/78] allow for serialization of custom page index providers --- parquet/src/file/metadata/page_index.rs | 21 +--- parquet/src/file/metadata/writer.rs | 121 ++++++++++++------------ parquet/src/file/writer.rs | 73 +++++++++++++- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9ded57d0a28e..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -460,22 +460,6 @@ impl PageIndex { pub fn into_builder(self) -> PageIndexBuilder { self.into() } - - /// Returns a reference to the raw column indexes structure - /// - /// This method provides access to the underlying column index data for serialization - /// and other low-level operations. - pub(crate) fn column_indexes_raw(&self) -> Option<&Vec>>> { - self.column_indexes.as_ref() - } - - /// Returns a reference to the raw offset indexes structure - /// - /// This method provides access to the underlying offset index data for serialization - /// and other low-level operations. - pub(crate) fn offset_indexes_raw(&self) -> Option<&Vec>>> { - self.offset_indexes.as_ref() - } } impl PageIndexProvider for PageIndex { @@ -536,7 +520,10 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { + pub(crate) fn empty_index( + num_row_groups: usize, + num_columns: usize, + ) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a1708ccf2c3a..d6de7e45c8c5 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::FileMeta; use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; @@ -54,8 +55,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { buf: &'a mut TrackedWrite, schema_descr: &'a SchemaDescPtr, row_groups: Vec, - column_indexes: Option>>>, - offset_indexes: Option>>>, + page_index: Option>, key_value_metadata: Option>, created_by: Option, object_writer: MetadataObjectWriter, @@ -71,14 +71,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized offset indexes. fn write_offset_indexes( &mut self, - offset_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut offset_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let offidx_vec = offset_indexes.as_mut().unwrap(); + + // we've already checked before calling that the offset indexes are populated + assert!(page_index.has_offset_indexes()); + // iter row group // iter each column // write offset index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(offset_index) = &offset_indexes[row_group_idx][column_idx] { + if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); self.object_writer.write_offset_index( offset_index, @@ -91,10 +98,11 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); + offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); } } } - Ok(()) + Ok(offset_indexes) } /// Serialize all the column indexes to the `self.buf` @@ -104,14 +112,21 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// of the serialized column indexes. fn write_column_indexes( &mut self, - column_indexes: &[Vec>], - ) -> Result<()> { + page_index: &Arc, + ) -> Result>>>> { + let mut column_indexes = + PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); + let colidx_vec = column_indexes.as_mut().unwrap(); + + // we've already checked before calling that the column indexes are populated + assert!(page_index.has_column_indexes()); + // iter row group // iter each column // write column index to the file for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() { for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() { - if let Some(column_index) = &column_indexes[row_group_idx][column_idx] { + if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) { let start_offset = self.buf.bytes_written(); // only update column_metadata if the write succeeds if self.object_writer.write_column_index( @@ -127,21 +142,28 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } + colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); } } } - Ok(()) + Ok(column_indexes) } - /// Serialize the column indexes and transform to `Option` - fn finalize_column_indexes(&mut self) -> Result>>>> { - let column_indexes = std::mem::take(&mut self.column_indexes); - - // Write column indexes to file - if let Some(column_indexes) = column_indexes.as_ref() { - self.write_column_indexes(column_indexes)?; + /// Serialize the column indexes and transform to `Option>>>` + fn finalize_column_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_column_indexes()) + { + return Ok(None); } + // Write column indexes to file while assembling a column index + let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; + // check to see if the index is `None` for every row group and column chunk let all_none = column_indexes .as_ref() @@ -155,13 +177,19 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes(&mut self) -> Result>>>> { - let offset_indexes = std::mem::take(&mut self.offset_indexes); + fn finalize_offset_indexes( + &mut self, + page_index: &Option>, + ) -> Result>>>> { + if page_index + .as_ref() + .is_none_or(|pi| !pi.has_offset_indexes()) + { + return Ok(None); + } // Write offset indexes to file - if let Some(offset_indexes) = offset_indexes.as_ref() { - self.write_offset_indexes(offset_indexes)?; - } + let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; // check to see if the index is `None` for every row group and column chunk let all_none = offset_indexes @@ -180,8 +208,9 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); // serialize page indexes and transform to the proper form for use in ParquetMetaData - let column_indexes = self.finalize_column_indexes()?; - let offset_indexes = self.finalize_offset_indexes()?; + let page_index = self.page_index.take(); + let column_indexes = self.finalize_column_indexes(&page_index)?; + let offset_indexes = self.finalize_offset_indexes(&page_index)?; // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -275,8 +304,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { buf, schema_descr, row_groups, - column_indexes: None, - offset_indexes: None, + page_index: None, key_value_metadata: None, created_by, object_writer: Default::default(), @@ -285,19 +313,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } } - pub fn with_column_indexes( - mut self, - column_indexes: Vec>>, - ) -> Self { - self.column_indexes = Some(column_indexes); - self - } - - pub fn with_offset_indexes( - mut self, - offset_indexes: Vec>>, - ) -> Self { - self.offset_indexes = Some(offset_indexes); + pub fn with_page_index(mut self, page_index: Arc) -> Self { + self.page_index = Some(page_index); self } @@ -333,17 +350,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -///
-/// -/// **NOTE:** -/// The serialization of custom [`PageIndexProvider`]s is not currently supported. -/// The only supported page index structure is [`PageIndex`]. If the metadata -/// contains any other [`PageIndexProvider`] implementation, the [`ColumnIndex`] -/// and [`OffsetIndex`] structures are silently omitted from the output. See -/// for more details. -/// -///
-/// /// # Output Format /// /// The format of the metadata is as follows: @@ -464,21 +470,10 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - // Downcast to PageIndex to access raw index structures for serialization. - // Page indexes from custom PageIndexProviders are not written. See - // if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && let Some(page_index) = page_index_arc - .as_any() - .downcast_ref::() + && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) { - if let Some(column_indexes) = page_index.column_indexes_raw() { - encoder = encoder.with_column_indexes(column_indexes.clone()); - } - - if let Some(offset_indexes) = page_index.offset_indexes_raw() { - encoder = encoder.with_offset_indexes(offset_indexes.clone()); - } + encoder = encoder.with_page_index(page_index_arc.clone()); } if let Some(key_value_metadata) = key_value_metadata { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 6c0f3e8dd283..18b7e6e9999e 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -18,6 +18,7 @@ //! [`SerializedFileWriter`]: Low level Parquet writer API use crate::bloom_filter::Sbbf; +use crate::file::metadata::page_index::PageIndex; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::offset_index::OffsetIndexMetaData; @@ -377,10 +378,27 @@ impl SerializedFileWriter { encoder = encoder.with_key_value_metadata(key_value_metadata) } - encoder = encoder.with_column_indexes(column_indexes); - if !self.props.offset_index_disabled() { - encoder = encoder.with_offset_indexes(offset_indexes); + // check for empty column index + let column_indexes = if column_indexes.is_empty() + || column_indexes + .iter() + .all(|cis| cis.iter().all(|ci| ci.is_none())) + { + None + } else { + Some(column_indexes) + }; + // offset index will always be created unless explicitly disabled + let offset_indexes = if self.props.offset_index_disabled() { + None + } else { + Some(offset_indexes) + }; + if column_indexes.is_some() || offset_indexes.is_some() { + let page_index = PageIndex::new(column_indexes, offset_indexes); + encoder = encoder.with_page_index(Arc::new(page_index)); } + encoder.finish() } @@ -2201,6 +2219,55 @@ mod tests { assert!(b_idx.is_none(), "{b_idx:?}"); } + #[test] + fn test_offset_index_disabled() { + let message_type = " + message test_schema { + REQUIRED INT32 a; + REQUIRED INT32 b; + } + "; + // write file with indexes disabled (including offset indexes) + let schema = Arc::new(parse_message_type(message_type).unwrap()); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::None) + .set_offset_index_disabled(true) + .build(); + let mut file = Vec::with_capacity(1024); + let mut file_writer = + SerializedFileWriter::new(&mut file, schema, Arc::new(props)).unwrap(); + + let mut row_group_writer = file_writer.next_row_group().unwrap(); + let mut a_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = a_writer.typed::(); + col_writer.write_batch(&[1, 2, 3], None, None).unwrap(); + a_writer.close().unwrap(); + + let mut b_writer = row_group_writer.next_column().unwrap().unwrap(); + let col_writer = b_writer.typed::(); + col_writer.write_batch(&[4, 5, 6], None, None).unwrap(); + b_writer.close().unwrap(); + row_group_writer.close().unwrap(); + + let metadata = file_writer.finish().unwrap(); + assert_eq!(metadata.num_row_groups(), 1); + let row_group = metadata.row_group(0); + assert_eq!(row_group.num_columns(), 2); + // no page indexes should exist + assert!(row_group.column(0).offset_index_offset().is_none()); + assert!(row_group.column(0).column_index_offset().is_none()); + assert!(row_group.column(1).offset_index_offset().is_none()); + assert!(row_group.column(1).column_index_offset().is_none()); + + drop(file_writer); + + // read file and request page index...should be `None` + let options = ReadOptionsBuilder::new().with_page_index().build(); + let reader = SerializedFileReader::new_with_options(Bytes::from(file), options).unwrap(); + + assert!(reader.metadata().page_index().is_none()); + } + #[test] fn test_byte_array_size_statistics() { let message_type = " From 979141a093023289d3041bc72a6dd4bbca23f5fd Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 8 Sep 2026 15:11:03 -0700 Subject: [PATCH 08/78] clippy --- parquet/src/file/metadata/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index d6de7e45c8c5..63cdf540d146 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -152,7 +152,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the column indexes and transform to `Option>>>` fn finalize_column_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -179,7 +179,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Serialize the offset indexes and transform to `Option` fn finalize_offset_indexes( &mut self, - page_index: &Option>, + page_index: Option<&Arc>, ) -> Result>>>> { if page_index .as_ref() @@ -209,8 +209,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // serialize page indexes and transform to the proper form for use in ParquetMetaData let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(&page_index)?; - let offset_indexes = self.finalize_offset_indexes(&page_index)?; + let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; + let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; // We only include ColumnOrder for leaf nodes. let column_orders = self From 3a8ee6ae551004b2993de8d3483e4ee309454cae Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 17:43:51 -0700 Subject: [PATCH 09/78] add plumbing for page index selective population --- parquet/benches/metadata.rs | 28 ++- parquet/src/arrow/arrow_reader/mod.rs | 56 ++++- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 16 ++ parquet/src/file/metadata/parser.rs | 68 +++--- parquet/src/file/metadata/push_decoder.rs | 221 +++++++++++++++++--- parquet/src/file/metadata/reader.rs | 108 ++++++++++ parquet/src/file/page_index/index_reader.rs | 11 - parquet/tests/arrow_reader/io/mod.rs | 4 +- 10 files changed, 441 insertions(+), 77 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 0ec1b3cd1733..d9fef3d1a5d0 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, ParquetMetaData, - ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, - RowGroupMetaData, + ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, + PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; use parquet::file::writer::TrackedWrite; @@ -191,6 +191,28 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&data) + .unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Required); + c.bench_function("read page index", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + c.bench_function("read page index reduced columns", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + let meta_data = get_footer_bytes(data.clone()); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata", |b| { diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 73bcb2048ad5..8f8953e69933 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,8 +45,8 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, - ParquetStatisticsPolicy, RowGroupMetaData, + PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; use crate::schema::types::SchemaDescriptor; @@ -593,8 +593,10 @@ pub struct ArrowReaderOptions { /// [ARROW_SCHEMA_META_KEY]: crate::arrow::ARROW_SCHEMA_META_KEY supplied_schema: Option, - pub(crate) column_index: PageIndexPolicy, - pub(crate) offset_index: PageIndexPolicy, + column_index: PageIndexPolicy, + offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -774,6 +776,32 @@ impl ArrowReaderOptions { self } + /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// + /// The column index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when filtering by a predicate + /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// + /// The offset index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when projecting a small subset + /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet /// footer will be skipped. /// @@ -924,6 +952,20 @@ impl ArrowReaderOptions { self.column_index } + /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// + /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. + pub fn offset_index_selection(&self) -> &PageIndexSelection { + &self.offset_index_selection + } + + /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// + /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. + pub fn column_index_selection(&self) -> &PageIndexSelection { + &self.column_index_selection + } + /// Retrieve the currently set metadata decoding options. pub fn metadata_options(&self) -> &ParquetMetaDataOptions { &self.metadata_options @@ -970,7 +1012,9 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self @@ -1019,6 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index 4bd65cfead59..a73d2c0d3fbe 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,7 +242,9 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 166a7eb7ab6a..65c084108c7f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, ParquetMetaDataReader}; +pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..6a5755fec3a7 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,6 +17,7 @@ //! Page Index structures for efficient page-level skipping +use crate::file::metadata::PageIndexSelection; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -558,6 +559,21 @@ impl PageIndexBuilder { } } + /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy + /// + /// For sparse indexes, this can save a great deal of memory + pub fn new_with_selection( + num_row_groups: usize, + num_columns: usize, + _column_index_selection: PageIndexSelection, + _offset_index_selection: PageIndexSelection, + ) -> Self { + Self { + column_indexes: Self::empty_index(num_row_groups, num_columns), + offset_indexes: Self::empty_index(num_row_groups, num_columns), + } + } + /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 861bb2e53b08..b2a35172e8af 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,13 +26,15 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use bytes::Bytes; +use crate::file::reader::ChunkReader; +use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -250,8 +252,9 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - bytes: &Bytes, - start_offset: u64, + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip { @@ -259,25 +262,28 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = PageIndexBuilder::new_with_selection( + num_row_groups, + num_columns, + column_index_selection.clone(), + offset_index_selection.clone(), + ); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, + column_index_selection, &mut builder, bytes, - start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, + offset_index_selection, &mut builder, bytes, - start_offset, )?; } @@ -293,27 +299,27 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_column_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -325,27 +331,27 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_offset_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index c5e77cc22468..b55468fd2980 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -21,8 +21,10 @@ use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; -use crate::file::metadata::{FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions}; -use crate::file::page_index::index_reader::acc_range; +use crate::file::metadata::{ + ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, +}; use crate::file::reader::ChunkReader; use bytes::Bytes; use std::ops::Range; @@ -226,6 +228,10 @@ pub struct ParquetMetaDataPushDecoder { column_index_policy: PageIndexPolicy, /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, + /// which rows and columns of the column index should be read + column_index_selection: PageIndexSelection, + /// which rows and columns of the offset index should be read + offset_index_selection: PageIndexSelection, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -250,6 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, + column_index_selection: PageIndexSelection::all(), + offset_index_selection: PageIndexSelection::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -300,6 +308,25 @@ impl ParquetMetaDataPushDecoder { self } + /// Select the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection.clone(); + self.offset_index_selection = selection; + self + } + + /// Select the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Select the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Set the options to use when decoding the Parquet metadata. pub fn with_metadata_options(mut self, options: Option>) -> Self { self.metadata_parser = self.metadata_parser.with_metadata_options(options); @@ -408,30 +435,36 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let range = range_for_page_index( + let ranges = ranges_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_selection, + &self.offset_index_selection, ); - let Some(page_index_range) = range else { + if ranges.is_empty() { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - }; + } + + let needed_ranges = ranges + .into_iter() + .filter(|r| !self.buffers.has_range(r)) + .collect::>(); - if !self.buffers.has_range(&page_index_range) { + if !needed_ranges.is_empty() { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + return Ok(DecodeResult::NeedsData(needed_ranges)); } - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, - &buffer, - offset, + &self.column_index_selection, + &self.offset_index_selection, + &self.buffers, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); @@ -481,25 +514,81 @@ enum DecodeState { Intermediate, } -/// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// Returns the minimum set of non-overlapping ranges needed to cover the requested +/// offset and column indexes. +/// +/// If no page indexes are present in the file, or none are actually +/// requested by the policies passed in, this will return an empty +/// vector. /// -/// Returns None if no page indexes are needed -pub fn range_for_page_index( +/// This may result in more ranges to fetch but may also result in fewer bytes +/// being read (and stored). To get a single range for all indexes, sort +/// the resultant vector by the range starts, and then create a single range +/// using `start` from the head and `end` from the tail. +/// +/// ```rust +/// # use core::ops::Range; +/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { +/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// let range = (ranges.first()?.start..ranges.last()?.end); +/// Some(range) +/// # } +/// ``` +pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, -) -> Option> { - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, +) -> Vec> { + let mut result = Vec::new(); + + fn add_ranges( + metadata: &ParquetMetaData, + selection: &PageIndexSelection, + ranges: &mut Vec>, + f: T, + ) where + T: Fn(&ColumnChunkMetaData) -> Option>, + { + for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { + if selection.includes_row_group(rg_idx) { + for (col_idx, col) in rg.columns().iter().enumerate() { + if selection.includes_column(col_idx) { + if let Some(range) = f(col) { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); + } + } + } + } + } } } - range + + if column_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + column_index_selection, + &mut result, + ColumnChunkMetaData::column_index_range, + ); + } + if offset_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + offset_index_selection, + &mut result, + ColumnChunkMetaData::offset_index_range, + ); + } + result } // These tests use the arrow writer to create a parquet file in memory @@ -651,6 +740,90 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } + /// Decode the metadata incrementally, with a partial page index + #[test] + fn test_metadata_decoder_incremental_partial_page_index() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + + /// Decode the metadata incrementally, with a partial page index, but coalesced + /// range request. + #[test] + fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let mut ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + // collapse ranges into a single range + ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + let range = ranges.first().unwrap().start..ranges.last().unwrap().end; + push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e4cec0fd0df5..c3ce6ab6ce23 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -27,6 +27,7 @@ use crate::file::metadata::{ use crate::file::reader::ChunkReader; use crate::schema::types::SchemaDescriptor; use bytes::Bytes; +use std::collections::BTreeSet; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -71,6 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -104,6 +107,89 @@ impl From for PageIndexPolicy { } } +/// Selects the row groups and columns for which page indexes are read. +/// +/// This is independent of [`PageIndexPolicy`], which controls whether selected +/// indexes are optional or required. By default, all row groups and columns are selected. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct PageIndexSelection { + // using i32 because that's how thrift vectors are sized + row_groups: Option>>, + columns: Option>>, +} + +// TODO(ets): add unit tests for PageIndexSelection +impl PageIndexSelection { + /// Select all row groups and columns. + pub fn all() -> Self { + Self::default() + } + + /// Select page indexes for only the listed columns. + /// + /// Any indices in `columns` that are less than zero will be ignored. + pub fn columns(columns: impl IntoIterator) -> Self { + Self { + row_groups: None, + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + /// Select page indexes for only the listed row groups. + /// + /// Any indices in `row_groups` that are less than zero will be ignored. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: None, + } + } + + /// Select page indexes for only the listed row groups and columns. + /// + /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + pub fn row_groups_and_columns( + row_groups: impl IntoIterator, + columns: impl IntoIterator, + ) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + // test if `idx` is in the row group set. returns false if idx > i32::MAX + pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.row_groups + .as_ref() + .is_none_or(|keep| keep.contains(&idx)) + } + + // test if `idx` is in the column set. returns false if idx > i32::MAX + pub(crate) fn includes_column(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + } + + // FIXME(ets): these will be used later + /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { + self.row_groups.as_deref() + } + + pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { + self.columns.as_deref() + }*/ +} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -137,6 +223,24 @@ impl ParquetMetaDataReader { self } + /// Selects the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { + self.with_column_index_selection(selection.clone()) + .with_offset_index_selection(selection) + } + + /// Selects the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Selects the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Sets the [`ParquetMetaDataOptions`] to use when decoding pub fn with_metadata_options(mut self, options: Option) -> Self { self.metadata_options = options.map(Arc::new); @@ -341,6 +445,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -488,6 +594,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/src/file/page_index/index_reader.rs b/parquet/src/file/page_index/index_reader.rs index 963ee0ea9782..235f0b241887 100644 --- a/parquet/src/file/page_index/index_reader.rs +++ b/parquet/src/file/page_index/index_reader.rs @@ -30,17 +30,6 @@ use crate::parquet_thrift::{ }; use crate::thrift_struct; use std::io::Write; -use std::ops::Range; - -/// Computes the covering range of two optional ranges -/// -/// For example `acc_range(Some(7..9), Some(1..3)) = Some(1..9)` -pub(crate) fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } -} /// Decode a Thrift [`OffsetIndex`] from the provided bytes. /// diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 3fb63ffdb87b..2f0348b835f5 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -132,7 +132,9 @@ impl AsyncFileReader for TestReader { if let Some(options) = options { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self.metadata = Some(Arc::new( From 1df722eb61f2a7b92f6a7f1e9c083228a0f99ea1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 13:23:30 -0700 Subject: [PATCH 10/78] more tests --- .../custom_page_index_provider.rs | 2 +- parquet/tests/arrow_reader/mod.rs | 1 + parquet/tests/arrow_reader/page_index.rs | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 parquet/tests/arrow_reader/page_index.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 2736b7bc35e2..5846db69b7a1 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -386,7 +386,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> NamedTempFile { +pub(super) fn create_test_file() -> NamedTempFile { let temp_file = tempfile::Builder::new() .prefix("custom_page_index_test") .suffix(".parquet") diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index 6c47a00b9b3d..867b69d7bbf5 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -47,6 +47,7 @@ mod int96_stats_roundtrip; mod invalid_utf8; mod io; mod large_string_overflow; +mod page_index; mod parquet_testing; #[cfg(feature = "async")] mod predicate_cache; diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs new file mode 100644 index 000000000000..821d1d4cc55d --- /dev/null +++ b/parquet/tests/arrow_reader/page_index.rs @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests of selective page index population + +use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use std::fs::File; + +use crate::custom_page_index_provider::create_test_file; + +#[test] +fn test_parse_selected_columns() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_none()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_none()); + } +} + +#[test] +fn test_parse_selected_columns_mixed() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_some()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups() { + // test populating PageIndex with a subset of row groups + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate indexes for row groups 0 and 2 + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) + .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + + for col in 0..num_cols { + // row group 0 has both indexes + assert!(page_index.column_index(0, col).is_some()); + assert!(page_index.offset_index(0, col).is_some()); + // row group 1 has no indexes + assert!(page_index.column_index(1, col).is_none()); + assert!(page_index.offset_index(1, col).is_none()); + // row group 2 has both indexes + assert!(page_index.column_index(2, col).is_some()); + assert!(page_index.offset_index(2, col).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups_and_columns() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + + // row groups 0 and 2 should have no indexes + [0, 2].into_iter().for_each(|i| { + let rg_idx = metadata.page_index_for_row_group(i); + for col in 0..num_cols { + assert!(rg_idx.column_index(col).is_none()); + assert!(rg_idx.offset_index(col).is_none()); + } + }); + + // row group 1 should have column index for column 0 and offset index for columns 0 & 2 + let rg_idx = metadata.page_index_for_row_group(1); + assert!(rg_idx.column_index(0).is_some()); + assert!(rg_idx.offset_index(0).is_some()); + assert!(rg_idx.column_index(1).is_none()); + assert!(rg_idx.offset_index(1).is_none()); + assert!(rg_idx.column_index(2).is_none()); + assert!(rg_idx.offset_index(2).is_some()); + assert!(rg_idx.column_index(3).is_none()); + assert!(rg_idx.offset_index(3).is_none()); +} + +#[test] +fn test_page_index_sizes() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // no index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_none()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 7393); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 7817); + + // full index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 13897); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 14321); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12776); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 13200); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12056); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 12480); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 11326); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 11750); +} From 9eacc746f0aa604387a4c7aa0bb50a60fed76128 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:05:59 -0700 Subject: [PATCH 11/78] rename PageIndexSelection to ColumnChunkMask --- parquet/benches/metadata.rs | 8 +-- parquet/src/arrow/arrow_reader/mod.rs | 46 +++++++------- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 8 +-- parquet/src/file/metadata/parser.rs | 29 +++++---- parquet/src/file/metadata/push_decoder.rs | 76 +++++++++++------------ parquet/src/file/metadata/reader.rs | 55 +++++++++------- parquet/tests/arrow_reader/io/mod.rs | 4 +- parquet/tests/arrow_reader/page_index.rs | 30 ++++----- 10 files changed, 135 insertions(+), 127 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index d9fef3d1a5d0..b5ee07c49530 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, - PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ColumnChunkMask, ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, + PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; @@ -205,8 +205,8 @@ fn criterion_benchmark(c: &mut Criterion) { let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); c.bench_function("read page index reduced columns", |b| { b.iter(|| { reader.read_page_indexes(&data).unwrap(); diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 8f8953e69933..492ea6063462 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,7 +45,7 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; @@ -595,8 +595,8 @@ pub struct ArrowReaderOptions { column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -776,29 +776,29 @@ impl ArrowReaderOptions { self } - /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when filtering by a predicate - /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } - /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [OffsetIndex] structure. /// /// The offset index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when projecting a small subset - /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -952,18 +952,18 @@ impl ArrowReaderOptions { self.column_index } - /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// Retrieve the currently set [`ColumnChunkMask`] for the offset index. /// - /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. - pub fn offset_index_selection(&self) -> &PageIndexSelection { - &self.offset_index_selection + /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask]. + pub fn offset_index_mask(&self) -> &ColumnChunkMask { + &self.offset_index_mask } - /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// Retrieve the currently set [`ColumnChunkMask`] for the column index. /// - /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. - pub fn column_index_selection(&self) -> &PageIndexSelection { - &self.column_index_selection + /// This can be set via [`with_column_index_mask`][Self::with_column_index_mask]. + pub fn column_index_mask(&self) -> &ColumnChunkMask { + &self.column_index_mask } /// Retrieve the currently set metadata decoding options. @@ -1013,8 +1013,8 @@ impl ParquetMetaDataReader { self = self .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self @@ -1063,8 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()) + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index a73d2c0d3fbe..c7db5d0abbc1 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -243,8 +243,8 @@ impl AsyncFileReader for ParquetObjectReader { metadata = metadata .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 65c084108c7f..12162a7f8f85 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 6a5755fec3a7..ea1b17f3af38 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,7 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::PageIndexSelection; +use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -562,11 +562,11 @@ impl PageIndexBuilder { /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy /// /// For sparse indexes, this can save a great deal of memory - pub fn new_with_selection( + pub fn new_with_mask( num_row_groups: usize, num_columns: usize, - _column_index_selection: PageIndexSelection, - _offset_index_selection: PageIndexSelection, + _column_index_mask: ColumnChunkMask, + _offset_index_mask: ColumnChunkMask, ) -> Self { Self { column_indexes: Self::empty_index(num_row_groups, num_columns), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index b2a35172e8af..47b93463147c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,8 +26,7 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; @@ -252,8 +251,8 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip @@ -262,17 +261,17 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_selection( + let mut builder = PageIndexBuilder::new_with_mask( num_row_groups, num_columns, - column_index_selection.clone(), - offset_index_selection.clone(), + column_index_mask.clone(), + offset_index_mask.clone(), ); if column_index_policy != PageIndexPolicy::Skip { parse_column_index( metadata, column_index_policy, - column_index_selection, + column_index_mask, &mut builder, bytes, )?; @@ -281,7 +280,7 @@ pub(crate) fn parse_page_index( parse_offset_index( metadata, offset_index_policy, - offset_index_selection, + offset_index_mask, &mut builder, bytes, )?; @@ -299,7 +298,7 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -307,12 +306,12 @@ fn parse_column_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); @@ -331,7 +330,7 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -339,12 +338,12 @@ fn parse_offset_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index b55468fd2980..3999fdf9a9fb 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,7 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; @@ -229,9 +229,9 @@ pub struct ParquetMetaDataPushDecoder { /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, /// which rows and columns of the column index should be read - column_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, /// which rows and columns of the offset index should be read - offset_index_selection: PageIndexSelection, + offset_index_mask: ColumnChunkMask, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -256,8 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, - column_index_selection: PageIndexSelection::all(), - offset_index_selection: PageIndexSelection::all(), + column_index_mask: ColumnChunkMask::all(), + offset_index_mask: ColumnChunkMask::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -309,21 +309,21 @@ impl ParquetMetaDataPushDecoder { } /// Select the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection.clone(); - self.offset_index_selection = selection; + pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask.clone(); + self.offset_index_mask = mask; self } /// Select the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Select the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -439,8 +439,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, ); if ranges.is_empty() { @@ -462,8 +462,8 @@ impl ParquetMetaDataPushDecoder { &mut metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, &self.buffers, )?; self.state = DecodeState::Finished; @@ -526,10 +526,10 @@ enum DecodeState { /// the resultant vector by the range starts, and then create a single range /// using `start` from the head and `end` from the tail. /// -/// ```rust +/// ```ignore /// # use core::ops::Range; /// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// ranges.sort_by_key(|r| r.start); /// let range = (ranges.first()?.start..ranges.last()?.end); /// Some(range) /// # } @@ -538,33 +538,33 @@ pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Vec> { let mut result = Vec::new(); fn add_ranges( metadata: &ParquetMetaData, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, ranges: &mut Vec>, f: T, ) where T: Fn(&ColumnChunkMetaData) -> Option>, { for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if selection.includes_row_group(rg_idx) { + if mask.includes_row_group(rg_idx) { for (col_idx, col) in rg.columns().iter().enumerate() { - if selection.includes_column(col_idx) { - if let Some(range) = f(col) { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } + if mask.includes_column(col_idx) + && let Some(range) = f(col) + { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); } } } @@ -575,7 +575,7 @@ pub fn ranges_for_page_index( if column_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - column_index_selection, + column_index_mask, &mut result, ColumnChunkMetaData::column_index_range, ); @@ -583,7 +583,7 @@ pub fn ranges_for_page_index( if offset_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - offset_index_selection, + offset_index_mask, &mut result, ColumnChunkMetaData::offset_index_range, ); @@ -747,7 +747,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -788,7 +788,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -803,7 +803,7 @@ mod tests { // one for column index and one for offset index for each row group assert_eq!(ranges.len(), 4); // collapse ranges into a single range - ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + ranges.sort_by_key(|r| r.start); let range = ranges.first().unwrap().start..ranges.last().unwrap().end; push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index c3ce6ab6ce23..cdaeb1855552 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -72,8 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -107,25 +107,34 @@ impl From for PageIndexPolicy { } } -/// Selects the row groups and columns for which page indexes are read. +/// Struct to specify column chunks for which metadata is required. /// -/// This is independent of [`PageIndexPolicy`], which controls whether selected -/// indexes are optional or required. By default, all row groups and columns are selected. +/// Column chunks are identified by row group index and column index. This struct +/// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), +/// horizontal slices (via [`Self::row_groups`]), or the intersection of the two +/// (via [`Self::row_groups_and_columns`]). +/// +/// # Example +/// TODO(ets): example +/// +/// At present this is only used to select elements of the [Page Index] for decoding. +/// +/// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct PageIndexSelection { +pub struct ColumnChunkMask { // using i32 because that's how thrift vectors are sized row_groups: Option>>, columns: Option>>, } -// TODO(ets): add unit tests for PageIndexSelection -impl PageIndexSelection { +// TODO(ets): add unit tests for ColumnChunkMask +impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { Self::default() } - /// Select page indexes for only the listed columns. + /// Select only the listed columns. /// /// Any indices in `columns` that are less than zero will be ignored. pub fn columns(columns: impl IntoIterator) -> Self { @@ -135,7 +144,7 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups. + /// Select only the listed row groups. /// /// Any indices in `row_groups` that are less than zero will be ignored. pub fn row_groups(row_groups: impl IntoIterator) -> Self { @@ -147,9 +156,9 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups and columns. + /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, @@ -224,20 +233,20 @@ impl ParquetMetaDataReader { } /// Selects the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { - self.with_column_index_selection(selection.clone()) - .with_offset_index_selection(selection) + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) } /// Selects the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Selects the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -445,8 +454,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -594,8 +603,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 2f0348b835f5..bb9b5aeba446 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -133,8 +133,8 @@ impl AsyncFileReader for TestReader { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self.metadata = Some(Arc::new( diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 821d1d4cc55d..e594e78e900d 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -17,7 +17,7 @@ //! Tests of selective page index population -use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -31,8 +31,8 @@ fn test_parse_selected_columns() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -66,8 +66,8 @@ fn test_parse_selected_columns_mixed() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -101,8 +101,8 @@ fn test_parse_selected_row_groups() { // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) - .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups([0, 2])) + .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -134,8 +134,8 @@ fn test_parse_selected_row_groups_and_columns() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -197,8 +197,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -212,8 +212,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -228,8 +228,8 @@ fn test_page_index_sizes() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); From 69ab61adc0adbeb6e3ac72c0d3db2c504cec1dca Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:17:29 -0700 Subject: [PATCH 12/78] docs and cleanup --- parquet/src/file/metadata/page_index.rs | 16 ---- parquet/src/file/metadata/parser.rs | 12 ++- parquet/src/file/metadata/push_decoder.rs | 2 + parquet/src/file/metadata/reader.rs | 94 +++++++++++++++++------ 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index ea1b17f3af38..83b595daf141 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,6 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -559,21 +558,6 @@ impl PageIndexBuilder { } } - /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy - /// - /// For sparse indexes, this can save a great deal of memory - pub fn new_with_mask( - num_row_groups: usize, - num_columns: usize, - _column_index_mask: ColumnChunkMask, - _offset_index_mask: ColumnChunkMask, - ) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } - } - /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 47b93463147c..a76eea9066c0 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,7 +245,8 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). -/// * `bytes` - The byte slice containing the page index data. +/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing +/// the page indexes. /// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, @@ -261,13 +262,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_mask( - num_row_groups, - num_columns, - column_index_mask.clone(), - offset_index_mask.clone(), - ); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -277,6 +274,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 3999fdf9a9fb..6b747b34aabf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -701,6 +701,8 @@ mod tests { // expect the second request to read the offset indexes let ranges = expect_needs_data(metadata_decoder.try_decode()); + // since we're reading all indexes, there should be a single range needed + assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); // expect the third request to read the actual data diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index cdaeb1855552..186ccfcce330 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -114,11 +114,28 @@ impl From for PageIndexPolicy { /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). /// -/// # Example -/// TODO(ets): example -/// /// At present this is only used to select elements of the [Page Index] for decoding. /// +/// # Examples +/// +/// To select columns 0 and 1 from all row groups: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::columns([0, 1]); +/// ``` +/// +/// To select all columns from row group 2: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups([2]); +/// ``` +/// +/// To select columns 1 and 3 from row group 0: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups_and_columns([0], [1, 3]); +/// ``` +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ColumnChunkMask { @@ -127,7 +144,6 @@ pub struct ColumnChunkMask { columns: Option>>, } -// TODO(ets): add unit tests for ColumnChunkMask impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { @@ -136,22 +152,22 @@ impl ColumnChunkMask { /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. + /// Any indices in `columns` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all columns. pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + columns: Self::iter_to_set(columns), } } /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. + /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all row groups. pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), + row_groups: Self::iter_to_set(row_groups), columns: None, } } @@ -159,20 +175,22 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. + /// Passing an empty set for `row_groups` is treated as selecting all row groups, and + /// an empty set for `columns` as selectiong all columns. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, ) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + row_groups: Self::iter_to_set(row_groups), + columns: Self::iter_to_set(columns), } } - // test if `idx` is in the row group set. returns false if idx > i32::MAX - pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + /// Test if `idx` is in the row group set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_row_group(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; @@ -181,22 +199,20 @@ impl ColumnChunkMask { .is_none_or(|keep| keep.contains(&idx)) } - // test if `idx` is in the column set. returns false if idx > i32::MAX - pub(crate) fn includes_column(&self, idx: usize) -> bool { + /// Test if `idx` is in the column set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_column(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) } - // FIXME(ets): these will be used later - /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { - self.row_groups.as_deref() + fn iter_to_set(indices: impl IntoIterator) -> Option>> { + let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); + (!set.is_empty()).then_some(Arc::new(set)) } - - pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { - self.columns.as_deref() - }*/ } impl ParquetMetaDataReader { @@ -1509,4 +1525,32 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } + + #[test] + fn test_chunk_mask() { + // basic test + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + // test that negative numbers are ignored + let mask = ColumnChunkMask::columns([-1, 0, 1]); + assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(!mask.includes_column(2)); + + // empty set should behave as "all" + let mask = ColumnChunkMask::columns([]); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(mask.includes_column(2)); + + // indexing beyond i32::MAX should return false + let mask = ColumnChunkMask::all(); + assert!(mask.includes_column(i32::MAX as usize)); + assert!(!mask.includes_column(u32::MAX as usize)); + } } From b7b529de8a2f95b66110092eb567cbc3930a7fd2 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 09:42:51 -0700 Subject: [PATCH 13/78] more miri friendly --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 2736b7bc35e2..09b41df1155f 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -408,7 +408,7 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = temp_file.reopen().unwrap(); + let file = File::create(temp_file.path()).unwrap(); let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); // Write 3 row groups with 50 rows each (150 rows total) From b682ebecff792b5d2efdd455633a4b7492335010 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 10:13:48 -0700 Subject: [PATCH 14/78] rework tests to do no file i/o --- .../custom_page_index_provider.rs | 28 ++++++------------- parquet/tests/arrow_reader/page_index.rs | 24 ++++++---------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 5846db69b7a1..b864a466001c 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -39,21 +39,17 @@ use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::fs::File; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::NamedTempFile; #[test] fn test_read_with_custom_page_index_provider() { // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + let file_bytes = create_test_file(); // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, + file_bytes.clone(), ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), ) .unwrap(); @@ -68,7 +64,7 @@ fn test_read_with_custom_page_index_provider() { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(file_bytes); + let mut provider = SelectivePageIndexProvider::new(file_bytes.clone()); // Populate indexes for row group 0, columns 0 and 1 provider.fetch_column_index(0, 0, metadata).unwrap(); @@ -105,8 +101,8 @@ fn test_read_with_custom_page_index_provider() { .unwrap(); // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + let builder = + ParquetRecordBatchReaderBuilder::new_with_metadata(file_bytes.clone(), arrow_metadata); // Create a RowSelection that: // - Selects rows 20-30 (in row group 0) @@ -386,13 +382,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -pub(super) fn create_test_file() -> NamedTempFile { - let temp_file = tempfile::Builder::new() - .prefix("custom_page_index_test") - .suffix(".parquet") - .tempfile() - .expect("tempfile creation"); - +pub(super) fn create_test_file() -> Bytes { let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("value", DataType::Int32, false), @@ -408,8 +398,8 @@ pub(super) fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = temp_file.reopen().unwrap(); - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + let mut buffer = Vec::with_capacity(1024); + let mut writer = ArrowWriter::try_new(&mut buffer, schema.clone(), Some(props)).unwrap(); // Write 3 row groups with 50 rows each (150 rows total) for row_group in 0..3 { @@ -456,5 +446,5 @@ pub(super) fn create_test_file() -> NamedTempFile { ); } } - temp_file + Bytes::from(buffer) } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e594e78e900d..c605eac59834 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -18,7 +18,6 @@ //! Tests of selective page index population use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; -use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -26,7 +25,6 @@ use crate::custom_page_index_provider::create_test_file; fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -35,7 +33,7 @@ fn test_parse_selected_columns() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -61,7 +59,6 @@ fn test_parse_selected_columns() { fn test_parse_selected_columns_mixed() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -70,7 +67,7 @@ fn test_parse_selected_columns_mixed() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -96,7 +93,6 @@ fn test_parse_selected_columns_mixed() { fn test_parse_selected_row_groups() { // test populating PageIndex with a subset of row groups let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() @@ -105,7 +101,7 @@ fn test_parse_selected_row_groups() { .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_cols = metadata.file_metadata().schema_descr().num_columns(); assert!(metadata.page_index().is_some()); @@ -128,7 +124,6 @@ fn test_parse_selected_row_groups() { fn test_parse_selected_row_groups_and_columns() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -138,7 +133,7 @@ fn test_parse_selected_row_groups_and_columns() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); let num_cols = metadata.file_metadata().schema_descr().num_columns(); @@ -168,13 +163,12 @@ fn test_parse_selected_row_groups_and_columns() { fn test_page_index_sizes() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // no index let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_none()); #[cfg(not(feature = "encryption"))] @@ -186,7 +180,7 @@ fn test_page_index_sizes() { let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -201,7 +195,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -216,7 +210,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -232,7 +226,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] From bd587b28de5b5577006f264cff8e7313570c795e Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 08:24:34 -0700 Subject: [PATCH 15/78] apply suggestions from review --- parquet/src/arrow/mod.rs | 107 ++++++++++++++++++++ parquet/src/file/metadata/page_index.rs | 5 +- parquet/src/file/metadata/writer.rs | 128 ++++++++---------------- 3 files changed, 149 insertions(+), 91 deletions(-) diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs index 8309a35e9b1d..b60f211a19e0 100644 --- a/parquet/src/arrow/mod.rs +++ b/parquet/src/arrow/mod.rs @@ -595,6 +595,113 @@ mod test { assert_eq!(original_metadata, roundtrip_metadata); } + #[test] + #[cfg_attr(miri, ignore)] // Takes too long + fn test_metadata_read_write_roundtrip_missing_page_index() { + let parquet_bytes = create_parquet_file(); + + // read the metadata from the file but skip the page indexes + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + let original_metadata = ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&parquet_bytes) + .unwrap(); + + // metadata_to_bytes should zero out the page index locations. if they aren't + // then reading metadata_bytes will fail with EOF + let metadata_bytes = metadata_to_bytes(&original_metadata); + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + let roundtrip_metadata = ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Optional) + .parse_and_finish(&metadata_bytes) + .expect("page index locations should have been cleared"); + + assert!(roundtrip_metadata.page_index().is_none()); + } + + #[test] + fn test_metadata_read_write_roundtrip_offset_index_only() { + // `Chunk` statistics: the file has an offset index, but no column index + let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let batch = RecordBatch::try_from_iter(vec![("id", array)]).unwrap(); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Chunk) + .build(); + let mut buf = vec![]; + let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let read = |bytes: &Bytes| { + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Optional) + .parse_and_finish(bytes) + .unwrap() + }; + let original = read(&Bytes::from(buf)); + let page_index = original.page_index().unwrap(); + assert!(!page_index.has_column_indexes() && page_index.has_offset_indexes()); + let roundtrip = read(&metadata_to_bytes(&original)); + assert_eq!( + normalize_locations(original), + normalize_locations(roundtrip) + ); + } + + #[test] + fn test_metadata_read_write_roundtrip_custom_page_index() { + use crate::file::metadata::page_index::PageIndexProvider; + use crate::file::page_index::{ + column_index::ColumnIndexMetaData, offset_index::OffsetIndexMetaData, + }; + + /// A custom provider that forwards to another provider + #[derive(Debug)] + struct Forward(Arc); + impl PageIndexProvider for Forward { + fn has_offset_indexes(&self) -> bool { + self.0.has_offset_indexes() + } + fn has_column_indexes(&self) -> bool { + self.0.has_column_indexes() + } + fn column_index(&self, rg: usize, col: usize) -> Option<&ColumnIndexMetaData> { + self.0.column_index(rg, col) + } + fn offset_index(&self, rg: usize, col: usize) -> Option<&OffsetIndexMetaData> { + self.0.offset_index(rg, col) + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + let read = |bytes: &Bytes| { + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + ParquetMetaDataReader::new() + .with_metadata_options(Some(options)) + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(bytes) + .unwrap() + }; + let original = read(&create_parquet_file()); + let provider = Forward(original.page_index().unwrap().clone()); + let custom = original + .clone() + .into_builder() + .set_page_index(Some(Arc::new(provider))) + .build(); + let roundtrip = read(&metadata_to_bytes(&custom)); + assert_eq!( + normalize_locations(original), + normalize_locations(roundtrip) + ); + } + /// Sets the page index offset locations in the metadata to `None` /// /// This is because the offsets are used to find the relative location of the index diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -520,10 +520,7 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - pub(crate) fn empty_index( - num_row_groups: usize, - num_columns: usize, - ) -> Option>>> { + fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { Some( (0..num_row_groups) .map(|_| { diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 63cdf540d146..596907d726fe 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; +use crate::file::metadata::page_index::PageIndexProvider; use crate::file::metadata::thrift::FileMeta; -use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData}; +use crate::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; use crate::schema::types::{SchemaDescPtr, SchemaDescriptor}; use crate::{ basic::ColumnOrder, @@ -69,14 +69,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Note: also updates the `ColumnChunk::offset_index_offset` and /// `ColumnChunk::offset_index_length` to reflect the position and length /// of the serialized offset indexes. - fn write_offset_indexes( - &mut self, - page_index: &Arc, - ) -> Result>>>> { - let mut offset_indexes = - PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); - let offidx_vec = offset_indexes.as_mut().unwrap(); - + fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { // we've already checked before calling that the offset indexes are populated assert!(page_index.has_offset_indexes()); @@ -98,11 +91,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); - offidx_vec[row_group_idx][column_idx] = Some(offset_index.clone()); + } else { + column_metadata.offset_index_offset = None; + column_metadata.offset_index_length = None; } } } - Ok(offset_indexes) + Ok(()) } /// Serialize all the column indexes to the `self.buf` @@ -110,17 +105,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// Note: also updates the `ColumnChunk::column_index_offset` and /// `ColumnChunk::column_index_length` to reflect the position and length /// of the serialized column indexes. - fn write_column_indexes( - &mut self, - page_index: &Arc, - ) -> Result>>>> { - let mut column_indexes = - PageIndexBuilder::empty_index(self.row_groups.len(), self.schema_descr.num_columns()); - let colidx_vec = column_indexes.as_mut().unwrap(); - - // we've already checked before calling that the column indexes are populated - assert!(page_index.has_column_indexes()); - + fn write_column_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { // iter row group // iter each column // write column index to the file @@ -142,64 +127,23 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } - colidx_vec[row_group_idx][column_idx] = Some(column_index.clone()); + } else { + column_metadata.column_index_offset = None; + column_metadata.column_index_length = None; } } } - Ok(column_indexes) - } - - /// Serialize the column indexes and transform to `Option>>>` - fn finalize_column_indexes( - &mut self, - page_index: Option<&Arc>, - ) -> Result>>>> { - if page_index - .as_ref() - .is_none_or(|pi| !pi.has_column_indexes()) - { - return Ok(None); - } - - // Write column indexes to file while assembling a column index - let column_indexes = self.write_column_indexes(page_index.as_ref().unwrap())?; - - // check to see if the index is `None` for every row group and column chunk - let all_none = column_indexes - .as_ref() - .is_some_and(|ci| ci.iter().all(|cii| cii.iter().all(|idx| idx.is_none()))); - - if all_none { - Ok(None) - } else { - Ok(column_indexes) - } + Ok(()) } - /// Serialize the offset indexes and transform to `Option` - fn finalize_offset_indexes( - &mut self, - page_index: Option<&Arc>, - ) -> Result>>>> { - if page_index - .as_ref() - .is_none_or(|pi| !pi.has_offset_indexes()) - { - return Ok(None); - } - - // Write offset indexes to file - let offset_indexes = self.write_offset_indexes(page_index.as_ref().unwrap())?; - - // check to see if the index is `None` for every row group and column chunk - let all_none = offset_indexes - .as_ref() - .is_some_and(|oi| oi.iter().all(|oii| oii.iter().all(|idx| idx.is_none()))); - - if all_none { - Ok(None) - } else { - Ok(offset_indexes) + fn clear_page_indexes(&mut self) { + for row_group in self.row_groups.iter_mut() { + for column_metadata in row_group.columns.iter_mut() { + column_metadata.column_index_offset = None; + column_metadata.column_index_length = None; + column_metadata.offset_index_offset = None; + column_metadata.offset_index_length = None; + } } } @@ -207,10 +151,15 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { pub fn finish(mut self) -> Result { let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum(); - // serialize page indexes and transform to the proper form for use in ParquetMetaData + // serialize page indexes and update index locations let page_index = self.page_index.take(); - let column_indexes = self.finalize_column_indexes(page_index.as_ref())?; - let offset_indexes = self.finalize_offset_indexes(page_index.as_ref())?; + if let Some(pi) = page_index.as_deref() { + self.write_column_indexes(pi)?; + self.write_offset_indexes(pi)?; + } else { + // no page indexes at all, then clear index locations + self.clear_page_indexes(); + } // We only include ColumnOrder for leaf nodes. let column_orders = self @@ -278,14 +227,17 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { self.buf.write_all(&metadata_len.to_le_bytes())?; self.buf.write_all(self.object_writer.get_file_magic())?; + // Note: we simply pass the original reference counted page index provider to the + // builder. Either it came from ParquetMetaDataWriter where it was cloned, and this + // reference will go away when ParquetMetaDataWriter::finish drops the newly created + // metadata, or it was passed an owned version from SerializedFileWriter which then + // takes it back via the returned metadata. + let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(page_index); + // If row group metadata was encrypted, we replace the encrypted row groups with // unencrypted metadata before it is returned to users. This allows the metadata // to be usable for retrieving the row group statistics for example, without users // needing to decrypt the metadata. - let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(Some(Arc::new( - PageIndex::new(column_indexes, offset_indexes), - ))); - Ok(match unencrypted_row_groups { Some(rg) => builder.set_row_groups(rg).build(), None => builder.set_row_groups(row_groups).build(), @@ -350,6 +302,10 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// +/// The writer serializes the page index of any [`PageIndexProvider`]. If the +/// provider has no index for a column chunk, the writer clears the offset and +/// length of that index in the [`ColumnChunkMetaData`]. +/// /// # Output Format /// /// The format of the metadata is as follows: @@ -470,10 +426,8 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { self.write_path_in_schema, ); - if let Some(page_index_arc) = self.metadata.page_index.as_ref() - && (page_index_arc.has_column_indexes() || page_index_arc.has_offset_indexes()) - { - encoder = encoder.with_page_index(page_index_arc.clone()); + if let Some(page_index_arc) = self.metadata.page_index.clone() { + encoder = encoder.with_page_index(page_index_arc); } if let Some(key_value_metadata) = key_value_metadata { From 997f184ac3a6269a02513dad3aba713cecad4de1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 08:27:40 -0700 Subject: [PATCH 16/78] remove assert --- parquet/src/file/metadata/writer.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 596907d726fe..cbba43261ec3 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -70,9 +70,6 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// `ColumnChunk::offset_index_length` to reflect the position and length /// of the serialized offset indexes. fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> { - // we've already checked before calling that the offset indexes are populated - assert!(page_index.has_offset_indexes()); - // iter row group // iter each column // write offset index to the file From 0969c2329209a7bab57a51cabb93415d7968ff92 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 09:00:52 -0700 Subject: [PATCH 17/78] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 09b41df1155f..f5b5bd32cd2a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -67,7 +67,7 @@ fn test_read_with_custom_page_index_provider() { // Simulate a scenario where: // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only + // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap let mut provider = SelectivePageIndexProvider::new(file_bytes); // Populate indexes for row group 0, columns 0 and 1 @@ -76,9 +76,11 @@ fn test_read_with_custom_page_index_provider() { provider.fetch_column_index(0, 1, metadata).unwrap(); provider.fetch_offset_index(0, 1, metadata).unwrap(); - // Populate indexes for row group 2, column 0 only + // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) provider.fetch_column_index(2, 0, metadata).unwrap(); provider.fetch_offset_index(2, 0, metadata).unwrap(); + provider.fetch_column_index(2, 2, metadata).unwrap(); + provider.fetch_offset_index(2, 2, metadata).unwrap(); // Verify the provider has the expected indexes assert!(provider.column_index(0, 0).is_some()); From d93796e9b51004d54055af58717f7415ab717989 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 09:06:13 -0700 Subject: [PATCH 18/78] add another assert --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index f5b5bd32cd2a..ca5359f08e5b 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -90,6 +90,7 @@ fn test_read_with_custom_page_index_provider() { assert!(provider.column_index(0, 2).is_none()); // Not populated assert!(provider.column_index(2, 0).is_some()); assert!(provider.column_index(2, 1).is_none()); // Not populated + assert!(provider.column_index(2, 2).is_some()); // Reset statistics so validation checks don't affect the final counts provider.reset_stats(); From 2a2fd09e34ae4b1521a87baa268830649408a070 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 09:35:38 -0700 Subject: [PATCH 19/78] clippy --- parquet/src/file/metadata/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index cbba43261ec3..766c3f1e787c 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -134,8 +134,8 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { } fn clear_page_indexes(&mut self) { - for row_group in self.row_groups.iter_mut() { - for column_metadata in row_group.columns.iter_mut() { + for row_group in &mut self.row_groups { + for column_metadata in &mut row_group.columns { column_metadata.column_index_offset = None; column_metadata.column_index_length = None; column_metadata.offset_index_offset = None; From c687e6ed9e1669c63fc884f7e02ed06a58017f92 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 11:24:27 -0700 Subject: [PATCH 20/78] restore old page index location behavior add flag to zero out page locations if desired --- parquet/src/arrow/mod.rs | 15 ++++++++++++--- parquet/src/file/metadata/writer.rs | 25 ++++++++++++++++++++++++- parquet/src/file/writer.rs | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs index 4b9a694e7304..e5cbafea5e64 100644 --- a/parquet/src/arrow/mod.rs +++ b/parquet/src/arrow/mod.rs @@ -596,7 +596,6 @@ mod test { } #[test] - #[cfg_attr(miri, ignore)] // Takes too long fn test_metadata_read_write_roundtrip_missing_page_index() { let parquet_bytes = create_parquet_file(); @@ -608,9 +607,9 @@ mod test { .parse_and_finish(&parquet_bytes) .unwrap(); - // metadata_to_bytes should zero out the page index locations. if they aren't + // metadata_to_bytes_no_page_idx should zero out the page index locations. if they aren't // then reading metadata_bytes will fail with EOF - let metadata_bytes = metadata_to_bytes(&original_metadata); + let metadata_bytes = metadata_to_bytes_no_page_idx(&original_metadata); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); let roundtrip_metadata = ParquetMetaDataReader::new() .with_metadata_options(Some(options)) @@ -754,6 +753,16 @@ mod test { Bytes::from(buf) } + // like metadata_to_bytes, but do not preserve page index location info + fn metadata_to_bytes_no_page_idx(metadata: &ParquetMetaData) -> Bytes { + let mut buf = vec![]; + ParquetMetaDataWriter::new(&mut buf, metadata) + .with_preserve_page_index_locations(false) + .finish() + .unwrap(); + Bytes::from(buf) + } + #[test] fn test_mask_from_column_names() { let schema = parse_schema( diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 766c3f1e787c..a439efc5e6da 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -61,6 +61,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { object_writer: MetadataObjectWriter, writer_version: i32, write_path_in_schema: bool, + preserve_page_index_locations: bool, } impl<'a, W: Write> ThriftMetadataWriter<'a, W> { @@ -153,7 +154,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { if let Some(pi) = page_index.as_deref() { self.write_column_indexes(pi)?; self.write_offset_indexes(pi)?; - } else { + } else if !self.preserve_page_index_locations { // no page indexes at all, then clear index locations self.clear_page_indexes(); } @@ -248,6 +249,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { created_by: Option, writer_version: i32, write_path_in_schema: bool, + preserve_page_index_locations: bool, ) -> Self { Self { buf, @@ -259,6 +261,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { object_writer: Default::default(), writer_version, write_path_in_schema, + preserve_page_index_locations, } } @@ -365,6 +368,7 @@ pub struct ParquetMetaDataWriter<'a, W: Write> { buf: TrackedWrite, metadata: &'a ParquetMetaData, write_path_in_schema: bool, + preserve_page_index_locations: bool, } impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { @@ -390,6 +394,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { buf, metadata, write_path_in_schema: true, + preserve_page_index_locations: true, } } @@ -402,6 +407,23 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { } } + /// Set whether or not to preserve the page index location metadata in the Thrift + /// `ColumnMetaData`. + /// + /// Because this struct is often used to externalize the footer metadata, it is + /// usually desirable to preserve this location information, even when the + /// page indexes are not duplicated (for instance if the provided `ParquetMetaData` + /// returns no `PageIndexProvider`). As such, this defaults to `true`. + /// + /// Set this to `false` to reset the location metadata if no page indexes are + /// present. + pub fn with_preserve_page_index_locations(self, val: bool) -> Self { + Self { + preserve_page_index_locations: val, + ..self + } + } + /// Write the metadata to the buffer pub fn finish(mut self) -> Result<()> { let file_metadata = self.metadata.file_metadata(); @@ -421,6 +443,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { created_by, file_metadata.version(), self.write_path_in_schema, + self.preserve_page_index_locations, ); if let Some(page_index_arc) = self.metadata.page_index.clone() { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 18b7e6e9999e..af60326ca387 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -367,6 +367,7 @@ impl SerializedFileWriter { Some(self.props.created_by().to_string()), self.props.writer_version().as_num(), write_path_in_schema, + false, ); #[cfg(feature = "encryption")] From 55eb9675b2b89d682023daa328a0f46091ac20c7 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 11:37:35 -0700 Subject: [PATCH 21/78] fix docs --- parquet/src/file/metadata/writer.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index a439efc5e6da..2ce8396f8c09 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -302,9 +302,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { /// metadata writer. Then set the corresponding `bloom_filter_offset` and /// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer. /// -/// The writer serializes the page index of any [`PageIndexProvider`]. If the -/// provider has no index for a column chunk, the writer clears the offset and -/// length of that index in the [`ColumnChunkMetaData`]. +/// The writer serializes the page index of any [`PageIndexProvider`] attached +/// to the input [`ParquetMetaData`]. By default, if the provider has no index +/// for a column chunk, the offset and length of that index in the [`ColumnChunkMetaData`] +/// are preserved in the output metadata. This is done so one can store the footer +/// metadata externally but keep the page indexes in the original file. If this +/// behavior is not desired, then [`Self::with_preserve_page_index_locations`] should +/// be set to `false`. /// /// # Output Format /// From 92fb20c5bf9f4238c15359f9de9763d7a5fe69ef Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 12:01:15 -0700 Subject: [PATCH 22/78] honor preserve_page_index_locations in write_x_indexes --- parquet/src/file/metadata/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 2ce8396f8c09..65f8e50a7270 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -89,7 +89,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { // set offset and index for offset index column_metadata.offset_index_offset = Some(start_offset as i64); column_metadata.offset_index_length = Some((end_offset - start_offset) as i32); - } else { + } else if !self.preserve_page_index_locations { column_metadata.offset_index_offset = None; column_metadata.offset_index_length = None; } @@ -125,7 +125,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { column_metadata.column_index_length = Some((end_offset - start_offset) as i32); } - } else { + } else if !self.preserve_page_index_locations { column_metadata.column_index_offset = None; column_metadata.column_index_length = None; } From 2489741dacebede25e2344b60b27a18b48820b67 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:45:29 -0700 Subject: [PATCH 23/78] implement review suggestions --- .../custom_page_index_provider.rs | 414 +++++++++--------- 1 file changed, 214 insertions(+), 200 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index ca5359f08e5b..6c93193f91c9 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -23,14 +23,17 @@ //! - Columns without page indexes fall back to whole-column-chunk fetching //! - Correct results are returned in both cases -use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow::compute::concat_batches; +use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use bytes::Bytes; +use parquet::DecodeResult; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelectionPolicy, }; +use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::file::metadata::page_index::PageIndexProvider; use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use parquet::file::page_index::column_index::ColumnIndexMetaData; @@ -38,22 +41,104 @@ use parquet::file::page_index::index_reader::{decode_column_index, decode_offset use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::fs::File; +use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::NamedTempFile; #[test] -fn test_read_with_custom_page_index_provider() { +fn test_read_with_custom_page_index_provider_sync() { + run_test(Reader::Sync); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_push() { + run_test(Reader::Push); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[cfg(feature = "async")] +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_async() { + run_test(Reader::Async); +} + +#[derive(Debug, Clone, Copy)] +enum Reader { + Sync, + Push, + #[cfg(feature = "async")] + Async, +} + +/// Read `file` with `reader`, using `metadata` and `selection` +fn read( + reader: Reader, + file: Bytes, + metadata: ArrowReaderMetadata, + selection: RowSelection, + requested: &mut Vec>, +) -> Vec { + match reader { + Reader::Sync => ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .collect::>() + .unwrap(), + Reader::Push => { + let mut decoder = ParquetPushDecoderBuilder::new_with_metadata(metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let mut batches = vec![]; + loop { + match decoder.try_decode().unwrap() { + DecodeResult::NeedsData(ranges) => { + requested.extend(ranges.iter().cloned()); + let data = ranges + .iter() + .map(|r| file.slice(r.start as usize..r.end as usize)) + .collect(); + decoder.push_ranges(ranges, data).unwrap(); + } + DecodeResult::Data(batch) => batches.push(batch), + DecodeResult::Finished => return batches, + } + } + } + #[cfg(feature = "async")] + Reader::Async => { + use futures::TryStreamExt; + let input = std::io::Cursor::new(file.to_vec()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + parquet::arrow::ParquetRecordBatchStreamBuilder::new_with_metadata(input, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .try_collect() + .await + .unwrap() + }) + } + } +} + +fn run_test(reader: Reader) { // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + let file_bytes = create_test_file(); // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, + file_bytes.clone(), ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), ) .unwrap(); @@ -68,32 +153,11 @@ fn test_read_with_custom_page_index_provider() { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap - let mut provider = SelectivePageIndexProvider::new(file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0, metadata).unwrap(); - provider.fetch_offset_index(0, 0, metadata).unwrap(); - provider.fetch_column_index(0, 1, metadata).unwrap(); - provider.fetch_offset_index(0, 1, metadata).unwrap(); - - // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) - provider.fetch_column_index(2, 0, metadata).unwrap(); - provider.fetch_offset_index(2, 0, metadata).unwrap(); - provider.fetch_column_index(2, 2, metadata).unwrap(); - provider.fetch_offset_index(2, 2, metadata).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - assert!(provider.column_index(2, 2).is_some()); - - // Reset statistics so validation checks don't affect the final counts - provider.reset_stats(); + let provider = SelectivePageIndexProvider::new( + file_bytes.clone(), + metadata, + &[(0, 0), (0, 1), (2, 0), (2, 2)], + ); // Step 4: Install the custom provider into metadata let mut metadata_builder = metadata.clone().into_builder(); @@ -108,8 +172,6 @@ fn test_read_with_custom_page_index_provider() { .unwrap(); // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); // Create a RowSelection that: // - Selects rows 20-30 (in row group 0) @@ -130,17 +192,33 @@ fn test_read_with_custom_page_index_provider() { parquet::arrow::arrow_reader::RowSelector::select(10), ]); - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); + let mut requested = Vec::>::new(); + let batches = read( + reader, + file_bytes.clone(), + arrow_metadata, + selection, + &mut requested, + ); // Step 7: Check statistics + + // only the push test will populate requested + if matches!(reader, Reader::Push) { + let fetched = |rg: usize, col: usize| -> u64 { + let (start, len) = metadata.row_group(rg).column(col).byte_range(); + requested + .iter() + .filter(|r| r.start >= start && r.end <= start + len) + .map(|r| r.end - r.start) + .sum() + }; + let chunk_len = |rg: usize, col: usize| metadata.row_group(rg).column(col).byte_range().1; + assert!(fetched(0, 0) < chunk_len(0, 0)); // offset index: selected pages only + assert_eq!(fetched(0, 2), chunk_len(0, 2)); // no offset index: whole chunk + } + // Note: We need to get the provider reference from the metadata to check stats let provider_ref = metadata_with_custom_index .page_index() @@ -168,61 +246,15 @@ fn test_read_with_custom_page_index_provider() { let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 30); - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - // Expected values for the selected rows: // rows 20-30: id=20..30, value=40..60, score=60..90 // rows 60-70: id=60..70, value=120..140, score=180..210 // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); + let ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected = make_batch(&ids); assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" + concat_batches(&expected.schema(), &batches).unwrap(), + expected ); } @@ -232,32 +264,46 @@ fn test_read_with_custom_page_index_provider() { /// page indexes selectively based on query predicates and projections. #[derive(Debug)] struct SelectivePageIndexProvider { - file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index - column_indexes: Option>>, - offset_indexes: Option>>, + column_indexes: Option>, + offset_indexes: Option>, // Usage statistics - column_index_hits: Arc, - column_index_misses: Arc, - offset_index_hits: Arc, - offset_index_misses: Arc, + column_index_hits: AtomicUsize, + column_index_misses: AtomicUsize, + offset_index_hits: AtomicUsize, + offset_index_misses: AtomicUsize, } impl SelectivePageIndexProvider { - fn new(file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes, metadata: &ParquetMetaData, chunks: &[(usize, usize)]) -> Self { + let mut column_indexes = HashMap::new(); + let mut offset_indexes = HashMap::new(); + for &(rg, col) in chunks { + column_indexes.insert( + (rg, col), + Self::fetch_column_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + offset_indexes.insert( + (rg, col), + Self::fetch_offset_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + } Self { - file_bytes, - column_indexes: None, - offset_indexes: None, - column_index_hits: Arc::new(AtomicUsize::new(0)), - column_index_misses: Arc::new(AtomicUsize::new(0)), - offset_index_hits: Arc::new(AtomicUsize::new(0)), - offset_index_misses: Arc::new(AtomicUsize::new(0)), + column_indexes: Some(column_indexes), + offset_indexes: Some(offset_indexes), + column_index_hits: AtomicUsize::new(0), + column_index_misses: AtomicUsize::new(0), + offset_index_hits: AtomicUsize::new(0), + offset_index_misses: AtomicUsize::new(0), } } /// Get statistics about column index usage - pub fn column_index_stats(&self) -> (usize, usize) { + fn column_index_stats(&self) -> (usize, usize) { ( self.column_index_hits.load(Ordering::Relaxed), self.column_index_misses.load(Ordering::Relaxed), @@ -265,7 +311,7 @@ impl SelectivePageIndexProvider { } /// Get statistics about offset index usage - pub fn offset_index_stats(&self) -> (usize, usize) { + fn offset_index_stats(&self) -> (usize, usize) { ( self.offset_index_hits.load(Ordering::Relaxed), self.offset_index_misses.load(Ordering::Relaxed), @@ -273,64 +319,44 @@ impl SelectivePageIndexProvider { } /// Get combined statistics as (hits, misses) for both index types - pub fn stats(&self) -> (usize, usize) { + fn stats(&self) -> (usize, usize) { let (col_hits, col_misses) = self.column_index_stats(); let (off_hits, off_misses) = self.offset_index_stats(); (col_hits + off_hits, col_misses + off_misses) } - /// Reset all statistics counters to zero - pub fn reset_stats(&self) { - self.column_index_hits.store(0, Ordering::Relaxed); - self.column_index_misses.store(0, Ordering::Relaxed); - self.offset_index_hits.store(0, Ordering::Relaxed); - self.offset_index_misses.store(0, Ordering::Relaxed); - } - /// Fetch and parse the column index for the given row group and column fn fetch_column_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.column_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.column_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_column_index(&idx_bytes, column.column_type())?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_column_index(&idx_bytes, column.column_type())?)) + } else { + Ok(None) } - Ok(()) } /// Fetch and parse the offset index for the given row group and column fn fetch_offset_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.offset_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.offset_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_offset_index(&idx_bytes)?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_offset_index(&idx_bytes)?)) + } else { + Ok(None) } - Ok(()) } } @@ -351,8 +377,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .column_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.column_index_hits.fetch_add(1, Ordering::Relaxed); @@ -371,8 +396,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .offset_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.offset_index_hits.fetch_add(1, Ordering::Relaxed); @@ -389,20 +413,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> NamedTempFile { - let temp_file = tempfile::Builder::new() - .prefix("custom_page_index_test") - .suffix(".parquet") - .tempfile() - .expect("tempfile creation"); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("value", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - Field::new("score", DataType::Int32, false), - ])); - +fn create_test_file() -> Bytes { // Create small pages and small row groups for testing let props = WriterProperties::builder() .set_statistics_enabled(EnabledStatistics::Page) @@ -411,37 +422,12 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = File::create(temp_file.path()).unwrap(); - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); - // Write 3 row groups with 50 rows each (150 rows total) - for row_group in 0..3 { - for batch_num in 0..5 { - let offset = (row_group * 50) + (batch_num * 10); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from( - (offset..offset + 10).collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 2).collect::>(), - )), - Arc::new(StringArray::from( - (offset..offset + 10) - .map(|x| format!("name{x}")) - .collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 3).collect::>(), - )), - ], - ) - .unwrap(); - writer.write(&batch).unwrap(); - } - writer.flush().unwrap(); - } + let batch = make_batch(&(0..150).collect::>().as_slice()); + let mut buffer = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); // check that file has properties we wanted (3 row groups, 5 pages per chunk) let metadata = writer.close().unwrap(); @@ -459,5 +445,33 @@ fn create_test_file() -> NamedTempFile { ); } } - temp_file + Bytes::from(buffer) +} + +fn make_batch(ids: &[i32]) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + // Write 3 row groups with 50 rows each (150 rows total) + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + ids.iter() + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap() } From 835170329bae2e3948ac900d676888e8484752af Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:56:14 -0700 Subject: [PATCH 24/78] clippy --- .../tests/arrow_reader/custom_page_index_provider.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 6c93193f91c9..8447146418f1 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -52,7 +52,9 @@ fn test_read_with_custom_page_index_provider_sync() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } @@ -60,7 +62,9 @@ fn test_read_with_custom_page_index_provider_push() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } @@ -423,7 +427,7 @@ fn create_test_file() -> Bytes { .build(); // Write 3 row groups with 50 rows each (150 rows total) - let batch = make_batch(&(0..150).collect::>().as_slice()); + let batch = make_batch((0..150).collect::>().as_slice()); let mut buffer = Vec::with_capacity(1024); let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); From 57d70307f880f932cdae1f8f6ddafd43e09a2fa8 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 15:23:34 -0700 Subject: [PATCH 25/78] tests now pass --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 8447146418f1..b6348ecd221a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -50,21 +50,13 @@ fn test_read_with_custom_page_index_provider_sync() { run_test(Reader::Sync); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } From 82087c5e8da815ded0e22aeb5f87962428d7d2b2 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 16:11:00 -0700 Subject: [PATCH 26/78] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/src/file/metadata/reader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 186ccfcce330..d8e3decef59e 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -109,7 +109,8 @@ impl From for PageIndexPolicy { /// Struct to specify column chunks for which metadata is required. /// -/// Column chunks are identified by row group index and column index. This struct +/// Column chunks are identified by row group index and leaf column index (the index of the +/// column in [`SchemaDescriptor::columns`], not the index of a root or Arrow field). This struct /// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). From 7b30fa0cf545e0c562b88c025bcb8e37dbefed28 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 07:53:10 -0700 Subject: [PATCH 27/78] revert most changes to push decoder --- parquet/src/file/metadata/push_decoder.rs | 189 +++------------------- 1 file changed, 24 insertions(+), 165 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 6b747b34aabf..4c8b514251ca 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,8 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; use bytes::Bytes; @@ -435,27 +434,20 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let ranges = ranges_for_page_index( + let range = range_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, ); - if ranges.is_empty() { + let Some(page_index_range) = range else { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - } - - let needed_ranges = ranges - .into_iter() - .filter(|r| !self.buffers.has_range(r)) - .collect::>(); + }; - if !needed_ranges.is_empty() { + if !self.buffers.has_range(&page_index_range) { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(DecodeResult::NeedsData(needed_ranges)); + return Ok(needs_range(page_index_range)); } parse_page_index( @@ -514,81 +506,32 @@ enum DecodeState { Intermediate, } -/// Returns the minimum set of non-overlapping ranges needed to cover the requested -/// offset and column indexes. +/// Returns the byte range needed to read the offset/page indexes, based on the +/// specified policies /// -/// If no page indexes are present in the file, or none are actually -/// requested by the policies passed in, this will return an empty -/// vector. -/// -/// This may result in more ranges to fetch but may also result in fewer bytes -/// being read (and stored). To get a single range for all indexes, sort -/// the resultant vector by the range starts, and then create a single range -/// using `start` from the head and `end` from the tail. -/// -/// ```ignore -/// # use core::ops::Range; -/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by_key(|r| r.start); -/// let range = (ranges.first()?.start..ranges.last()?.end); -/// Some(range) -/// # } -/// ``` -pub fn ranges_for_page_index( +/// Returns None if no page indexes are needed +pub fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_mask: &ColumnChunkMask, - offset_index_mask: &ColumnChunkMask, -) -> Vec> { - let mut result = Vec::new(); - - fn add_ranges( - metadata: &ParquetMetaData, - mask: &ColumnChunkMask, - ranges: &mut Vec>, - f: T, - ) where - T: Fn(&ColumnChunkMetaData) -> Option>, - { - for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if mask.includes_row_group(rg_idx) { - for (col_idx, col) in rg.columns().iter().enumerate() { - if mask.includes_column(col_idx) - && let Some(range) = f(col) - { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } - } - } - } +) -> Option> { + fn acc_range(a: Option>, b: Option>) -> Option> { + match (a, b) { + (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), + (None, x) | (x, None) => x, } } - if column_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - column_index_mask, - &mut result, - ColumnChunkMetaData::column_index_range, - ); - } - if offset_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - offset_index_mask, - &mut result, - ColumnChunkMetaData::offset_index_range, - ); + let mut range = None; + for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { + if column_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.column_index_range()); + } + if offset_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.offset_index_range()); + } } - result + range } // These tests use the arrow writer to create a parquet file in memory @@ -742,90 +685,6 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } - /// Decode the metadata incrementally, with a partial page index - #[test] - fn test_metadata_decoder_incremental_partial_page_index() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - - /// Decode the metadata incrementally, with a partial page index, but coalesced - /// range request. - #[test] - fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let mut ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - // collapse ranges into a single range - ranges.sort_by_key(|r| r.start); - let range = ranges.first().unwrap().start..ranges.last().unwrap().end; - push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) From 3cefc59282e1fdd31eeeb0082f9db9732ab715b4 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 09:33:16 -0700 Subject: [PATCH 28/78] address a number of issues found in review, primarily documentation changes, and changes to chunk mask semantics and storage. Also fixes the preloading reader issue. --- parquet/src/arrow/arrow_reader/mod.rs | 28 ++- parquet/src/arrow/async_reader/store.rs | 12 +- parquet/src/file/metadata/reader.rs | 183 ++++++++++++------ parquet/tests/arrow_reader/page_index.rs | 123 ++++++++++++ .../arrow_reader/partial_offset_index.rs | 67 ++++++- 5 files changed, 333 insertions(+), 80 deletions(-) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 492ea6063462..9788f597d95b 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -776,6 +776,12 @@ impl ArrowReaderOptions { self } + /// Sets the same [`ColumnChunkMask`] for both page-index structures. + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) + } + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed @@ -783,6 +789,11 @@ impl ArrowReaderOptions { /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// The mask applies only if the column-index policy is not [`PageIndexPolicy::Skip`] + /// (the default), or an underlying reader is configured to preload the index. It is + /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; + /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. + /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -796,6 +807,10 @@ impl ArrowReaderOptions { /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// Page pruning also needs the offset index for predicate columns, so callers should + /// include those columns in addition to projected columns. The same loading and policy + /// qualifications as [`Self::with_column_index_mask`] apply. + /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; @@ -1012,9 +1027,16 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + + // Preload settings on the underlying reader may enable an index even when the + // corresponding options policy is `Skip`, so apply non-default masks independently. + if !options.column_index_mask().is_all() { + self = self.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + self = self.with_offset_index_mask(options.offset_index_mask().clone()); } self diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index c7db5d0abbc1..5528cc05b2bd 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,9 +242,15 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + if let Some(options) = options { + if !options.column_index_mask().is_all() { + metadata = metadata.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + metadata = metadata.with_offset_index_mask(options.offset_index_mask().clone()); + } } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index d8e3decef59e..8987912499fb 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -91,10 +91,12 @@ pub enum PageIndexPolicy { Skip, /// Read the page index if it exists, otherwise do not error. Optional, - /// Require the page index to exist, and error if it does not. + /// Require offset indexes for every selected column chunk when any selected + /// page index ranges exist. /// - /// Only enforced for the offset index; this is the same as [`Self::Optional`] - /// for the column index. + /// For compatibility, a file with no page index ranges is accepted. This is only + /// enforced for the offset index; this is the same as [`Self::Optional`] for the + /// column index. Required, } @@ -138,11 +140,12 @@ impl From for PageIndexPolicy { /// ``` /// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct ColumnChunkMask { - // using i32 because that's how thrift vectors are sized - row_groups: Option>>, - columns: Option>>, + // `None` means all, while `Some(empty)` means none. Store u32 because + // Parquet/Thrift collections cannot contain more than i32::MAX entries. + row_groups: Option>, + columns: Option>, } impl ColumnChunkMask { @@ -151,11 +154,18 @@ impl ColumnChunkMask { Self::default() } + /// Select no row groups or columns. + pub fn none() -> Self { + Self { + row_groups: Some(Arc::from([])), + columns: Some(Arc::from([])), + } + } + /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all columns. - pub fn columns(columns: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no columns. + pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, columns: Self::iter_to_set(columns), @@ -164,9 +174,8 @@ impl ColumnChunkMask { /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all row groups. - pub fn row_groups(row_groups: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no row groups. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { row_groups: Self::iter_to_set(row_groups), columns: None, @@ -175,12 +184,10 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. - /// Passing an empty set for `row_groups` is treated as selecting all row groups, and - /// an empty set for `columns` as selectiong all columns. + /// An empty iterator for either dimension selects no column chunks. pub fn row_groups_and_columns( - row_groups: impl IntoIterator, - columns: impl IntoIterator, + row_groups: impl IntoIterator, + columns: impl IntoIterator, ) -> Self { Self { row_groups: Self::iter_to_set(row_groups), @@ -189,30 +196,79 @@ impl ColumnChunkMask { } /// Test if `idx` is in the row group set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_row_group(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { - return false; - }; - self.row_groups - .as_ref() - .is_none_or(|keep| keep.contains(&idx)) + Self::includes_index(self.row_groups.as_ref(), idx) } /// Test if `idx` is in the column set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_column(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { + Self::includes_index(self.columns.as_ref(), idx) + } + + fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { + // return false for out-of-bounds index + let Ok(idx) = u32::try_from(idx) else { return false; }; - self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + keep.is_none_or(|keep| keep.binary_search(&idx).is_ok()) + } + + /// Returns `true` when this mask selects every column chunk. + pub fn is_all(&self) -> bool { + self.row_groups.is_none() && self.columns.is_none() + } + + /// Returns selected row groups, or `None` when all row groups are selected. + pub fn selected_row_groups(&self) -> Option<&[u32]> { + self.row_groups.as_deref() + } + + /// Returns selected leaf columns, or `None` when all columns are selected. + pub fn selected_columns(&self) -> Option<&[u32]> { + self.columns.as_deref() + } + + /// Creates a mask selecting the leaf columns in an Arrow projection. + #[cfg(feature = "arrow")] + pub fn from_projection( + projection: &crate::arrow::ProjectionMask, + schema: &SchemaDescriptor, + ) -> Self { + Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } - fn iter_to_set(indices: impl IntoIterator) -> Option>> { - let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); - (!set.is_empty()).then_some(Arc::new(set)) + /// Returns an iterator over the row group indices selected by this mask + pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + Self::axis_indices(self.row_groups.as_deref(), num_row_groups) + } + + /// Returns an iterator over the column indices selected by this mask + pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + Self::axis_indices(self.columns.as_deref(), num_columns) + } + + fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + match axis { + None => Box::new(0..len), + Some(indices) => Box::new( + indices + .iter() + .map(|&i| i as usize) + .take_while(move |&i| i < len), + ), + } + } + + fn iter_to_set(indices: impl IntoIterator) -> Option> { + Some( + indices + .into_iter() + .filter_map(|i| u32::try_from(i).ok()) + .collect::>() + .into_iter() + .collect::>() + .into(), + ) } } @@ -249,19 +305,29 @@ impl ParquetMetaDataReader { self } - /// Selects the row groups and columns for which both page index structures are read. + /// Selects the row groups and leaf columns for which both page index structures are read. + /// + /// The mask applies only to index structures whose policy is not [`PageIndexPolicy::Skip`]. + /// This reader fetches one byte range covering the selected indexes, so a sparse mask + /// primarily reduces decoding and memory rather than necessarily reducing I/O. pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { self.with_column_index_mask(mask.clone()) .with_offset_index_mask(mask) } - /// Selects the row groups and columns for which column indexes are read. + /// Selects the row groups and leaf columns for which column indexes are read. + /// + /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self } - /// Selects the row groups and columns for which offset indexes are read. + /// Selects the row groups and leaf columns for which offset indexes are read. + /// + /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self @@ -1121,6 +1187,23 @@ mod tests { "EOF: Parquet file too small. Size is 1728 but need 1729" ); } + + #[test] + fn test_chunk_mask() { + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + let mask = ColumnChunkMask::columns([]); + assert!(!mask.includes_column(0)); + assert_eq!(mask.selected_columns(), Some([].as_slice())); + + let all = ColumnChunkMask::all(); + assert!(all.is_all()); + assert!(all.includes_column(u32::MAX as usize)); + } } #[cfg(all(feature = "async", feature = "arrow", test))] @@ -1526,32 +1609,4 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } - - #[test] - fn test_chunk_mask() { - // basic test - let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); - assert!(mask.includes_row_group(0)); - assert!(!mask.includes_row_group(1)); - assert!(!mask.includes_column(0)); - assert!(mask.includes_column(1)); - - // test that negative numbers are ignored - let mask = ColumnChunkMask::columns([-1, 0, 1]); - assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(!mask.includes_column(2)); - - // empty set should behave as "all" - let mask = ColumnChunkMask::columns([]); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(mask.includes_column(2)); - - // indexing beyond i32::MAX should return false - let mask = ColumnChunkMask::all(); - assert!(mask.includes_column(i32::MAX as usize)); - assert!(!mask.includes_column(u32::MAX as usize)); - } } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c605eac59834..c84e1c7b09c9 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -21,6 +21,129 @@ use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataR use crate::custom_page_index_provider::create_test_file; +fn assert_page_index_cells( + metadata: &parquet::file::metadata::ParquetMetaData, + expect_ci: impl Fn(usize, usize) -> bool, + expect_oi: impl Fn(usize, usize) -> bool, +) { + let page_index = metadata.page_index().expect("page index should be loaded"); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + for rg in 0..metadata.num_row_groups() { + for col in 0..num_cols { + assert_eq!( + page_index.column_index(rg, col).is_some(), + expect_ci(rg, col), + "column index rg={rg} col={col}" + ); + assert_eq!( + page_index.offset_index(rg, col).is_some(), + expect_oi(rg, col), + "offset index rg={rg} col={col}" + ); + } + } +} + +#[test] +fn test_arrow_reader_options_page_index_masks() { + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let column_mask = ColumnChunkMask::row_groups_and_columns([1], [0]); + let offset_mask = ColumnChunkMask::columns([2]); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(column_mask.clone()) + .with_offset_index_mask(offset_mask.clone()); + assert_eq!(options.column_index_mask(), &column_mask); + assert_eq!(options.offset_index_mask(), &offset_mask); + + let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap(); + assert_page_index_cells( + metadata.metadata(), + |rg, col| rg == 1 && col == 0, + |_, col| col == 2, + ); + + let metadata = ParquetMetaDataReader::new() + .with_arrow_reader_options(Some(&options)) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells(&metadata, |rg, col| rg == 1 && col == 0, |_, col| col == 2); +} + +#[test] +fn test_parse_with_page_index_mask() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([2], [1, 3])) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| rg == 2 && (col == 1 || col == 3), + |rg, col| rg == 2 && (col == 1 || col == 3), + ); +} + +/*#[test] +// C11 coverage +fn test_repeated_page_index_reads_merge_cells() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + ); +} + +#[test] +// C3 coverage +fn test_partial_page_statistics_remain_aligned() { + use arrow_array::Array; + use parquet::arrow::arrow_reader::statistics::StatisticsConverter; + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([1])); + let metadata = ArrowReaderMetadata::load(&file, options).unwrap(); + let parquet_metadata = metadata.metadata(); + let page_index = parquet_metadata.page_index().unwrap().as_ref(); + let row_groups = parquet_metadata.row_groups(); + let row_group_indices = [0, 1, 2]; + let converter = StatisticsConverter::try_new( + "id", + metadata.schema(), + parquet_metadata.file_metadata().schema_descr(), + ) + .unwrap(); + + let mins = converter + .data_page_mins(page_index, row_group_indices.iter()) + .unwrap(); + let row_counts = converter + .data_page_row_counts(page_index, row_groups, row_group_indices.iter()) + .unwrap() + .unwrap(); + assert_eq!(mins.len(), row_counts.len()); + assert_eq!(row_counts.null_count(), row_counts.len()); +}*/ + #[test] fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns diff --git a/parquet/tests/arrow_reader/partial_offset_index.rs b/parquet/tests/arrow_reader/partial_offset_index.rs index 72489da5c64a..28aa4bb18568 100644 --- a/parquet/tests/arrow_reader/partial_offset_index.rs +++ b/parquet/tests/arrow_reader/partial_offset_index.rs @@ -30,7 +30,7 @@ use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::arrow::{ArrowWriter, ProjectionMask}; use parquet::column::writer::ColumnCloseResult; use parquet::errors::Result; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use parquet::file::properties::WriterProperties; use parquet::file::writer::SerializedFileWriter; use std::sync::Arc; @@ -144,15 +144,23 @@ fn configure(builder: ArrowReaderBuilder, trigger: Trigger) -> ArrowReader } } -fn read_sync(file: &Bytes, trigger: Trigger) -> Result> { - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options())?; +fn read_sync( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options)?; Ok(configure(builder, trigger) .build()? .collect::>()?) } #[cfg(feature = "async")] -fn read_async(file: &Bytes, trigger: Trigger) -> Result> { +fn read_async( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { use futures::TryStreamExt; use parquet::arrow::ParquetRecordBatchStreamBuilder; @@ -161,13 +169,17 @@ fn read_async(file: &Bytes, trigger: Trigger) -> Result> { .unwrap(); runtime.block_on(async { let input = std::io::Cursor::new(file.clone()); - let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options()).await?; + let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options).await?; configure(builder, trigger).build()?.try_collect().await }) } -fn read_push_decoder(file: &Bytes, trigger: Trigger) -> Result> { - let metadata = ArrowReaderMetadata::load(file, options())?; +fn read_push_decoder( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let metadata = ArrowReaderMetadata::load(file, options)?; let builder = ParquetPushDecoderBuilder::new_with_metadata(metadata); let mut decoder = configure(builder, trigger).build()?; let mut batches = vec![]; @@ -202,10 +214,45 @@ fn check(has_index: impl Fn(usize, usize) -> bool) { let expected = test_batch().slice(150, 100); for trigger in [Trigger::RowSelection, Trigger::RowFilter] { let results = [ - ("sync", read_sync(&file, trigger)), - ("push decoder", read_push_decoder(&file, trigger)), + ("sync", read_sync(&file, trigger, options())), + ("push decoder", read_push_decoder(&file, trigger, options())), #[cfg(feature = "async")] - ("async", read_async(&file, trigger)), + ("async", read_async(&file, trigger, options())), + ]; + for (reader, result) in results { + let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); + let actual = concat_batches(&expected.schema(), &batches).unwrap(); + assert_eq!(actual, expected, "{reader} reader, {trigger:?}"); + } + } +} + +#[test] +fn test_read_with_offset_index_mask() { + let file = test_file(|_, _| true); + let masked_options = + || options().with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])); + let metadata = ArrowReaderMetadata::load(&file, masked_options()).unwrap(); + let page_index = metadata.metadata().page_index().unwrap(); + for row_group in 0..2 { + for column in 0..3 { + assert_eq!( + page_index.offset_index(row_group, column).is_some(), + row_group == 1 && column == 0 + ); + } + } + + let expected = test_batch().slice(150, 100); + for trigger in [Trigger::RowSelection, Trigger::RowFilter] { + let results = [ + ("sync", read_sync(&file, trigger, masked_options())), + ( + "push decoder", + read_push_decoder(&file, trigger, masked_options()), + ), + #[cfg(feature = "async")] + ("async", read_async(&file, trigger, masked_options())), ]; for (reader, result) in results { let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); From 4ee4125c5da460172d858cdd34039044dd131484 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:06:06 -0700 Subject: [PATCH 29/78] modify range_for_page_index to take masks --- parquet/src/file/metadata/push_decoder.rs | 71 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 4c8b514251ca..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -438,6 +438,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, ); let Some(page_index_range) = range else { @@ -507,28 +509,38 @@ enum DecodeState { } /// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// specified policies and masks /// /// Returns None if no page indexes are needed -pub fn range_for_page_index( +pub(crate) fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Option> { - fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } - } - - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + let mut range: Option> = None; + for (policy, mask, column_index) in [ + (column_index_policy, column_index_mask, true), + (offset_index_policy, offset_index_mask, false), + ] { + if policy != PageIndexPolicy::Skip { + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); + } + } + } } } range @@ -564,6 +576,33 @@ mod tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[test] + fn test_range_for_page_index_respects_column_mask() { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); + push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]); + let metadata = expect_data(decoder.try_decode()); + + let all = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::all(), + &ColumnChunkMask::all(), + ) + .unwrap(); + let masked = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::columns([0]), + &ColumnChunkMask::all(), + ) + .unwrap(); + + assert!(all.start <= masked.start && masked.end <= all.end); + assert!(masked.end - masked.start < all.end - all.start); + } + /// It is possible to feed some, but not all, of the footer into the metadata decoder /// before asked. This avoids multiple IO requests #[test] From f6a8fde4e2ab2a2b8ff63a15b0b89b2486a04a06 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:30:47 -0700 Subject: [PATCH 30/78] more efficient page index parser traversal --- parquet/src/file/metadata/parser.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index a76eea9066c0..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,7 +247,6 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. -/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -303,15 +302,9 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -335,15 +328,9 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; From 682edbfc0a7ccf4321bf6bcfd8ad2384a822f181 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:36:06 -0700 Subject: [PATCH 31/78] address StatisticsConverter misalignment --- parquet/src/arrow/arrow_reader/statistics.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/statistics.rs b/parquet/src/arrow/arrow_reader/statistics.rs index 486e937a79de..32684b2b1210 100644 --- a/parquet/src/arrow/arrow_reader/statistics.rs +++ b/parquet/src/arrow/arrow_reader/statistics.rs @@ -2042,6 +2042,15 @@ impl<'a> StatisticsConverter<'a> { let mut nulls = NullBufferBuilder::new(0); for rg_idx in row_group_indices { let Some(offset_index) = page_index.offset_index(*rg_idx, parquet_index) else { + // A partial page index may contain the ColumnIndex but not the OffsetIndex. + // Preserve page-wise alignment with data_page_{mins,maxes,null_counts} by + // emitting one null row count for every page known from the ColumnIndex. + let num_pages = page_index + .column_index(*rg_idx, parquet_index) + .map(|index| index.num_pages() as usize) + .unwrap_or(0); + row_counts.resize(row_counts.len() + num_pages, 0); + nulls.append_n_nulls(num_pages); continue; }; let page_locations = offset_index.page_locations(); From 18f34a8d36b46ba0ae7c732bc50f5f22583cd227 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:41:44 -0700 Subject: [PATCH 32/78] enable C3 test --- parquet/tests/arrow_reader/page_index.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c84e1c7b09c9..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -107,10 +107,9 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} +}*/ #[test] -// C3 coverage fn test_partial_page_statistics_remain_aligned() { use arrow_array::Array; use parquet::arrow::arrow_reader::statistics::StatisticsConverter; @@ -142,7 +141,7 @@ fn test_partial_page_statistics_remain_aligned() { .unwrap(); assert_eq!(mins.len(), row_counts.len()); assert_eq!(row_counts.null_count(), row_counts.len()); -}*/ +} #[test] fn test_parse_selected_columns() { From 81693113b564e9997c4518e0098bec0e4f5c7757 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:17:05 -0700 Subject: [PATCH 33/78] fix C11: preserve existing indexes across multiple parse calls --- parquet/src/file/metadata/page_index.rs | 42 +++++++++++++++++++++++ parquet/src/file/metadata/parser.rs | 11 ++++-- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 ++++- parquet/tests/arrow_reader/page_index.rs | 22 ++++++++++-- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 83b595daf141..18e3d4c85851 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,6 +513,36 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { + /// Creates a new [`PageIndexBuilder`] populated from an existing provider. + pub(crate) fn new_from_provider( + page_index: &dyn PageIndexProvider, + num_row_groups: usize, + num_columns: usize, + ) -> Self { + let mut builder = Self::default(); + if page_index.has_column_indexes() { + builder.allocate_column_indexes(num_row_groups, num_columns); + } + if page_index.has_offset_indexes() { + builder.allocate_offset_indexes(num_row_groups, num_columns); + } + for row_group in 0..num_row_groups { + for column in 0..num_columns { + if builder.column_indexes.is_some() + && let Some(index) = page_index.column_index(row_group, column) + { + builder.put_column_index(index.clone(), row_group, column); + } + if builder.offset_indexes.is_some() + && let Some(index) = page_index.offset_index(row_group, column) + { + builder.put_offset_index(index.clone(), row_group, column); + } + } + } + builder + } + /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -570,6 +600,12 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.column_indexes.is_none() { + self.allocate_column_indexes(num_row_groups, num_columns); + } + } + /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -582,6 +618,12 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.offset_indexes.is_none() { + self.allocate_offset_indexes(num_row_groups, num_columns); + } + } + /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..d146c89434a7 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,9 +261,14 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = metadata + .page_index() + .map(|index| { + PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) + }) + .unwrap_or_default(); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); + builder.ensure_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -273,7 +278,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); + builder.ensure_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..8e396fc8e441 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,7 +272,8 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. + /// after the metadata has already been decoded. Newly decoded page indexes + /// are merged with any page indexes already present in `metadata`. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 8987912499fb..4a4a3b9a894d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -514,6 +514,10 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// Newly read indexes are merged with any existing page index. Existing entries outside + /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are + /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -523,6 +527,7 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -664,7 +669,8 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. + /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as + /// described by [`Self::read_page_indexes()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..4c2353fb49c5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,8 +87,7 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage +#[test] fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -107,7 +106,24 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -}*/ +} + +#[test] +fn test_repeated_page_index_reads_preserve_skipped_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells(&metadata, |_, _| true, |_, _| true); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 09337c842f721327d973a5fc4255f0197021d1ab Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:28:41 -0700 Subject: [PATCH 34/78] Fix page index loading with suffix prefetch --- parquet/src/file/metadata/reader.rs | 37 +++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e4cec0fd0df5..088f3281db11 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -688,10 +688,9 @@ impl ParquetMetaDataReader { } else { let metadata_start = suffix_len - metadata_offset; let slice = suffix.slice(metadata_start..suffix_len - FOOTER_SIZE); - Ok(( - self.decode_footer_metadata(slice, file_size, footer)?, - Some((0, suffix.slice(..metadata_start))), - )) + // `MetadataSuffixFetch` does not expose the file size, so the file offset of + // the bytes preceding the metadata is unknown and they cannot be reused safely. + Ok((self.decode_footer_metadata(slice, file_size, footer)?, None)) } } @@ -1347,6 +1346,36 @@ mod async_tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[tokio::test] + async fn test_page_index_via_suffix_with_prefetch() { + let mut file = get_test_file("alltypes_tiny_pages.parquet"); + let mut suffix_file = file.try_clone().unwrap(); + let len = file.len(); + let fetch_count = AtomicUsize::new(0); + let suffix_fetch_count = AtomicUsize::new(0); + + let mut fetch = |range| { + fetch_count.fetch_add(1, Ordering::SeqCst); + futures::future::ready(read_range(&mut file, range)) + }; + let mut suffix_fetch = |suffix| { + suffix_fetch_count.fetch_add(1, Ordering::SeqCst); + futures::future::ready(read_suffix(&mut suffix_file, suffix)) + }; + + let input = MetadataSuffixFetchFn(&mut fetch, &mut suffix_fetch); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_prefetch_hint(Some((len - 1000) as usize)) + .load_via_suffix_and_finish(input) + .await + .unwrap(); + + assert_eq!(suffix_fetch_count.load(Ordering::SeqCst), 1); + assert_eq!(fetch_count.load(Ordering::SeqCst), 1); + assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); + } + fn write_parquet_file(offset_index_disabled: bool) -> Result { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); let batch = RecordBatch::try_new( From 5b9c07d9ccbb0ff3d6c69554a5c406945f4215ef Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 17:43:51 -0700 Subject: [PATCH 35/78] add plumbing for page index selective population --- parquet/benches/metadata.rs | 28 ++- parquet/src/arrow/arrow_reader/mod.rs | 56 ++++- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 16 ++ parquet/src/file/metadata/parser.rs | 68 +++--- parquet/src/file/metadata/push_decoder.rs | 221 +++++++++++++++++--- parquet/src/file/metadata/reader.rs | 108 ++++++++++ parquet/src/file/page_index/index_reader.rs | 11 - parquet/tests/arrow_reader/io/mod.rs | 4 +- 10 files changed, 441 insertions(+), 77 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 0ec1b3cd1733..d9fef3d1a5d0 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, ParquetMetaData, - ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, - RowGroupMetaData, + ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, + PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; use parquet::file::writer::TrackedWrite; @@ -191,6 +191,28 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&data) + .unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Required); + c.bench_function("read page index", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + c.bench_function("read page index reduced columns", |b| { + b.iter(|| { + reader.read_page_indexes(&data).unwrap(); + }) + }); + let meta_data = get_footer_bytes(data.clone()); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata", |b| { diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 73bcb2048ad5..8f8953e69933 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,8 +45,8 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, - ParquetStatisticsPolicy, RowGroupMetaData, + PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; use crate::schema::types::SchemaDescriptor; @@ -593,8 +593,10 @@ pub struct ArrowReaderOptions { /// [ARROW_SCHEMA_META_KEY]: crate::arrow::ARROW_SCHEMA_META_KEY supplied_schema: Option, - pub(crate) column_index: PageIndexPolicy, - pub(crate) offset_index: PageIndexPolicy, + column_index: PageIndexPolicy, + offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -774,6 +776,32 @@ impl ArrowReaderOptions { self } + /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// + /// The column index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when filtering by a predicate + /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// + /// The offset index can be costly to decode and store, especially when it is needed + /// only for a subset of row groups or columns (such as when projecting a small subset + /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// the time needed to decode this metadata. + /// + /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet /// footer will be skipped. /// @@ -924,6 +952,20 @@ impl ArrowReaderOptions { self.column_index } + /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// + /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. + pub fn offset_index_selection(&self) -> &PageIndexSelection { + &self.offset_index_selection + } + + /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// + /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. + pub fn column_index_selection(&self) -> &PageIndexSelection { + &self.column_index_selection + } + /// Retrieve the currently set metadata decoding options. pub fn metadata_options(&self) -> &ParquetMetaDataOptions { &self.metadata_options @@ -970,7 +1012,9 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self @@ -1019,6 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index 4bd65cfead59..a73d2c0d3fbe 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,7 +242,9 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 166a7eb7ab6a..65c084108c7f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, ParquetMetaDataReader}; +pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..46254777e668 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,6 +17,7 @@ //! Page Index structures for efficient page-level skipping +use crate::file::metadata::PageIndexSelection; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -555,6 +556,21 @@ impl PageIndexBuilder { } } + /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy + /// + /// For sparse indexes, this can save a great deal of memory + pub fn new_with_selection( + num_row_groups: usize, + num_columns: usize, + _column_index_selection: PageIndexSelection, + _offset_index_selection: PageIndexSelection, + ) -> Self { + Self { + column_indexes: Self::empty_index(num_row_groups, num_columns), + offset_indexes: Self::empty_index(num_row_groups, num_columns), + } + } + /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 861bb2e53b08..b2a35172e8af 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,13 +26,15 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use bytes::Bytes; +use crate::file::reader::ChunkReader; +use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -250,8 +252,9 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - bytes: &Bytes, - start_offset: u64, + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip { @@ -259,25 +262,28 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = PageIndexBuilder::new_with_selection( + num_row_groups, + num_columns, + column_index_selection.clone(), + offset_index_selection.clone(), + ); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, + column_index_selection, &mut builder, bytes, - start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, + offset_index_selection, &mut builder, bytes, - start_offset, )?; } @@ -293,27 +299,27 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_column_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -325,27 +331,27 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, + selection: &PageIndexSelection, page_index_builder: &mut PageIndexBuilder, - bytes: &Bytes, - start_offset: u64, + bytes: &PushBuffers, ) -> crate::errors::Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { + if !selection.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { + if !selection.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let r_start = usize::try_from(r.start - start_offset)?; - let r_end = usize::try_from(r.end - start_offset)?; - let idx = inner::parse_single_offset_index( - &bytes[r_start..r_end], - metadata, - col, - rg_idx, - col_idx, - )?; + let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx = + inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index c5e77cc22468..b55468fd2980 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -21,8 +21,10 @@ use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; -use crate::file::metadata::{FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions}; -use crate::file::page_index::index_reader::acc_range; +use crate::file::metadata::{ + ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ParquetMetaDataOptions, +}; use crate::file::reader::ChunkReader; use bytes::Bytes; use std::ops::Range; @@ -226,6 +228,10 @@ pub struct ParquetMetaDataPushDecoder { column_index_policy: PageIndexPolicy, /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, + /// which rows and columns of the column index should be read + column_index_selection: PageIndexSelection, + /// which rows and columns of the offset index should be read + offset_index_selection: PageIndexSelection, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -250,6 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, + column_index_selection: PageIndexSelection::all(), + offset_index_selection: PageIndexSelection::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -300,6 +308,25 @@ impl ParquetMetaDataPushDecoder { self } + /// Select the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection.clone(); + self.offset_index_selection = selection; + self + } + + /// Select the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Select the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Set the options to use when decoding the Parquet metadata. pub fn with_metadata_options(mut self, options: Option>) -> Self { self.metadata_parser = self.metadata_parser.with_metadata_options(options); @@ -408,30 +435,36 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let range = range_for_page_index( + let ranges = ranges_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_selection, + &self.offset_index_selection, ); - let Some(page_index_range) = range else { + if ranges.is_empty() { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - }; + } + + let needed_ranges = ranges + .into_iter() + .filter(|r| !self.buffers.has_range(r)) + .collect::>(); - if !self.buffers.has_range(&page_index_range) { + if !needed_ranges.is_empty() { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + return Ok(DecodeResult::NeedsData(needed_ranges)); } - let buffer = self.get_bytes(&page_index_range)?; - let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, - &buffer, - offset, + &self.column_index_selection, + &self.offset_index_selection, + &self.buffers, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); @@ -481,25 +514,81 @@ enum DecodeState { Intermediate, } -/// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// Returns the minimum set of non-overlapping ranges needed to cover the requested +/// offset and column indexes. +/// +/// If no page indexes are present in the file, or none are actually +/// requested by the policies passed in, this will return an empty +/// vector. /// -/// Returns None if no page indexes are needed -pub fn range_for_page_index( +/// This may result in more ranges to fetch but may also result in fewer bytes +/// being read (and stored). To get a single range for all indexes, sort +/// the resultant vector by the range starts, and then create a single range +/// using `start` from the head and `end` from the tail. +/// +/// ```rust +/// # use core::ops::Range; +/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { +/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// let range = (ranges.first()?.start..ranges.last()?.end); +/// Some(range) +/// # } +/// ``` +pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, -) -> Option> { - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + column_index_selection: &PageIndexSelection, + offset_index_selection: &PageIndexSelection, +) -> Vec> { + let mut result = Vec::new(); + + fn add_ranges( + metadata: &ParquetMetaData, + selection: &PageIndexSelection, + ranges: &mut Vec>, + f: T, + ) where + T: Fn(&ColumnChunkMetaData) -> Option>, + { + for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { + if selection.includes_row_group(rg_idx) { + for (col_idx, col) in rg.columns().iter().enumerate() { + if selection.includes_column(col_idx) { + if let Some(range) = f(col) { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); + } + } + } + } + } } } - range + + if column_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + column_index_selection, + &mut result, + ColumnChunkMetaData::column_index_range, + ); + } + if offset_index_policy != PageIndexPolicy::Skip { + add_ranges( + metadata, + offset_index_selection, + &mut result, + ColumnChunkMetaData::offset_index_range, + ); + } + result } // These tests use the arrow writer to create a parquet file in memory @@ -651,6 +740,90 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } + /// Decode the metadata incrementally, with a partial page index + #[test] + fn test_metadata_decoder_incremental_partial_page_index() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + + /// Decode the metadata incrementally, with a partial page index, but coalesced + /// range request. + #[test] + fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { + let file_len = TEST_FILE_DATA.len() as u64; + let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) + .unwrap() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_selection(PageIndexSelection::columns([0])); + let ranges = expect_needs_data(metadata_decoder.try_decode()); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the first request to read the footer + let ranges = expect_needs_data(metadata_decoder.try_decode()); + push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); + + // expect the second request to read the page indexes + let mut ranges = expect_needs_data(metadata_decoder.try_decode()); + // one for column index and one for offset index for each row group + assert_eq!(ranges.len(), 4); + // collapse ranges into a single range + ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + let range = ranges.first().unwrap().start..ranges.last().unwrap().end; + push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); + + // expect the third request to read the actual data + let metadata = expect_data(metadata_decoder.try_decode()); + expect_finished(metadata_decoder.try_decode()); + + assert_eq!(metadata.num_row_groups(), 2); + assert_eq!(metadata.row_group(0).num_rows(), 200); + assert_eq!(metadata.row_group(1).num_rows(), 200); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for rg in 0..metadata.num_row_groups() { + for col in 0..metadata.file_metadata().schema_descr().num_columns() { + assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); + assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); + } + } + } + static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 088f3281db11..9f31c250c33b 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -27,6 +27,7 @@ use crate::file::metadata::{ use crate::file::reader::ChunkReader; use crate::schema::types::SchemaDescriptor; use bytes::Bytes; +use std::collections::BTreeSet; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -71,6 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, + column_index_selection: PageIndexSelection, + offset_index_selection: PageIndexSelection, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -104,6 +107,89 @@ impl From for PageIndexPolicy { } } +/// Selects the row groups and columns for which page indexes are read. +/// +/// This is independent of [`PageIndexPolicy`], which controls whether selected +/// indexes are optional or required. By default, all row groups and columns are selected. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct PageIndexSelection { + // using i32 because that's how thrift vectors are sized + row_groups: Option>>, + columns: Option>>, +} + +// TODO(ets): add unit tests for PageIndexSelection +impl PageIndexSelection { + /// Select all row groups and columns. + pub fn all() -> Self { + Self::default() + } + + /// Select page indexes for only the listed columns. + /// + /// Any indices in `columns` that are less than zero will be ignored. + pub fn columns(columns: impl IntoIterator) -> Self { + Self { + row_groups: None, + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + /// Select page indexes for only the listed row groups. + /// + /// Any indices in `row_groups` that are less than zero will be ignored. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: None, + } + } + + /// Select page indexes for only the listed row groups and columns. + /// + /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + pub fn row_groups_and_columns( + row_groups: impl IntoIterator, + columns: impl IntoIterator, + ) -> Self { + Self { + row_groups: Some(Arc::new( + row_groups.into_iter().filter(|&i| i >= 0).collect(), + )), + columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + } + } + + // test if `idx` is in the row group set. returns false if idx > i32::MAX + pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.row_groups + .as_ref() + .is_none_or(|keep| keep.contains(&idx)) + } + + // test if `idx` is in the column set. returns false if idx > i32::MAX + pub(crate) fn includes_column(&self, idx: usize) -> bool { + let Ok(idx) = i32::try_from(idx) else { + return false; + }; + self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + } + + // FIXME(ets): these will be used later + /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { + self.row_groups.as_deref() + } + + pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { + self.columns.as_deref() + }*/ +} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -137,6 +223,24 @@ impl ParquetMetaDataReader { self } + /// Selects the row groups and columns for which both page index structures are read. + pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { + self.with_column_index_selection(selection.clone()) + .with_offset_index_selection(selection) + } + + /// Selects the row groups and columns for which column indexes are read. + pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.column_index_selection = selection; + self + } + + /// Selects the row groups and columns for which offset indexes are read. + pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { + self.offset_index_selection = selection; + self + } + /// Sets the [`ParquetMetaDataOptions`] to use when decoding pub fn with_metadata_options(mut self, options: Option) -> Self { self.metadata_options = options.map(Arc::new); @@ -341,6 +445,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -488,6 +594,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) + .with_offset_index_selection(self.offset_index_selection.clone()) + .with_column_index_selection(self.column_index_selection.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/src/file/page_index/index_reader.rs b/parquet/src/file/page_index/index_reader.rs index 963ee0ea9782..235f0b241887 100644 --- a/parquet/src/file/page_index/index_reader.rs +++ b/parquet/src/file/page_index/index_reader.rs @@ -30,17 +30,6 @@ use crate::parquet_thrift::{ }; use crate::thrift_struct; use std::io::Write; -use std::ops::Range; - -/// Computes the covering range of two optional ranges -/// -/// For example `acc_range(Some(7..9), Some(1..3)) = Some(1..9)` -pub(crate) fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } -} /// Decode a Thrift [`OffsetIndex`] from the provided bytes. /// diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 3fb63ffdb87b..2f0348b835f5 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -132,7 +132,9 @@ impl AsyncFileReader for TestReader { if let Some(options) = options { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()); + .with_offset_index_policy(options.offset_index_policy()) + .with_column_index_selection(options.column_index_selection().clone()) + .with_offset_index_selection(options.offset_index_selection().clone()); } self.metadata = Some(Arc::new( From 1c30c6a2f590cfee466a0fb81aa179c673230a24 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 13:23:30 -0700 Subject: [PATCH 36/78] more tests --- .../custom_page_index_provider.rs | 2 +- parquet/tests/arrow_reader/mod.rs | 1 + parquet/tests/arrow_reader/page_index.rs | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 parquet/tests/arrow_reader/page_index.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 147c9a7336c0..026637742798 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -404,7 +404,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> Bytes { +pub(super) fn create_test_file() -> Bytes { // Create small pages and small row groups for testing let props = WriterProperties::builder() .set_statistics_enabled(EnabledStatistics::Page) diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index 1dfcfb690160..680524f64e35 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -47,6 +47,7 @@ mod int96_stats_roundtrip; mod invalid_utf8; mod io; mod large_string_overflow; +mod page_index; mod parquet_testing; mod partial_offset_index; #[cfg(feature = "async")] diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs new file mode 100644 index 000000000000..821d1d4cc55d --- /dev/null +++ b/parquet/tests/arrow_reader/page_index.rs @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests of selective page index population + +use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use std::fs::File; + +use crate::custom_page_index_provider::create_test_file; + +#[test] +fn test_parse_selected_columns() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_none()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_none()); + } +} + +#[test] +fn test_parse_selected_columns_mixed() { + // test populating PageIndex with a subset of columns + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_rg = metadata.num_row_groups(); + + // test page indexes + for rg in 0..num_rg { + let idx = metadata.page_index_for_row_group(rg); + // column 0 has both indexes + assert!(idx.column_index(0).is_some()); + assert!(idx.offset_index(0).is_some()); + // column 1 has no indexes + assert!(idx.column_index(1).is_none()); + assert!(idx.offset_index(1).is_some()); + // column 2 has offset index + assert!(idx.column_index(2).is_none()); + assert!(idx.offset_index(2).is_some()); + // column 3 has no indexes + assert!(idx.column_index(3).is_none()); + assert!(idx.offset_index(3).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups() { + // test populating PageIndex with a subset of row groups + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate indexes for row groups 0 and 2 + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) + .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + + for col in 0..num_cols { + // row group 0 has both indexes + assert!(page_index.column_index(0, col).is_some()); + assert!(page_index.offset_index(0, col).is_some()); + // row group 1 has no indexes + assert!(page_index.column_index(1, col).is_none()); + assert!(page_index.offset_index(1, col).is_none()); + // row group 2 has both indexes + assert!(page_index.column_index(2, col).is_some()); + assert!(page_index.offset_index(2, col).is_some()); + } +} + +#[test] +fn test_parse_selected_row_groups_and_columns() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + + // row groups 0 and 2 should have no indexes + [0, 2].into_iter().for_each(|i| { + let rg_idx = metadata.page_index_for_row_group(i); + for col in 0..num_cols { + assert!(rg_idx.column_index(col).is_none()); + assert!(rg_idx.offset_index(col).is_none()); + } + }); + + // row group 1 should have column index for column 0 and offset index for columns 0 & 2 + let rg_idx = metadata.page_index_for_row_group(1); + assert!(rg_idx.column_index(0).is_some()); + assert!(rg_idx.offset_index(0).is_some()); + assert!(rg_idx.column_index(1).is_none()); + assert!(rg_idx.offset_index(1).is_none()); + assert!(rg_idx.column_index(2).is_none()); + assert!(rg_idx.offset_index(2).is_some()); + assert!(rg_idx.column_index(3).is_none()); + assert!(rg_idx.offset_index(3).is_none()); +} + +#[test] +fn test_page_index_sizes() { + // test populating PageIndex by row group and column + let temp_file = create_test_file(); + let file = File::open(temp_file.path()).unwrap(); + + // no index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_none()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 7393); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 7817); + + // full index + let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 13897); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 14321); + + // populate column 0 for column index and all columns for offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::all()); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12776); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 13200); + + // populate column 0 for column index and columns 0 & 2 for the offset index + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::columns([0])) + .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 12056); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 12480); + + // populate only row group 1, column index gets column 0, offset index gets + // columns 0 and 2. + let mut reader = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) + .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + + // parse metadata + reader.try_parse(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert!(metadata.page_index().is_some()); + #[cfg(not(feature = "encryption"))] + assert_eq!(metadata.memory_size(), 11326); + #[cfg(feature = "encryption")] + assert_eq!(metadata.memory_size(), 11750); +} From 3177084ca648afcfc553afd7ac15ebf9eab82878 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:05:59 -0700 Subject: [PATCH 37/78] rename PageIndexSelection to ColumnChunkMask --- parquet/benches/metadata.rs | 8 +-- parquet/src/arrow/arrow_reader/mod.rs | 46 +++++++------- parquet/src/arrow/async_reader/store.rs | 4 +- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 8 +-- parquet/src/file/metadata/parser.rs | 29 +++++---- parquet/src/file/metadata/push_decoder.rs | 76 +++++++++++------------ parquet/src/file/metadata/reader.rs | 55 +++++++++------- parquet/tests/arrow_reader/io/mod.rs | 4 +- parquet/tests/arrow_reader/page_index.rs | 30 ++++----- 10 files changed, 135 insertions(+), 127 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index d9fef3d1a5d0..b5ee07c49530 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use parquet::basic::{Encoding, PageType, Type as PhysicalType}; use parquet::file::metadata::{ - ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, PageIndexPolicy, - PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, + ColumnChunkMask, ColumnChunkMetaData, FileMetaData, LevelHistogram, PageEncodingStats, + PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetMetaDataWriter, ParquetStatisticsPolicy, RowGroupMetaData, }; use parquet::file::statistics::Statistics; @@ -205,8 +205,8 @@ fn criterion_benchmark(c: &mut Criterion) { let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 1, 4])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 1, 4])); c.bench_function("read page index reduced columns", |b| { b.iter(|| { reader.read_page_indexes(&data).unwrap(); diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 8f8953e69933..492ea6063462 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -45,7 +45,7 @@ use crate::column::page::{PageIterator, PageReader}; use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; use crate::file::metadata::{ - PageIndexPolicy, PageIndexSelection, ParquetMetaData, ParquetMetaDataOptions, + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData, }; use crate::file::reader::{ChunkReader, SerializedPageReader}; @@ -595,8 +595,8 @@ pub struct ArrowReaderOptions { column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, /// Options to control reading of Parquet metadata metadata_options: ParquetMetaDataOptions, @@ -776,29 +776,29 @@ impl ArrowReaderOptions { self } - /// Sets the [`PageIndexSelection`] for the Parquet [ColumnIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when filtering by a predicate - /// on a single column). Providing a [`PageIndexSelection`] can greatly decrease + /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } - /// Sets the [`PageIndexSelection`] for the Parquet [OffsetIndex] structure. + /// Sets the [`ColumnChunkMask`] for the Parquet [OffsetIndex] structure. /// /// The offset index can be costly to decode and store, especially when it is needed /// only for a subset of row groups or columns (such as when projecting a small subset - /// of columns). Providing a [`PageIndexSelection`] can greatly decrease + /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -952,18 +952,18 @@ impl ArrowReaderOptions { self.column_index } - /// Retrieve the currently set [`PageIndexSelection`] for the offset index. + /// Retrieve the currently set [`ColumnChunkMask`] for the offset index. /// - /// This can be set via [`with_offset_index_selection`][Self::with_offset_index_selection]. - pub fn offset_index_selection(&self) -> &PageIndexSelection { - &self.offset_index_selection + /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask]. + pub fn offset_index_mask(&self) -> &ColumnChunkMask { + &self.offset_index_mask } - /// Retrieve the currently set [`PageIndexSelection`] for the column index. + /// Retrieve the currently set [`ColumnChunkMask`] for the column index. /// - /// This can be set via [`with_column_index_selection`][Self::with_column_index_selection]. - pub fn column_index_selection(&self) -> &PageIndexSelection { - &self.column_index_selection + /// This can be set via [`with_column_index_mask`][Self::with_column_index_mask]. + pub fn column_index_mask(&self) -> &ColumnChunkMask { + &self.column_index_mask } /// Retrieve the currently set metadata decoding options. @@ -1013,8 +1013,8 @@ impl ParquetMetaDataReader { self = self .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self @@ -1063,8 +1063,8 @@ impl ArrowReaderMetadata { let metadata = ParquetMetaDataReader::new() .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()) + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()) .with_metadata_options(Some(options.metadata_options.clone())); #[cfg(feature = "encryption")] let metadata = metadata.with_decryption_properties( diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index a73d2c0d3fbe..c7db5d0abbc1 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -243,8 +243,8 @@ impl AsyncFileReader for ParquetObjectReader { metadata = metadata .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 65c084108c7f..12162a7f8f85 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -92,7 +92,7 @@ use crate::{ pub use footer_tail::FooterTail; pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy}; pub use push_decoder::ParquetMetaDataPushDecoder; -pub use reader::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::io::Write; use std::ops::Range; use std::sync::Arc; diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 46254777e668..2e70b6ab7650 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,7 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::PageIndexSelection; +use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -559,11 +559,11 @@ impl PageIndexBuilder { /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy /// /// For sparse indexes, this can save a great deal of memory - pub fn new_with_selection( + pub fn new_with_mask( num_row_groups: usize, num_columns: usize, - _column_index_selection: PageIndexSelection, - _offset_index_selection: PageIndexSelection, + _column_index_mask: ColumnChunkMask, + _offset_index_mask: ColumnChunkMask, ) -> Self { Self { column_indexes: Self::empty_index(num_row_groups, num_columns), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index b2a35172e8af..47b93463147c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -26,8 +26,7 @@ use crate::errors::ParquetError; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ - ColumnChunkMetaData, PageIndexPolicy, PageIndexSelection, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::page_index::column_index::ColumnIndexMetaData; @@ -252,8 +251,8 @@ pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip @@ -262,17 +261,17 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_selection( + let mut builder = PageIndexBuilder::new_with_mask( num_row_groups, num_columns, - column_index_selection.clone(), - offset_index_selection.clone(), + column_index_mask.clone(), + offset_index_mask.clone(), ); if column_index_policy != PageIndexPolicy::Skip { parse_column_index( metadata, column_index_policy, - column_index_selection, + column_index_mask, &mut builder, bytes, )?; @@ -281,7 +280,7 @@ pub(crate) fn parse_page_index( parse_offset_index( metadata, offset_index_policy, - offset_index_selection, + offset_index_mask, &mut builder, bytes, )?; @@ -299,7 +298,7 @@ pub(crate) fn parse_page_index( fn parse_column_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -307,12 +306,12 @@ fn parse_column_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); @@ -331,7 +330,7 @@ fn parse_column_index( fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, bytes: &PushBuffers, ) -> crate::errors::Result<()> { @@ -339,12 +338,12 @@ fn parse_offset_index( return Ok(()); } for rg_idx in 0..metadata.num_row_groups() { - if !selection.includes_row_group(rg_idx) { + if !mask.includes_row_group(rg_idx) { continue; } let rg = metadata.row_group(rg_idx); for col_idx in 0..rg.num_columns() { - if !selection.includes_column(col_idx) { + if !mask.includes_column(col_idx) { continue; } let col = rg.column(col_idx); diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index b55468fd2980..3999fdf9a9fb 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,7 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMetaData, FooterTail, PageIndexPolicy, PageIndexSelection, ParquetMetaData, + ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; @@ -229,9 +229,9 @@ pub struct ParquetMetaDataPushDecoder { /// policy for loading OffsetIndex (part of the PageIndex) offset_index_policy: PageIndexPolicy, /// which rows and columns of the column index should be read - column_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, /// which rows and columns of the offset index should be read - offset_index_selection: PageIndexSelection, + offset_index_mask: ColumnChunkMask, /// Underlying buffers buffers: crate::util::push_buffers::PushBuffers, /// Encryption API @@ -256,8 +256,8 @@ impl ParquetMetaDataPushDecoder { state: DecodeState::ReadingFooter, column_index_policy: PageIndexPolicy::Optional, offset_index_policy: PageIndexPolicy::Optional, - column_index_selection: PageIndexSelection::all(), - offset_index_selection: PageIndexSelection::all(), + column_index_mask: ColumnChunkMask::all(), + offset_index_mask: ColumnChunkMask::all(), buffers: crate::util::push_buffers::PushBuffers::new(file_len), metadata_parser: MetadataParser::new(), }) @@ -309,21 +309,21 @@ impl ParquetMetaDataPushDecoder { } /// Select the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection.clone(); - self.offset_index_selection = selection; + pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask.clone(); + self.offset_index_mask = mask; self } /// Select the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Select the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -439,8 +439,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, ); if ranges.is_empty() { @@ -462,8 +462,8 @@ impl ParquetMetaDataPushDecoder { &mut metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_selection, - &self.offset_index_selection, + &self.column_index_mask, + &self.offset_index_mask, &self.buffers, )?; self.state = DecodeState::Finished; @@ -526,10 +526,10 @@ enum DecodeState { /// the resultant vector by the range starts, and then create a single range /// using `start` from the head and `end` from the tail. /// -/// ```rust +/// ```ignore /// # use core::ops::Range; /// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); +/// ranges.sort_by_key(|r| r.start); /// let range = (ranges.first()?.start..ranges.last()?.end); /// Some(range) /// # } @@ -538,33 +538,33 @@ pub fn ranges_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_selection: &PageIndexSelection, - offset_index_selection: &PageIndexSelection, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Vec> { let mut result = Vec::new(); fn add_ranges( metadata: &ParquetMetaData, - selection: &PageIndexSelection, + mask: &ColumnChunkMask, ranges: &mut Vec>, f: T, ) where T: Fn(&ColumnChunkMetaData) -> Option>, { for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if selection.includes_row_group(rg_idx) { + if mask.includes_row_group(rg_idx) { for (col_idx, col) in rg.columns().iter().enumerate() { - if selection.includes_column(col_idx) { - if let Some(range) = f(col) { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } + if mask.includes_column(col_idx) + && let Some(range) = f(col) + { + // ranges shouldn't overlap, so only check for contiguous ranges + // [s1..e1], [s2..e2] where e1 == s2 + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + } else { + ranges.push(range); } } } @@ -575,7 +575,7 @@ pub fn ranges_for_page_index( if column_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - column_index_selection, + column_index_mask, &mut result, ColumnChunkMetaData::column_index_range, ); @@ -583,7 +583,7 @@ pub fn ranges_for_page_index( if offset_index_policy != PageIndexPolicy::Skip { add_ranges( metadata, - offset_index_selection, + offset_index_mask, &mut result, ColumnChunkMetaData::offset_index_range, ); @@ -747,7 +747,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -788,7 +788,7 @@ mod tests { let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) .unwrap() .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_selection(PageIndexSelection::columns([0])); + .with_page_index_mask(ColumnChunkMask::columns([0])); let ranges = expect_needs_data(metadata_decoder.try_decode()); assert_eq!(ranges.len(), 1); assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); @@ -803,7 +803,7 @@ mod tests { // one for column index and one for offset index for each row group assert_eq!(ranges.len(), 4); // collapse ranges into a single range - ranges.sort_by(|r1, r2| r1.start.cmp(&r2.start)); + ranges.sort_by_key(|r| r.start); let range = ranges.first().unwrap().start..ranges.last().unwrap().end; push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 9f31c250c33b..6e0b5f011ee0 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -72,8 +72,8 @@ pub struct ParquetMetaDataReader { metadata: Option, column_index: PageIndexPolicy, offset_index: PageIndexPolicy, - column_index_selection: PageIndexSelection, - offset_index_selection: PageIndexSelection, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, prefetch_hint: Option, metadata_options: Option>, // Size of the serialized thrift metadata plus the 8 byte footer. Only set if @@ -107,25 +107,34 @@ impl From for PageIndexPolicy { } } -/// Selects the row groups and columns for which page indexes are read. +/// Struct to specify column chunks for which metadata is required. /// -/// This is independent of [`PageIndexPolicy`], which controls whether selected -/// indexes are optional or required. By default, all row groups and columns are selected. +/// Column chunks are identified by row group index and column index. This struct +/// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), +/// horizontal slices (via [`Self::row_groups`]), or the intersection of the two +/// (via [`Self::row_groups_and_columns`]). +/// +/// # Example +/// TODO(ets): example +/// +/// At present this is only used to select elements of the [Page Index] for decoding. +/// +/// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct PageIndexSelection { +pub struct ColumnChunkMask { // using i32 because that's how thrift vectors are sized row_groups: Option>>, columns: Option>>, } -// TODO(ets): add unit tests for PageIndexSelection -impl PageIndexSelection { +// TODO(ets): add unit tests for ColumnChunkMask +impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { Self::default() } - /// Select page indexes for only the listed columns. + /// Select only the listed columns. /// /// Any indices in `columns` that are less than zero will be ignored. pub fn columns(columns: impl IntoIterator) -> Self { @@ -135,7 +144,7 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups. + /// Select only the listed row groups. /// /// Any indices in `row_groups` that are less than zero will be ignored. pub fn row_groups(row_groups: impl IntoIterator) -> Self { @@ -147,9 +156,9 @@ impl PageIndexSelection { } } - /// Select page indexes for only the listed row groups and columns. + /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns that are less than zero will be ignored. + /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, @@ -224,20 +233,20 @@ impl ParquetMetaDataReader { } /// Selects the row groups and columns for which both page index structures are read. - pub fn with_page_index_selection(self, selection: PageIndexSelection) -> Self { - self.with_column_index_selection(selection.clone()) - .with_offset_index_selection(selection) + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) } /// Selects the row groups and columns for which column indexes are read. - pub fn with_column_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.column_index_selection = selection; + pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.column_index_mask = mask; self } /// Selects the row groups and columns for which offset indexes are read. - pub fn with_offset_index_selection(mut self, selection: PageIndexSelection) -> Self { - self.offset_index_selection = selection; + pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { + self.offset_index_mask = mask; self } @@ -445,8 +454,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); @@ -594,8 +603,8 @@ impl ParquetMetaDataReader { let push_decoder = ParquetMetaDataPushDecoder::try_new_with_metadata(file_size, metadata)? .with_offset_index_policy(self.offset_index) .with_column_index_policy(self.column_index) - .with_offset_index_selection(self.offset_index_selection.clone()) - .with_column_index_selection(self.column_index_selection.clone()) + .with_offset_index_mask(self.offset_index_mask.clone()) + .with_column_index_mask(self.column_index_mask.clone()) .with_metadata_options(self.metadata_options.clone()); let mut push_decoder = self.prepare_push_decoder(push_decoder); diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs index 2f0348b835f5..bb9b5aeba446 100644 --- a/parquet/tests/arrow_reader/io/mod.rs +++ b/parquet/tests/arrow_reader/io/mod.rs @@ -133,8 +133,8 @@ impl AsyncFileReader for TestReader { metadata_reader = metadata_reader .with_column_index_policy(options.column_index_policy()) .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_selection(options.column_index_selection().clone()) - .with_offset_index_selection(options.offset_index_selection().clone()); + .with_column_index_mask(options.column_index_mask().clone()) + .with_offset_index_mask(options.offset_index_mask().clone()); } self.metadata = Some(Arc::new( diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 821d1d4cc55d..e594e78e900d 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -17,7 +17,7 @@ //! Tests of selective page index population -use parquet::file::metadata::{PageIndexPolicy, PageIndexSelection, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -31,8 +31,8 @@ fn test_parse_selected_columns() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -66,8 +66,8 @@ fn test_parse_selected_columns_mixed() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -101,8 +101,8 @@ fn test_parse_selected_row_groups() { // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups([0, 2])) - .with_offset_index_selection(PageIndexSelection::row_groups([0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups([0, 2])) + .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -134,8 +134,8 @@ fn test_parse_selected_row_groups_and_columns() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -197,8 +197,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::all()); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata reader.try_parse(&file).unwrap(); @@ -212,8 +212,8 @@ fn test_page_index_sizes() { // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::columns([0])) - .with_offset_index_selection(PageIndexSelection::columns([0, 2])); + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata reader.try_parse(&file).unwrap(); @@ -228,8 +228,8 @@ fn test_page_index_sizes() { // columns 0 and 2. let mut reader = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Optional) - .with_column_index_selection(PageIndexSelection::row_groups_and_columns([1], [0])) - .with_offset_index_selection(PageIndexSelection::row_groups_and_columns([1], [0, 2])); + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])) + .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata reader.try_parse(&file).unwrap(); From 942f2bfe0ece4a0c1bf6f2c88eaea10fd8735266 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Sat, 19 Sep 2026 22:17:29 -0700 Subject: [PATCH 38/78] docs and cleanup --- parquet/src/file/metadata/page_index.rs | 16 ---- parquet/src/file/metadata/parser.rs | 12 ++- parquet/src/file/metadata/push_decoder.rs | 2 + parquet/src/file/metadata/reader.rs | 94 +++++++++++++++++------ 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 2e70b6ab7650..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,7 +17,6 @@ //! Page Index structures for efficient page-level skipping -use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -556,21 +555,6 @@ impl PageIndexBuilder { } } - /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy - /// - /// For sparse indexes, this can save a great deal of memory - pub fn new_with_mask( - num_row_groups: usize, - num_columns: usize, - _column_index_mask: ColumnChunkMask, - _offset_index_mask: ColumnChunkMask, - ) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } - } - /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 47b93463147c..a76eea9066c0 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,7 +245,8 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). -/// * `bytes` - The byte slice containing the page index data. +/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing +/// the page indexes. /// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, @@ -261,13 +262,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_mask( - num_row_groups, - num_columns, - column_index_mask.clone(), - offset_index_mask.clone(), - ); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -277,6 +274,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 3999fdf9a9fb..6b747b34aabf 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -701,6 +701,8 @@ mod tests { // expect the second request to read the offset indexes let ranges = expect_needs_data(metadata_decoder.try_decode()); + // since we're reading all indexes, there should be a single range needed + assert_eq!(ranges.len(), 1); push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); // expect the third request to read the actual data diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 6e0b5f011ee0..993554f1087d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -114,11 +114,28 @@ impl From for PageIndexPolicy { /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). /// -/// # Example -/// TODO(ets): example -/// /// At present this is only used to select elements of the [Page Index] for decoding. /// +/// # Examples +/// +/// To select columns 0 and 1 from all row groups: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::columns([0, 1]); +/// ``` +/// +/// To select all columns from row group 2: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups([2]); +/// ``` +/// +/// To select columns 1 and 3 from row group 0: +/// ```rust +/// # use parquet::file::metadata::ColumnChunkMask; +/// let mask = ColumnChunkMask::row_groups_and_columns([0], [1, 3]); +/// ``` +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ColumnChunkMask { @@ -127,7 +144,6 @@ pub struct ColumnChunkMask { columns: Option>>, } -// TODO(ets): add unit tests for ColumnChunkMask impl ColumnChunkMask { /// Select all row groups and columns. pub fn all() -> Self { @@ -136,22 +152,22 @@ impl ColumnChunkMask { /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. + /// Any indices in `columns` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all columns. pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + columns: Self::iter_to_set(columns), } } /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. + /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty + /// set is treated the same as selecting all row groups. pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), + row_groups: Self::iter_to_set(row_groups), columns: None, } } @@ -159,20 +175,22 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. + /// Passing an empty set for `row_groups` is treated as selecting all row groups, and + /// an empty set for `columns` as selectiong all columns. pub fn row_groups_and_columns( row_groups: impl IntoIterator, columns: impl IntoIterator, ) -> Self { Self { - row_groups: Some(Arc::new( - row_groups.into_iter().filter(|&i| i >= 0).collect(), - )), - columns: Some(Arc::new(columns.into_iter().filter(|&i| i >= 0).collect())), + row_groups: Self::iter_to_set(row_groups), + columns: Self::iter_to_set(columns), } } - // test if `idx` is in the row group set. returns false if idx > i32::MAX - pub(crate) fn includes_row_group(&self, idx: usize) -> bool { + /// Test if `idx` is in the row group set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_row_group(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; @@ -181,22 +199,20 @@ impl ColumnChunkMask { .is_none_or(|keep| keep.contains(&idx)) } - // test if `idx` is in the column set. returns false if idx > i32::MAX - pub(crate) fn includes_column(&self, idx: usize) -> bool { + /// Test if `idx` is in the column set. + /// + /// Returns `false` if `idx > `[`i32::MAX`]. + pub fn includes_column(&self, idx: usize) -> bool { let Ok(idx) = i32::try_from(idx) else { return false; }; self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) } - // FIXME(ets): these will be used later - /*pub(crate) fn selected_row_groups(&self) -> Option<&BTreeSet> { - self.row_groups.as_deref() + fn iter_to_set(indices: impl IntoIterator) -> Option>> { + let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); + (!set.is_empty()).then_some(Arc::new(set)) } - - pub(crate) fn selected_columns(&self) -> Option<&BTreeSet> { - self.columns.as_deref() - }*/ } impl ParquetMetaDataReader { @@ -1538,4 +1554,32 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } + + #[test] + fn test_chunk_mask() { + // basic test + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + // test that negative numbers are ignored + let mask = ColumnChunkMask::columns([-1, 0, 1]); + assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(!mask.includes_column(2)); + + // empty set should behave as "all" + let mask = ColumnChunkMask::columns([]); + assert!(mask.includes_column(0)); + assert!(mask.includes_column(1)); + assert!(mask.includes_column(2)); + + // indexing beyond i32::MAX should return false + let mask = ColumnChunkMask::all(); + assert!(mask.includes_column(i32::MAX as usize)); + assert!(!mask.includes_column(u32::MAX as usize)); + } } From 199a41d2690668ce67f1b236bcdc0c6d8d4f0c08 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 10:13:48 -0700 Subject: [PATCH 39/78] rework tests to do no file i/o --- parquet/tests/arrow_reader/page_index.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e594e78e900d..c605eac59834 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -18,7 +18,6 @@ //! Tests of selective page index population use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; -use std::fs::File; use crate::custom_page_index_provider::create_test_file; @@ -26,7 +25,6 @@ use crate::custom_page_index_provider::create_test_file; fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -35,7 +33,7 @@ fn test_parse_selected_columns() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -61,7 +59,6 @@ fn test_parse_selected_columns() { fn test_parse_selected_columns_mixed() { // test populating PageIndex with a subset of columns let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -70,7 +67,7 @@ fn test_parse_selected_columns_mixed() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_rg = metadata.num_row_groups(); @@ -96,7 +93,6 @@ fn test_parse_selected_columns_mixed() { fn test_parse_selected_row_groups() { // test populating PageIndex with a subset of row groups let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate indexes for row groups 0 and 2 let mut reader = ParquetMetaDataReader::new() @@ -105,7 +101,7 @@ fn test_parse_selected_row_groups() { .with_offset_index_mask(ColumnChunkMask::row_groups([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); let num_cols = metadata.file_metadata().schema_descr().num_columns(); assert!(metadata.page_index().is_some()); @@ -128,7 +124,6 @@ fn test_parse_selected_row_groups() { fn test_parse_selected_row_groups_and_columns() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -138,7 +133,7 @@ fn test_parse_selected_row_groups_and_columns() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); let num_cols = metadata.file_metadata().schema_descr().num_columns(); @@ -168,13 +163,12 @@ fn test_parse_selected_row_groups_and_columns() { fn test_page_index_sizes() { // test populating PageIndex by row group and column let temp_file = create_test_file(); - let file = File::open(temp_file.path()).unwrap(); // no index let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Skip); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_none()); #[cfg(not(feature = "encryption"))] @@ -186,7 +180,7 @@ fn test_page_index_sizes() { let mut reader = ParquetMetaDataReader::new().with_page_index_policy(PageIndexPolicy::Required); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -201,7 +195,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::all()); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -216,7 +210,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::columns([0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] @@ -232,7 +226,7 @@ fn test_page_index_sizes() { .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2])); // parse metadata - reader.try_parse(&file).unwrap(); + reader.try_parse(&temp_file).unwrap(); let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] From 3dfeeca12defd2b6ea40de025580b1e917a32711 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 16:11:00 -0700 Subject: [PATCH 40/78] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/src/file/metadata/reader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 993554f1087d..86f13e5fe8d6 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -109,7 +109,8 @@ impl From for PageIndexPolicy { /// Struct to specify column chunks for which metadata is required. /// -/// Column chunks are identified by row group index and column index. This struct +/// Column chunks are identified by row group index and leaf column index (the index of the +/// column in [`SchemaDescriptor::columns`], not the index of a root or Arrow field). This struct /// allows for specifying vertical slices of column chunk data (via [`Self::columns`]), /// horizontal slices (via [`Self::row_groups`]), or the intersection of the two /// (via [`Self::row_groups_and_columns`]). From 26962603f9a90979952052c5887dec37616bcb51 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 07:53:10 -0700 Subject: [PATCH 41/78] revert most changes to push decoder --- parquet/src/file/metadata/push_decoder.rs | 189 +++------------------- 1 file changed, 24 insertions(+), 165 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 6b747b34aabf..4c8b514251ca 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -22,8 +22,7 @@ use crate::errors::{ParquetError, Result}; use crate::file::FOOTER_SIZE; use crate::file::metadata::parser::{MetadataParser, parse_page_index}; use crate::file::metadata::{ - ColumnChunkMask, ColumnChunkMetaData, FooterTail, PageIndexPolicy, ParquetMetaData, - ParquetMetaDataOptions, + ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, }; use crate::file::reader::ChunkReader; use bytes::Bytes; @@ -435,27 +434,20 @@ impl ParquetMetaDataPushDecoder { DecodeState::ReadingPageIndex(mut metadata) => { // First determine if any page indexes are needed based on // the specified policies - let ranges = ranges_for_page_index( + let range = range_for_page_index( &metadata, self.column_index_policy, self.offset_index_policy, - &self.column_index_mask, - &self.offset_index_mask, ); - if ranges.is_empty() { + let Some(page_index_range) = range else { self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); - } - - let needed_ranges = ranges - .into_iter() - .filter(|r| !self.buffers.has_range(r)) - .collect::>(); + }; - if !needed_ranges.is_empty() { + if !self.buffers.has_range(&page_index_range) { self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(DecodeResult::NeedsData(needed_ranges)); + return Ok(needs_range(page_index_range)); } parse_page_index( @@ -514,81 +506,32 @@ enum DecodeState { Intermediate, } -/// Returns the minimum set of non-overlapping ranges needed to cover the requested -/// offset and column indexes. +/// Returns the byte range needed to read the offset/page indexes, based on the +/// specified policies /// -/// If no page indexes are present in the file, or none are actually -/// requested by the policies passed in, this will return an empty -/// vector. -/// -/// This may result in more ranges to fetch but may also result in fewer bytes -/// being read (and stored). To get a single range for all indexes, sort -/// the resultant vector by the range starts, and then create a single range -/// using `start` from the head and `end` from the tail. -/// -/// ```ignore -/// # use core::ops::Range; -/// # fn coalesce_page_index_ranges(ranges: &mut Vec>) -> Option> { -/// ranges.sort_by_key(|r| r.start); -/// let range = (ranges.first()?.start..ranges.last()?.end); -/// Some(range) -/// # } -/// ``` -pub fn ranges_for_page_index( +/// Returns None if no page indexes are needed +pub fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, - column_index_mask: &ColumnChunkMask, - offset_index_mask: &ColumnChunkMask, -) -> Vec> { - let mut result = Vec::new(); - - fn add_ranges( - metadata: &ParquetMetaData, - mask: &ColumnChunkMask, - ranges: &mut Vec>, - f: T, - ) where - T: Fn(&ColumnChunkMetaData) -> Option>, - { - for (rg_idx, rg) in metadata.row_groups().iter().enumerate() { - if mask.includes_row_group(rg_idx) { - for (col_idx, col) in rg.columns().iter().enumerate() { - if mask.includes_column(col_idx) - && let Some(range) = f(col) - { - // ranges shouldn't overlap, so only check for contiguous ranges - // [s1..e1], [s2..e2] where e1 == s2 - if let Some(last) = ranges.last_mut() - && last.end == range.start - { - last.end = range.end; - } else { - ranges.push(range); - } - } - } - } +) -> Option> { + fn acc_range(a: Option>, b: Option>) -> Option> { + match (a, b) { + (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), + (None, x) | (x, None) => x, } } - if column_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - column_index_mask, - &mut result, - ColumnChunkMetaData::column_index_range, - ); - } - if offset_index_policy != PageIndexPolicy::Skip { - add_ranges( - metadata, - offset_index_mask, - &mut result, - ColumnChunkMetaData::offset_index_range, - ); + let mut range = None; + for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { + if column_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.column_index_range()); + } + if offset_index_policy != PageIndexPolicy::Skip { + range = acc_range(range, c.offset_index_range()); + } } - result + range } // These tests use the arrow writer to create a parquet file in memory @@ -742,90 +685,6 @@ mod tests { assert!(metadata.page_index().is_none()); // of course, we did not read the page index } - /// Decode the metadata incrementally, with a partial page index - #[test] - fn test_metadata_decoder_incremental_partial_page_index() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - - /// Decode the metadata incrementally, with a partial page index, but coalesced - /// range request. - #[test] - fn test_metadata_decoder_incremental_partial_page_index_one_fetch() { - let file_len = TEST_FILE_DATA.len() as u64; - let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len) - .unwrap() - .with_page_index_policy(PageIndexPolicy::Required) - .with_page_index_mask(ColumnChunkMask::columns([0])); - let ranges = expect_needs_data(metadata_decoder.try_decode()); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0], test_file_len() - 8..test_file_len()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the first request to read the footer - let ranges = expect_needs_data(metadata_decoder.try_decode()); - push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges); - - // expect the second request to read the page indexes - let mut ranges = expect_needs_data(metadata_decoder.try_decode()); - // one for column index and one for offset index for each row group - assert_eq!(ranges.len(), 4); - // collapse ranges into a single range - ranges.sort_by_key(|r| r.start); - let range = ranges.first().unwrap().start..ranges.last().unwrap().end; - push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![range]); - - // expect the third request to read the actual data - let metadata = expect_data(metadata_decoder.try_decode()); - expect_finished(metadata_decoder.try_decode()); - - assert_eq!(metadata.num_row_groups(), 2); - assert_eq!(metadata.row_group(0).num_rows(), 200); - assert_eq!(metadata.row_group(1).num_rows(), 200); - assert!(metadata.page_index().is_some()); - let page_index = metadata.page_index().unwrap(); - for rg in 0..metadata.num_row_groups() { - for col in 0..metadata.file_metadata().schema_descr().num_columns() { - assert_eq!(page_index.column_index(rg, col).is_some(), col == 0); - assert_eq!(page_index.offset_index(rg, col).is_some(), col == 0); - } - } - } - static TEST_BATCH: LazyLock = LazyLock::new(|| { // Input batch has 400 rows, with 3 columns: "a", "b", "c" // Note c is a different types (so the data page sizes will be different) From e303c539ecb6c304c2970c2ea0c65e1de43343c6 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 09:33:16 -0700 Subject: [PATCH 42/78] address a number of issues found in review, primarily documentation changes, and changes to chunk mask semantics and storage. Also fixes the preloading reader issue. --- parquet/src/arrow/arrow_reader/mod.rs | 28 ++- parquet/src/arrow/async_reader/store.rs | 12 +- parquet/src/file/metadata/reader.rs | 183 ++++++++++++------ parquet/tests/arrow_reader/page_index.rs | 123 ++++++++++++ .../arrow_reader/partial_offset_index.rs | 67 ++++++- 5 files changed, 333 insertions(+), 80 deletions(-) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 492ea6063462..9788f597d95b 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -776,6 +776,12 @@ impl ArrowReaderOptions { self } + /// Sets the same [`ColumnChunkMask`] for both page-index structures. + pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { + self.with_column_index_mask(mask.clone()) + .with_offset_index_mask(mask) + } + /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure. /// /// The column index can be costly to decode and store, especially when it is needed @@ -783,6 +789,11 @@ impl ArrowReaderOptions { /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// The mask applies only if the column-index policy is not [`PageIndexPolicy::Skip`] + /// (the default), or an underlying reader is configured to preload the index. It is + /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; + /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. + /// /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -796,6 +807,10 @@ impl ArrowReaderOptions { /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease /// the time needed to decode this metadata. /// + /// Page pruning also needs the offset index for predicate columns, so callers should + /// include those columns in addition to projected columns. The same loading and policy + /// qualifications as [`Self::with_column_index_mask`] apply. + /// /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; @@ -1012,9 +1027,16 @@ impl ParquetMetaDataReader { { self = self .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + + // Preload settings on the underlying reader may enable an index even when the + // corresponding options policy is `Skip`, so apply non-default masks independently. + if !options.column_index_mask().is_all() { + self = self.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + self = self.with_offset_index_mask(options.offset_index_mask().clone()); } self diff --git a/parquet/src/arrow/async_reader/store.rs b/parquet/src/arrow/async_reader/store.rs index c7db5d0abbc1..5528cc05b2bd 100644 --- a/parquet/src/arrow/async_reader/store.rs +++ b/parquet/src/arrow/async_reader/store.rs @@ -242,9 +242,15 @@ impl AsyncFileReader for ParquetObjectReader { { metadata = metadata .with_column_index_policy(options.column_index_policy()) - .with_offset_index_policy(options.offset_index_policy()) - .with_column_index_mask(options.column_index_mask().clone()) - .with_offset_index_mask(options.offset_index_mask().clone()); + .with_offset_index_policy(options.offset_index_policy()); + } + if let Some(options) = options { + if !options.column_index_mask().is_all() { + metadata = metadata.with_column_index_mask(options.column_index_mask().clone()); + } + if !options.offset_index_mask().is_all() { + metadata = metadata.with_offset_index_mask(options.offset_index_mask().clone()); + } } let metadata = if let Some(file_size) = self.file_size { diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 86f13e5fe8d6..af345d7193fd 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -91,10 +91,12 @@ pub enum PageIndexPolicy { Skip, /// Read the page index if it exists, otherwise do not error. Optional, - /// Require the page index to exist, and error if it does not. + /// Require offset indexes for every selected column chunk when any selected + /// page index ranges exist. /// - /// Only enforced for the offset index; this is the same as [`Self::Optional`] - /// for the column index. + /// For compatibility, a file with no page index ranges is accepted. This is only + /// enforced for the offset index; this is the same as [`Self::Optional`] for the + /// column index. Required, } @@ -138,11 +140,12 @@ impl From for PageIndexPolicy { /// ``` /// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct ColumnChunkMask { - // using i32 because that's how thrift vectors are sized - row_groups: Option>>, - columns: Option>>, + // `None` means all, while `Some(empty)` means none. Store u32 because + // Parquet/Thrift collections cannot contain more than i32::MAX entries. + row_groups: Option>, + columns: Option>, } impl ColumnChunkMask { @@ -151,11 +154,18 @@ impl ColumnChunkMask { Self::default() } + /// Select no row groups or columns. + pub fn none() -> Self { + Self { + row_groups: Some(Arc::from([])), + columns: Some(Arc::from([])), + } + } + /// Select only the listed columns. /// - /// Any indices in `columns` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all columns. - pub fn columns(columns: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no columns. + pub fn columns(columns: impl IntoIterator) -> Self { Self { row_groups: None, columns: Self::iter_to_set(columns), @@ -164,9 +174,8 @@ impl ColumnChunkMask { /// Select only the listed row groups. /// - /// Any indices in `row_groups` that are less than zero will be ignored. Passing an empty - /// set is treated the same as selecting all row groups. - pub fn row_groups(row_groups: impl IntoIterator) -> Self { + /// Passing an empty iterator selects no row groups. + pub fn row_groups(row_groups: impl IntoIterator) -> Self { Self { row_groups: Self::iter_to_set(row_groups), columns: None, @@ -175,12 +184,10 @@ impl ColumnChunkMask { /// Select only the listed row groups and columns. /// - /// Any indices in `row_groups` or `columns` that are less than zero will be ignored. - /// Passing an empty set for `row_groups` is treated as selecting all row groups, and - /// an empty set for `columns` as selectiong all columns. + /// An empty iterator for either dimension selects no column chunks. pub fn row_groups_and_columns( - row_groups: impl IntoIterator, - columns: impl IntoIterator, + row_groups: impl IntoIterator, + columns: impl IntoIterator, ) -> Self { Self { row_groups: Self::iter_to_set(row_groups), @@ -189,30 +196,79 @@ impl ColumnChunkMask { } /// Test if `idx` is in the row group set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_row_group(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { - return false; - }; - self.row_groups - .as_ref() - .is_none_or(|keep| keep.contains(&idx)) + Self::includes_index(self.row_groups.as_ref(), idx) } /// Test if `idx` is in the column set. - /// - /// Returns `false` if `idx > `[`i32::MAX`]. pub fn includes_column(&self, idx: usize) -> bool { - let Ok(idx) = i32::try_from(idx) else { + Self::includes_index(self.columns.as_ref(), idx) + } + + fn includes_index(keep: Option<&Arc<[u32]>>, idx: usize) -> bool { + // return false for out-of-bounds index + let Ok(idx) = u32::try_from(idx) else { return false; }; - self.columns.as_ref().is_none_or(|keep| keep.contains(&idx)) + keep.is_none_or(|keep| keep.binary_search(&idx).is_ok()) + } + + /// Returns `true` when this mask selects every column chunk. + pub fn is_all(&self) -> bool { + self.row_groups.is_none() && self.columns.is_none() + } + + /// Returns selected row groups, or `None` when all row groups are selected. + pub fn selected_row_groups(&self) -> Option<&[u32]> { + self.row_groups.as_deref() + } + + /// Returns selected leaf columns, or `None` when all columns are selected. + pub fn selected_columns(&self) -> Option<&[u32]> { + self.columns.as_deref() + } + + /// Creates a mask selecting the leaf columns in an Arrow projection. + #[cfg(feature = "arrow")] + pub fn from_projection( + projection: &crate::arrow::ProjectionMask, + schema: &SchemaDescriptor, + ) -> Self { + Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } - fn iter_to_set(indices: impl IntoIterator) -> Option>> { - let set: BTreeSet = indices.into_iter().filter(|&i| i >= 0).collect(); - (!set.is_empty()).then_some(Arc::new(set)) + /// Returns an iterator over the row group indices selected by this mask + pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + Self::axis_indices(self.row_groups.as_deref(), num_row_groups) + } + + /// Returns an iterator over the column indices selected by this mask + pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + Self::axis_indices(self.columns.as_deref(), num_columns) + } + + fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + match axis { + None => Box::new(0..len), + Some(indices) => Box::new( + indices + .iter() + .map(|&i| i as usize) + .take_while(move |&i| i < len), + ), + } + } + + fn iter_to_set(indices: impl IntoIterator) -> Option> { + Some( + indices + .into_iter() + .filter_map(|i| u32::try_from(i).ok()) + .collect::>() + .into_iter() + .collect::>() + .into(), + ) } } @@ -249,19 +305,29 @@ impl ParquetMetaDataReader { self } - /// Selects the row groups and columns for which both page index structures are read. + /// Selects the row groups and leaf columns for which both page index structures are read. + /// + /// The mask applies only to index structures whose policy is not [`PageIndexPolicy::Skip`]. + /// This reader fetches one byte range covering the selected indexes, so a sparse mask + /// primarily reduces decoding and memory rather than necessarily reducing I/O. pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self { self.with_column_index_mask(mask.clone()) .with_offset_index_mask(mask) } - /// Selects the row groups and columns for which column indexes are read. + /// Selects the row groups and leaf columns for which column indexes are read. + /// + /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self } - /// Selects the row groups and columns for which offset indexes are read. + /// Selects the row groups and leaf columns for which offset indexes are read. + /// + /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. + /// [`PageIndexPolicy::Required`] validates only selected column chunks. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self @@ -1120,6 +1186,23 @@ mod tests { "EOF: Parquet file too small. Size is 1728 but need 1729" ); } + + #[test] + fn test_chunk_mask() { + let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); + assert!(mask.includes_row_group(0)); + assert!(!mask.includes_row_group(1)); + assert!(!mask.includes_column(0)); + assert!(mask.includes_column(1)); + + let mask = ColumnChunkMask::columns([]); + assert!(!mask.includes_column(0)); + assert_eq!(mask.selected_columns(), Some([].as_slice())); + + let all = ColumnChunkMask::all(); + assert!(all.is_all()); + assert!(all.includes_column(u32::MAX as usize)); + } } #[cfg(all(feature = "async", feature = "arrow", test))] @@ -1555,32 +1638,4 @@ mod async_tests { read_and_check(f.as_file(), PageIndexPolicy::Optional).unwrap(); read_and_check(f.as_file(), PageIndexPolicy::Skip).unwrap(); } - - #[test] - fn test_chunk_mask() { - // basic test - let mask = ColumnChunkMask::row_groups_and_columns([0], [1]); - assert!(mask.includes_row_group(0)); - assert!(!mask.includes_row_group(1)); - assert!(!mask.includes_column(0)); - assert!(mask.includes_column(1)); - - // test that negative numbers are ignored - let mask = ColumnChunkMask::columns([-1, 0, 1]); - assert!(mask.columns.as_ref().is_some_and(|c| c.len() == 2)); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(!mask.includes_column(2)); - - // empty set should behave as "all" - let mask = ColumnChunkMask::columns([]); - assert!(mask.includes_column(0)); - assert!(mask.includes_column(1)); - assert!(mask.includes_column(2)); - - // indexing beyond i32::MAX should return false - let mask = ColumnChunkMask::all(); - assert!(mask.includes_column(i32::MAX as usize)); - assert!(!mask.includes_column(u32::MAX as usize)); - } } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c605eac59834..c84e1c7b09c9 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -21,6 +21,129 @@ use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataR use crate::custom_page_index_provider::create_test_file; +fn assert_page_index_cells( + metadata: &parquet::file::metadata::ParquetMetaData, + expect_ci: impl Fn(usize, usize) -> bool, + expect_oi: impl Fn(usize, usize) -> bool, +) { + let page_index = metadata.page_index().expect("page index should be loaded"); + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + for rg in 0..metadata.num_row_groups() { + for col in 0..num_cols { + assert_eq!( + page_index.column_index(rg, col).is_some(), + expect_ci(rg, col), + "column index rg={rg} col={col}" + ); + assert_eq!( + page_index.offset_index(rg, col).is_some(), + expect_oi(rg, col), + "offset index rg={rg} col={col}" + ); + } + } +} + +#[test] +fn test_arrow_reader_options_page_index_masks() { + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let column_mask = ColumnChunkMask::row_groups_and_columns([1], [0]); + let offset_mask = ColumnChunkMask::columns([2]); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(column_mask.clone()) + .with_offset_index_mask(offset_mask.clone()); + assert_eq!(options.column_index_mask(), &column_mask); + assert_eq!(options.offset_index_mask(), &offset_mask); + + let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap(); + assert_page_index_cells( + metadata.metadata(), + |rg, col| rg == 1 && col == 0, + |_, col| col == 2, + ); + + let metadata = ParquetMetaDataReader::new() + .with_arrow_reader_options(Some(&options)) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells(&metadata, |rg, col| rg == 1 && col == 0, |_, col| col == 2); +} + +#[test] +fn test_parse_with_page_index_mask() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([2], [1, 3])) + .parse_and_finish(&file) + .unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| rg == 2 && (col == 1 || col == 3), + |rg, col| rg == 2 && (col == 1 || col == 3), + ); +} + +/*#[test] +// C11 coverage +fn test_repeated_page_index_reads_merge_cells() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells( + &metadata, + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + ); +} + +#[test] +// C3 coverage +fn test_partial_page_statistics_remain_aligned() { + use arrow_array::Array; + use parquet::arrow::arrow_reader::statistics::StatisticsConverter; + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + + let file = create_test_file(); + let options = ArrowReaderOptions::new() + .with_page_index_policy(PageIndexPolicy::Required) + .with_column_index_mask(ColumnChunkMask::columns([0])) + .with_offset_index_mask(ColumnChunkMask::columns([1])); + let metadata = ArrowReaderMetadata::load(&file, options).unwrap(); + let parquet_metadata = metadata.metadata(); + let page_index = parquet_metadata.page_index().unwrap().as_ref(); + let row_groups = parquet_metadata.row_groups(); + let row_group_indices = [0, 1, 2]; + let converter = StatisticsConverter::try_new( + "id", + metadata.schema(), + parquet_metadata.file_metadata().schema_descr(), + ) + .unwrap(); + + let mins = converter + .data_page_mins(page_index, row_group_indices.iter()) + .unwrap(); + let row_counts = converter + .data_page_row_counts(page_index, row_groups, row_group_indices.iter()) + .unwrap() + .unwrap(); + assert_eq!(mins.len(), row_counts.len()); + assert_eq!(row_counts.null_count(), row_counts.len()); +}*/ + #[test] fn test_parse_selected_columns() { // test populating PageIndex with a subset of columns diff --git a/parquet/tests/arrow_reader/partial_offset_index.rs b/parquet/tests/arrow_reader/partial_offset_index.rs index 72489da5c64a..28aa4bb18568 100644 --- a/parquet/tests/arrow_reader/partial_offset_index.rs +++ b/parquet/tests/arrow_reader/partial_offset_index.rs @@ -30,7 +30,7 @@ use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::arrow::{ArrowWriter, ProjectionMask}; use parquet::column::writer::ColumnCloseResult; use parquet::errors::Result; -use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; use parquet::file::properties::WriterProperties; use parquet::file::writer::SerializedFileWriter; use std::sync::Arc; @@ -144,15 +144,23 @@ fn configure(builder: ArrowReaderBuilder, trigger: Trigger) -> ArrowReader } } -fn read_sync(file: &Bytes, trigger: Trigger) -> Result> { - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options())?; +fn read_sync( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file.clone(), options)?; Ok(configure(builder, trigger) .build()? .collect::>()?) } #[cfg(feature = "async")] -fn read_async(file: &Bytes, trigger: Trigger) -> Result> { +fn read_async( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { use futures::TryStreamExt; use parquet::arrow::ParquetRecordBatchStreamBuilder; @@ -161,13 +169,17 @@ fn read_async(file: &Bytes, trigger: Trigger) -> Result> { .unwrap(); runtime.block_on(async { let input = std::io::Cursor::new(file.clone()); - let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options()).await?; + let builder = ParquetRecordBatchStreamBuilder::new_with_options(input, options).await?; configure(builder, trigger).build()?.try_collect().await }) } -fn read_push_decoder(file: &Bytes, trigger: Trigger) -> Result> { - let metadata = ArrowReaderMetadata::load(file, options())?; +fn read_push_decoder( + file: &Bytes, + trigger: Trigger, + options: ArrowReaderOptions, +) -> Result> { + let metadata = ArrowReaderMetadata::load(file, options)?; let builder = ParquetPushDecoderBuilder::new_with_metadata(metadata); let mut decoder = configure(builder, trigger).build()?; let mut batches = vec![]; @@ -202,10 +214,45 @@ fn check(has_index: impl Fn(usize, usize) -> bool) { let expected = test_batch().slice(150, 100); for trigger in [Trigger::RowSelection, Trigger::RowFilter] { let results = [ - ("sync", read_sync(&file, trigger)), - ("push decoder", read_push_decoder(&file, trigger)), + ("sync", read_sync(&file, trigger, options())), + ("push decoder", read_push_decoder(&file, trigger, options())), #[cfg(feature = "async")] - ("async", read_async(&file, trigger)), + ("async", read_async(&file, trigger, options())), + ]; + for (reader, result) in results { + let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); + let actual = concat_batches(&expected.schema(), &batches).unwrap(); + assert_eq!(actual, expected, "{reader} reader, {trigger:?}"); + } + } +} + +#[test] +fn test_read_with_offset_index_mask() { + let file = test_file(|_, _| true); + let masked_options = + || options().with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0])); + let metadata = ArrowReaderMetadata::load(&file, masked_options()).unwrap(); + let page_index = metadata.metadata().page_index().unwrap(); + for row_group in 0..2 { + for column in 0..3 { + assert_eq!( + page_index.offset_index(row_group, column).is_some(), + row_group == 1 && column == 0 + ); + } + } + + let expected = test_batch().slice(150, 100); + for trigger in [Trigger::RowSelection, Trigger::RowFilter] { + let results = [ + ("sync", read_sync(&file, trigger, masked_options())), + ( + "push decoder", + read_push_decoder(&file, trigger, masked_options()), + ), + #[cfg(feature = "async")] + ("async", read_async(&file, trigger, masked_options())), ]; for (reader, result) in results { let batches = result.unwrap_or_else(|e| panic!("{reader} reader, {trigger:?}: {e}")); From 9672e90836025fd1fc1f96cc2419a1a46a687852 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:06:06 -0700 Subject: [PATCH 43/78] modify range_for_page_index to take masks --- parquet/src/file/metadata/push_decoder.rs | 71 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 4c8b514251ca..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -438,6 +438,8 @@ impl ParquetMetaDataPushDecoder { &metadata, self.column_index_policy, self.offset_index_policy, + &self.column_index_mask, + &self.offset_index_mask, ); let Some(page_index_range) = range else { @@ -507,28 +509,38 @@ enum DecodeState { } /// Returns the byte range needed to read the offset/page indexes, based on the -/// specified policies +/// specified policies and masks /// /// Returns None if no page indexes are needed -pub fn range_for_page_index( +pub(crate) fn range_for_page_index( metadata: &ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, ) -> Option> { - fn acc_range(a: Option>, b: Option>) -> Option> { - match (a, b) { - (Some(a), Some(b)) => Some(a.start.min(b.start)..a.end.max(b.end)), - (None, x) | (x, None) => x, - } - } - - let mut range = None; - for c in metadata.row_groups().iter().flat_map(|r| r.columns()) { - if column_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.column_index_range()); - } - if offset_index_policy != PageIndexPolicy::Skip { - range = acc_range(range, c.offset_index_range()); + let mut range: Option> = None; + for (policy, mask, column_index) in [ + (column_index_policy, column_index_mask, true), + (offset_index_policy, offset_index_mask, false), + ] { + if policy != PageIndexPolicy::Skip { + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); + } + } + } } } range @@ -564,6 +576,33 @@ mod tests { assert!(metadata.page_index().is_some_and(|idx| idx.is_complete())); } + #[test] + fn test_range_for_page_index_respects_column_mask() { + let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap(); + push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]); + let metadata = expect_data(decoder.try_decode()); + + let all = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::all(), + &ColumnChunkMask::all(), + ) + .unwrap(); + let masked = range_for_page_index( + &metadata, + PageIndexPolicy::Required, + PageIndexPolicy::Skip, + &ColumnChunkMask::columns([0]), + &ColumnChunkMask::all(), + ) + .unwrap(); + + assert!(all.start <= masked.start && masked.end <= all.end); + assert!(masked.end - masked.start < all.end - all.start); + } + /// It is possible to feed some, but not all, of the footer into the metadata decoder /// before asked. This avoids multiple IO requests #[test] From 1c5a1351bd07c6f3f6a36b6f7c2812a90c7400ca Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:30:47 -0700 Subject: [PATCH 44/78] more efficient page index parser traversal --- parquet/src/file/metadata/parser.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index a76eea9066c0..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,7 +247,6 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. -/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -303,15 +302,9 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -335,15 +328,9 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; From 2c597fb80308fdf1db21fc86d128b79ed01b83e1 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:36:06 -0700 Subject: [PATCH 45/78] address StatisticsConverter misalignment --- parquet/src/arrow/arrow_reader/statistics.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/statistics.rs b/parquet/src/arrow/arrow_reader/statistics.rs index 486e937a79de..32684b2b1210 100644 --- a/parquet/src/arrow/arrow_reader/statistics.rs +++ b/parquet/src/arrow/arrow_reader/statistics.rs @@ -2042,6 +2042,15 @@ impl<'a> StatisticsConverter<'a> { let mut nulls = NullBufferBuilder::new(0); for rg_idx in row_group_indices { let Some(offset_index) = page_index.offset_index(*rg_idx, parquet_index) else { + // A partial page index may contain the ColumnIndex but not the OffsetIndex. + // Preserve page-wise alignment with data_page_{mins,maxes,null_counts} by + // emitting one null row count for every page known from the ColumnIndex. + let num_pages = page_index + .column_index(*rg_idx, parquet_index) + .map(|index| index.num_pages() as usize) + .unwrap_or(0); + row_counts.resize(row_counts.len() + num_pages, 0); + nulls.append_n_nulls(num_pages); continue; }; let page_locations = offset_index.page_locations(); From 134a3b91c95aee64baa6212535664f6151d4ad4e Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 10:41:44 -0700 Subject: [PATCH 46/78] enable C3 test --- parquet/tests/arrow_reader/page_index.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index c84e1c7b09c9..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -107,10 +107,9 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} +}*/ #[test] -// C3 coverage fn test_partial_page_statistics_remain_aligned() { use arrow_array::Array; use parquet::arrow::arrow_reader::statistics::StatisticsConverter; @@ -142,7 +141,7 @@ fn test_partial_page_statistics_remain_aligned() { .unwrap(); assert_eq!(mins.len(), row_counts.len()); assert_eq!(row_counts.null_count(), row_counts.len()); -}*/ +} #[test] fn test_parse_selected_columns() { From dc23d994fab26476c3d7b177cad68d8a150b2dd1 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 15:17:05 -0700 Subject: [PATCH 47/78] fix C11: preserve existing indexes across multiple parse calls --- parquet/src/file/metadata/page_index.rs | 42 +++++++++++++++++++++++ parquet/src/file/metadata/parser.rs | 11 ++++-- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 ++++- parquet/tests/arrow_reader/page_index.rs | 22 ++++++++++-- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..5f94fe3c9d88 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,6 +513,36 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { + /// Creates a new [`PageIndexBuilder`] populated from an existing provider. + pub(crate) fn new_from_provider( + page_index: &dyn PageIndexProvider, + num_row_groups: usize, + num_columns: usize, + ) -> Self { + let mut builder = Self::default(); + if page_index.has_column_indexes() { + builder.allocate_column_indexes(num_row_groups, num_columns); + } + if page_index.has_offset_indexes() { + builder.allocate_offset_indexes(num_row_groups, num_columns); + } + for row_group in 0..num_row_groups { + for column in 0..num_columns { + if builder.column_indexes.is_some() + && let Some(index) = page_index.column_index(row_group, column) + { + builder.put_column_index(index.clone(), row_group, column); + } + if builder.offset_indexes.is_some() + && let Some(index) = page_index.offset_index(row_group, column) + { + builder.put_offset_index(index.clone(), row_group, column); + } + } + } + builder + } + /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -567,6 +597,12 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.column_indexes.is_none() { + self.allocate_column_indexes(num_row_groups, num_columns); + } + } + /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -579,6 +615,12 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } + pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + if self.offset_indexes.is_none() { + self.allocate_offset_indexes(num_row_groups, num_columns); + } + } + /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..d146c89434a7 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,9 +261,14 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); + let mut builder = metadata + .page_index() + .map(|index| { + PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) + }) + .unwrap_or_default(); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); + builder.ensure_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -273,7 +278,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); + builder.ensure_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..8e396fc8e441 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,7 +272,8 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. + /// after the metadata has already been decoded. Newly decoded page indexes + /// are merged with any page indexes already present in `metadata`. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index af345d7193fd..cb8b9457d91f 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -514,6 +514,10 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// Newly read indexes are merged with any existing page index. Existing entries outside + /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are + /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -523,6 +527,7 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -664,7 +669,8 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. + /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as + /// described by [`Self::read_page_indexes()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..4c2353fb49c5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,8 +87,7 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage +#[test] fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -107,7 +106,24 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -}*/ +} + +#[test] +fn test_repeated_page_index_reads_preserve_skipped_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + assert_page_index_cells(&metadata, |_, _| true, |_, _| true); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 78917edcc93308cc4020de6c3d56177c0fce9aea Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 24 Sep 2026 21:10:38 -0700 Subject: [PATCH 48/78] empty_index no longer needs pub(crate) --- parquet/src/file/metadata/page_index.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 18e3d4c85851..5f94fe3c9d88 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -550,10 +550,7 @@ impl PageIndexBuilder { /// /// # Type Parameters /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - pub(crate) fn empty_index( - num_row_groups: usize, - num_columns: usize, - ) -> Option>>> { + fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { Some( (0..num_row_groups) .map(|_| { From 9fd6d8a68fe22dad63f29f759b9ad6c1fccf62f0 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 12:32:45 -0700 Subject: [PATCH 49/78] Revert "fix C11: preserve existing indexes across multiple parse calls" This reverts commit dc23d994fab26476c3d7b177cad68d8a150b2dd1. --- parquet/src/file/metadata/page_index.rs | 42 ----------------------- parquet/src/file/metadata/parser.rs | 11 ++---- parquet/src/file/metadata/push_decoder.rs | 3 +- parquet/src/file/metadata/reader.rs | 8 +---- parquet/tests/arrow_reader/page_index.rs | 22 ++---------- 5 files changed, 8 insertions(+), 78 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 5f94fe3c9d88..78c3dc2792bb 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -513,36 +513,6 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { - /// Creates a new [`PageIndexBuilder`] populated from an existing provider. - pub(crate) fn new_from_provider( - page_index: &dyn PageIndexProvider, - num_row_groups: usize, - num_columns: usize, - ) -> Self { - let mut builder = Self::default(); - if page_index.has_column_indexes() { - builder.allocate_column_indexes(num_row_groups, num_columns); - } - if page_index.has_offset_indexes() { - builder.allocate_offset_indexes(num_row_groups, num_columns); - } - for row_group in 0..num_row_groups { - for column in 0..num_columns { - if builder.column_indexes.is_some() - && let Some(index) = page_index.column_index(row_group, column) - { - builder.put_column_index(index.clone(), row_group, column); - } - if builder.offset_indexes.is_some() - && let Some(index) = page_index.offset_index(row_group, column) - { - builder.put_offset_index(index.clone(), row_group, column); - } - } - } - builder - } - /// Creates an empty index structure with space for the specified number of row groups and columns /// /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. @@ -597,12 +567,6 @@ impl PageIndexBuilder { self.column_indexes = Self::empty_index(num_row_groups, num_columns); } - pub(crate) fn ensure_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - if self.column_indexes.is_none() { - self.allocate_column_indexes(num_row_groups, num_columns); - } - } - /// Allocates space for offset indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -615,12 +579,6 @@ impl PageIndexBuilder { self.offset_indexes = Self::empty_index(num_row_groups, num_columns); } - pub(crate) fn ensure_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - if self.offset_indexes.is_none() { - self.allocate_offset_indexes(num_row_groups, num_columns); - } - } - /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index d146c89434a7..22feffd2e613 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -261,14 +261,9 @@ pub(crate) fn parse_page_index( } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = metadata - .page_index() - .map(|index| { - PageIndexBuilder::new_from_provider(index.as_ref(), num_row_groups, num_columns) - }) - .unwrap_or_default(); + let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { - builder.ensure_column_indexes(num_row_groups, num_columns); + builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -278,7 +273,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.ensure_offset_indexes(num_row_groups, num_columns); + builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 8e396fc8e441..89cca4785962 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -272,8 +272,7 @@ impl ParquetMetaDataPushDecoder { /// Create a decoder with the given `ParquetMetaData` already known. /// /// This can be used to parse and populate the page index structures - /// after the metadata has already been decoded. Newly decoded page indexes - /// are merged with any page indexes already present in `metadata`. + /// after the metadata has already been decoded. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 8c7d07840ec0..cee752f4dd79 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -518,10 +518,6 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. - /// - /// Newly read indexes are merged with any existing page index. Existing entries outside - /// the configured masks, or for an index whose policy is [`PageIndexPolicy::Skip`], are - /// preserved. Newly read entries replace existing entries for the same column chunk. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -531,7 +527,6 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. - /// Newly read indexes are merged as described by [`Self::read_page_indexes()`]. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -673,8 +668,7 @@ impl ParquetMetaDataReader { } /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already - /// been obtained. See [`Self::new_with_metadata()`]. Newly read indexes are merged as - /// described by [`Self::read_page_indexes()`]. + /// been obtained. See [`Self::new_with_metadata()`]. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 4c2353fb49c5..57062c73d2e0 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,7 +87,8 @@ fn test_parse_with_page_index_mask() { ); } -#[test] +/*#[test] +// C11 coverage fn test_repeated_page_index_reads_merge_cells() { let file = create_test_file(); let metadata = ParquetMetaDataReader::new() @@ -106,24 +107,7 @@ fn test_repeated_page_index_reads_merge_cells() { |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), ); -} - -#[test] -fn test_repeated_page_index_reads_preserve_skipped_index() { - let file = create_test_file(); - let metadata = ParquetMetaDataReader::new() - .with_column_index_policy(PageIndexPolicy::Required) - .with_offset_index_policy(PageIndexPolicy::Skip) - .parse_and_finish(&file) - .unwrap(); - - let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) - .with_column_index_policy(PageIndexPolicy::Skip) - .with_offset_index_policy(PageIndexPolicy::Required); - reader.read_page_indexes(&file).unwrap(); - let metadata = reader.finish().unwrap(); - assert_page_index_cells(&metadata, |_, _| true, |_, _| true); -} +}*/ #[test] fn test_partial_page_statistics_remain_aligned() { From bc975c40ee82dd29c37a74ddf6cc45ec72f50cb9 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 15:01:35 -0700 Subject: [PATCH 50/78] replace path for C11 --- parquet/src/file/metadata/parser.rs | 21 +++--- parquet/src/file/metadata/push_decoder.rs | 21 +++--- parquet/src/file/metadata/reader.rs | 13 ++++ parquet/tests/arrow_reader/page_index.rs | 88 +++++++++++++++++++++-- 4 files changed, 119 insertions(+), 24 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 22feffd2e613..84645687540d 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -237,10 +237,10 @@ pub(crate) fn decode_metadata( parquet_metadata_from_bytes(buf, options) } -/// Parses page index from the provided bytes and adds it to the metadata. +/// Parses page indexes from the provided bytes and replaces those in the metadata. /// /// Arguments -/// * `metadata` - The ParquetMetaData to which the parsed column index will be added. +/// * `metadata` - The ParquetMetaData whose page index will be replaced. /// * `column_index_policy` - The policy for handling column index parsing (e.g., /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., @@ -255,13 +255,10 @@ pub(crate) fn parse_page_index( offset_index_mask: &ColumnChunkMask, bytes: &PushBuffers, ) -> crate::errors::Result<()> { - if column_index_policy == PageIndexPolicy::Skip && offset_index_policy == PageIndexPolicy::Skip - { - return Ok(()); - } let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); let mut builder = PageIndexBuilder::default(); + if column_index_policy != PageIndexPolicy::Skip { builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( @@ -283,12 +280,16 @@ pub(crate) fn parse_page_index( )?; } + // Always replace the page index, even if both policies are Skip. + // This ensures that repeated calls to read_page_indexes replace the existing index + // rather than preserving it. let page_index = builder.build(); - // if both indexes are missing from the file, return without modifying `metadata` - if !page_index.has_column_indexes() && !page_index.has_offset_indexes() { - return Ok(()); + if page_index.has_column_indexes() || page_index.has_offset_indexes() { + metadata.set_page_index(Some(Arc::new(page_index))); + } else { + // If no indexes were read, clear the page index + metadata.set_page_index(None); } - metadata.set_page_index(Some(Arc::new(page_index))); Ok(()) } diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 89cca4785962..af106dd1f5e0 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -273,6 +273,11 @@ impl ParquetMetaDataPushDecoder { /// /// This can be used to parse and populate the page index structures /// after the metadata has already been decoded. + /// + /// When decoding page indexes, any existing page indexes in `metadata` will be + /// replaced based on the configured policies. See + /// [`crate::file::metadata::ParquetMetaDataReader::read_page_indexes`] for details + /// on the replacement behavior. pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result { let mut new_self = Self::try_new(file_len)?; new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata)); @@ -442,14 +447,14 @@ impl ParquetMetaDataPushDecoder { &self.offset_index_mask, ); - let Some(page_index_range) = range else { - self.state = DecodeState::Finished; - return Ok(DecodeResult::Data(*metadata)); - }; - - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); + // Even if there is no range to read (for example, both policies are Skip, + // the masks select nothing, or the file has no indexes), parse the result to + // replace any existing indexes. + if let Some(page_index_range) = range { + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(needs_range(page_index_range)); + } } parse_page_index( diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index cee752f4dd79..efd73a8b38fd 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -518,6 +518,13 @@ impl ParquetMetaDataReader { /// Read the page index structures when a [`ParquetMetaData`] has already been obtained. /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. + /// + /// # Behavior on multiple calls + /// + /// On success, this method replaces any existing page index in the metadata. Index types + /// configured as [`PageIndexPolicy::Skip`] are cleared; other index types contain only the + /// indexes selected by their configured masks and available according to their policies. If + /// neither index type is populated, the page index is cleared. pub fn read_page_indexes(&mut self, reader: &R) -> Result<()> { self.read_page_indexes_sized(reader, reader.len()) } @@ -527,6 +534,9 @@ impl ParquetMetaDataReader { /// a [`Bytes`] struct containing the tail of the file). /// See [`Self::new_with_metadata()`] and [`Self::has_metadata()`]. Like /// [`Self::try_parse_sized()`] this function may return [`ParquetError::NeedMoreData`]. + /// + /// See [`Self::read_page_indexes()`] for a description of the replacement behavior on + /// multiple calls. pub fn read_page_indexes_sized( &mut self, reader: &R, @@ -669,6 +679,9 @@ impl ParquetMetaDataReader { /// Asynchronously fetch the page index structures when a [`ParquetMetaData`] has already /// been obtained. See [`Self::new_with_metadata()`]. + /// + /// See [`Self::read_page_indexes()`] for a description of the replacement behavior on + /// multiple calls. #[cfg(all(feature = "async", feature = "arrow"))] pub async fn load_page_index(&mut self, fetch: F) -> Result<()> { self.load_page_index_with_remainder(fetch, None).await diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 57062c73d2e0..471f69f6201c 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,27 +87,103 @@ fn test_parse_with_page_index_mask() { ); } -/*#[test] -// C11 coverage -fn test_repeated_page_index_reads_merge_cells() { +#[test] +// C11 coverage: verify that repeated calls to read_page_indexes replace existing indexes +fn test_repeated_page_index_reads_replace() { let file = create_test_file(); + // First read: load indexes for row group 0, column 0 let metadata = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Required) .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [0])) .parse_and_finish(&file) .unwrap(); + // Second read: load indexes for row group 1, column 1 + // This should REPLACE the previous indexes, not merge with them let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) .with_page_index_policy(PageIndexPolicy::Required) .with_page_index_mask(ColumnChunkMask::row_groups_and_columns([1], [1])); reader.read_page_indexes(&file).unwrap(); let metadata = reader.finish().unwrap(); + + // After the second read, only row group 1, column 1 should have indexes + // The indexes from the first read (row group 0, column 0) should be gone assert_page_index_cells( &metadata, - |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), - |rg, col| (rg == 0 && col == 0) || (rg == 1 && col == 1), + |rg, col| rg == 1 && col == 1, + |rg, col| rg == 1 && col == 1, + ); +} + +#[test] +// C11 coverage: verify that Skip policy clears the corresponding index type +fn test_page_index_skip_policy_clears_index() { + let file = create_test_file(); + // First read: load both column and offset indexes + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + + // Verify both indexes are present + let page_index = metadata.page_index().expect("page index should be loaded"); + assert!( + page_index.has_column_indexes(), + "column indexes should be present" + ); + assert!( + page_index.has_offset_indexes(), + "offset indexes should be present" + ); + + // Second read: Skip column index but load offset index + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Required); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + + // After the second read, only offset indexes should be present + let page_index = metadata.page_index().expect("page index should be loaded"); + assert!( + !page_index.has_column_indexes(), + "column indexes should be cleared" + ); + assert!( + page_index.has_offset_indexes(), + "offset indexes should be present" ); -}*/ + + // Third read: Skip both indexes + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Skip) + .with_offset_index_policy(PageIndexPolicy::Skip); + reader.read_page_indexes(&file).unwrap(); + let metadata = reader.finish().unwrap(); + + // After the third read, no page index should be present + assert!( + metadata.page_index().is_none(), + "page index should be cleared when both policies are Skip" + ); +} + +#[test] +fn test_empty_page_index_mask_clears_index() { + let file = create_test_file(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + assert!(metadata.page_index().is_some()); + + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required) + .with_page_index_mask(ColumnChunkMask::none()); + reader.read_page_indexes(&file).unwrap(); + + assert!(reader.finish().unwrap().page_index().is_none()); +} #[test] fn test_partial_page_statistics_remain_aligned() { From 0a9dab932a54dfc9ef3f055fdb658456bfb13652 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 15:52:49 -0700 Subject: [PATCH 51/78] restore previous compatibility exception for PageIndexPolicy::Required --- parquet/src/file/metadata/push_decoder.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index af106dd1f5e0..a631e153f992 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -447,14 +447,19 @@ impl ParquetMetaDataPushDecoder { &self.offset_index_mask, ); - // Even if there is no range to read (for example, both policies are Skip, - // the masks select nothing, or the file has no indexes), parse the result to - // replace any existing indexes. - if let Some(page_index_range) = range { - if !self.buffers.has_range(&page_index_range) { - self.state = DecodeState::ReadingPageIndex(metadata); - return Ok(needs_range(page_index_range)); - } + let Some(page_index_range) = range else { + // There is nothing to decode because both policies are Skip, the masks + // select nothing, or the file has no index ranges. Clear any previously + // loaded indexes while preserving the compatibility behavior that + // PageIndexPolicy::Required accepts files with no page indexes at all. + metadata.set_page_index(None); + self.state = DecodeState::Finished; + return Ok(DecodeResult::Data(*metadata)); + }; + + if !self.buffers.has_range(&page_index_range) { + self.state = DecodeState::ReadingPageIndex(metadata); + return Ok(needs_range(page_index_range)); } parse_page_index( From 372d06515e53e87bad0864e1002eddd712b6ff4d Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 16:10:25 -0700 Subject: [PATCH 52/78] remove mention of the C11 tag --- parquet/tests/arrow_reader/page_index.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 471f69f6201c..e1a830028647 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -88,8 +88,8 @@ fn test_parse_with_page_index_mask() { } #[test] -// C11 coverage: verify that repeated calls to read_page_indexes replace existing indexes fn test_repeated_page_index_reads_replace() { + // verify that repeated calls to read_page_indexes replace existing indexes let file = create_test_file(); // First read: load indexes for row group 0, column 0 let metadata = ParquetMetaDataReader::new() @@ -116,8 +116,8 @@ fn test_repeated_page_index_reads_replace() { } #[test] -// C11 coverage: verify that Skip policy clears the corresponding index type fn test_page_index_skip_policy_clears_index() { + // verify that Skip policy clears the corresponding index type let file = create_test_file(); // First read: load both column and offset indexes let metadata = ParquetMetaDataReader::new() From ec46a88cf119a1de9b435a092fcc78252f65de11 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 12:52:04 -0700 Subject: [PATCH 53/78] switch to grid storage for page index --- parquet/src/arrow/async_reader/mod.rs | 4 +- parquet/src/file/metadata/mod.rs | 4 +- parquet/src/file/metadata/page_index.rs | 348 ++++++++++++++++++++--- parquet/src/file/metadata/parser.rs | 31 +- parquet/src/file/writer.rs | 6 +- parquet/tests/arrow_reader/page_index.rs | 16 +- 6 files changed, 341 insertions(+), 68 deletions(-) diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs index 7d5a07f36985..b24a475a673d 100644 --- a/parquet/src/arrow/async_reader/mod.rs +++ b/parquet/src/arrow/async_reader/mod.rs @@ -1008,7 +1008,7 @@ mod tests { use crate::basic::Encoding; use crate::file::metadata::PageIndexPolicy; use crate::file::metadata::ParquetMetaDataReader; - use crate::file::metadata::page_index::PageIndex; + use crate::file::metadata::page_index::{Grid, PageIndex}; use crate::file::properties::WriterProperties; use arrow::compute::kernels::cmp::eq; use arrow::error::Result as ArrowResult; @@ -1970,7 +1970,7 @@ mod tests { .await .unwrap(); - let page_index = PageIndex::new(None, Some(vec![])); + let page_index = PageIndex::new(None, Some(Grid::new_dense(0, 0))); metadata.set_page_index(Some(Arc::new(page_index))); let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap(); diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index f5ee3b20a24c..dfe85e2df47f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -2122,9 +2122,9 @@ mod tests { // Size with page index (includes Arc overhead plus PageIndex heap size) #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3280; + let bigger_expected_size = 3328; #[cfg(feature = "encryption")] - let bigger_expected_size = 3448; + let bigger_expected_size = 3496; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..da0080e0ad15 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,11 +17,13 @@ //! Page Index structures for efficient page-level skipping +use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, offset_index::{OffsetIndexMetaData, PageLocation}, }; +use std::collections::BTreeSet; use std::sync::Arc; /// Trait for accessing Parquet [Page Index] data for efficient page-level skipping @@ -363,6 +365,172 @@ impl RowGroupPageIndex { } } +/// A memory-efficient sparse set representation storing sorted deduplicated indexes +/// +/// Stores which positions are set in a sparse vector. For example: +/// `[None, None, Some(...), Some(...), None, Some(...)]` becomes `[2, 3, 5]` +/// +/// Position checking uses binary search for O(log n) lookup. +/// +/// Implementation note: we can downsize to `u32` here because thrift encodes vector +/// sizes with an `i32`. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Keep { + /// Sorted, deduplicated indexes of set positions + /// None means all positions in the span are set + kept: Option>, + /// Total span of positions (0..span) + span: u32, +} + +// FIXME(ets): since chunk mask is i32 keep should be as well +impl Keep { + pub(crate) fn new(set: &BTreeSet, span: usize) -> Self { + // TODO: need to error if span > max(u32) + let kept = set + .iter() + .filter(|&&idx| idx < span as i32) + .map(|&idx| idx as u32) + .collect::>(); + let kept = if kept.len() == span { + None + } else { + Some(Arc::from(kept)) + }; + + Self { + kept, + span: span as u32, + } + } + + pub(crate) fn new_full(span: usize) -> Self { + Self { + kept: None, + span: span as u32, + } + } + + /// Retrieve a position if set + fn position(&self, idx: usize) -> Option { + // below CUTOFF elements, use linear search + const CUTOFF: usize = 32; + match self.kept.as_ref() { + None => (idx < self.span as usize).then_some(idx), + Some(k) if k.len() > CUTOFF => k.binary_search(&u32::try_from(idx).ok()?).ok(), + Some(k) => k.iter().position(|&i| i == idx as u32), + } + } + + /// Returns the number of set positions + fn len(&self) -> usize { + match &self.kept { + None => self.span as usize, + Some(indexes) => indexes.len(), + } + } +} + +impl HeapSize for Arc<[u32]> { + fn heap_size(&self) -> usize { + // Arc stores weak and strong counts on the heap alongside an instance of T + // T = [u32], so that should be the size of a pointer + the size of the allocation + 2 * std::mem::size_of::() + + std::mem::size_of::<*mut u32>() + + std::mem::size_of_val(self.as_ref()) + } +} + +impl HeapSize for Keep { + fn heap_size(&self) -> usize { + self.kept.heap_size() + } +} + +/// A memory-efficient 2D sparse grid using Keep structures for rows and columns +/// +/// Maps (row_group_idx, column_idx) to values efficiently for sparse access patterns. +/// This is particularly useful when only a few columns are accessed from wide schemas. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Grid { + /// Set of row group indexes that have any values + rows: Keep, + /// Set of column indexes that have any values + cols: Keep, + /// Flattened cells stored in row-major order + /// cells[row_offset * cols.len() + col_offset] = value at (row, col) + /// where row_offset = position of row in rows.kept + /// and col_offset = position of col in cols.kept + cells: Vec>, +} + +impl Grid { + /// Creates a new empty Grid with the specified dimensions + fn new(rows: Keep, cols: Keep) -> Self { + let size = rows.len() * cols.len(); + let cells = vec![None; size]; + Self { rows, cols, cells } + } + + pub(crate) fn new_dense(num_row_groups: usize, num_columns: usize) -> Self { + let rows = Keep::new_full(num_row_groups); + let cols = Keep::new_full(num_columns); + Self::new(rows, cols) + } + + pub(crate) fn from_vec(index: Vec>>) -> Self { + let num_row_groups = index.len(); + let num_columns = if index.is_empty() { 0 } else { index[0].len() }; + let mut result = Self::new_dense(num_row_groups, num_columns); + for (rg_idx, row_group) in index.into_iter().enumerate() { + for (col_idx, idx) in row_group.into_iter().enumerate() { + if let Some(idx) = idx { + result.insert(rg_idx, col_idx, idx); + } + } + } + + result + } + + /// Gets a value at the specified row and column + pub(crate) fn get(&self, row: usize, col: usize) -> Option<&T> { + // Find the offset of this row in the kept rows + let row_offset = self.rows.position(row)?; + let col_offset = self.cols.position(col)?; + + let index = row_offset * self.cols.len() + col_offset; + self.cells.get(index)?.as_ref() + } + + /// Sets a value at the specified row and column + pub(crate) fn insert(&mut self, row: usize, col: usize, value: T) { + let row_offset = self.rows.position(row); + let col_offset = self.cols.position(col); + if let Some(row_offset) = row_offset + && let Some(col_offset) = col_offset + { + // update the existing cell + let index = row_offset * self.cols.len() + col_offset; + + if index < self.cells.len() { + self.cells[index] = Some(value); + } + } + } + + /// Returns true if the grid has no values + pub(crate) fn is_empty(&self) -> bool { + self.cells.iter().all(|cell| cell.is_none()) + } +} + +impl HeapSize for Grid { + fn heap_size(&self) -> usize { + self.rows.heap_size() + self.cols.heap_size() + self.cells.heap_size() + } +} + /// Struct to encapsulate the Parquet [Page Index] /// /// This struct provides a dense representation of the Page Index. It is @@ -441,14 +609,14 @@ impl RowGroupPageIndex { /// [`ParquetMetaData`]: crate::file::metadata::ParquetMetaData #[derive(Debug, Clone, PartialEq)] pub struct PageIndex { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndex { pub(crate) fn new( - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, ) -> Self { Self { column_indexes, @@ -476,8 +644,7 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - let rg = self.column_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.column_indexes.as_ref()?.get(row_group_idx, column_idx) } fn offset_index( @@ -485,8 +652,7 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - let rg = self.offset_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.offset_indexes.as_ref()?.get(row_group_idx, column_idx) } fn as_any(&self) -> &dyn std::any::Any { @@ -508,28 +674,25 @@ impl HeapSize for PageIndex { /// - Automatic conversion of empty structures to `None` to save memory #[derive(Default)] pub struct PageIndexBuilder { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndexBuilder { - /// Creates an empty index structure with space for the specified number of row groups and columns - /// - /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. - /// The outer vector has one entry per row group, and each inner vector has one entry per column. - /// - /// # Type Parameters - /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { - Some( - (0..num_row_groups) - .map(|_| { - let mut idx = Vec::with_capacity(num_columns); - idx.resize_with(num_columns, || None); - idx - }) - .collect(), - ) + fn storage_for_selection( + num_row_groups: usize, + num_columns: usize, + mask: &ColumnChunkMask, + ) -> Option> { + let keep_rows = mask.selected_row_groups().map_or_else( + || Keep::new_full(num_row_groups), + |rows| Keep::new(rows, num_row_groups), + ); + let keep_cols = mask.selected_columns().map_or_else( + || Keep::new_full(num_columns), + |cols| Keep::new(cols, num_columns), + ); + Some(Grid::::new(keep_rows, keep_cols)) } /// Creates a new [`PageIndexBuilder`] with space allocated for both column and offset indexes @@ -538,9 +701,34 @@ impl PageIndexBuilder { /// All index entries are initialized to `None` and can be populated using /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). pub fn new(num_row_groups: usize, num_columns: usize) -> Self { + let keep_cols = Keep::new_full(num_columns); + let keep_rows = Keep::new_full(num_row_groups); + Self { + column_indexes: Some(Grid::new(keep_rows.clone(), keep_cols.clone())), + offset_indexes: Some(Grid::new(keep_rows, keep_cols)), + } + } + + /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy + /// + /// For sparse indexes, this can save a great deal of memory + pub fn new_with_mask( + num_row_groups: usize, + num_columns: usize, + column_index_mask: ColumnChunkMask, + offset_index_mask: ColumnChunkMask, + ) -> Self { Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), + column_indexes: Self::storage_for_selection( + num_row_groups, + num_columns, + &column_index_mask, + ), + offset_indexes: Self::storage_for_selection( + num_row_groups, + num_columns, + &offset_index_mask, + ), } } @@ -564,7 +752,9 @@ impl PageIndexBuilder { /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.column_indexes = Self::empty_index(num_row_groups, num_columns); + let keep_cols = Keep::new_full(num_columns); + let keep_rows = Keep::new_full(num_row_groups); + self.column_indexes = Some(Grid::new(keep_rows, keep_cols)); } /// Allocates space for offset indexes @@ -576,7 +766,9 @@ impl PageIndexBuilder { /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.offset_indexes = Self::empty_index(num_row_groups, num_columns); + let keep_cols = Keep::new_full(num_columns); + let keep_rows = Keep::new_full(num_row_groups); + self.offset_indexes = Some(Grid::new(keep_rows, keep_cols)); } /// Sets the column index for a specific row group and column @@ -589,11 +781,8 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.column_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(column_index); + if let Some(ref mut indexes) = self.column_indexes { + indexes.insert(row_group_idx, column_idx, column_index); } } @@ -607,21 +796,16 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.offset_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(offset_index); + if let Some(ref mut indexes) = self.offset_indexes { + indexes.insert(row_group_idx, column_idx, offset_index); } } /// Checks if an index structure is entirely empty (all entries are None) - fn is_empty_index(index: Option<&Vec>>>) -> bool { + fn is_empty_index(index: Option<&Grid>) -> bool { match index { None => true, - Some(row_groups) => row_groups - .iter() - .all(|columns| columns.iter().all(|entry| entry.is_none())), + Some(index) => index.is_empty(), } } @@ -654,3 +838,77 @@ impl From for PageIndexBuilder { Self::new_from(page_index) } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::{Grid, Keep}; + use crate::{ + basic::BoundaryOrder, + file::page_index::column_index::{ColumnIndexMetaData, PrimitiveColumnIndex}, + }; + + fn colidx_for_test() -> ColumnIndexMetaData { + let ci = PrimitiveColumnIndex::::try_new( + vec![false; 3], + BoundaryOrder::ASCENDING, + Some(vec![0; 3]), + Some(vec![0; 3]), + None, + None, + vec![&[0, 0, 0, 0]; 3], + vec![&[1, 0, 0, 0]; 3], + ) + .unwrap(); + ColumnIndexMetaData::INT32(ci) + } + + #[test] + fn test_sparse_get_put() { + let ci = colidx_for_test(); + + let keep_rows = Keep::new(&BTreeSet::from_iter([0, 3, 7]), 10); + let keep_cols = Keep::new(&BTreeSet::from_iter([5, 10, 99]), 100); + let mut storage = Grid::new(keep_rows, keep_cols); + + // Test insertion and retrieval + storage.insert(0, 5, ci.clone()); + storage.insert(3, 10, ci.clone()); + storage.insert(7, 99, ci.clone()); + + // Test successful retrievals + assert!(storage.get(0, 5).is_some()); + assert!(storage.get(3, 10).is_some()); + assert!(storage.get(7, 99).is_some()); + + // Test missing entries + assert!(storage.get(0, 0).is_none()); + assert!(storage.get(1, 5).is_none()); + assert!(storage.get(0, 10).is_none()); + + // Test out of bounds + assert!(storage.get(20, 5).is_none()); + assert!(storage.get(0, 200).is_none()); + } + + #[test] + fn test_empty_keep_selects_nothing() { + let keep = Keep::new(&BTreeSet::new(), 10); + assert_eq!(keep.len(), 0); + assert_eq!(keep.position(0), None); + } + + #[test] + fn test_sparse_is_empty() { + let ci = colidx_for_test(); + + let keep_rows = Keep::new(&BTreeSet::from_iter([0, 3, 7]), 10); + let keep_cols = Keep::new(&BTreeSet::from_iter([5, 10, 99]), 100); + let mut storage = Grid::new(keep_rows, keep_cols); + assert!(storage.is_empty()); + + storage.insert(0, 5, ci.clone()); + assert!(!storage.is_empty()); + } +} diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 84645687540d..8a8f6d7dc51e 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,6 +247,7 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. +/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -257,10 +258,13 @@ pub(crate) fn parse_page_index( ) -> crate::errors::Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); - + let mut builder = PageIndexBuilder::new_with_mask( + num_row_groups, + num_columns, + column_index_mask.clone(), + offset_index_mask.clone(), + ); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); parse_column_index( metadata, column_index_policy, @@ -270,7 +274,6 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); parse_offset_index( metadata, offset_index_policy, @@ -303,9 +306,15 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { + for rg_idx in 0..metadata.num_row_groups() { + if !mask.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); - for col_idx in mask.column_indices(rg.num_columns()) { + for col_idx in 0..rg.num_columns() { + if !mask.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -329,9 +338,15 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { + for rg_idx in 0..metadata.num_row_groups() { + if !mask.includes_row_group(rg_idx) { + continue; + } let rg = metadata.row_group(rg_idx); - for col_idx in mask.column_indices(rg.num_columns()) { + for col_idx in 0..rg.num_columns() { + if !mask.includes_column(col_idx) { + continue; + } let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index af60326ca387..2ab709589ce5 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -18,7 +18,7 @@ //! [`SerializedFileWriter`]: Low level Parquet writer API use crate::bloom_filter::Sbbf; -use crate::file::metadata::page_index::PageIndex; +use crate::file::metadata::page_index::{Grid, PageIndex}; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::offset_index::OffsetIndexMetaData; @@ -387,13 +387,13 @@ impl SerializedFileWriter { { None } else { - Some(column_indexes) + Some(Grid::from_vec(column_indexes)) }; // offset index will always be created unless explicitly disabled let offset_indexes = if self.props.offset_index_disabled() { None } else { - Some(offset_indexes) + Some(Grid::from_vec(offset_indexes)) }; if column_indexes.is_some() || offset_indexes.is_some() { let page_index = PageIndex::new(column_indexes, offset_indexes); diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e1a830028647..1eab0a89b93b 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -382,9 +382,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 13897); + assert_eq!(metadata.memory_size(), 13849); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 14321); + assert_eq!(metadata.memory_size(), 14273); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -397,9 +397,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12776); + assert_eq!(metadata.memory_size(), 10668); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 13200); + assert_eq!(metadata.memory_size(), 11092); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -412,9 +412,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12056); + assert_eq!(metadata.memory_size(), 9692); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 12480); + assert_eq!(metadata.memory_size(), 10116); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -428,7 +428,7 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 11326); + assert_eq!(metadata.memory_size(), 8362); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 11750); + assert_eq!(metadata.memory_size(), 8786); } From f56a82d3b9be29fff00d0bd47f982a9fbd0fa1b0 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 14:10:30 -0700 Subject: [PATCH 54/78] rework Keep to use i32 internally --- parquet/src/file/metadata/page_index.rs | 136 +++++++++++++++++++----- parquet/src/file/metadata/parser.rs | 8 +- 2 files changed, 114 insertions(+), 30 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index da0080e0ad15..b7a6d5ba8c15 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -23,6 +23,7 @@ use crate::file::page_index::{ column_index::ColumnIndexMetaData, offset_index::{OffsetIndexMetaData, PageLocation}, }; +use std::borrow::Borrow; use std::collections::BTreeSet; use std::sync::Arc; @@ -372,25 +373,33 @@ impl RowGroupPageIndex { /// /// Position checking uses binary search for O(log n) lookup. /// -/// Implementation note: we can downsize to `u32` here because thrift encodes vector +/// Implementation note: we can downsize to `i32` here because thrift encodes vector /// sizes with an `i32`. #[derive(Debug, Clone, PartialEq)] pub(crate) struct Keep { /// Sorted, deduplicated indexes of set positions /// None means all positions in the span are set - kept: Option>, + kept: Option>, /// Total span of positions (0..span) - span: u32, + span: i32, } -// FIXME(ets): since chunk mask is i32 keep should be as well impl Keep { - pub(crate) fn new(set: &BTreeSet, span: usize) -> Self { - // TODO: need to error if span > max(u32) + pub(crate) fn new(set: I, span: usize) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + assert!( + i32::try_from(span).is_ok(), + "Keep cannot have a span that exceeds the storage of an i32: got {span}" + ); + // this should preserve the BTreeSet ordering let kept = set - .iter() - .filter(|&&idx| idx < span as i32) - .map(|&idx| idx as u32) + .into_iter() + .map(|idx| *idx.borrow()) + .filter(|&idx| idx < span) + .map(|idx| idx as i32) .collect::>(); let kept = if kept.len() == span { None @@ -400,25 +409,31 @@ impl Keep { Self { kept, - span: span as u32, + span: span as i32, } } + // shortened version for a full keep set pub(crate) fn new_full(span: usize) -> Self { + assert!( + i32::try_from(span).is_ok(), + "Keep cannot have a span that exceeds the storage of an i32: got {span}" + ); Self { kept: None, - span: span as u32, + span: span as i32, } } /// Retrieve a position if set fn position(&self, idx: usize) -> Option { + let needle = i32::try_from(idx).ok()?; // below CUTOFF elements, use linear search const CUTOFF: usize = 32; match self.kept.as_ref() { None => (idx < self.span as usize).then_some(idx), - Some(k) if k.len() > CUTOFF => k.binary_search(&u32::try_from(idx).ok()?).ok(), - Some(k) => k.iter().position(|&i| i == idx as u32), + Some(k) if k.len() > CUTOFF => k.binary_search(&needle).ok(), + Some(k) => k.iter().position(|&i| i == needle), } } @@ -431,12 +446,12 @@ impl Keep { } } -impl HeapSize for Arc<[u32]> { +impl HeapSize for Arc<[i32]> { fn heap_size(&self) -> usize { // Arc stores weak and strong counts on the heap alongside an instance of T - // T = [u32], so that should be the size of a pointer + the size of the allocation + // T = [i32], so that should be the size of a pointer + the size of the allocation 2 * std::mem::size_of::() - + std::mem::size_of::<*mut u32>() + + std::mem::size_of::<*mut i32>() + std::mem::size_of_val(self.as_ref()) } } @@ -679,20 +694,87 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { + fn storage_for_update( + num_row_groups: usize, + num_columns: usize, + mask: Option<&ColumnChunkMask>, + existing: Vec<(usize, usize, T)>, + ) -> Option> { + if mask.is_none() && existing.is_empty() { + return None; + } + + let rows = match mask.map(ColumnChunkMask::selected_row_groups) { + Some(None) => Keep::new_full(num_row_groups), + selected => { + let mut rows: BTreeSet<_> = existing.iter().map(|(row, _, _)| *row).collect(); + if let Some(Some(selected)) = selected { + rows.extend(selected.iter().map(|&row| row as usize)); + } + Keep::new(rows, num_row_groups) + } + }; + let cols = match mask.map(ColumnChunkMask::selected_columns) { + Some(None) => Keep::new_full(num_columns), + selected => { + let mut cols: BTreeSet<_> = existing.iter().map(|(_, col, _)| *col).collect(); + if let Some(Some(selected)) = selected { + cols.extend(selected.iter().map(|&col| col as usize)); + } + Keep::new(cols, num_columns) + } + }; + let mut grid = Grid::new(rows, cols); + for (row, col, value) in existing { + grid.insert(row, col, value); + } + Some(grid) + } + fn storage_for_selection( num_row_groups: usize, num_columns: usize, mask: &ColumnChunkMask, ) -> Option> { - let keep_rows = mask.selected_row_groups().map_or_else( - || Keep::new_full(num_row_groups), - |rows| Keep::new(rows, num_row_groups), - ); - let keep_cols = mask.selected_columns().map_or_else( - || Keep::new_full(num_columns), - |cols| Keep::new(cols, num_columns), - ); - Some(Grid::::new(keep_rows, keep_cols)) + Self::storage_for_update(num_row_groups, num_columns, Some(mask), vec![]) + } + + pub(crate) fn new_for_update( + page_index: Option<&dyn PageIndexProvider>, + num_row_groups: usize, + num_columns: usize, + column_index_mask: Option<&ColumnChunkMask>, + offset_index_mask: Option<&ColumnChunkMask>, + ) -> Self { + let mut column_indexes = vec![]; + let mut offset_indexes = vec![]; + if let Some(page_index) = page_index { + for row in 0..num_row_groups { + for col in 0..num_columns { + if let Some(index) = page_index.column_index(row, col) { + column_indexes.push((row, col, index.clone())); + } + if let Some(index) = page_index.offset_index(row, col) { + offset_indexes.push((row, col, index.clone())); + } + } + } + } + + Self { + column_indexes: Self::storage_for_update( + num_row_groups, + num_columns, + column_index_mask, + column_indexes, + ), + offset_indexes: Self::storage_for_update( + num_row_groups, + num_columns, + offset_index_mask, + offset_indexes, + ), + } } /// Creates a new [`PageIndexBuilder`] with space allocated for both column and offset indexes @@ -897,10 +979,12 @@ mod tests { let keep = Keep::new(&BTreeSet::new(), 10); assert_eq!(keep.len(), 0); assert_eq!(keep.position(0), None); + assert_eq!(keep.position(9), None); + assert_eq!(keep.position(10), None); } #[test] - fn test_sparse_is_empty() { + fn test_grid_is_empty() { let ci = colidx_for_test(); let keep_rows = Keep::new(&BTreeSet::from_iter([0, 3, 7]), 10); diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 8a8f6d7dc51e..5e23d7f55716 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -247,7 +247,6 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. -/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, @@ -258,11 +257,12 @@ pub(crate) fn parse_page_index( ) -> crate::errors::Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_with_mask( + let mut builder = PageIndexBuilder::new_for_update( + metadata.page_index().map(|index| index.as_ref()), num_row_groups, num_columns, - column_index_mask.clone(), - offset_index_mask.clone(), + (column_index_policy != PageIndexPolicy::Skip).then_some(column_index_mask), + (offset_index_policy != PageIndexPolicy::Skip).then_some(offset_index_mask), ); if column_index_policy != PageIndexPolicy::Skip { parse_column_index( From c11421ee000b85dc719a92bc0ac6bbbf8d7e1acf Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 16:34:39 -0700 Subject: [PATCH 55/78] revert change to i32 --- parquet/src/file/metadata/page_index.rs | 64 +++++++++---------------- 1 file changed, 23 insertions(+), 41 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index b7a6d5ba8c15..39b9991dcf2d 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -23,7 +23,6 @@ use crate::file::page_index::{ column_index::ColumnIndexMetaData, offset_index::{OffsetIndexMetaData, PageLocation}, }; -use std::borrow::Borrow; use std::collections::BTreeSet; use std::sync::Arc; @@ -372,66 +371,51 @@ impl RowGroupPageIndex { /// `[None, None, Some(...), Some(...), None, Some(...)]` becomes `[2, 3, 5]` /// /// Position checking uses binary search for O(log n) lookup. -/// -/// Implementation note: we can downsize to `i32` here because thrift encodes vector -/// sizes with an `i32`. +/// Stores indexes as `u32` because Parquet/Thrift collections cannot contain more than +/// `i32::MAX` entries. #[derive(Debug, Clone, PartialEq)] pub(crate) struct Keep { /// Sorted, deduplicated indexes of set positions /// None means all positions in the span are set - kept: Option>, + kept: Option>, /// Total span of positions (0..span) - span: i32, + span: u32, } impl Keep { - pub(crate) fn new(set: I, span: usize) -> Self - where - I: IntoIterator, - I::Item: Borrow, - { - assert!( - i32::try_from(span).is_ok(), - "Keep cannot have a span that exceeds the storage of an i32: got {span}" - ); - // this should preserve the BTreeSet ordering - let kept = set + pub(crate) fn new(set: impl IntoIterator, span: usize) -> Self { + let span = u32::try_from(span).expect("Keep span exceeds u32::MAX"); + let mut kept = set .into_iter() - .map(|idx| *idx.borrow()) + .filter_map(|idx| u32::try_from(idx).ok()) .filter(|&idx| idx < span) - .map(|idx| idx as i32) .collect::>(); - let kept = if kept.len() == span { + kept.sort_unstable(); + kept.dedup(); + let kept = if kept.len() == span as usize { None } else { Some(Arc::from(kept)) }; - Self { - kept, - span: span as i32, - } + Self { kept, span } } // shortened version for a full keep set pub(crate) fn new_full(span: usize) -> Self { - assert!( - i32::try_from(span).is_ok(), - "Keep cannot have a span that exceeds the storage of an i32: got {span}" - ); Self { kept: None, - span: span as i32, + span: u32::try_from(span).expect("Keep span exceeds u32::MAX"), } } /// Retrieve a position if set fn position(&self, idx: usize) -> Option { - let needle = i32::try_from(idx).ok()?; + let needle = u32::try_from(idx).ok()?; // below CUTOFF elements, use linear search const CUTOFF: usize = 32; match self.kept.as_ref() { - None => (idx < self.span as usize).then_some(idx), + None => (needle < self.span).then_some(idx), Some(k) if k.len() > CUTOFF => k.binary_search(&needle).ok(), Some(k) => k.iter().position(|&i| i == needle), } @@ -446,12 +430,12 @@ impl Keep { } } -impl HeapSize for Arc<[i32]> { +impl HeapSize for Arc<[u32]> { fn heap_size(&self) -> usize { // Arc stores weak and strong counts on the heap alongside an instance of T - // T = [i32], so that should be the size of a pointer + the size of the allocation + // T = [u32], so that should be the size of a pointer + the size of the allocation 2 * std::mem::size_of::() - + std::mem::size_of::<*mut i32>() + + std::mem::size_of::<*mut u32>() + std::mem::size_of_val(self.as_ref()) } } @@ -923,8 +907,6 @@ impl From for PageIndexBuilder { #[cfg(test)] mod tests { - use std::collections::BTreeSet; - use super::{Grid, Keep}; use crate::{ basic::BoundaryOrder, @@ -950,8 +932,8 @@ mod tests { fn test_sparse_get_put() { let ci = colidx_for_test(); - let keep_rows = Keep::new(&BTreeSet::from_iter([0, 3, 7]), 10); - let keep_cols = Keep::new(&BTreeSet::from_iter([5, 10, 99]), 100); + let keep_rows = Keep::new([7, 0, 3, 7], 10); + let keep_cols = Keep::new([5, 10, 99], 100); let mut storage = Grid::new(keep_rows, keep_cols); // Test insertion and retrieval @@ -976,7 +958,7 @@ mod tests { #[test] fn test_empty_keep_selects_nothing() { - let keep = Keep::new(&BTreeSet::new(), 10); + let keep = Keep::new([], 10); assert_eq!(keep.len(), 0); assert_eq!(keep.position(0), None); assert_eq!(keep.position(9), None); @@ -987,8 +969,8 @@ mod tests { fn test_grid_is_empty() { let ci = colidx_for_test(); - let keep_rows = Keep::new(&BTreeSet::from_iter([0, 3, 7]), 10); - let keep_cols = Keep::new(&BTreeSet::from_iter([5, 10, 99]), 100); + let keep_rows = Keep::new([0, 3, 7], 10); + let keep_cols = Keep::new([5, 10, 99], 100); let mut storage = Grid::new(keep_rows, keep_cols); assert!(storage.is_empty()); From c2844e682136219697fb7ccda762b7803c58277e Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 17:01:52 -0700 Subject: [PATCH 56/78] remove PageIndexBuilder::new_with_mask and signal when put_X fails --- parquet/src/file/metadata/page_index.rs | 111 ++++++++++-------------- 1 file changed, 48 insertions(+), 63 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 39b9991dcf2d..9d52ce12ca09 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -502,20 +502,18 @@ impl Grid { self.cells.get(index)?.as_ref() } - /// Sets a value at the specified row and column - pub(crate) fn insert(&mut self, row: usize, col: usize, value: T) { - let row_offset = self.rows.position(row); - let col_offset = self.cols.position(col); - if let Some(row_offset) = row_offset - && let Some(col_offset) = col_offset - { - // update the existing cell - let index = row_offset * self.cols.len() + col_offset; - - if index < self.cells.len() { - self.cells[index] = Some(value); - } - } + /// Sets a value at the specified row and column. + /// + /// Returns `false`, and drops `value`, if the grid has no storage for the position. + pub(crate) fn insert(&mut self, row: usize, col: usize, value: T) -> bool { + let (Some(row_offset), Some(col_offset)) = + (self.rows.position(row), self.cols.position(col)) + else { + return false; + }; + + self.cells[row_offset * self.cols.len() + col_offset] = Some(value); + true } /// Returns true if the grid has no values @@ -623,7 +621,11 @@ impl PageIndex { } } - /// Convert this `PageIndex` into a [`PageIndexBuilder`] + /// Convert this `PageIndex` into a [`PageIndexBuilder`]. + /// + /// The builder retains the storage shape of this index. Consequently, its `put_*` + /// methods return `false` for positions outside that shape. The `allocate_*` methods + /// can replace it with dense storage, but discard existing entries for that index type. pub fn into_builder(self) -> PageIndexBuilder { self.into() } @@ -715,14 +717,6 @@ impl PageIndexBuilder { Some(grid) } - fn storage_for_selection( - num_row_groups: usize, - num_columns: usize, - mask: &ColumnChunkMask, - ) -> Option> { - Self::storage_for_update(num_row_groups, num_columns, Some(mask), vec![]) - } - pub(crate) fn new_for_update( page_index: Option<&dyn PageIndexProvider>, num_row_groups: usize, @@ -775,29 +769,6 @@ impl PageIndexBuilder { } } - /// Creates a new [`PageIndexBuilder`] where storage is defined by the policy - /// - /// For sparse indexes, this can save a great deal of memory - pub fn new_with_mask( - num_row_groups: usize, - num_columns: usize, - column_index_mask: ColumnChunkMask, - offset_index_mask: ColumnChunkMask, - ) -> Self { - Self { - column_indexes: Self::storage_for_selection( - num_row_groups, - num_columns, - &column_index_mask, - ), - offset_indexes: Self::storage_for_selection( - num_row_groups, - num_columns, - &offset_index_mask, - ), - } - } - /// Creates a new [`PageIndexBuilder`] from an existing [`PageIndex`] /// /// This takes ownership of the index structures from the provided [`PageIndex`], @@ -817,6 +788,7 @@ impl PageIndexBuilder { /// /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). + /// This replaces any existing column index storage and discards its entries. pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { let keep_cols = Keep::new_full(num_columns); let keep_rows = Keep::new_full(num_row_groups); @@ -831,6 +803,7 @@ impl PageIndexBuilder { /// /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). + /// This replaces any existing offset index storage and discards its entries. pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { let keep_cols = Keep::new_full(num_columns); let keep_rows = Keep::new_full(num_row_groups); @@ -839,32 +812,32 @@ impl PageIndexBuilder { /// Sets the column index for a specific row group and column /// - /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), - /// or the row group or column index is out of bounds, this method does nothing. + /// Returns `false`, and drops `column_index`, if column indexes were not allocated + /// (see [`Self::allocate_column_indexes`]) or the grid has no storage for the position. pub fn put_column_index( &mut self, column_index: ColumnIndexMetaData, row_group_idx: usize, column_idx: usize, - ) { - if let Some(ref mut indexes) = self.column_indexes { - indexes.insert(row_group_idx, column_idx, column_index); - } + ) -> bool { + self.column_indexes + .as_mut() + .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, column_index)) } /// Sets the offset index for a specific row group and column /// - /// If offset indexes were not allocated (see [`Self::allocate_offset_indexes`]), - /// or the row group or column index is out of bounds, this method does nothing. + /// Returns `false`, and drops `offset_index`, if offset indexes were not allocated + /// (see [`Self::allocate_offset_indexes`]) or the grid has no storage for the position. pub fn put_offset_index( &mut self, offset_index: OffsetIndexMetaData, row_group_idx: usize, column_idx: usize, - ) { - if let Some(ref mut indexes) = self.offset_indexes { - indexes.insert(row_group_idx, column_idx, offset_index); - } + ) -> bool { + self.offset_indexes + .as_mut() + .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, offset_index)) } /// Checks if an index structure is entirely empty (all entries are None) @@ -907,7 +880,7 @@ impl From for PageIndexBuilder { #[cfg(test)] mod tests { - use super::{Grid, Keep}; + use super::{Grid, Keep, PageIndex}; use crate::{ basic::BoundaryOrder, file::page_index::column_index::{ColumnIndexMetaData, PrimitiveColumnIndex}, @@ -937,9 +910,10 @@ mod tests { let mut storage = Grid::new(keep_rows, keep_cols); // Test insertion and retrieval - storage.insert(0, 5, ci.clone()); - storage.insert(3, 10, ci.clone()); - storage.insert(7, 99, ci.clone()); + assert!(storage.insert(0, 5, ci.clone())); + assert!(storage.insert(3, 10, ci.clone())); + assert!(storage.insert(7, 99, ci.clone())); + assert!(!storage.insert(1, 5, ci.clone())); // Test successful retrievals assert!(storage.get(0, 5).is_some()); @@ -965,6 +939,17 @@ mod tests { assert_eq!(keep.position(10), None); } + #[test] + fn test_builder_put_reports_missing_storage() { + let ci = colidx_for_test(); + let mut grid = Grid::new(Keep::new([0], 1), Keep::new([0], 2)); + assert!(grid.insert(0, 0, ci.clone())); + + let mut builder = PageIndex::new(Some(grid), None).into_builder(); + assert!(builder.put_column_index(ci.clone(), 0, 0)); + assert!(!builder.put_column_index(ci, 0, 1)); + } + #[test] fn test_grid_is_empty() { let ci = colidx_for_test(); @@ -974,7 +959,7 @@ mod tests { let mut storage = Grid::new(keep_rows, keep_cols); assert!(storage.is_empty()); - storage.insert(0, 5, ci.clone()); + assert!(storage.insert(0, 5, ci.clone())); assert!(!storage.is_empty()); } } From 25dca4c7bd2f10db11acf62248411632ee047ce9 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 17:08:59 -0700 Subject: [PATCH 57/78] fix heap_size --- parquet/src/file/metadata/page_index.rs | 9 ++++----- parquet/tests/arrow_reader/page_index.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9d52ce12ca09..5b92b4ac2ba3 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -432,11 +432,10 @@ impl Keep { impl HeapSize for Arc<[u32]> { fn heap_size(&self) -> usize { - // Arc stores weak and strong counts on the heap alongside an instance of T - // T = [u32], so that should be the size of a pointer + the size of the allocation - 2 * std::mem::size_of::() - + std::mem::size_of::<*mut u32>() - + std::mem::size_of_val(self.as_ref()) + // The heap block contains the strong and weak counts followed by the slice, + // padded to the alignment of usize. The fat pointer itself is stored inline. + (2 * std::mem::size_of::() + std::mem::size_of_val(self.as_ref())) + .next_multiple_of(std::mem::align_of::()) } } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 1eab0a89b93b..4b32990545a7 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -397,9 +397,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 10668); + assert_eq!(metadata.memory_size(), 10664); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 11092); + assert_eq!(metadata.memory_size(), 11088); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -412,9 +412,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 9692); + assert_eq!(metadata.memory_size(), 9680); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 10116); + assert_eq!(metadata.memory_size(), 10104); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -428,7 +428,7 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 8362); + assert_eq!(metadata.memory_size(), 8342); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 8786); + assert_eq!(metadata.memory_size(), 8766); } From 7053422dafcac579effd75a438ff70fd82eea3bd Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 08:29:02 -0700 Subject: [PATCH 58/78] make new fallible --- parquet/src/arrow/async_reader/mod.rs | 2 +- .../arrow/push_decoder/reader_builder/mod.rs | 4 +- parquet/src/file/metadata/mod.rs | 8 +- parquet/src/file/metadata/page_index.rs | 132 ++++++++++++------ parquet/src/file/metadata/parser.rs | 2 +- parquet/src/file/writer.rs | 4 +- 6 files changed, 97 insertions(+), 55 deletions(-) diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs index b24a475a673d..e2e49b130c59 100644 --- a/parquet/src/arrow/async_reader/mod.rs +++ b/parquet/src/arrow/async_reader/mod.rs @@ -1970,7 +1970,7 @@ mod tests { .await .unwrap(); - let page_index = PageIndex::new(None, Some(Grid::new_dense(0, 0))); + let page_index = PageIndex::new(None, Some(Grid::new_dense(0, 0).unwrap())); metadata.set_page_index(Some(Arc::new(page_index))); let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap(); diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 0e6a0e0a6629..ecdea1e8cd9d 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -969,7 +969,7 @@ mod tests { #[test] fn test_loaded_row_ranges_intersect_column_page_boundaries() { - let mut page_index = PageIndexBuilder::new(1, 2); + let mut page_index = PageIndexBuilder::new(1, 2).unwrap(); let column = |first_rows: &[i64]| OffsetIndexMetaData { page_locations: first_rows .iter() @@ -1050,7 +1050,7 @@ mod tests { #[test] fn test_auto_keeps_mask_when_page_pruning_skips_pages() { - let mut page_index = PageIndexBuilder::new(1, 1); + let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); page_index.put_offset_index( OffsetIndexMetaData { page_locations: [0, 2, 4, 6, 8, 10] diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index dfe85e2df47f..85dd770a8469 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -2091,7 +2091,7 @@ mod tests { assert_eq!(parquet_meta.memory_size(), base_expected_size); - let mut page_index = PageIndexBuilder::new(1, 1); + let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); let mut column_index = ColumnIndexBuilder::new(Type::BOOLEAN); column_index.append(false, vec![1u8], vec![2u8, 3u8], 4, None); @@ -2229,7 +2229,7 @@ mod tests { #[test] fn test_page_index_builder_partial_population() { - let mut builder = PageIndexBuilder::new(1, 1); + let mut builder = PageIndexBuilder::new(1, 1).unwrap(); // Add an offset index let mut offset_idx_builder = OffsetIndexBuilder::new(); @@ -2249,7 +2249,7 @@ mod tests { #[test] fn test_page_index_builder_empty_to_none() { // Create builder but don't populate any indexes - let builder = PageIndexBuilder::new(2, 2); + let builder = PageIndexBuilder::new(2, 2).unwrap(); let page_index = builder.build(); @@ -2261,7 +2261,7 @@ mod tests { #[test] fn test_rebuild_page_index() { // Create builder with one rowgroup and two columns - let mut builder = PageIndexBuilder::new(1, 2); + let mut builder = PageIndexBuilder::new(1, 2).unwrap(); // Add column index for first column let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32); diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 5b92b4ac2ba3..9f714935f13a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -17,6 +17,7 @@ //! Page Index structures for efficient page-level skipping +use crate::errors::{ParquetError, Result}; use crate::file::metadata::ColumnChunkMask; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ @@ -383,8 +384,8 @@ pub(crate) struct Keep { } impl Keep { - pub(crate) fn new(set: impl IntoIterator, span: usize) -> Self { - let span = u32::try_from(span).expect("Keep span exceeds u32::MAX"); + pub(crate) fn new(set: impl IntoIterator, span: usize) -> Result { + let span = Self::try_span(span)?; let mut kept = set .into_iter() .filter_map(|idx| u32::try_from(idx).ok()) @@ -398,15 +399,21 @@ impl Keep { Some(Arc::from(kept)) }; - Self { kept, span } + Ok(Self { kept, span }) } // shortened version for a full keep set - pub(crate) fn new_full(span: usize) -> Self { - Self { + pub(crate) fn new_full(span: usize) -> Result { + Ok(Self { kept: None, - span: u32::try_from(span).expect("Keep span exceeds u32::MAX"), - } + span: Self::try_span(span)?, + }) + } + + fn try_span(span: usize) -> Result { + u32::try_from(span).map_err(|_| { + ParquetError::General(format!("page index dimension exceeds u32::MAX: {span}")) + }) } /// Retrieve a position if set @@ -470,16 +477,16 @@ impl Grid { Self { rows, cols, cells } } - pub(crate) fn new_dense(num_row_groups: usize, num_columns: usize) -> Self { - let rows = Keep::new_full(num_row_groups); - let cols = Keep::new_full(num_columns); - Self::new(rows, cols) + pub(crate) fn new_dense(num_row_groups: usize, num_columns: usize) -> Result { + let rows = Keep::new_full(num_row_groups)?; + let cols = Keep::new_full(num_columns)?; + Ok(Self::new(rows, cols)) } - pub(crate) fn from_vec(index: Vec>>) -> Self { + pub(crate) fn from_vec(index: Vec>>) -> Result { let num_row_groups = index.len(); let num_columns = if index.is_empty() { 0 } else { index[0].len() }; - let mut result = Self::new_dense(num_row_groups, num_columns); + let mut result = Self::new_dense(num_row_groups, num_columns)?; for (rg_idx, row_group) in index.into_iter().enumerate() { for (col_idx, idx) in row_group.into_iter().enumerate() { if let Some(idx) = idx { @@ -488,7 +495,7 @@ impl Grid { } } - result + Ok(result) } /// Gets a value at the specified row and column @@ -588,7 +595,7 @@ impl HeapSize for Grid { /// /// // Assemble the PageIndex (one entry per row group, each with one /// // entry per column) and attach it to the metadata -/// let mut page_index = PageIndexBuilder::new(1, 1); +/// let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); /// page_index.put_column_index(column_index, 0, 0); /// page_index.put_offset_index(offset_index, 0, 0); /// let page_index = page_index.build(); @@ -684,36 +691,36 @@ impl PageIndexBuilder { num_columns: usize, mask: Option<&ColumnChunkMask>, existing: Vec<(usize, usize, T)>, - ) -> Option> { + ) -> Result>> { if mask.is_none() && existing.is_empty() { - return None; + return Ok(None); } let rows = match mask.map(ColumnChunkMask::selected_row_groups) { - Some(None) => Keep::new_full(num_row_groups), + Some(None) => Keep::new_full(num_row_groups)?, selected => { let mut rows: BTreeSet<_> = existing.iter().map(|(row, _, _)| *row).collect(); if let Some(Some(selected)) = selected { rows.extend(selected.iter().map(|&row| row as usize)); } - Keep::new(rows, num_row_groups) + Keep::new(rows, num_row_groups)? } }; let cols = match mask.map(ColumnChunkMask::selected_columns) { - Some(None) => Keep::new_full(num_columns), + Some(None) => Keep::new_full(num_columns)?, selected => { let mut cols: BTreeSet<_> = existing.iter().map(|(_, col, _)| *col).collect(); if let Some(Some(selected)) = selected { cols.extend(selected.iter().map(|&col| col as usize)); } - Keep::new(cols, num_columns) + Keep::new(cols, num_columns)? } }; let mut grid = Grid::new(rows, cols); for (row, col, value) in existing { grid.insert(row, col, value); } - Some(grid) + Ok(Some(grid)) } pub(crate) fn new_for_update( @@ -722,7 +729,7 @@ impl PageIndexBuilder { num_columns: usize, column_index_mask: Option<&ColumnChunkMask>, offset_index_mask: Option<&ColumnChunkMask>, - ) -> Self { + ) -> Result { let mut column_indexes = vec![]; let mut offset_indexes = vec![]; if let Some(page_index) = page_index { @@ -738,20 +745,20 @@ impl PageIndexBuilder { } } - Self { + Ok(Self { column_indexes: Self::storage_for_update( num_row_groups, num_columns, column_index_mask, column_indexes, - ), + )?, offset_indexes: Self::storage_for_update( num_row_groups, num_columns, offset_index_mask, offset_indexes, - ), - } + )?, + }) } /// Creates a new [`PageIndexBuilder`] with space allocated for both column and offset indexes @@ -759,13 +766,17 @@ impl PageIndexBuilder { /// This allocates empty index structures for the specified number of row groups and columns. /// All index entries are initialized to `None` and can be populated using /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). - pub fn new(num_row_groups: usize, num_columns: usize) -> Self { - let keep_cols = Keep::new_full(num_columns); - let keep_rows = Keep::new_full(num_row_groups); - Self { + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn new(num_row_groups: usize, num_columns: usize) -> Result { + let keep_cols = Keep::new_full(num_columns)?; + let keep_rows = Keep::new_full(num_row_groups)?; + Ok(Self { column_indexes: Some(Grid::new(keep_rows.clone(), keep_cols.clone())), offset_indexes: Some(Grid::new(keep_rows, keep_cols)), - } + }) } /// Creates a new [`PageIndexBuilder`] from an existing [`PageIndex`] @@ -788,10 +799,19 @@ impl PageIndexBuilder { /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). /// This replaces any existing column index storage and discards its entries. - pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - let keep_cols = Keep::new_full(num_columns); - let keep_rows = Keep::new_full(num_row_groups); + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn allocate_column_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + let keep_cols = Keep::new_full(num_columns)?; + let keep_rows = Keep::new_full(num_row_groups)?; self.column_indexes = Some(Grid::new(keep_rows, keep_cols)); + Ok(()) } /// Allocates space for offset indexes @@ -803,10 +823,19 @@ impl PageIndexBuilder { /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). /// This replaces any existing offset index storage and discards its entries. - pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - let keep_cols = Keep::new_full(num_columns); - let keep_rows = Keep::new_full(num_row_groups); + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn allocate_offset_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + let keep_cols = Keep::new_full(num_columns)?; + let keep_rows = Keep::new_full(num_row_groups)?; self.offset_indexes = Some(Grid::new(keep_rows, keep_cols)); + Ok(()) } /// Sets the column index for a specific row group and column @@ -879,7 +908,7 @@ impl From for PageIndexBuilder { #[cfg(test)] mod tests { - use super::{Grid, Keep, PageIndex}; + use super::{Grid, Keep, PageIndex, PageIndexBuilder}; use crate::{ basic::BoundaryOrder, file::page_index::column_index::{ColumnIndexMetaData, PrimitiveColumnIndex}, @@ -904,8 +933,8 @@ mod tests { fn test_sparse_get_put() { let ci = colidx_for_test(); - let keep_rows = Keep::new([7, 0, 3, 7], 10); - let keep_cols = Keep::new([5, 10, 99], 100); + let keep_rows = Keep::new([7, 0, 3, 7], 10).unwrap(); + let keep_cols = Keep::new([5, 10, 99], 100).unwrap(); let mut storage = Grid::new(keep_rows, keep_cols); // Test insertion and retrieval @@ -931,17 +960,30 @@ mod tests { #[test] fn test_empty_keep_selects_nothing() { - let keep = Keep::new([], 10); + let keep = Keep::new([], 10).unwrap(); assert_eq!(keep.len(), 0); assert_eq!(keep.position(0), None); assert_eq!(keep.position(9), None); assert_eq!(keep.position(10), None); } + #[cfg(target_pointer_width = "64")] + #[test] + fn test_oversized_dimensions_return_error() { + let span = u32::MAX as usize + 1; + assert!(Keep::new([], span).is_err()); + assert!(Keep::new_full(span).is_err()); + assert!(PageIndexBuilder::new(0, span).is_err()); + + let mut builder = PageIndexBuilder::default(); + assert!(builder.allocate_column_indexes(0, span).is_err()); + assert!(builder.allocate_offset_indexes(0, span).is_err()); + } + #[test] fn test_builder_put_reports_missing_storage() { let ci = colidx_for_test(); - let mut grid = Grid::new(Keep::new([0], 1), Keep::new([0], 2)); + let mut grid = Grid::new(Keep::new([0], 1).unwrap(), Keep::new([0], 2).unwrap()); assert!(grid.insert(0, 0, ci.clone())); let mut builder = PageIndex::new(Some(grid), None).into_builder(); @@ -953,8 +995,8 @@ mod tests { fn test_grid_is_empty() { let ci = colidx_for_test(); - let keep_rows = Keep::new([0, 3, 7], 10); - let keep_cols = Keep::new([5, 10, 99], 100); + let keep_rows = Keep::new([0, 3, 7], 10).unwrap(); + let keep_cols = Keep::new([5, 10, 99], 100).unwrap(); let mut storage = Grid::new(keep_rows, keep_cols); assert!(storage.is_empty()); diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 5e23d7f55716..9d10c48204ab 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -263,7 +263,7 @@ pub(crate) fn parse_page_index( num_columns, (column_index_policy != PageIndexPolicy::Skip).then_some(column_index_mask), (offset_index_policy != PageIndexPolicy::Skip).then_some(offset_index_mask), - ); + )?; if column_index_policy != PageIndexPolicy::Skip { parse_column_index( metadata, diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 2ab709589ce5..302234cc1c89 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -387,13 +387,13 @@ impl SerializedFileWriter { { None } else { - Some(Grid::from_vec(column_indexes)) + Some(Grid::from_vec(column_indexes)?) }; // offset index will always be created unless explicitly disabled let offset_indexes = if self.props.offset_index_disabled() { None } else { - Some(Grid::from_vec(offset_indexes)) + Some(Grid::from_vec(offset_indexes)?) }; if column_indexes.is_some() || offset_indexes.is_some() { let page_index = PageIndex::new(column_indexes, offset_indexes); From 5c7719be69b176dab72a1dbd75641dadc5cf0c13 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 16:58:40 -0700 Subject: [PATCH 59/78] fix some merge problems --- parquet/src/file/metadata/page_index.rs | 62 ++++++------------------- parquet/src/file/metadata/parser.rs | 23 ++------- 2 files changed, 18 insertions(+), 67 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 9f714935f13a..ee628c67d4ee 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -24,7 +24,6 @@ use crate::file::page_index::{ column_index::ColumnIndexMetaData, offset_index::{OffsetIndexMetaData, PageLocation}, }; -use std::collections::BTreeSet; use std::sync::Arc; /// Trait for accessing Parquet [Page Index] data for efficient page-level skipping @@ -686,77 +685,42 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { - fn storage_for_update( + fn storage_for_selection( num_row_groups: usize, num_columns: usize, mask: Option<&ColumnChunkMask>, - existing: Vec<(usize, usize, T)>, ) -> Result>> { - if mask.is_none() && existing.is_empty() { + let Some(mask) = mask else { return Ok(None); - } + }; - let rows = match mask.map(ColumnChunkMask::selected_row_groups) { - Some(None) => Keep::new_full(num_row_groups)?, - selected => { - let mut rows: BTreeSet<_> = existing.iter().map(|(row, _, _)| *row).collect(); - if let Some(Some(selected)) = selected { - rows.extend(selected.iter().map(|&row| row as usize)); - } - Keep::new(rows, num_row_groups)? - } + let rows = match mask.selected_row_groups() { + None => Keep::new_full(num_row_groups)?, + Some(selected) => Keep::new(selected.iter().map(|&row| row as usize), num_row_groups)?, }; - let cols = match mask.map(ColumnChunkMask::selected_columns) { - Some(None) => Keep::new_full(num_columns)?, - selected => { - let mut cols: BTreeSet<_> = existing.iter().map(|(_, col, _)| *col).collect(); - if let Some(Some(selected)) = selected { - cols.extend(selected.iter().map(|&col| col as usize)); - } - Keep::new(cols, num_columns)? - } + let cols = match mask.selected_columns() { + None => Keep::new_full(num_columns)?, + Some(selected) => Keep::new(selected.iter().map(|&col| col as usize), num_columns)?, }; - let mut grid = Grid::new(rows, cols); - for (row, col, value) in existing { - grid.insert(row, col, value); - } - Ok(Some(grid)) + Ok(Some(Grid::new(rows, cols))) } - pub(crate) fn new_for_update( - page_index: Option<&dyn PageIndexProvider>, + pub(crate) fn new_for_read( num_row_groups: usize, num_columns: usize, column_index_mask: Option<&ColumnChunkMask>, offset_index_mask: Option<&ColumnChunkMask>, ) -> Result { - let mut column_indexes = vec![]; - let mut offset_indexes = vec![]; - if let Some(page_index) = page_index { - for row in 0..num_row_groups { - for col in 0..num_columns { - if let Some(index) = page_index.column_index(row, col) { - column_indexes.push((row, col, index.clone())); - } - if let Some(index) = page_index.offset_index(row, col) { - offset_indexes.push((row, col, index.clone())); - } - } - } - } - Ok(Self { - column_indexes: Self::storage_for_update( + column_indexes: Self::storage_for_selection( num_row_groups, num_columns, column_index_mask, - column_indexes, )?, - offset_indexes: Self::storage_for_update( + offset_indexes: Self::storage_for_selection( num_row_groups, num_columns, offset_index_mask, - offset_indexes, )?, }) } diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 9d10c48204ab..d75ee3f0ea84 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -257,8 +257,7 @@ pub(crate) fn parse_page_index( ) -> crate::errors::Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_for_update( - metadata.page_index().map(|index| index.as_ref()), + let mut builder = PageIndexBuilder::new_for_read( num_row_groups, num_columns, (column_index_policy != PageIndexPolicy::Skip).then_some(column_index_mask), @@ -306,15 +305,9 @@ fn parse_column_index( if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; @@ -338,15 +331,9 @@ fn parse_offset_index( if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } - for rg_idx in 0..metadata.num_row_groups() { - if !mask.includes_row_group(rg_idx) { - continue; - } + for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); - for col_idx in 0..rg.num_columns() { - if !mask.includes_column(col_idx) { - continue; - } + for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; From 9b87ed6fc77bcfc2fe91c4bfee58f9b0aad28df4 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:14:34 -0700 Subject: [PATCH 60/78] D5: partialeq for grid --- parquet/src/file/metadata/page_index.rs | 39 +++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index ee628c67d4ee..014d3c5d1bde 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -455,7 +455,7 @@ impl HeapSize for Keep { /// /// Maps (row_group_idx, column_idx) to values efficiently for sparse access patterns. /// This is particularly useful when only a few columns are accessed from wide schemas. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub(crate) struct Grid { /// Set of row group indexes that have any values rows: Keep, @@ -468,11 +468,26 @@ pub(crate) struct Grid { cells: Vec>, } -impl Grid { +impl PartialEq for Grid { + fn eq(&self, other: &Self) -> bool { + if self.rows == other.rows && self.cols == other.cols { + return self.cells == other.cells; + } + + let num_rows = self.rows.span.max(other.rows.span) as usize; + let num_columns = self.cols.span.max(other.cols.span) as usize; + (0..num_rows).all(|row| { + (0..num_columns).all(|column| self.get(row, column) == other.get(row, column)) + }) + } +} + +impl Grid { /// Creates a new empty Grid with the specified dimensions fn new(rows: Keep, cols: Keep) -> Self { let size = rows.len() * cols.len(); - let cells = vec![None; size]; + let mut cells = Vec::with_capacity(size); + cells.resize_with(size, || None); Self { rows, cols, cells } } @@ -685,7 +700,7 @@ pub struct PageIndexBuilder { } impl PageIndexBuilder { - fn storage_for_selection( + fn storage_for_selection( num_row_groups: usize, num_columns: usize, mask: Option<&ColumnChunkMask>, @@ -833,7 +848,7 @@ impl PageIndexBuilder { } /// Checks if an index structure is entirely empty (all entries are None) - fn is_empty_index(index: Option<&Grid>) -> bool { + fn is_empty_index(index: Option<&Grid>) -> bool { match index { None => true, Some(index) => index.is_empty(), @@ -967,4 +982,18 @@ mod tests { assert!(storage.insert(0, 5, ci.clone())); assert!(!storage.is_empty()); } + + #[test] + fn test_grid_equality_ignores_storage_shape() { + let ci = colidx_for_test(); + let mut dense = Grid::new_dense(2, 3).unwrap(); + assert!(dense.insert(0, 0, ci.clone())); + + let mut sparse = Grid::new(Keep::new([0], 2).unwrap(), Keep::new([0], 3).unwrap()); + assert!(sparse.insert(0, 0, ci.clone())); + assert_eq!(dense, sparse); + + assert!(dense.insert(1, 2, ci)); + assert_ne!(dense, sparse); + } } From 455b342e2360ac8446ca51dd4a995940f2ff423f Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:17:52 -0700 Subject: [PATCH 61/78] D6: fix Grid::from_vec --- parquet/src/file/metadata/page_index.rs | 48 +++++++++++++++++-------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 014d3c5d1bde..f4b090e2f74a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -491,25 +491,25 @@ impl Grid { Self { rows, cols, cells } } - pub(crate) fn new_dense(num_row_groups: usize, num_columns: usize) -> Result { - let rows = Keep::new_full(num_row_groups)?; - let cols = Keep::new_full(num_columns)?; - Ok(Self::new(rows, cols)) - } - pub(crate) fn from_vec(index: Vec>>) -> Result { let num_row_groups = index.len(); - let num_columns = if index.is_empty() { 0 } else { index[0].len() }; - let mut result = Self::new_dense(num_row_groups, num_columns)?; - for (rg_idx, row_group) in index.into_iter().enumerate() { - for (col_idx, idx) in row_group.into_iter().enumerate() { - if let Some(idx) = idx { - result.insert(rg_idx, col_idx, idx); - } + let num_columns = index.first().map_or(0, Vec::len); + let mut cells = Vec::with_capacity(num_row_groups * num_columns); + for (row_group_idx, row_group) in index.into_iter().enumerate() { + if row_group.len() != num_columns { + return Err(ParquetError::General(format!( + "ragged page index: row group {row_group_idx} has {} columns, expected {num_columns}", + row_group.len() + ))); } + cells.extend(row_group); } - Ok(result) + Ok(Self { + rows: Keep::new_full(num_row_groups)?, + cols: Keep::new_full(num_columns)?, + cells, + }) } /// Gets a value at the specified row and column @@ -986,7 +986,7 @@ mod tests { #[test] fn test_grid_equality_ignores_storage_shape() { let ci = colidx_for_test(); - let mut dense = Grid::new_dense(2, 3).unwrap(); + let mut dense = Grid::new(Keep::new_full(2).unwrap(), Keep::new_full(3).unwrap()); assert!(dense.insert(0, 0, ci.clone())); let mut sparse = Grid::new(Keep::new([0], 2).unwrap(), Keep::new([0], 3).unwrap()); @@ -996,4 +996,22 @@ mod tests { assert!(dense.insert(1, 2, ci)); assert_ne!(dense, sparse); } + + #[test] + fn test_grid_from_vec() { + let grid = Grid::from_vec(vec![vec![Some(1), None], vec![None, Some(2)]]).unwrap(); + assert_eq!(grid.get(0, 0), Some(&1)); + assert_eq!(grid.get(0, 1), None); + assert_eq!(grid.get(1, 0), None); + assert_eq!(grid.get(1, 1), Some(&2)); + } + + #[test] + fn test_grid_from_vec_rejects_ragged_rows() { + let error = Grid::from_vec(vec![vec![None::], vec![None, Some(1)]]).unwrap_err(); + assert_eq!( + error.to_string(), + "Parquet error: ragged page index: row group 1 has 2 columns, expected 1" + ); + } } From 3b7a0780872bc13d8a9b5621d740558fff91e9dd Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:20:28 -0700 Subject: [PATCH 62/78] D10: doc fixes --- parquet/src/file/metadata/page_index.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index f4b090e2f74a..96410e42de62 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -370,7 +370,7 @@ impl RowGroupPageIndex { /// Stores which positions are set in a sparse vector. For example: /// `[None, None, Some(...), Some(...), None, Some(...)]` becomes `[2, 3, 5]` /// -/// Position checking uses binary search for O(log n) lookup. +/// Position checking uses a linear scan for up to 32 indexes and binary search above that. /// Stores indexes as `u32` because Parquet/Thrift collections cannot contain more than /// `i32::MAX` entries. #[derive(Debug, Clone, PartialEq)] @@ -457,9 +457,9 @@ impl HeapSize for Keep { /// This is particularly useful when only a few columns are accessed from wide schemas. #[derive(Debug, Clone)] pub(crate) struct Grid { - /// Set of row group indexes that have any values + /// Row group indexes that have storage; a cell can still be `None` rows: Keep, - /// Set of column indexes that have any values + /// Column indexes that have storage; a cell can still be `None` cols: Keep, /// Flattened cells stored in row-major order /// cells[row_offset * cols.len() + col_offset] = value at (row, col) @@ -550,10 +550,10 @@ impl HeapSize for Grid { /// Struct to encapsulate the Parquet [Page Index] /// -/// This struct provides a dense representation of the Page Index. It is -/// used internally by this crate when assembling and writing the Page -/// Index. It is also the default implementation of the [`PageIndexProvider`] -/// contained in the [`ParquetMetaData`]. +/// This struct provides a sparse representation of the Page Index: it has storage only for +/// the column chunks selected when it was built (all chunks by default). It is used internally +/// by this crate when assembling and writing the Page Index. It is also the default +/// implementation of the [`PageIndexProvider`] contained in the [`ParquetMetaData`]. /// /// # Example: Constructing a synthetic `PageIndex` /// From 8e8494ed63093666cd28a3e25e60ad363cf50fcb Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:27:11 -0700 Subject: [PATCH 63/78] add ability to create Keep from a chunk mask --- parquet/src/file/metadata/page_index.rs | 92 ++++++++++++++++--------- parquet/src/file/metadata/reader.rs | 8 +++ 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 96410e42de62..e4addf57b1c3 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -383,24 +383,6 @@ pub(crate) struct Keep { } impl Keep { - pub(crate) fn new(set: impl IntoIterator, span: usize) -> Result { - let span = Self::try_span(span)?; - let mut kept = set - .into_iter() - .filter_map(|idx| u32::try_from(idx).ok()) - .filter(|&idx| idx < span) - .collect::>(); - kept.sort_unstable(); - kept.dedup(); - let kept = if kept.len() == span as usize { - None - } else { - Some(Arc::from(kept)) - }; - - Ok(Self { kept, span }) - } - // shortened version for a full keep set pub(crate) fn new_full(span: usize) -> Result { Ok(Self { @@ -409,6 +391,26 @@ impl Keep { }) } + /// Creates a keep set from an already sorted and deduplicated mask axis. + /// + /// The mask allocation is shared when all of its indexes are in range. + fn from_mask(kept: Option>, span: usize) -> Result { + let span = Self::try_span(span)?; + let Some(kept) = kept else { + return Ok(Self { kept: None, span }); + }; + + let in_range = kept.partition_point(|&idx| idx < span); + let kept = if in_range == span as usize { + None + } else if in_range == kept.len() { + Some(kept) + } else { + Some(Arc::from(&kept[..in_range])) + }; + Ok(Self { kept, span }) + } + fn try_span(span: usize) -> Result { u32::try_from(span).map_err(|_| { ParquetError::General(format!("page index dimension exceeds u32::MAX: {span}")) @@ -709,14 +711,8 @@ impl PageIndexBuilder { return Ok(None); }; - let rows = match mask.selected_row_groups() { - None => Keep::new_full(num_row_groups)?, - Some(selected) => Keep::new(selected.iter().map(|&row| row as usize), num_row_groups)?, - }; - let cols = match mask.selected_columns() { - None => Keep::new_full(num_columns)?, - Some(selected) => Keep::new(selected.iter().map(|&col| col as usize), num_columns)?, - }; + let rows = Keep::from_mask(mask.selected_row_groups_shared(), num_row_groups)?; + let cols = Keep::from_mask(mask.selected_columns_shared(), num_columns)?; Ok(Some(Grid::new(rows, cols))) } @@ -890,8 +886,10 @@ mod tests { use super::{Grid, Keep, PageIndex, PageIndexBuilder}; use crate::{ basic::BoundaryOrder, + file::metadata::ColumnChunkMask, file::page_index::column_index::{ColumnIndexMetaData, PrimitiveColumnIndex}, }; + use std::sync::Arc; fn colidx_for_test() -> ColumnIndexMetaData { let ci = PrimitiveColumnIndex::::try_new( @@ -912,8 +910,8 @@ mod tests { fn test_sparse_get_put() { let ci = colidx_for_test(); - let keep_rows = Keep::new([7, 0, 3, 7], 10).unwrap(); - let keep_cols = Keep::new([5, 10, 99], 100).unwrap(); + let keep_rows = Keep::from_mask(Some(Arc::from([0, 3, 7])), 10).unwrap(); + let keep_cols = Keep::from_mask(Some(Arc::from([5, 10, 99])), 100).unwrap(); let mut storage = Grid::new(keep_rows, keep_cols); // Test insertion and retrieval @@ -939,18 +937,38 @@ mod tests { #[test] fn test_empty_keep_selects_nothing() { - let keep = Keep::new([], 10).unwrap(); + let keep = Keep::from_mask(Some(Arc::from([])), 10).unwrap(); assert_eq!(keep.len(), 0); assert_eq!(keep.position(0), None); assert_eq!(keep.position(9), None); assert_eq!(keep.position(10), None); } + #[test] + fn test_keep_shares_mask_storage() { + let mask = ColumnChunkMask::columns([1, 3]); + let selected = mask.selected_columns_shared().unwrap(); + let keep = Keep::from_mask(Some(Arc::clone(&selected)), 5).unwrap(); + + assert!(Arc::ptr_eq(keep.kept.as_ref().unwrap(), &selected)); + assert_eq!(keep.position(1), Some(0)); + assert_eq!(keep.position(3), Some(1)); + } + + #[test] + fn test_keep_from_mask_discards_out_of_range_indexes() { + let mask = ColumnChunkMask::columns([1, 3, 10]); + let keep = Keep::from_mask(mask.selected_columns_shared(), 5).unwrap(); + + assert_eq!(keep.kept.as_deref(), Some([1, 3].as_slice())); + assert_eq!(keep.position(10), None); + } + #[cfg(target_pointer_width = "64")] #[test] fn test_oversized_dimensions_return_error() { let span = u32::MAX as usize + 1; - assert!(Keep::new([], span).is_err()); + assert!(Keep::from_mask(Some(Arc::from([])), span).is_err()); assert!(Keep::new_full(span).is_err()); assert!(PageIndexBuilder::new(0, span).is_err()); @@ -962,7 +980,10 @@ mod tests { #[test] fn test_builder_put_reports_missing_storage() { let ci = colidx_for_test(); - let mut grid = Grid::new(Keep::new([0], 1).unwrap(), Keep::new([0], 2).unwrap()); + let mut grid = Grid::new( + Keep::from_mask(Some(Arc::from([0])), 1).unwrap(), + Keep::from_mask(Some(Arc::from([0])), 2).unwrap(), + ); assert!(grid.insert(0, 0, ci.clone())); let mut builder = PageIndex::new(Some(grid), None).into_builder(); @@ -974,8 +995,8 @@ mod tests { fn test_grid_is_empty() { let ci = colidx_for_test(); - let keep_rows = Keep::new([0, 3, 7], 10).unwrap(); - let keep_cols = Keep::new([5, 10, 99], 100).unwrap(); + let keep_rows = Keep::from_mask(Some(Arc::from([0, 3, 7])), 10).unwrap(); + let keep_cols = Keep::from_mask(Some(Arc::from([5, 10, 99])), 100).unwrap(); let mut storage = Grid::new(keep_rows, keep_cols); assert!(storage.is_empty()); @@ -989,7 +1010,10 @@ mod tests { let mut dense = Grid::new(Keep::new_full(2).unwrap(), Keep::new_full(3).unwrap()); assert!(dense.insert(0, 0, ci.clone())); - let mut sparse = Grid::new(Keep::new([0], 2).unwrap(), Keep::new([0], 3).unwrap()); + let mut sparse = Grid::new( + Keep::from_mask(Some(Arc::from([0])), 2).unwrap(), + Keep::from_mask(Some(Arc::from([0])), 3).unwrap(), + ); assert!(sparse.insert(0, 0, ci.clone())); assert_eq!(dense, sparse); diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index efd73a8b38fd..456459d5f288 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -232,6 +232,14 @@ impl ColumnChunkMask { self.columns.as_deref() } + pub(crate) fn selected_row_groups_shared(&self) -> Option> { + self.row_groups.clone() + } + + pub(crate) fn selected_columns_shared(&self) -> Option> { + self.columns.clone() + } + /// Creates a mask selecting the leaf columns in an Arrow projection. #[cfg(feature = "arrow")] pub fn from_projection( From 8334423d4cfdf4dd36efc6d8a3171f241207eb49 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:30:48 -0700 Subject: [PATCH 64/78] add tests --- parquet/src/file/metadata/page_index.rs | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index e4addf57b1c3..40746b932de8 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -964,6 +964,18 @@ mod tests { assert_eq!(keep.position(10), None); } + #[test] + fn test_keep_normalizes_full_selection() { + let mask = ColumnChunkMask::columns([2, 0, 1, 2]); + let keep = Keep::from_mask(mask.selected_columns_shared(), 3).unwrap(); + + assert!(keep.kept.is_none()); + assert_eq!(keep.len(), 3); + assert_eq!(keep.position(0), Some(0)); + assert_eq!(keep.position(2), Some(2)); + assert_eq!(keep.position(3), None); + } + #[cfg(target_pointer_width = "64")] #[test] fn test_oversized_dimensions_return_error() { @@ -991,6 +1003,40 @@ mod tests { assert!(!builder.put_column_index(ci, 0, 1)); } + #[test] + fn test_builder_sparse_storage_shape() { + let mask = ColumnChunkMask::columns([0, 2]); + let builder = PageIndexBuilder::new_for_read(4, 5, Some(&mask), None).unwrap(); + + let column_indexes = builder.column_indexes.unwrap(); + assert_eq!(column_indexes.rows.len(), 4); + assert_eq!(column_indexes.cols.len(), 2); + assert_eq!(column_indexes.cells.len(), 8); + assert!(builder.offset_indexes.is_none()); + } + + #[test] + fn test_builder_allocate_indexes_creates_dense_empty_storage() { + let ci = colidx_for_test(); + let mut builder = PageIndexBuilder::default(); + + builder.allocate_column_indexes(2, 3).unwrap(); + assert!(builder.put_column_index(ci, 1, 2)); + builder.allocate_column_indexes(2, 3).unwrap(); + let column_indexes = builder.column_indexes.as_ref().unwrap(); + assert_eq!(column_indexes.rows.len(), 2); + assert_eq!(column_indexes.cols.len(), 3); + assert_eq!(column_indexes.cells.len(), 6); + assert!(column_indexes.is_empty()); + + builder.allocate_offset_indexes(2, 3).unwrap(); + let offset_indexes = builder.offset_indexes.as_ref().unwrap(); + assert_eq!(offset_indexes.rows.len(), 2); + assert_eq!(offset_indexes.cols.len(), 3); + assert_eq!(offset_indexes.cells.len(), 6); + assert!(offset_indexes.is_empty()); + } + #[test] fn test_grid_is_empty() { let ci = colidx_for_test(); From e1e320172c707e6e97380bcb49338043203f1a6e Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:44:05 -0700 Subject: [PATCH 65/78] revert breaking api changes --- parquet/src/arrow/async_reader/mod.rs | 2 +- .../arrow/push_decoder/reader_builder/mod.rs | 4 +- parquet/src/file/metadata/mod.rs | 8 +- parquet/src/file/metadata/page_index.rs | 76 ++++++++++--------- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs index e2e49b130c59..7bee2de2d816 100644 --- a/parquet/src/arrow/async_reader/mod.rs +++ b/parquet/src/arrow/async_reader/mod.rs @@ -1970,7 +1970,7 @@ mod tests { .await .unwrap(); - let page_index = PageIndex::new(None, Some(Grid::new_dense(0, 0).unwrap())); + let page_index = PageIndex::new(None, Some(Grid::from_vec(vec![]).unwrap())); metadata.set_page_index(Some(Arc::new(page_index))); let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap(); diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index ecdea1e8cd9d..0e6a0e0a6629 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -969,7 +969,7 @@ mod tests { #[test] fn test_loaded_row_ranges_intersect_column_page_boundaries() { - let mut page_index = PageIndexBuilder::new(1, 2).unwrap(); + let mut page_index = PageIndexBuilder::new(1, 2); let column = |first_rows: &[i64]| OffsetIndexMetaData { page_locations: first_rows .iter() @@ -1050,7 +1050,7 @@ mod tests { #[test] fn test_auto_keeps_mask_when_page_pruning_skips_pages() { - let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); + let mut page_index = PageIndexBuilder::new(1, 1); page_index.put_offset_index( OffsetIndexMetaData { page_locations: [0, 2, 4, 6, 8, 10] diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 85dd770a8469..dfe85e2df47f 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -2091,7 +2091,7 @@ mod tests { assert_eq!(parquet_meta.memory_size(), base_expected_size); - let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); + let mut page_index = PageIndexBuilder::new(1, 1); let mut column_index = ColumnIndexBuilder::new(Type::BOOLEAN); column_index.append(false, vec![1u8], vec![2u8, 3u8], 4, None); @@ -2229,7 +2229,7 @@ mod tests { #[test] fn test_page_index_builder_partial_population() { - let mut builder = PageIndexBuilder::new(1, 1).unwrap(); + let mut builder = PageIndexBuilder::new(1, 1); // Add an offset index let mut offset_idx_builder = OffsetIndexBuilder::new(); @@ -2249,7 +2249,7 @@ mod tests { #[test] fn test_page_index_builder_empty_to_none() { // Create builder but don't populate any indexes - let builder = PageIndexBuilder::new(2, 2).unwrap(); + let builder = PageIndexBuilder::new(2, 2); let page_index = builder.build(); @@ -2261,7 +2261,7 @@ mod tests { #[test] fn test_rebuild_page_index() { // Create builder with one rowgroup and two columns - let mut builder = PageIndexBuilder::new(1, 2).unwrap(); + let mut builder = PageIndexBuilder::new(1, 2); // Add column index for first column let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32); diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 40746b932de8..0c9ef7ef6f0a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -611,7 +611,7 @@ impl HeapSize for Grid { /// /// // Assemble the PageIndex (one entry per row group, each with one /// // entry per column) and attach it to the metadata -/// let mut page_index = PageIndexBuilder::new(1, 1).unwrap(); +/// let mut page_index = PageIndexBuilder::new(1, 1); /// page_index.put_column_index(column_index, 0, 0); /// page_index.put_offset_index(offset_index, 0, 0); /// let page_index = page_index.build(); @@ -742,16 +742,18 @@ impl PageIndexBuilder { /// All index entries are initialized to `None` and can be populated using /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). /// - /// # Errors + /// # Panics /// - /// Returns an error if either dimension exceeds `u32::MAX`. - pub fn new(num_row_groups: usize, num_columns: usize) -> Result { - let keep_cols = Keep::new_full(num_columns)?; - let keep_rows = Keep::new_full(num_row_groups)?; - Ok(Self { + /// Panics if either dimension exceeds `u32::MAX`. + pub fn new(num_row_groups: usize, num_columns: usize) -> Self { + let keep_cols = + Keep::new_full(num_columns).expect("page index column count exceeds u32::MAX"); + let keep_rows = + Keep::new_full(num_row_groups).expect("page index row group count exceeds u32::MAX"); + Self { column_indexes: Some(Grid::new(keep_rows.clone(), keep_cols.clone())), offset_indexes: Some(Grid::new(keep_rows, keep_cols)), - }) + } } /// Creates a new [`PageIndexBuilder`] from an existing [`PageIndex`] @@ -775,18 +777,15 @@ impl PageIndexBuilder { /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). /// This replaces any existing column index storage and discards its entries. /// - /// # Errors + /// # Panics /// - /// Returns an error if either dimension exceeds `u32::MAX`. - pub fn allocate_column_indexes( - &mut self, - num_row_groups: usize, - num_columns: usize, - ) -> Result<()> { - let keep_cols = Keep::new_full(num_columns)?; - let keep_rows = Keep::new_full(num_row_groups)?; + /// Panics if either dimension exceeds `u32::MAX`. + pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + let keep_cols = + Keep::new_full(num_columns).expect("page index column count exceeds u32::MAX"); + let keep_rows = + Keep::new_full(num_row_groups).expect("page index row group count exceeds u32::MAX"); self.column_indexes = Some(Grid::new(keep_rows, keep_cols)); - Ok(()) } /// Allocates space for offset indexes @@ -799,18 +798,15 @@ impl PageIndexBuilder { /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). /// This replaces any existing offset index storage and discards its entries. /// - /// # Errors + /// # Panics /// - /// Returns an error if either dimension exceeds `u32::MAX`. - pub fn allocate_offset_indexes( - &mut self, - num_row_groups: usize, - num_columns: usize, - ) -> Result<()> { - let keep_cols = Keep::new_full(num_columns)?; - let keep_rows = Keep::new_full(num_row_groups)?; + /// Panics if either dimension exceeds `u32::MAX`. + pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { + let keep_cols = + Keep::new_full(num_columns).expect("page index column count exceeds u32::MAX"); + let keep_rows = + Keep::new_full(num_row_groups).expect("page index row group count exceeds u32::MAX"); self.offset_indexes = Some(Grid::new(keep_rows, keep_cols)); - Ok(()) } /// Sets the column index for a specific row group and column @@ -982,11 +978,19 @@ mod tests { let span = u32::MAX as usize + 1; assert!(Keep::from_mask(Some(Arc::from([])), span).is_err()); assert!(Keep::new_full(span).is_err()); - assert!(PageIndexBuilder::new(0, span).is_err()); - - let mut builder = PageIndexBuilder::default(); - assert!(builder.allocate_column_indexes(0, span).is_err()); - assert!(builder.allocate_offset_indexes(0, span).is_err()); + assert!(std::panic::catch_unwind(|| PageIndexBuilder::new(0, span)).is_err()); + assert!( + std::panic::catch_unwind(|| { + PageIndexBuilder::default().allocate_column_indexes(0, span) + }) + .is_err() + ); + assert!( + std::panic::catch_unwind(|| { + PageIndexBuilder::default().allocate_offset_indexes(0, span) + }) + .is_err() + ); } #[test] @@ -1020,16 +1024,16 @@ mod tests { let ci = colidx_for_test(); let mut builder = PageIndexBuilder::default(); - builder.allocate_column_indexes(2, 3).unwrap(); + builder.allocate_column_indexes(2, 3); assert!(builder.put_column_index(ci, 1, 2)); - builder.allocate_column_indexes(2, 3).unwrap(); + builder.allocate_column_indexes(2, 3); let column_indexes = builder.column_indexes.as_ref().unwrap(); assert_eq!(column_indexes.rows.len(), 2); assert_eq!(column_indexes.cols.len(), 3); assert_eq!(column_indexes.cells.len(), 6); assert!(column_indexes.is_empty()); - builder.allocate_offset_indexes(2, 3).unwrap(); + builder.allocate_offset_indexes(2, 3); let offset_indexes = builder.offset_indexes.as_ref().unwrap(); assert_eq!(offset_indexes.rows.len(), 2); assert_eq!(offset_indexes.cols.len(), 3); From 9dc6483481ea91876dbcc56c357e5a392c522592 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:55:39 -0700 Subject: [PATCH 66/78] check that index insertion succeeds --- parquet/src/file/metadata/parser.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index d75ee3f0ea84..044a6f40638c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -313,7 +313,11 @@ fn parse_column_index( let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; let idx = inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_column_index(idx, rg_idx, col_idx); + if !page_index_builder.put_column_index(idx, rg_idx, col_idx) { + return Err(general_err!( + "page index builder has no column-index storage for row group {rg_idx}, column {col_idx}" + )); + } } } } @@ -339,7 +343,11 @@ fn parse_offset_index( let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; let idx = inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_offset_index(idx, rg_idx, col_idx); + if !page_index_builder.put_offset_index(idx, rg_idx, col_idx) { + return Err(general_err!( + "page index builder has no offset-index storage for row group {rg_idx}, column {col_idx}" + )); + } } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); } From 0accf0d77f39fe1126bef195903c0ca5bd5a3685 Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 25 Sep 2026 17:59:58 -0700 Subject: [PATCH 67/78] document possible double counting in PageIndex::heap_size --- parquet/src/file/metadata/page_index.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 0c9ef7ef6f0a..b07ee33e42f1 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -685,6 +685,9 @@ impl PageIndexProvider for PageIndex { impl HeapSize for PageIndex { fn heap_size(&self) -> usize { + // Column and offset grids created from the same ColumnChunkMask may share their + // Keep allocations. HeapSize accounts for each Arc independently, so those shared + // allocations may be counted more than once. self.column_indexes.heap_size() + self.offset_indexes.heap_size() } } From 81af27bf66727a65d0b27238f67761e6a5ebf450 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 25 Sep 2026 22:10:17 -0700 Subject: [PATCH 68/78] revert changes to the put_x_index fns --- parquet/src/file/metadata/page_index.rs | 29 ++++++++++++++----------- parquet/src/file/metadata/parser.rs | 12 ++-------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index b07ee33e42f1..280a7bf477ec 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -814,32 +814,32 @@ impl PageIndexBuilder { /// Sets the column index for a specific row group and column /// - /// Returns `false`, and drops `column_index`, if column indexes were not allocated - /// (see [`Self::allocate_column_indexes`]) or the grid has no storage for the position. + /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), + /// or the row group or column index is out of bounds, this method does nothing. pub fn put_column_index( &mut self, column_index: ColumnIndexMetaData, row_group_idx: usize, column_idx: usize, - ) -> bool { - self.column_indexes - .as_mut() - .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, column_index)) + ) { + if let Some(ref mut indexes) = self.column_indexes { + indexes.insert(row_group_idx, column_idx, column_index); + } } /// Sets the offset index for a specific row group and column /// - /// Returns `false`, and drops `offset_index`, if offset indexes were not allocated - /// (see [`Self::allocate_offset_indexes`]) or the grid has no storage for the position. + /// If offset indexes were not allocated (see [`Self::allocate_offset_indexes`]), + /// or the row group or column index is out of bounds, this method does nothing. pub fn put_offset_index( &mut self, offset_index: OffsetIndexMetaData, row_group_idx: usize, column_idx: usize, - ) -> bool { - self.offset_indexes - .as_mut() - .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, offset_index)) + ) { + if let Some(ref mut indexes) = self.offset_indexes { + indexes.insert(row_group_idx, column_idx, offset_index); + } } /// Checks if an index structure is entirely empty (all entries are None) @@ -1005,9 +1005,11 @@ mod tests { ); assert!(grid.insert(0, 0, ci.clone())); + /* TODO(ets): add a fallible put let mut builder = PageIndex::new(Some(grid), None).into_builder(); assert!(builder.put_column_index(ci.clone(), 0, 0)); assert!(!builder.put_column_index(ci, 0, 1)); + */ } #[test] @@ -1028,7 +1030,8 @@ mod tests { let mut builder = PageIndexBuilder::default(); builder.allocate_column_indexes(2, 3); - assert!(builder.put_column_index(ci, 1, 2)); + // TODO(ets): need a fallible put + //assert!(builder.put_column_index(ci, 1, 2)); builder.allocate_column_indexes(2, 3); let column_indexes = builder.column_indexes.as_ref().unwrap(); assert_eq!(column_indexes.rows.len(), 2); diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 044a6f40638c..d75ee3f0ea84 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -313,11 +313,7 @@ fn parse_column_index( let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; let idx = inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; - if !page_index_builder.put_column_index(idx, rg_idx, col_idx) { - return Err(general_err!( - "page index builder has no column-index storage for row group {rg_idx}, column {col_idx}" - )); - } + page_index_builder.put_column_index(idx, rg_idx, col_idx); } } } @@ -343,11 +339,7 @@ fn parse_offset_index( let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; let idx = inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; - if !page_index_builder.put_offset_index(idx, rg_idx, col_idx) { - return Err(general_err!( - "page index builder has no offset-index storage for row group {rg_idx}, column {col_idx}" - )); - } + page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); } From 3f421e0ba5126b1e42165d4bbf681ca56039e37c Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 25 Sep 2026 22:28:13 -0700 Subject: [PATCH 69/78] clippy --- parquet/src/file/metadata/page_index.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 280a7bf477ec..b53ee007b4f0 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -882,7 +882,7 @@ impl From for PageIndexBuilder { #[cfg(test)] mod tests { - use super::{Grid, Keep, PageIndex, PageIndexBuilder}; + use super::{Grid, Keep, PageIndexBuilder}; use crate::{ basic::BoundaryOrder, file::metadata::ColumnChunkMask, @@ -1032,6 +1032,7 @@ mod tests { builder.allocate_column_indexes(2, 3); // TODO(ets): need a fallible put //assert!(builder.put_column_index(ci, 1, 2)); + builder.put_column_index(ci, 1, 2); builder.allocate_column_indexes(2, 3); let column_indexes = builder.column_indexes.as_ref().unwrap(); assert_eq!(column_indexes.rows.len(), 2); From 220c6c453516c39ef07b020663273710e670878b Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 11:04:31 -0700 Subject: [PATCH 70/78] add reshape APIs to the PageIndex and document --- parquet/src/file/metadata/page_index.rs | 239 +++++++++++++++++++++++- parquet/src/file/metadata/parser.rs | 2 +- 2 files changed, 235 insertions(+), 6 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index b53ee007b4f0..6f32f2d0697f 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -436,6 +436,14 @@ impl Keep { Some(indexes) => indexes.len(), } } + + /// Returns the logical index at the specified storage position. + fn index(&self, position: usize) -> usize { + match &self.kept { + None => position, + Some(indexes) => indexes[position] as usize, + } + } } impl HeapSize for Arc<[u32]> { @@ -538,6 +546,27 @@ impl Grid { true } + /// Moves values into a new grid, preserving those whose logical coordinates + /// are selected by the new grid. + fn reshape(self, mut reshaped: Self) -> Self { + let old_num_cols = self.cols.len(); + + for (cell_idx, value) in self.cells.into_iter().enumerate() { + let Some(value) = value else { + continue; + }; + + let row_offset = cell_idx / old_num_cols; + let col_offset = cell_idx % old_num_cols; + let row = self.rows.index(row_offset); + let col = self.cols.index(col_offset); + // A smaller shape or sparse mask may intentionally omit this coordinate. + let _ = reshaped.insert(row, col, value); + } + + reshaped + } + /// Returns true if the grid has no values pub(crate) fn is_empty(&self) -> bool { self.cells.iter().all(|cell| cell.is_none()) @@ -557,6 +586,11 @@ impl HeapSize for Grid { /// by this crate when assembling and writing the Page Index. It is also the default /// implementation of the [`PageIndexProvider`] contained in the [`ParquetMetaData`]. /// +/// Use [`PageIndexBuilder::new`] to create dense storage for both index types, or +/// [`PageIndexBuilder::try_new_with_masks`] to select storage independently for column and +/// offset indexes. An existing index can be reshaped while retaining selected entries with +/// [`PageIndex::try_into_builder_with_shape`]. +/// /// # Example: Constructing a synthetic `PageIndex` /// /// This example builds a [`ParquetMetaData`] for a file with a single row @@ -622,6 +656,66 @@ impl HeapSize for Grid { /// assert!(metadata.page_index().unwrap().is_complete()); /// ``` /// +/// # Example: Expanding a cached sparse `PageIndex` +/// +/// A query engine may cache only the page indexes needed by an initial query. If a later +/// query projects another column, the cached index can be reshaped to add storage for that +/// column without discarding the existing entries. +/// +/// ``` +/// # use parquet::errors::Result; +/// # use parquet::file::metadata::{ColumnChunkMask, OffsetIndexBuilder}; +/// # use parquet::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; +/// # use parquet::file::page_index::offset_index::OffsetIndexMetaData; +/// +/// # fn main() -> Result<()> { +/// fn offset_index(offset: i64) -> OffsetIndexMetaData { +/// // create an `OffsetIndexMetaData` with the given offset +/// # let mut builder = OffsetIndexBuilder::new(); +/// # builder.append_row_count(10); +/// # builder.append_offset_and_size(offset, 100); +/// # builder.build() +/// } +/// +/// // The file has two row groups and five columns. The first query needs offset +/// // indexes for columns 0 and 2 in row group 0 only. +/// // The column index is not used so it is left unallocated. +/// let initial_projection = ColumnChunkMask::row_groups_and_columns([0], [0, 2]); +/// let mut builder = PageIndexBuilder::try_new_with_masks( +/// 2, +/// 5, +/// None, +/// Some(&initial_projection), +/// )?; +/// builder.put_offset_index(offset_index(100), 0, 0); +/// builder.put_offset_index(offset_index(200), 0, 2); +/// let cached = builder.build(); +/// +/// assert!(cached.offset_index(0, 0).is_some()); +/// assert!(cached.offset_index(0, 1).is_none()); +/// assert!(cached.offset_index(0, 2).is_some()); +/// assert!(cached.offset_index(1, 0).is_none()); +/// +/// // A later query also projects column 4. Reshape the offset-index grid to +/// // include it; entries for columns 0 and 2 are retained. +/// let expanded_projection = ColumnChunkMask::row_groups_and_columns([0], [0, 2, 4]); +/// let mut builder = cached.try_into_builder_with_shape( +/// 2, +/// 5, +/// None, +/// Some(&expanded_projection), +/// )?; +/// builder.put_offset_index(offset_index(400), 0, 4); +/// let expanded = builder.build(); +/// +/// assert!(expanded.offset_index(0, 0).is_some()); +/// assert!(expanded.offset_index(0, 2).is_some()); +/// assert!(expanded.offset_index(0, 4).is_some()); +/// assert!(expanded.offset_index(0, 1).is_none()); +/// # Ok(()) +/// # } +/// ``` +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ /// [`ColumnIndex`]: crate::file::page_index::column_index::ColumnIndexMetaData /// [`OffsetIndex`]: crate::file::page_index::offset_index::OffsetIndexMetaData @@ -646,11 +740,39 @@ impl PageIndex { /// Convert this `PageIndex` into a [`PageIndexBuilder`]. /// /// The builder retains the storage shape of this index. Consequently, its `put_*` - /// methods return `false` for positions outside that shape. The `allocate_*` methods - /// can replace it with dense storage, but discard existing entries for that index type. + /// methods ignore positions outside that shape. The `allocate_*` methods can replace it + /// with dense storage, but discard existing entries for that index type. Use + /// [`Self::try_into_builder_with_shape`] to change the shape while preserving entries. pub fn into_builder(self) -> PageIndexBuilder { self.into() } + + /// Convert this `PageIndex` into a [`PageIndexBuilder`] with the requested shape. + /// + /// Existing entries selected by the new masks are preserved. Passing `Some` with a + /// [`ColumnChunkMask::all`] mask creates dense storage for that index type, passing + /// `Some` with another mask creates sparse storage, and passing `None` removes storage + /// for that index type. + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn try_into_builder_with_shape( + self, + num_row_groups: usize, + num_columns: usize, + column_index_mask: Option<&ColumnChunkMask>, + offset_index_mask: Option<&ColumnChunkMask>, + ) -> Result { + let mut builder = self.into_builder(); + builder.try_reshape( + num_row_groups, + num_columns, + column_index_mask, + offset_index_mask, + )?; + Ok(builder) + } } impl PageIndexProvider for PageIndex { @@ -719,7 +841,16 @@ impl PageIndexBuilder { Ok(Some(Grid::new(rows, cols))) } - pub(crate) fn new_for_read( + /// Creates a new builder with storage selected independently for each index type. + /// + /// Passing `Some` with a [`ColumnChunkMask::all`] mask creates dense storage for that + /// index type, passing `Some` with another mask creates sparse storage, and passing + /// `None` leaves that index type unallocated. + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn try_new_with_masks( num_row_groups: usize, num_columns: usize, column_index_mask: Option<&ColumnChunkMask>, @@ -770,6 +901,39 @@ impl PageIndexBuilder { } } + /// Changes the shape of the index storage while preserving selected entries. + /// + /// Existing entries selected by the new masks are preserved. Passing `Some` with a + /// [`ColumnChunkMask::all`] mask creates dense storage for that index type, passing + /// `Some` with another mask creates sparse storage, and passing `None` removes storage + /// for that index type. A mask may also allocate a previously absent index type. + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `u32::MAX`. + pub fn try_reshape( + &mut self, + num_row_groups: usize, + num_columns: usize, + column_index_mask: Option<&ColumnChunkMask>, + offset_index_mask: Option<&ColumnChunkMask>, + ) -> Result<()> { + let column_indexes = + Self::storage_for_selection(num_row_groups, num_columns, column_index_mask)?; + let offset_indexes = + Self::storage_for_selection(num_row_groups, num_columns, offset_index_mask)?; + + self.column_indexes = match (self.column_indexes.take(), column_indexes) { + (Some(old), Some(new)) => Some(old.reshape(new)), + (_, new) => new, + }; + self.offset_indexes = match (self.offset_indexes.take(), offset_indexes) { + (Some(old), Some(new)) => Some(old.reshape(new)), + (_, new) => new, + }; + Ok(()) + } + /// Allocates space for column indexes /// /// This allocates an empty index structure for the specified number of row groups and columns. @@ -882,7 +1046,7 @@ impl From for PageIndexBuilder { #[cfg(test)] mod tests { - use super::{Grid, Keep, PageIndexBuilder}; + use super::{Grid, Keep, PageIndex, PageIndexBuilder}; use crate::{ basic::BoundaryOrder, file::metadata::ColumnChunkMask, @@ -994,6 +1158,11 @@ mod tests { }) .is_err() ); + assert!( + PageIndexBuilder::default() + .try_reshape(0, span, Some(&ColumnChunkMask::all()), None) + .is_err() + ); } #[test] @@ -1015,7 +1184,7 @@ mod tests { #[test] fn test_builder_sparse_storage_shape() { let mask = ColumnChunkMask::columns([0, 2]); - let builder = PageIndexBuilder::new_for_read(4, 5, Some(&mask), None).unwrap(); + let builder = PageIndexBuilder::try_new_with_masks(4, 5, Some(&mask), None).unwrap(); let column_indexes = builder.column_indexes.unwrap(); assert_eq!(column_indexes.rows.len(), 4); @@ -1048,6 +1217,66 @@ mod tests { assert!(offset_indexes.is_empty()); } + #[test] + fn test_builder_reshape_preserves_entries_with_dense_and_sparse_storage() { + let ci = colidx_for_test(); + let mut grid = Grid::new( + Keep::from_mask(Some(Arc::from([0, 2])), 3).unwrap(), + Keep::from_mask(Some(Arc::from([1])), 3).unwrap(), + ); + assert!(grid.insert(0, 1, ci.clone())); + assert!(grid.insert(2, 1, ci.clone())); + + let page_index = PageIndex::new(Some(grid), None); + let all = ColumnChunkMask::all(); + let mut builder = page_index + .try_into_builder_with_shape(4, 4, Some(&all), None) + .unwrap(); + + let column_indexes = builder.column_indexes.as_ref().unwrap(); + assert!(column_indexes.rows.kept.is_none()); + assert!(column_indexes.cols.kept.is_none()); + assert_eq!(column_indexes.cells.len(), 16); + assert_eq!(column_indexes.get(0, 1), Some(&ci)); + assert_eq!(column_indexes.get(2, 1), Some(&ci)); + assert!(builder.offset_indexes.is_none()); + + // A position omitted from the original sparse grid is now available. + builder.put_column_index(ci.clone(), 3, 3); + assert_eq!( + builder.column_indexes.as_ref().unwrap().get(3, 3), + Some(&ci) + ); + + // Sparse reshaping retains selected entries, drops unselected entries, and can + // allocate storage for an index type that was previously absent. + let column_mask = ColumnChunkMask::row_groups_and_columns([0, 3], [1, 3]); + let offset_mask = ColumnChunkMask::row_groups_and_columns([1], [2]); + builder + .try_reshape(4, 4, Some(&column_mask), Some(&offset_mask)) + .unwrap(); + let column_indexes = builder.column_indexes.as_ref().unwrap(); + assert_eq!(column_indexes.cells.len(), 4); + assert_eq!(column_indexes.get(0, 1), Some(&ci)); + assert!(column_indexes.get(2, 1).is_none()); + assert_eq!(column_indexes.get(3, 3), Some(&ci)); + let offset_indexes = builder.offset_indexes.as_ref().unwrap(); + assert_eq!(offset_indexes.cells.len(), 1); + assert!(offset_indexes.get(1, 2).is_none()); + + // Changing dimensions preserves only entries that remain in bounds. + builder.try_reshape(2, 2, Some(&all), None).unwrap(); + let column_indexes = builder.column_indexes.as_ref().unwrap(); + assert_eq!(column_indexes.cells.len(), 4); + assert_eq!(column_indexes.get(0, 1), Some(&ci)); + assert!(column_indexes.get(3, 3).is_none()); + + // Passing None removes storage for that index type. + builder.try_reshape(4, 4, None, Some(&all)).unwrap(); + assert!(builder.column_indexes.is_none()); + assert_eq!(builder.offset_indexes.as_ref().unwrap().cells.len(), 16); + } + #[test] fn test_grid_is_empty() { let ci = colidx_for_test(); diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index d75ee3f0ea84..59ebd783eafd 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -257,7 +257,7 @@ pub(crate) fn parse_page_index( ) -> crate::errors::Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::new_for_read( + let mut builder = PageIndexBuilder::try_new_with_masks( num_row_groups, num_columns, (column_index_policy != PageIndexPolicy::Skip).then_some(column_index_mask), From c217708947cc536c393d9eaed62233028ebd9f60 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:05:49 -0700 Subject: [PATCH 71/78] remove linear search for small spans --- parquet/src/file/metadata/page_index.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 6f32f2d0697f..96b0f50ff676 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -370,7 +370,7 @@ impl RowGroupPageIndex { /// Stores which positions are set in a sparse vector. For example: /// `[None, None, Some(...), Some(...), None, Some(...)]` becomes `[2, 3, 5]` /// -/// Position checking uses a linear scan for up to 32 indexes and binary search above that. +/// Position checking uses binary search. /// Stores indexes as `u32` because Parquet/Thrift collections cannot contain more than /// `i32::MAX` entries. #[derive(Debug, Clone, PartialEq)] @@ -420,12 +420,9 @@ impl Keep { /// Retrieve a position if set fn position(&self, idx: usize) -> Option { let needle = u32::try_from(idx).ok()?; - // below CUTOFF elements, use linear search - const CUTOFF: usize = 32; match self.kept.as_ref() { None => (needle < self.span).then_some(idx), - Some(k) if k.len() > CUTOFF => k.binary_search(&needle).ok(), - Some(k) => k.iter().position(|&i| i == needle), + Some(k) => k.binary_search(&needle).ok(), } } From 19f7125ccab7a0c985ced15c8944d0dc39e6baa4 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:29:11 -0700 Subject: [PATCH 72/78] add fallible put functions to PageIndexBuilder --- parquet/src/file/metadata/page_index.rs | 50 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 96b0f50ff676..cf5c519c401a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -983,9 +983,22 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.column_indexes { - indexes.insert(row_group_idx, column_idx, column_index); - } + let _ = self.try_put_column_index(column_index, row_group_idx, column_idx); + } + + /// Attempts to set the column index for a specific row group and column. + /// + /// Returns `false`, and drops `column_index`, if column indexes were not allocated + /// (see [`Self::allocate_column_indexes`]) or the grid has no storage for the position. + pub fn try_put_column_index( + &mut self, + column_index: ColumnIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> bool { + self.column_indexes + .as_mut() + .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, column_index)) } /// Sets the offset index for a specific row group and column @@ -998,9 +1011,22 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.offset_indexes { - indexes.insert(row_group_idx, column_idx, offset_index); - } + let _ = self.try_put_offset_index(offset_index, row_group_idx, column_idx); + } + + /// Attempts to set the offset index for a specific row group and column. + /// + /// Returns `false`, and drops `offset_index`, if offset indexes were not allocated + /// (see [`Self::allocate_offset_indexes`]) or the grid has no storage for the position. + pub fn try_put_offset_index( + &mut self, + offset_index: OffsetIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> bool { + self.offset_indexes + .as_mut() + .is_some_and(|indexes| indexes.insert(row_group_idx, column_idx, offset_index)) } /// Checks if an index structure is entirely empty (all entries are None) @@ -1163,7 +1189,7 @@ mod tests { } #[test] - fn test_builder_put_reports_missing_storage() { + fn test_builder_try_put_reports_missing_storage() { let ci = colidx_for_test(); let mut grid = Grid::new( Keep::from_mask(Some(Arc::from([0])), 1).unwrap(), @@ -1171,11 +1197,9 @@ mod tests { ); assert!(grid.insert(0, 0, ci.clone())); - /* TODO(ets): add a fallible put let mut builder = PageIndex::new(Some(grid), None).into_builder(); - assert!(builder.put_column_index(ci.clone(), 0, 0)); - assert!(!builder.put_column_index(ci, 0, 1)); - */ + assert!(builder.try_put_column_index(ci.clone(), 0, 0)); + assert!(!builder.try_put_column_index(ci, 0, 1)); } #[test] @@ -1196,9 +1220,7 @@ mod tests { let mut builder = PageIndexBuilder::default(); builder.allocate_column_indexes(2, 3); - // TODO(ets): need a fallible put - //assert!(builder.put_column_index(ci, 1, 2)); - builder.put_column_index(ci, 1, 2); + assert!(builder.try_put_column_index(ci, 1, 2)); builder.allocate_column_indexes(2, 3); let column_indexes = builder.column_indexes.as_ref().unwrap(); assert_eq!(column_indexes.rows.len(), 2); From 3ab2fd4dd9e1f16c15f623088c03c3596f115bd5 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:39:46 -0700 Subject: [PATCH 73/78] finish docs for parse_page_index --- parquet/src/file/metadata/parser.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 84645687540d..3f2c0649089c 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -245,6 +245,10 @@ pub(crate) fn decode_metadata( /// Required, Optional, Skip). /// * `offset_index_policy` - The policy for handling offset index parsing (e.g., /// Required, Optional, Skip). +/// * `column_index_mask` - The row groups and leaf columns whose column indexes are parsed. +/// Ignored when `column_index_policy` is [`PageIndexPolicy::Skip`]. +/// * `offset_index_mask` - The row groups and leaf columns whose offset indexes are parsed. +/// Ignored when `offset_index_policy` is [`PageIndexPolicy::Skip`]. /// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing /// the page indexes. pub(crate) fn parse_page_index( From c47bb75b7e9c95fdf04c418bfb6bd57b7d1dfa17 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 28 Sep 2026 13:53:00 -0700 Subject: [PATCH 74/78] import ProjectionMask --- parquet/src/file/metadata/reader.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index efd73a8b38fd..e144f37fdb2d 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "arrow")] +use crate::arrow::ProjectionMask; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptionProperties; use crate::errors::{ParquetError, Result}; @@ -234,10 +236,7 @@ impl ColumnChunkMask { /// Creates a mask selecting the leaf columns in an Arrow projection. #[cfg(feature = "arrow")] - pub fn from_projection( - projection: &crate::arrow::ProjectionMask, - schema: &SchemaDescriptor, - ) -> Self { + pub fn from_projection(projection: &ProjectionMask, schema: &SchemaDescriptor) -> Self { Self::columns((0..schema.num_columns()).filter(|&i| projection.leaf_included(i))) } From b9fd8847c44454e1fa3ec570e02774b8fd3962d0 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:04:40 -0700 Subject: [PATCH 75/78] address nit --- parquet/src/file/metadata/push_decoder.rs | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index a631e153f992..1019e808c3c1 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -534,21 +534,22 @@ pub(crate) fn range_for_page_index( (column_index_policy, column_index_mask, true), (offset_index_policy, offset_index_mask, false), ] { - if policy != PageIndexPolicy::Skip { - for row_group in mask.row_group_indices(metadata.num_row_groups()) { - for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { - let column = metadata.row_group(row_group).column(column); - let index = if column_index { - column.column_index_range() - } else { - column.offset_index_range() - }; - if let Some(index) = index { - range = Some(match range { - Some(range) => range.start.min(index.start)..range.end.max(index.end), - None => index, - }); - } + if policy == PageIndexPolicy::Skip { + continue; + } + for row_group in mask.row_group_indices(metadata.num_row_groups()) { + for column in mask.column_indices(metadata.row_group(row_group).num_columns()) { + let column = metadata.row_group(row_group).column(column); + let index = if column_index { + column.column_index_range() + } else { + column.offset_index_range() + }; + if let Some(index) = index { + range = Some(match range { + Some(range) => range.start.min(index.start)..range.end.max(index.end), + None => index, + }); } } } From ad865184537abeab2463330e0cc26bd39905072d Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:09:45 -0700 Subject: [PATCH 76/78] replace boxed iterator with more targeted concrete iterator impl --- parquet/src/file/metadata/reader.rs | 58 ++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index e144f37fdb2d..7f7eb81c3090 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -34,6 +34,8 @@ use crate::schema::types::SchemaDescriptor; use arrow_array::ArrayRef; use bytes::Bytes; use std::collections::BTreeSet; +use std::iter::FusedIterator; +use std::slice::Iter; use std::sync::Arc; use std::{io::Read, ops::Range}; @@ -241,24 +243,22 @@ impl ColumnChunkMask { } /// Returns an iterator over the row group indices selected by this mask - pub fn row_group_indices(&self, num_row_groups: usize) -> Box + '_> { + pub fn row_group_indices(&self, num_row_groups: usize) -> impl Iterator + '_ { Self::axis_indices(self.row_groups.as_deref(), num_row_groups) } /// Returns an iterator over the column indices selected by this mask - pub fn column_indices(&self, num_columns: usize) -> Box + '_> { + pub fn column_indices(&self, num_columns: usize) -> impl Iterator + '_ { Self::axis_indices(self.columns.as_deref(), num_columns) } - fn axis_indices(axis: Option<&[u32]>, len: usize) -> Box + '_> { + fn axis_indices(axis: Option<&[u32]>, len: usize) -> AxisIndices<'_> { match axis { - None => Box::new(0..len), - Some(indices) => Box::new( - indices - .iter() - .map(|&i| i as usize) - .take_while(move |&i| i < len), - ), + None => AxisIndices::All(0..len), + Some(indices) => { + let end = indices.partition_point(|&i| (i as usize) < len); + AxisIndices::Selected(indices[..end].iter()) + } } } @@ -275,6 +275,33 @@ impl ColumnChunkMask { } } +// helper for ColumnChunkMask selected index traversal +enum AxisIndices<'a> { + All(Range), + Selected(Iter<'a, u32>), +} + +impl Iterator for AxisIndices<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + match self { + Self::All(indices) => indices.next(), + Self::Selected(indices) => indices.next().map(|&i| i as usize), + } + } + + fn size_hint(&self) -> (usize, Option) { + match self { + Self::All(indices) => indices.size_hint(), + Self::Selected(indices) => indices.size_hint(), + } + } +} + +impl ExactSizeIterator for AxisIndices<'_> {} +impl FusedIterator for AxisIndices<'_> {} + impl ParquetMetaDataReader { /// Create a new [`ParquetMetaDataReader`] pub fn new() -> Self { @@ -1301,6 +1328,17 @@ mod tests { let all = ColumnChunkMask::all(); assert!(all.is_all()); assert!(all.includes_column(u32::MAX as usize)); + + assert_eq!(all.row_group_indices(3).collect::>(), [0, 1, 2]); + assert_eq!(all.column_indices(2).collect::>(), [0, 1]); + + let none = ColumnChunkMask::none(); + assert_eq!(none.row_group_indices(3).count(), 0); + assert_eq!(none.column_indices(3).count(), 0); + + let sparse = ColumnChunkMask::row_groups_and_columns([0, 2, 4], [1, 3, 5]); + assert_eq!(sparse.row_group_indices(3).collect::>(), [0, 2]); + assert_eq!(sparse.column_indices(4).collect::>(), [1, 3]); } } From 4823984ecc27cb9291ae96a85e331a40334bc8c9 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 29 Sep 2026 16:43:51 -0700 Subject: [PATCH 77/78] revert to using single passed in Bytes --- parquet/src/file/metadata/parser.rs | 55 +++++++++++++---------- parquet/src/file/metadata/push_decoder.rs | 5 ++- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 3f2c0649089c..22f0564a9c4f 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -20,9 +20,11 @@ //! These functions parse thrift-encoded metadata from a byte slice //! into the corresponding Rust structures -use std::sync::Arc; +use std::{ops::Range, sync::Arc}; -use crate::errors::ParquetError; +use bytes::Bytes; + +use crate::errors::{ParquetError, Result}; use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ @@ -32,8 +34,6 @@ use crate::file::metadata::{ use crate::file::page_index::column_index::ColumnIndexMetaData; use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index}; use crate::file::page_index::offset_index::OffsetIndexMetaData; -use crate::file::reader::ChunkReader; -use crate::util::push_buffers::PushBuffers; /// Helper struct for metadata parsing /// @@ -50,7 +50,6 @@ mod inner { use super::*; use crate::encryption::decrypt::FileDecryptionProperties; - use crate::errors::Result; /// API for decoding metadata that may be encrypted #[derive(Debug, Default)] @@ -108,7 +107,7 @@ mod inner { column: &ColumnChunkMetaData, row_group_index: usize, col_index: usize, - ) -> crate::errors::Result { + ) -> Result { use crate::encryption::decrypt::CryptoContext; match &column.column_crypto_metadata { Some(crypto_metadata) => { @@ -136,7 +135,7 @@ mod inner { column: &ColumnChunkMetaData, row_group_index: usize, col_index: usize, - ) -> crate::errors::Result { + ) -> Result { use crate::encryption::decrypt::CryptoContext; match &column.column_crypto_metadata { Some(crypto_metadata) => { @@ -162,7 +161,6 @@ mod inner { #[cfg(not(feature = "encryption"))] mod inner { use super::*; - use crate::errors::Result; use std::sync::Arc; /// parallel implementation when encryption feature is not enabled /// @@ -208,7 +206,7 @@ mod inner { column: &ColumnChunkMetaData, _row_group_index: usize, _col_index: usize, - ) -> crate::errors::Result { + ) -> Result { decode_column_index(bytes, column.column_type()) } @@ -218,7 +216,7 @@ mod inner { _column: &ColumnChunkMetaData, _row_group_index: usize, _col_index: usize, - ) -> crate::errors::Result { + ) -> Result { decode_offset_index(bytes) } } @@ -233,7 +231,7 @@ mod inner { pub(crate) fn decode_metadata( buf: &[u8], options: Option<&ParquetMetaDataOptions>, -) -> crate::errors::Result { +) -> Result { parquet_metadata_from_bytes(buf, options) } @@ -249,16 +247,17 @@ pub(crate) fn decode_metadata( /// Ignored when `column_index_policy` is [`PageIndexPolicy::Skip`]. /// * `offset_index_mask` - The row groups and leaf columns whose offset indexes are parsed. /// Ignored when `offset_index_policy` is [`PageIndexPolicy::Skip`]. -/// * `bytes` - [`PushBuffers`] that should have already been populated with the bytes containing -/// the page indexes. +/// * `bytes` - The byte slice containing the page index data. +/// * `start_offset` - The offset where `bytes` begin in the file. pub(crate) fn parse_page_index( metadata: &mut ParquetMetaData, column_index_policy: PageIndexPolicy, offset_index_policy: PageIndexPolicy, column_index_mask: &ColumnChunkMask, offset_index_mask: &ColumnChunkMask, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { let num_row_groups = metadata.num_row_groups(); let num_columns = metadata.file_metadata().schema_descr().num_columns(); let mut builder = PageIndexBuilder::default(); @@ -271,6 +270,7 @@ pub(crate) fn parse_page_index( column_index_mask, &mut builder, bytes, + start_offset, )?; } if offset_index_policy != PageIndexPolicy::Skip { @@ -281,6 +281,7 @@ pub(crate) fn parse_page_index( offset_index_mask, &mut builder, bytes, + start_offset, )?; } @@ -302,8 +303,9 @@ fn parse_column_index( column_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { if column_index_policy == PageIndexPolicy::Skip { return Ok(()); } @@ -312,9 +314,9 @@ fn parse_column_index( for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.column_index_range() { - let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = - inner::parse_single_column_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; + inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_column_index(idx, rg_idx, col_idx); } } @@ -328,8 +330,9 @@ fn parse_offset_index( offset_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, page_index_builder: &mut PageIndexBuilder, - bytes: &PushBuffers, -) -> crate::errors::Result<()> { + bytes: &Bytes, + start_offset: u64, +) -> Result<()> { if offset_index_policy == PageIndexPolicy::Skip { return Ok(()); } @@ -338,9 +341,9 @@ fn parse_offset_index( for col_idx in mask.column_indices(rg.num_columns()) { let col = rg.column(col_idx); if let Some(r) = col.offset_index_range() { - let idx_bytes = bytes.get_bytes(r.start, (r.end - r.start) as usize)?; + let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = - inner::parse_single_offset_index(&idx_bytes, metadata, col, rg_idx, col_idx)?; + inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?; page_index_builder.put_offset_index(idx, rg_idx, col_idx); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); @@ -350,3 +353,9 @@ fn parse_offset_index( Ok(()) } + +fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range) -> Result<&[u8]> { + let start = usize::try_from(range.start - start_offset)?; + let end = usize::try_from(range.end - start_offset)?; + Ok(&bytes[start..end]) +} diff --git a/parquet/src/file/metadata/push_decoder.rs b/parquet/src/file/metadata/push_decoder.rs index 1019e808c3c1..78179528047e 100644 --- a/parquet/src/file/metadata/push_decoder.rs +++ b/parquet/src/file/metadata/push_decoder.rs @@ -462,13 +462,16 @@ impl ParquetMetaDataPushDecoder { return Ok(needs_range(page_index_range)); } + let buffer = self.get_bytes(&page_index_range)?; + let offset = page_index_range.start; parse_page_index( &mut metadata, self.column_index_policy, self.offset_index_policy, &self.column_index_mask, &self.offset_index_mask, - &self.buffers, + &buffer, + offset, )?; self.state = DecodeState::Finished; return Ok(DecodeResult::Data(*metadata)); From 7ff415c158bd35707235a530f892c93ec05101f5 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Tue, 6 Oct 2026 22:32:50 -0700 Subject: [PATCH 78/78] remove functions re-introduced during merge --- parquet/src/file/metadata/reader.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 7a29371c24ae..b22b447346bd 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -237,16 +237,6 @@ impl ColumnChunkMask { self.row_groups.is_none() && self.columns.is_none() } - /// Returns selected row groups, or `None` when all row groups are selected. - pub fn selected_row_groups(&self) -> Option<&[u32]> { - self.row_groups.as_deref() - } - - /// Returns selected leaf columns, or `None` when all columns are selected. - pub fn selected_columns(&self) -> Option<&[u32]> { - self.columns.as_deref() - } - pub(crate) fn selected_row_groups_shared(&self) -> Option> { self.row_groups.clone() }