From 976bb35eeea2fe07ed647911ee86a227dc7bb77b Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 30 Oct 2026 21:23:43 -0700 Subject: [PATCH 1/9] improved page index storage --- parquet/Cargo.toml | 5 + parquet/examples/page_index_cache.rs | 309 ++++++++++++ parquet/src/arrow/async_reader/mod.rs | 2 +- parquet/src/file/metadata/column_chunk.rs | 317 ++++++++++++ parquet/src/file/metadata/mod.rs | 19 +- parquet/src/file/metadata/page_index.rs | 579 +++++++++++++++++++--- parquet/src/file/metadata/parser.rs | 8 +- parquet/src/file/metadata/reader.rs | 28 ++ parquet/src/file/writer.rs | 2 +- parquet/tests/arrow_reader/page_index.rs | 16 +- 10 files changed, 1189 insertions(+), 96 deletions(-) create mode 100644 parquet/examples/page_index_cache.rs create mode 100644 parquet/src/file/metadata/column_chunk.rs diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index 7e66a21e7024..68098b96ce21 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -177,6 +177,11 @@ name = "custom_page_index" required-features = ["arrow"] path = "./examples/custom_page_index.rs" +[[example]] +name = "page_index_cache" +required-features = ["arrow"] +path = "./examples/page_index_cache.rs" + [[test]] name = "arrow_reader" required-features = ["arrow"] diff --git a/parquet/examples/page_index_cache.rs b/parquet/examples/page_index_cache.rs new file mode 100644 index 000000000000..34b7c340844a --- /dev/null +++ b/parquet/examples/page_index_cache.rs @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Incrementally caching page indexes with [`ParquetMetaDataReader`]. +//! +//! This example keeps footer metadata and decoded page indexes in separate caches. For each +//! query it: +//! +//! 1. Finds the requested column chunks that are absent from the page-index cache +//! 2. Uses [`ParquetMetaDataReader::read_page_index_and_finish`] to request and decode those misses +//! 3. Moves the decoded entries into the cache and wraps them in [`Arc`] +//! 4. Builds a query-specific [`PageIndexProvider`] from shared cache entries +//! +//! A production cache shared by multiple files would also include a stable file identity and +//! freshness information in its keys. This example also caches only indexes that are present; +//! it does not record negative results. Consequently, it may request a missing index again on a +//! later query. A production cache should track coordinates known not to have an index. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +use arrow::array::{Int32Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::errors::Result; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{ + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, +}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; + +type ColumnChunk = (usize, usize); + +/// A query-specific provider backed by handles to cached entries. +#[derive(Debug, Default)] +struct CachedPageIndexProvider { + column_indexes: HashMap>, + offset_indexes: HashMap>, +} + +impl PageIndexProvider for CachedPageIndexProvider { + fn has_offset_indexes(&self) -> bool { + !self.offset_indexes.is_empty() + } + + fn has_column_indexes(&self) -> bool { + !self.column_indexes.is_empty() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .get(&(row_group_idx, column_idx)) + .map(Arc::as_ref) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .get(&(row_group_idx, column_idx)) + .map(Arc::as_ref) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Decoded page indexes cached independently of query-specific metadata. +#[derive(Default)] +struct PageIndexCache { + column_indexes: HashMap>, + offset_indexes: HashMap>, +} + +impl PageIndexCache { + /// Returns the footer metadata and a provider containing exactly the entries requested for + /// one query, loading any cache misses through `ParquetMetaDataReader` first. + fn page_index_for_query( + &mut self, + mut metadata: ParquetMetaData, + file_bytes: &Bytes, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, + ) -> Result<(ParquetMetaData, CachedPageIndexProvider)> { + let num_row_groups = metadata.num_row_groups(); + let num_columns = metadata.file_metadata().schema_descr().num_columns(); + + let missing_column_indexes = column_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter(|key| !self.column_indexes.contains_key(key)) + .collect::>(); + let missing_offset_indexes = offset_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter(|key| !self.offset_indexes.contains_key(key)) + .collect::>(); + + println!( + " cache misses: {} column indexes, {} offset indexes", + missing_column_indexes.len(), + missing_offset_indexes.len() + ); + + if !missing_column_indexes.is_empty() || !missing_offset_indexes.is_empty() { + // ColumnChunkMask describes a Cartesian product. This produces the smallest such mask + // covering the misses and may therefore decode additional entries for irregular miss + // sets. Those additional entries are useful cache population and are retained below. + let column_miss_mask = covering_mask(&missing_column_indexes); + let offset_miss_mask = covering_mask(&missing_offset_indexes); + let reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Optional) + .with_offset_index_policy(PageIndexPolicy::Optional) + .with_column_index_mask(column_miss_mask) + .with_offset_index_mask(offset_miss_mask); + + let (returned_metadata, page_index) = reader.read_page_index_and_finish(file_bytes)?; + metadata = returned_metadata; + if let Some(page_index) = page_index { + let (column_indexes, offset_indexes) = page_index.into_index_entries(); + self.column_indexes + .extend(column_indexes.map(|(key, index)| (key, Arc::new(index)))); + self.offset_indexes + .extend(offset_indexes.map(|(key, index)| (key, Arc::new(index)))); + } + } + + // Build an immutable provider containing only handles needed by this query. + let column_indexes = column_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter_map(|key| { + self.column_indexes + .get(&key) + .map(|index| (key, Arc::clone(index))) + }) + .collect(); + let offset_indexes = offset_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter_map(|key| { + self.offset_indexes + .get(&key) + .map(|index| (key, Arc::clone(index))) + }) + .collect(); + + Ok(( + metadata, + CachedPageIndexProvider { + column_indexes, + offset_indexes, + }, + )) + } +} + +/// Returns the smallest Cartesian mask covering `chunks`. +fn covering_mask(chunks: &[ColumnChunk]) -> ColumnChunkMask { + if chunks.is_empty() { + return ColumnChunkMask::none(); + } + + let row_groups = chunks.iter().map(|&(row_group, _)| row_group); + let columns = chunks.iter().map(|&(_, column)| column); + ColumnChunkMask::row_groups_and_columns( + row_groups.collect::>(), + columns.collect::>(), + ) +} + +/// Reads and caches footer metadata without decoding page indexes. +fn decode_footer(file_bytes: &Bytes) -> Result { + ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(file_bytes) +} + +fn main() -> Result<()> { + let file_bytes = create_sample_file()?; + let cached_metadata = decode_footer(&file_bytes)?; + let mut cache = PageIndexCache::default(); + + // Query 1 filters column 0 and projects columns 0, 1, and 4 in row group 0. + let column_mask = ColumnChunkMask::row_groups_and_columns([0], [0]); + let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 1, 4]); + println!("Query 1:"); + let (cached_metadata, provider) = + cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?; + assert_query_indexes(&provider, 0, &[0], &[0, 1, 4]); + + // Attach the query-specific provider to a cheap clone of the cached footer metadata. + let query_metadata = cached_metadata + .clone() + .into_builder() + .set_page_index(Some(Arc::new(provider))) + .build(); + assert!(query_metadata.page_index().is_some()); + + // Query 2 reuses the column index and offset indexes for columns 0 and 4. Only the offset + // index for column 2 is decoded and added to the cache. + let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 2, 4]); + println!("Query 2:"); + let (_cached_metadata, provider) = + cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?; + assert_query_indexes(&provider, 0, &[0], &[0, 2, 4]); + + println!( + "Cache now contains {} column indexes and {} offset indexes", + cache.column_indexes.len(), + cache.offset_indexes.len() + ); + Ok(()) +} + +fn assert_query_indexes( + page_index: &dyn PageIndexProvider, + row_group_idx: usize, + column_indexes: &[usize], + offset_indexes: &[usize], +) { + for column_idx in 0..6 { + assert_eq!( + page_index.column_index(row_group_idx, column_idx).is_some(), + column_indexes.contains(&column_idx) + ); + assert_eq!( + page_index.offset_index(row_group_idx, column_idx).is_some(), + offset_indexes.contains(&column_idx) + ); + } +} + +fn create_sample_file() -> Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + Field::new("category", DataType::Utf8, false), + Field::new("amount", DataType::Int32, false), + ])); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_size_limit(100) + .set_write_batch_size(10) + .build(); + let mut buffer = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut buffer, Arc::clone(&schema), Some(props))?; + + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = row_group * 50 + batch_num * 10; + let values = (offset..offset + 10).collect::>(); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(values.clone())), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 2).collect::>(), + )), + Arc::new(StringArray::from( + values + .iter() + .map(|value| format!("name{value}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 3).collect::>(), + )), + Arc::new(StringArray::from( + values + .iter() + .map(|value| if value % 2 == 0 { "even" } else { "odd" }) + .collect::>(), + )), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 4).collect::>(), + )), + ], + )?; + writer.write(&batch)?; + } + writer.flush()?; + } + writer.close()?; + Ok(Bytes::from(buffer)) +} diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs index 7a1e78d96289..1f093a9d3c76 100644 --- a/parquet/src/arrow/async_reader/mod.rs +++ b/parquet/src/arrow/async_reader/mod.rs @@ -1247,7 +1247,7 @@ mod tests { .await .unwrap(); - let page_index = PageIndex::new(None, Some(vec![])); + let page_index = PageIndex::try_new(None, Some(vec![])).unwrap(); metadata.set_page_index(Some(Arc::new(page_index))); let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap(); diff --git a/parquet/src/file/metadata/column_chunk.rs b/parquet/src/file/metadata/column_chunk.rs new file mode 100644 index 000000000000..d8ca67a0a4c2 --- /dev/null +++ b/parquet/src/file/metadata/column_chunk.rs @@ -0,0 +1,317 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Sparse storage keyed by row-group and column coordinates. + +use crate::errors::{ParquetError, Result}; +use crate::file::metadata::memory::HeapSize; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct ColumnChunkKey(u64); + +impl ColumnChunkKey { + pub(crate) fn try_new(row_group_idx: usize, column_idx: usize) -> Result { + let row_group_idx = u32::try_from(row_group_idx).map_err(|_| { + ParquetError::General(format!( + "row group index exceeds packed coordinate limit: {row_group_idx}" + )) + })?; + let column_idx = u32::try_from(column_idx).map_err(|_| { + ParquetError::General(format!( + "column index exceeds packed coordinate limit: {column_idx}" + )) + })?; + Ok(Self( + (u64::from(row_group_idx) << u32::BITS) | u64::from(column_idx), + )) + } + + pub(crate) fn coordinates(self) -> (usize, usize) { + ((self.0 >> u32::BITS) as usize, (self.0 as u32) as usize) + } +} + +impl HeapSize for ColumnChunkKey { + fn heap_size(&self) -> usize { + 0 + } +} + +fn validate_dimensions(num_row_groups: usize, num_columns: usize) -> Result<()> { + // Parquet metadata collection lengths are represented by signed 32-bit Thrift values. + i32::try_from(num_row_groups).map_err(|_| { + ParquetError::General(format!( + "row group count exceeds i32::MAX: {num_row_groups}" + )) + })?; + i32::try_from(num_columns).map_err(|_| { + ParquetError::General(format!("column count exceeds i32::MAX: {num_columns}")) + })?; + Ok(()) +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnChunkMap { + num_row_groups: usize, + num_columns: usize, + entries: HashMap, +} + +impl ColumnChunkMap { + pub(crate) fn try_new(num_row_groups: usize, num_columns: usize) -> Result { + validate_dimensions(num_row_groups, num_columns)?; + Ok(Self { + num_row_groups, + num_columns, + entries: HashMap::new(), + }) + } + + pub(crate) fn num_row_groups(&self) -> usize { + self.num_row_groups + } + + pub(crate) fn num_columns(&self) -> usize { + self.num_columns + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn key(&self, row_group_idx: usize, column_idx: usize) -> Result { + if row_group_idx >= self.num_row_groups || column_idx >= self.num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {} by {}", + self.num_row_groups, self.num_columns + ))); + } + ColumnChunkKey::try_new(row_group_idx, column_idx) + } + + /// Inserts or replaces an entry. + pub(crate) fn try_insert( + &mut self, + row_group_idx: usize, + column_idx: usize, + value: T, + ) -> Result> { + let key = self.key(row_group_idx, column_idx)?; + if let Some(existing) = self.entries.get_mut(&key) { + return Ok(Some(std::mem::replace(existing, value))); + } + + self.entries.try_reserve(1).map_err(|err| { + ParquetError::General(format!( + "failed to reserve storage for column chunk at row group {row_group_idx}, \ + column {column_idx}: {err}" + )) + })?; + Ok(self.entries.insert(key, value)) + } + + pub(crate) fn try_freeze(self) -> Result> { + validate_dimensions(self.num_row_groups, self.num_columns)?; + let num_entries = self.entries.len(); + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + for entry in self.entries { + entries.push(entry); + } + entries.sort_unstable_by_key(|(key, _)| *key); + Ok(ColumnChunkEntries { + num_row_groups: self.num_row_groups, + num_columns: self.num_columns, + entries: entries.into_boxed_slice(), + }) + } +} + +impl HeapSize for ColumnChunkMap { + fn heap_size(&self) -> usize { + self.entries.heap_size() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnChunkEntries { + num_row_groups: usize, + num_columns: usize, + entries: Box<[(ColumnChunkKey, T)]>, +} + +impl ColumnChunkEntries { + pub(crate) fn try_from_rows(rows: Vec>>) -> Result { + let num_row_groups = rows.len(); + let num_columns = rows.iter().map(Vec::len).max().unwrap_or_default(); + validate_dimensions(num_row_groups, num_columns)?; + let num_entries = rows + .iter() + .map(|columns| columns.iter().filter(|value| value.is_some()).count()) + .try_fold(0_usize, |total, count| total.checked_add(count)) + .ok_or_else(|| { + ParquetError::General("column chunk entry count exceeds usize::MAX".to_string()) + })?; + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + for (row_group_idx, columns) in rows.into_iter().enumerate() { + for (column_idx, value) in columns.into_iter().enumerate() { + if let Some(value) = value { + entries.push((ColumnChunkKey::try_new(row_group_idx, column_idx)?, value)); + } + } + } + Ok(Self { + num_row_groups, + num_columns, + entries: entries.into_boxed_slice(), + }) + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn key(&self, row_group_idx: usize, column_idx: usize) -> Result { + if row_group_idx >= self.num_row_groups || column_idx >= self.num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {} by {}", + self.num_row_groups, self.num_columns + ))); + } + ColumnChunkKey::try_new(row_group_idx, column_idx) + } + + pub(crate) fn get(&self, row_group_idx: usize, column_idx: usize) -> Result> { + let key = self.key(row_group_idx, column_idx)?; + let Ok(entry_idx) = self + .entries + .binary_search_by_key(&key, |(entry_key, _)| *entry_key) + else { + return Ok(None); + }; + let Some((_, value)) = self.entries.get(entry_idx) else { + return Err(ParquetError::General(format!( + "column chunk binary search returned invalid entry index {entry_idx}" + ))); + }; + Ok(Some(value)) + } + + #[cfg_attr(not(test), expect(unused))] + pub(crate) fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(key, value)| (*key, value)) + } + + pub(crate) fn into_iter(self) -> impl Iterator { + self.entries.into_vec().into_iter() + } + + pub(crate) fn try_into_map(self) -> Result> { + let mut map = ColumnChunkMap::try_new(self.num_row_groups, self.num_columns)?; + for (key, value) in self.entries { + let (row_group_idx, column_idx) = key.coordinates(); + map.try_insert(row_group_idx, column_idx, value)?; + } + Ok(map) + } +} + +impl HeapSize for ColumnChunkEntries { + fn heap_size(&self) -> usize { + self.entries.len() * std::mem::size_of::<(ColumnChunkKey, V)>() + + self + .entries + .iter() + .map(|(_, value)| value.heap_size()) + .sum::() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sparse_storage_is_sorted_and_validates_coordinates() { + let mut map = ColumnChunkMap::try_new(2, 3).unwrap(); + map.try_insert(1, 2, 12).unwrap(); + map.try_insert(0, 1, 1).unwrap(); + let entries = map.try_freeze().unwrap(); + + let coordinates = entries + .iter() + .map(|(key, _)| key.coordinates()) + .collect::>(); + assert_eq!(coordinates, vec![(0, 1), (1, 2)]); + assert_eq!(entries.get(0, 1).unwrap(), Some(&1)); + assert_eq!(entries.get(0, 2).unwrap(), None); + assert!(entries.get(2, 0).is_err()); + assert!(entries.get(0, 3).is_err()); + } + + #[test] + fn zero_and_invalid_dimensions() { + assert!(ColumnChunkMap::<()>::try_new(0, 0).is_ok()); + assert!(ColumnChunkMap::<()>::try_new(i32::MAX as usize + 1, 1).is_err()); + assert!(ColumnChunkMap::<()>::try_new(1, i32::MAX as usize + 1).is_err()); + } + + #[test] + fn insertion_replaces_existing_value_without_growing() { + let mut map = ColumnChunkMap::try_new(1, 1).unwrap(); + assert_eq!(map.try_insert(0, 0, "first").unwrap(), None); + let capacity = map.entries.capacity(); + + assert_eq!(map.try_insert(0, 0, "replacement").unwrap(), Some("first")); + assert_eq!(map.entries.len(), 1); + assert_eq!(map.entries.capacity(), capacity); + } + + #[test] + fn freeze_and_thaw_preserve_entries() { + let mut map = ColumnChunkMap::try_new(2, 2).unwrap(); + map.try_insert(1, 0, 3).unwrap(); + let expected_entries = map.try_freeze().unwrap(); + + let mut map = ColumnChunkMap::try_new(2, 2).unwrap(); + map.try_insert(1, 0, 3).unwrap(); + let entries = map.try_freeze().unwrap(); + let thawed = entries.try_into_map().unwrap(); + assert_eq!(thawed.try_freeze().unwrap(), expected_entries); + } + + #[test] + fn frozen_heap_size_uses_packed_key_layout() { + let entries = ColumnChunkEntries::try_from_rows(vec![vec![Some(1_i32)]]).unwrap(); + assert_eq!( + entries.heap_size(), + std::mem::size_of::<(ColumnChunkKey, i32)>() + ); + } +} diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 99b6bd6acdde..2cc81f325eca 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -50,6 +50,7 @@ //! //! [`external_metadata.rs`]: https://github.com/apache/arrow-rs/tree/master/parquet/examples/external_metadata.rs //! +mod column_chunk; #[cfg(feature = "arrow")] mod dictionary; mod footer_tail; @@ -222,14 +223,16 @@ impl ParquetMetaData { /// /// 1. Includes size of self /// - /// 2. Includes heap memory for sub fields such as [`FileMetaData`] and - /// [`RowGroupMetaData`]. + /// 2. Includes heap memory for sub fields such as [`FileMetaData`], + /// [`RowGroupMetaData`], and the built-in [`PageIndex`] provider. /// - /// 3. Includes memory from shared pointers (e.g. [`SchemaDescPtr`]). This - /// means `memory_size` will over estimate the memory size if such pointers - /// are shared. + /// 3. Includes memory reachable through shared pointers, such as [`SchemaDescPtr`] and the + /// [`Arc`]-wrapped page index provider. Therefore, the same allocation may be counted by + /// multiple metadata values or when the provider is also retained in an external cache. /// - /// 4. Does not include any allocator overheads + /// 4. Does not include heap memory used by custom [`PageIndexProvider`] implementations. + /// + /// 5. Does not include any allocator overheads. pub fn memory_size(&self) -> usize { #[cfg(feature = "encryption")] let encryption_size = self.file_decryptor.heap_size(); @@ -2206,9 +2209,9 @@ mod tests { // Size with page index (includes Arc overhead plus PageIndex heap size) #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3312; + let bigger_expected_size = 3296; #[cfg(feature = "encryption")] - let bigger_expected_size = 3480; + let bigger_expected_size = 3464; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 78c3dc2792bb..6b27ea412427 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -16,7 +16,13 @@ // under the License. //! Page Index structures for efficient page-level skipping +//! +//! Row-group and column counts are each limited to `i32::MAX`, consistent with the +//! Parquet/Thrift metadata format. Populated entries are stored sparsely, subject to platform +//! address-space and available-memory limits. +use crate::errors::{ParquetError, Result}; +use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkMap}; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -365,10 +371,11 @@ impl RowGroupPageIndex { /// Struct to encapsulate the Parquet [Page Index] /// -/// This struct provides a dense representation of the Page Index. It is -/// used internally by this crate when assembling and writing the Page -/// Index. It is also the default implementation of the [`PageIndexProvider`] -/// contained in the [`ParquetMetaData`]. +/// This struct provides a sparse, immutable representation of the Page Index, stored in row-major +/// order and keyed by row group and column index. It owns its populated column and offset index +/// metadata. Use [`Self::into_index_entries`] to move that metadata into an external cache. +/// It is used internally by this crate when assembling and writing the Page Index, and is the +/// default implementation of the [`PageIndexProvider`] contained in the [`ParquetMetaData`]. /// /// # Example: Constructing a synthetic `PageIndex` /// @@ -424,9 +431,13 @@ impl RowGroupPageIndex { /// /// // Assemble the PageIndex (one entry per row group, each with one /// // entry per column) and attach it to the metadata -/// let mut page_index = PageIndexBuilder::new(1, 1); -/// page_index.put_column_index(column_index, 0, 0); -/// page_index.put_offset_index(offset_index, 0, 0); +/// let mut page_index = PageIndexBuilder::try_new(1, 1).unwrap(); +/// page_index +/// .try_put_column_index(column_index, 0, 0) +/// .unwrap(); +/// page_index +/// .try_put_offset_index(offset_index, 0, 0) +/// .unwrap(); /// let page_index = page_index.build(); /// let metadata = metadata /// .into_builder() @@ -441,24 +452,65 @@ impl RowGroupPageIndex { /// [`ParquetMetaData`]: crate::file::metadata::ParquetMetaData #[derive(Debug, Clone, PartialEq)] pub struct PageIndex { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndex { - pub(crate) fn new( + pub(crate) fn try_new( column_indexes: Option>>>, offset_indexes: Option>>>, - ) -> Self { - Self { + ) -> Result { + let column_indexes = column_indexes + .map(ColumnChunkEntries::try_from_rows) + .transpose()? + .filter(|indexes| !indexes.is_empty()); + let offset_indexes = offset_indexes + .map(ColumnChunkEntries::try_from_rows) + .transpose()? + .filter(|indexes| !indexes.is_empty()); + + Ok(Self { column_indexes, offset_indexes, - } + }) } /// Convert this `PageIndex` into a [`PageIndexBuilder`] pub fn into_builder(self) -> PageIndexBuilder { - self.into() + self.try_into_builder() + .unwrap_or_else(|err| panic!("failed to thaw page index: {err}")) + } + + /// Fallibly converts this `PageIndex` into a [`PageIndexBuilder`]. + pub fn try_into_builder(self) -> Result { + PageIndexBuilder::try_new_from(self) + } + + /// Consumes this page index and returns its populated column and offset index entries. + /// + /// Each entry contains its `(row_group_index, column_index)` coordinate and owned index + /// metadata. This can be used to move parsed indexes into a cache without cloning their + /// metadata. Entries are returned in row-major order. + #[expect(clippy::type_complexity)] + pub fn into_index_entries( + self, + ) -> ( + impl Iterator, + impl Iterator, + ) { + let column_indexes = self + .column_indexes + .into_iter() + .flat_map(ColumnChunkEntries::into_iter) + .map(|(key, index)| (key.coordinates(), index)); + let offset_indexes = self + .offset_indexes + .into_iter() + .flat_map(ColumnChunkEntries::into_iter) + .map(|(key, index)| (key.coordinates(), index)); + + (column_indexes, offset_indexes) } } @@ -476,8 +528,10 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - let rg = self.column_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.column_indexes + .as_ref()? + .get(row_group_idx, column_idx) + .ok()? } fn offset_index( @@ -485,8 +539,10 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - let rg = self.offset_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.offset_indexes + .as_ref()? + .get(row_group_idx, column_idx) + .ok()? } fn as_any(&self) -> &dyn std::any::Any { @@ -506,42 +562,40 @@ impl HeapSize for PageIndex { /// - Populating column indexes for predicate columns (for page filtering) /// - Populating offset indexes for projected columns (for direct I/O) /// - Automatic conversion of empty structures to `None` to save memory -#[derive(Default)] +#[derive(Default, Debug)] pub struct PageIndexBuilder { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndexBuilder { - /// Creates an empty index structure with space for the specified number of row groups and columns + /// Creates a new [`PageIndexBuilder`] for the specified number of row groups and columns. /// - /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. - /// The outer vector has one entry per row group, and each inner vector has one entry per column. + /// The dimensions are used to validate inserted coordinates. Storage is allocated only for + /// entries populated using + /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). /// - /// # Type Parameters - /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { - Some( - (0..num_row_groups) - .map(|_| { - let mut idx = Vec::with_capacity(num_columns); - idx.resize_with(num_columns, || None); - idx - }) - .collect(), - ) + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_new`](Self::try_new) for a fallible alternative. + pub fn new(num_row_groups: usize, num_columns: usize) -> Self { + Self::try_new(num_row_groups, num_columns).expect("invalid page index dimensions") } - /// Creates a new [`PageIndexBuilder`] with space allocated for both column and offset indexes + /// Attempts to create a new [`PageIndexBuilder`] for the specified number of row groups and columns. /// - /// This allocates empty index structures for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using - /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). - pub fn new(num_row_groups: usize, num_columns: usize) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_new(num_row_groups: usize, num_columns: usize) -> Result { + Ok(Self { + column_indexes: Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?), + offset_indexes: Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?), + }) } /// Creates a new [`PageIndexBuilder`] from an existing [`PageIndex`] @@ -549,34 +603,97 @@ impl PageIndexBuilder { /// This takes ownership of the index structures from the provided [`PageIndex`], /// allowing them to be modified and rebuilt. Useful for updating existing page indexes. pub(crate) fn new_from(page_index: PageIndex) -> Self { - Self { - column_indexes: page_index.column_indexes, - offset_indexes: page_index.offset_indexes, - } + Self::try_new_from(page_index) + .unwrap_or_else(|err| panic!("failed to thaw page index: {err}")) + } + + fn try_new_from(page_index: PageIndex) -> Result { + Ok(Self { + column_indexes: page_index + .column_indexes + .map(ColumnChunkEntries::try_into_map) + .transpose()?, + offset_indexes: page_index + .offset_indexes + .map(ColumnChunkEntries::try_into_map) + .transpose()?, + }) } /// Allocates space for column indexes /// - /// This allocates an empty index structure for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using + /// This replaces any existing column indexes with an empty sparse map having the specified + /// dimensions. It can then be populated using /// [`put_column_index`](Self::put_column_index). /// /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). + /// + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_allocate_column_indexes`](Self::try_allocate_column_indexes) for a + /// fallible alternative. pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.column_indexes = Self::empty_index(num_row_groups, num_columns); + self.try_allocate_column_indexes(num_row_groups, num_columns) + .expect("invalid page index dimensions"); + } + + /// Attempts to allocate space for column indexes + /// + /// This is a fallible version of [`allocate_column_indexes`](Self::allocate_column_indexes). + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_allocate_column_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + self.column_indexes = Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?); + Ok(()) } /// Allocates space for offset indexes /// - /// This allocates an empty index structure for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using + /// This replaces any existing offset indexes with an empty sparse map having the specified + /// dimensions. It can then be populated using /// [`put_offset_index`](Self::put_offset_index). /// /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). + /// + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_allocate_offset_indexes`](Self::try_allocate_offset_indexes) for a + /// fallible alternative. pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.offset_indexes = Self::empty_index(num_row_groups, num_columns); + self.try_allocate_offset_indexes(num_row_groups, num_columns) + .expect("invalid page index dimensions"); + } + + /// Attempts to allocate space for offset indexes + /// + /// This is a fallible version of [`allocate_offset_indexes`](Self::allocate_offset_indexes). + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_allocate_offset_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + self.offset_indexes = Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?); + Ok(()) } /// Sets the column index for a specific row group and column @@ -589,12 +706,37 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.column_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(column_index); + // preserve stated contract that this does nothing when indexes were not allocated + // or passed in indices are out of bounds + let Some(indexes) = self.column_indexes.as_ref() else { + return; + }; + if row_group_idx >= indexes.num_row_groups() || column_idx >= indexes.num_columns() { + return; } + self.try_put_column_index(column_index, row_group_idx, column_idx) + .unwrap_or_else(|err| panic!("{err}")); + } + + /// Attempts to set the column index for a specific row group and column. + /// + /// # Errors + /// + /// Returns an error, and drops `column_index`, if column indexes were not allocated or the + /// coordinate is out of bounds. + pub fn try_put_column_index( + &mut self, + column_index: ColumnIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> Result<()> { + let indexes = self + .column_indexes + .as_mut() + .ok_or_else(|| ParquetError::General("column indexes are not allocated".to_string()))?; + indexes + .try_insert(row_group_idx, column_idx, column_index) + .map(|_| ()) } /// Sets the offset index for a specific row group and column @@ -607,22 +749,42 @@ impl PageIndexBuilder { row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.offset_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(offset_index); + // preserve stated contract that this does nothing when indexes were not allocated + // or passed in indices are out of bounds + let Some(indexes) = self.offset_indexes.as_ref() else { + return; + }; + if row_group_idx >= indexes.num_row_groups() || column_idx >= indexes.num_columns() { + return; } + self.try_put_offset_index(offset_index, row_group_idx, column_idx) + .unwrap_or_else(|err| panic!("{err}")); } - /// Checks if an index structure is entirely empty (all entries are None) - fn is_empty_index(index: Option<&Vec>>>) -> bool { - match index { - None => true, - Some(row_groups) => row_groups - .iter() - .all(|columns| columns.iter().all(|entry| entry.is_none())), - } + /// Attempts to set the offset index for a specific row group and column. + /// + /// # Errors + /// + /// Returns an error, and drops `offset_index`, if offset indexes were not allocated or the + /// coordinate is out of bounds. + pub fn try_put_offset_index( + &mut self, + offset_index: OffsetIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> Result<()> { + let indexes = self + .offset_indexes + .as_mut() + .ok_or_else(|| ParquetError::General("offset indexes are not allocated".to_string()))?; + indexes + .try_insert(row_group_idx, column_idx, offset_index) + .map(|_| ()) + } + + /// Checks if an index structure is entirely empty. + fn is_empty_index(index: Option<&ColumnChunkMap>) -> bool { + index.is_none_or(ColumnChunkMap::is_empty) } /// Consumes the builder and returns a [`PageIndex`] @@ -633,6 +795,11 @@ impl PageIndexBuilder { /// - [`PageIndex::has_column_indexes()`] and [`PageIndex::has_offset_indexes()`] /// correctly return `false` for unpopulated indexes pub fn build(self) -> PageIndex { + self.try_build().expect("failed to build page index") + } + + /// Fallibly consumes the builder and returns a [`PageIndex`]. + pub fn try_build(self) -> Result { let column_indexes = if Self::is_empty_index(self.column_indexes.as_ref()) { None } else { @@ -645,7 +812,10 @@ impl PageIndexBuilder { self.offset_indexes }; - PageIndex::new(column_indexes, offset_indexes) + Ok(PageIndex { + column_indexes: column_indexes.map(ColumnChunkMap::try_freeze).transpose()?, + offset_indexes: offset_indexes.map(ColumnChunkMap::try_freeze).transpose()?, + }) } } @@ -654,3 +824,264 @@ impl From for PageIndexBuilder { Self::new_from(page_index) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::basic::Type as PhysicalType; + use crate::file::metadata::{ColumnIndexBuilder, OffsetIndexBuilder}; + + #[test] + fn test_dimensions_within_i32_max() { + // Valid dimensions should work + let builder = PageIndexBuilder::new(100, 50); + assert!(builder.column_indexes.is_some()); + assert!(builder.offset_indexes.is_some()); + + let result = PageIndexBuilder::try_new(100, 50); + assert!(result.is_ok()); + } + + #[test] + fn test_try_new_validates_dimensions() { + // Test row group count exceeding i32::MAX + let oversized = i32::MAX as usize + 1; + let result = PageIndexBuilder::try_new(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + // Test column count exceeding i32::MAX + let result = PageIndexBuilder::try_new(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_new_panics_on_oversized_row_groups() { + let oversized = i32::MAX as usize + 1; + PageIndexBuilder::new(oversized, 10); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_new_panics_on_oversized_columns() { + let oversized = i32::MAX as usize + 1; + PageIndexBuilder::new(10, oversized); + } + + #[test] + fn test_try_allocate_column_indexes_validates_dimensions() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + + let result = builder.try_allocate_column_indexes(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + let result = builder.try_allocate_column_indexes(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_allocate_column_indexes_panics_on_oversized() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + builder.allocate_column_indexes(oversized, 10); + } + + #[test] + fn test_try_allocate_offset_indexes_validates_dimensions() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + + let result = builder.try_allocate_offset_indexes(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + let result = builder.try_allocate_offset_indexes(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_allocate_offset_indexes_panics_on_oversized() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + builder.allocate_offset_indexes(oversized, 10); + } + + #[test] + fn test_page_index_builder_basic_operations() { + let mut builder = PageIndexBuilder::new(2, 3); + + // Create a simple column index + let mut col_index = ColumnIndexBuilder::new(PhysicalType::INT32); + col_index.append( + false, + 1i32.to_le_bytes().to_vec(), + 10i32.to_le_bytes().to_vec(), + 0, + None, + ); + let col_index = col_index.build().unwrap(); + + // Create a simple offset index + let mut off_index = OffsetIndexBuilder::new(); + off_index.append_row_count(100); + off_index.append_offset_and_size(1000, 500); + let off_index = off_index.build(); + + // Insert indexes and report coordinates that cannot be populated + assert!( + builder + .try_put_column_index(col_index.clone(), 0, 0) + .is_ok() + ); + assert!( + builder + .try_put_offset_index(off_index.clone(), 0, 0) + .is_ok() + ); + assert!( + builder + .try_put_column_index(col_index.clone(), 2, 0) + .is_err() + ); + assert!( + builder + .try_put_offset_index(off_index.clone(), 0, 3) + .is_err() + ); + + // Legacy insertion remains a no-op for unallocated storage and invalid coordinates. + let mut legacy_unallocated = PageIndexBuilder::default(); + legacy_unallocated.put_column_index(col_index.clone(), 0, 0); + legacy_unallocated.put_offset_index(off_index.clone(), 0, 0); + let empty_page_index = legacy_unallocated.build(); + assert!(!empty_page_index.has_column_indexes()); + assert!(!empty_page_index.has_offset_indexes()); + + builder.put_column_index(col_index.clone(), 2, 0); + builder.put_column_index(col_index.clone(), 0, 3); + builder.put_offset_index(off_index.clone(), 2, 0); + builder.put_offset_index(off_index.clone(), 0, 3); + + let mut unallocated = PageIndexBuilder::default(); + assert!(unallocated.try_put_column_index(col_index, 0, 0).is_err()); + assert!(unallocated.try_put_offset_index(off_index, 0, 0).is_err()); + + // Build and verify + let page_index = builder.build(); + assert!(page_index.has_column_indexes()); + assert!(page_index.has_offset_indexes()); + assert!(page_index.column_index(0, 0).is_some()); + assert!(page_index.offset_index(0, 0).is_some()); + assert!(page_index.column_index(0, 1).is_none()); + assert!(page_index.offset_index(1, 0).is_none()); + } + + #[test] + fn test_into_index_entries_moves_owned_values_in_row_major_order() { + fn column_index(seed: i32, num_pages: usize) -> ColumnIndexMetaData { + let mut builder = ColumnIndexBuilder::new(PhysicalType::INT32); + for page in 0..num_pages { + let min = seed + page as i32; + builder.append( + false, + min.to_le_bytes().to_vec(), + (min + 10).to_le_bytes().to_vec(), + 0, + None, + ); + } + builder.build().unwrap() + } + + fn offset_index(seed: i64, num_pages: usize) -> OffsetIndexMetaData { + let mut builder = OffsetIndexBuilder::new(); + for page in 0..num_pages { + builder.append_row_count(10); + builder.append_offset_and_size(seed + page as i64 * 100, 100); + } + builder.build() + } + + let first_column = column_index(1, 1); + let second_column = column_index(2, 2); + let column_values = match &second_column { + ColumnIndexMetaData::INT32(index) => index.min_values().as_ptr(), + _ => unreachable!(), + }; + let offset = offset_index(10, 3); + let page_locations = offset.page_locations().as_ptr(); + + let mut builder = PageIndexBuilder::new(3, 3); + builder.try_put_column_index(second_column, 2, 0).unwrap(); + builder.try_put_offset_index(offset, 1, 1).unwrap(); + builder + .try_put_column_index(first_column.clone(), 0, 2) + .unwrap(); + + let (column_entries, offset_entries) = builder.build().into_index_entries(); + let column_entries = column_entries.collect::>(); + let offset_entries = offset_entries.collect::>(); + assert_eq!( + column_entries, + vec![((0, 2), first_column), ((2, 0), column_index(2, 2))] + ); + assert_eq!(offset_entries, vec![((1, 1), offset_index(10, 3))]); + match &column_entries[1].1 { + ColumnIndexMetaData::INT32(index) => { + assert_eq!(index.min_values().as_ptr(), column_values) + } + _ => unreachable!(), + } + assert_eq!( + offset_entries[0].1.page_locations().as_ptr(), + page_locations + ); + + let column = column_index(3, 1); + let mut builder = PageIndexBuilder::default(); + builder.try_allocate_column_indexes(1, 1).unwrap(); + builder.try_put_column_index(column.clone(), 0, 0).unwrap(); + let (column_entries, offset_entries) = builder.build().into_index_entries(); + assert_eq!(column_entries.collect::>(), vec![((0, 0), column)]); + assert_eq!(offset_entries.count(), 0); + } + + #[test] + fn test_page_index_try_new_discards_empty_indexes() { + let page_index = PageIndex::try_new(Some(vec![vec![None]]), Some(vec![])).unwrap(); + assert!(!page_index.has_column_indexes()); + assert!(!page_index.has_offset_indexes()); + } +} diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 4599452187bb..0a4139a26321 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -263,7 +263,7 @@ pub(crate) fn parse_page_index( let mut builder = PageIndexBuilder::default(); if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); + builder.try_allocate_column_indexes(num_row_groups, num_columns)?; parse_column_index( metadata, column_index_policy, @@ -274,7 +274,7 @@ pub(crate) fn parse_page_index( )?; } if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); + builder.try_allocate_offset_indexes(num_row_groups, num_columns)?; parse_offset_index( metadata, offset_index_policy, @@ -312,7 +312,7 @@ fn parse_column_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_column_index(idx, rg_idx, col_idx); + page_index_builder.try_put_column_index(idx, rg_idx, col_idx)?; } } } @@ -339,7 +339,7 @@ fn parse_offset_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_offset_index(idx, rg_idx, col_idx); + page_index_builder.try_put_offset_index(idx, rg_idx, col_idx)?; } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); } diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 7f1092a700ca..90fc8971206c 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -266,6 +266,23 @@ impl ColumnChunkMask { Self::axis_indices(self.columns.as_deref(), num_columns) } + /// Returns an iterator over the `(row_group_index, column_index)` coordinates selected by + /// this mask. + /// + /// Coordinates are returned in row-major order and are bounded by `num_row_groups` and + /// `num_columns`. + pub fn column_chunk_indices( + &self, + num_row_groups: usize, + num_columns: usize, + ) -> impl Iterator + '_ { + self.row_group_indices(num_row_groups) + .flat_map(move |row_group_idx| { + self.column_indices(num_columns) + .map(move |column_idx| (row_group_idx, column_idx)) + }) + } + fn axis_indices(axis: Option<&[u32]>, len: usize) -> AxisIndices<'_> { match axis { None => AxisIndices::All(0..len), @@ -1492,6 +1509,17 @@ mod tests { let sparse = ColumnChunkMask::row_groups_and_columns([0, 2, 4], [1, 3, 5]); assert_eq!(sparse.row_group_indices(3).collect::>(), [0, 2]); assert_eq!(sparse.column_indices(4).collect::>(), [1, 3]); + assert_eq!( + sparse.column_chunk_indices(3, 4).collect::>(), + [(0, 1), (0, 3), (2, 1), (2, 3)] + ); + assert_eq!( + all.column_chunk_indices(2, 3).collect::>(), + [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)] + ); + assert_eq!(none.column_chunk_indices(3, 3).count(), 0); + assert_eq!(sparse.column_chunk_indices(0, 4).count(), 0); + assert_eq!(sparse.column_chunk_indices(3, 0).count(), 0); } } diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 7030cb28c671..4940bfec42d2 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -396,7 +396,7 @@ impl SerializedFileWriter { Some(offset_indexes) }; if column_indexes.is_some() || offset_indexes.is_some() { - let page_index = PageIndex::new(column_indexes, offset_indexes); + let page_index = PageIndex::try_new(column_indexes, offset_indexes)?; encoder = encoder.with_page_index(Arc::new(page_index)); } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index de3bae70c133..8bc5116c33e8 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -382,9 +382,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 13929); + assert_eq!(metadata.memory_size(), 13993); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 14353); + assert_eq!(metadata.memory_size(), 14417); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -397,9 +397,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12808); + assert_eq!(metadata.memory_size(), 10712); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 13232); + assert_eq!(metadata.memory_size(), 11136); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -412,9 +412,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12088); + assert_eq!(metadata.memory_size(), 9656); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 12512); + assert_eq!(metadata.memory_size(), 10080); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -428,7 +428,7 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 11358); + assert_eq!(metadata.memory_size(), 8222); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 11782); + assert_eq!(metadata.memory_size(), 8646); } From b09c64cf927bcedef2a2dd31effcd422bdedbdb2 Mon Sep 17 00:00:00 2001 From: seidl Date: Tue, 3 Nov 2026 15:17:44 -0800 Subject: [PATCH 2/9] bypass builder when constructing PageIndex from thrift --- parquet/src/file/metadata/column_chunk.rs | 83 ++++++++++++++++--- parquet/src/file/metadata/page_index.rs | 21 +++-- parquet/src/file/metadata/parser.rs | 98 +++++++++++++++-------- parquet/tests/arrow_reader/page_index.rs | 86 ++++++++++++++++++++ 4 files changed, 236 insertions(+), 52 deletions(-) diff --git a/parquet/src/file/metadata/column_chunk.rs b/parquet/src/file/metadata/column_chunk.rs index d8ca67a0a4c2..09852135e55d 100644 --- a/parquet/src/file/metadata/column_chunk.rs +++ b/parquet/src/file/metadata/column_chunk.rs @@ -139,11 +139,7 @@ impl ColumnChunkMap { entries.push(entry); } entries.sort_unstable_by_key(|(key, _)| *key); - Ok(ColumnChunkEntries { - num_row_groups: self.num_row_groups, - num_columns: self.num_columns, - entries: entries.into_boxed_slice(), - }) + ColumnChunkEntries::try_from_sorted_entries(self.num_row_groups, self.num_columns, entries) } } @@ -161,6 +157,37 @@ pub(crate) struct ColumnChunkEntries { } impl ColumnChunkEntries { + /// Constructs immutable storage from entries in strictly increasing row-major order. + /// Missing coordinates need no placeholder; dimensions only bound the stored keys. + pub(crate) fn try_from_sorted_entries( + num_row_groups: usize, + num_columns: usize, + entries: Vec<(ColumnChunkKey, T)>, + ) -> Result { + validate_dimensions(num_row_groups, num_columns)?; + let mut previous = None; + for (key, _) in &entries { + let (row_group_idx, column_idx) = key.coordinates(); + if row_group_idx >= num_row_groups || column_idx >= num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {num_row_groups} by {num_columns}" + ))); + } + if previous.is_some_and(|previous| previous >= *key) { + return Err(ParquetError::General( + "column chunk entries must be unique and in row-major order".to_string(), + )); + } + previous = Some(*key); + } + Ok(Self { + num_row_groups, + num_columns, + entries: entries.into_boxed_slice(), + }) + } + pub(crate) fn try_from_rows(rows: Vec>>) -> Result { let num_row_groups = rows.len(); let num_columns = rows.iter().map(Vec::len).max().unwrap_or_default(); @@ -185,11 +212,7 @@ impl ColumnChunkEntries { } } } - Ok(Self { - num_row_groups, - num_columns, - entries: entries.into_boxed_slice(), - }) + Self::try_from_sorted_entries(num_row_groups, num_columns, entries) } pub(crate) fn is_empty(&self) -> bool { @@ -275,6 +298,46 @@ mod tests { assert!(entries.get(0, 3).is_err()); } + #[test] + fn sorted_entries_preserve_sparse_coordinates() { + let entries = ColumnChunkEntries::try_from_sorted_entries( + 100, + 1_000, + vec![ + (ColumnChunkKey::try_new(0, 3).unwrap(), 3_i32), + (ColumnChunkKey::try_new(99, 999).unwrap(), 999), + ], + ) + .unwrap(); + assert_eq!(entries.get(0, 3).unwrap(), Some(&3)); + assert_eq!(entries.get(0, 2).unwrap(), None); + assert_eq!(entries.get(99, 999).unwrap(), Some(&999)); + assert_eq!( + entries.heap_size(), + 2 * std::mem::size_of::<(ColumnChunkKey, i32)>() + ); + assert!( + ColumnChunkEntries::::try_from_sorted_entries(0, 0, vec![]) + .unwrap() + .is_empty() + ); + } + + #[test] + fn sorted_entries_reject_invalid_coordinates_and_order() { + let first = ColumnChunkKey::try_new(0, 0).unwrap(); + let second = ColumnChunkKey::try_new(1, 1).unwrap(); + for entries in [vec![(first, 1), (first, 2)], vec![(second, 1), (first, 2)]] { + assert!(ColumnChunkEntries::try_from_sorted_entries(2, 2, entries).is_err()); + } + assert!(ColumnChunkEntries::try_from_sorted_entries(1, 2, vec![(second, 1)]).is_err()); + assert!(ColumnChunkEntries::try_from_sorted_entries(2, 1, vec![(second, 1)]).is_err()); + assert!( + ColumnChunkEntries::::try_from_sorted_entries(i32::MAX as usize + 1, 1, vec![],) + .is_err() + ); + } + #[test] fn zero_and_invalid_dimensions() { assert!(ColumnChunkMap::<()>::try_new(0, 0).is_ok()); diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 6b27ea412427..130446fc247a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -463,17 +463,22 @@ impl PageIndex { ) -> Result { let column_indexes = column_indexes .map(ColumnChunkEntries::try_from_rows) - .transpose()? - .filter(|indexes| !indexes.is_empty()); + .transpose()?; let offset_indexes = offset_indexes .map(ColumnChunkEntries::try_from_rows) - .transpose()? - .filter(|indexes| !indexes.is_empty()); + .transpose()?; - Ok(Self { - column_indexes, - offset_indexes, - }) + Ok(Self::from_entries(column_indexes, offset_indexes)) + } + + pub(crate) fn from_entries( + column_indexes: Option>, + offset_indexes: Option>, + ) -> Self { + Self { + column_indexes: column_indexes.filter(|indexes| !indexes.is_empty()), + offset_indexes: offset_indexes.filter(|indexes| !indexes.is_empty()), + } } /// Convert this `PageIndex` into a [`PageIndexBuilder`] diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index 0a4139a26321..a8c8c0818af4 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -25,7 +25,8 @@ use std::ops::Range; use bytes::Bytes; use crate::errors::{ParquetError, Result}; -use crate::file::metadata::page_index::{PageIndex, PageIndexBuilder, PageIndexProvider}; +use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkKey}; +use crate::file::metadata::page_index::{PageIndex, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, @@ -258,34 +259,29 @@ pub(crate) fn parse_page_index( bytes: &Bytes, start_offset: u64, ) -> Result> { - let num_row_groups = metadata.num_row_groups(); - let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); - - if column_index_policy != PageIndexPolicy::Skip { - builder.try_allocate_column_indexes(num_row_groups, num_columns)?; - parse_column_index( + let column_indexes = if column_index_policy == PageIndexPolicy::Skip { + None + } else { + Some(parse_column_index( metadata, - column_index_policy, column_index_mask, - &mut builder, bytes, start_offset, - )?; - } - if offset_index_policy != PageIndexPolicy::Skip { - builder.try_allocate_offset_indexes(num_row_groups, num_columns)?; - parse_offset_index( + )?) + }; + let offset_indexes = if offset_index_policy == PageIndexPolicy::Skip { + None + } else { + Some(parse_offset_index( metadata, offset_index_policy, offset_index_mask, - &mut builder, bytes, start_offset, - )?; - } + )?) + }; - let page_index = builder.build(); + let page_index = PageIndex::from_entries(column_indexes, offset_indexes); if page_index.has_column_indexes() || page_index.has_offset_indexes() { Ok(Some(page_index)) } else { @@ -293,17 +289,45 @@ pub(crate) fn parse_page_index( } } +/// Reserve only for selected chunks that actually have an index range. An extra +/// metadata-only pass avoids both vector growth for dense reads and reserving a +/// full rectangular grid for sparse reads or files with missing indexes. +fn reserve_index_entries( + metadata: &ParquetMetaData, + mask: &ColumnChunkMask, + index_range: impl Fn(&ColumnChunkMetaData) -> Option>, +) -> Result> { + let num_entries = + mask.row_group_indices(metadata.num_row_groups()) + .try_fold(0_usize, |count, rg_idx| { + let rg = metadata.row_group(rg_idx); + let present = mask + .column_indices(rg.num_columns()) + .filter(|&col_idx| index_range(rg.column(col_idx)).is_some()) + .count(); + count.checked_add(present).ok_or_else(|| { + ParquetError::General("column chunk entry count exceeds usize::MAX".to_string()) + }) + })?; + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + Ok(entries) +} + fn parse_column_index( metadata: &ParquetMetaData, - column_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, - page_index_builder: &mut PageIndexBuilder, bytes: &Bytes, start_offset: u64, -) -> Result<()> { - if column_index_policy == PageIndexPolicy::Skip { - return Ok(()); - } +) -> Result> { + let mut entries = + reserve_index_entries(metadata, mask, ColumnChunkMetaData::column_index_range)?; + // Mask axes are sorted and unique, so present entries remain in row-major + // order even when selected chunks have no index. for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); for col_idx in mask.column_indices(rg.num_columns()) { @@ -312,25 +336,27 @@ fn parse_column_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.try_put_column_index(idx, rg_idx, col_idx)?; + entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx)); } } } - Ok(()) + ColumnChunkEntries::try_from_sorted_entries( + metadata.num_row_groups(), + metadata.file_metadata().schema_descr().num_columns(), + entries, + ) } fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, - page_index_builder: &mut PageIndexBuilder, bytes: &Bytes, start_offset: u64, -) -> Result<()> { - if offset_index_policy == PageIndexPolicy::Skip { - return Ok(()); - } +) -> Result> { + let mut entries = + reserve_index_entries(metadata, mask, ColumnChunkMetaData::offset_index_range)?; for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); for col_idx in mask.column_indices(rg.num_columns()) { @@ -339,14 +365,18 @@ fn parse_offset_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.try_put_offset_index(idx, rg_idx, col_idx)?; + entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx)); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); } } } - Ok(()) + ColumnChunkEntries::try_from_sorted_entries( + metadata.num_row_groups(), + metadata.file_metadata().schema_descr().num_columns(), + entries, + ) } fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range) -> Result<&[u8]> { diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 8bc5116c33e8..509ddcfc2c5d 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -87,6 +87,92 @@ fn test_parse_with_page_index_mask() { ); } +#[test] +fn test_sparse_page_index_missing_ranges_preserve_coordinates() { + let file = create_test_file(); + let original = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + let mut builder = metadata.into_builder(); + let row_groups = builder + .take_row_groups() + .into_iter() + .enumerate() + .map(|(rg_idx, rg)| { + let columns = rg + .columns() + .iter() + .enumerate() + .map(|(col_idx, column)| { + let mut column = column.clone().into_builder(); + if matches!((rg_idx, col_idx), (0, 0) | (2, 3)) { + column = column + .set_column_index_offset(None) + .set_column_index_length(None); + } + if matches!((rg_idx, col_idx), (0, 1) | (1, 2)) { + column = column + .set_offset_index_offset(None) + .set_offset_index_length(None); + } + column.build().unwrap() + }) + .collect(); + rg.into_builder() + .set_column_metadata(columns) + .build() + .unwrap() + }) + .collect(); + let metadata = builder.set_row_groups(row_groups).build(); + + // Masks canonicalize unordered, duplicate, and out-of-bounds coordinates. + // Missing ranges must leave holes in coordinate space, not vector positions. + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns( + [2, 0, 2, 99], + [3, 0, 1, 3, 99], + )); + reader.read_page_indexes(&file).unwrap(); + let read = reader.finish().unwrap(); + assert_page_index_cells( + &read, + |rg, col| { + matches!(rg, 0 | 2) && matches!(col, 0 | 1 | 3) && !matches!((rg, col), (0, 0) | (2, 3)) + }, + |rg, col| !matches!((rg, col), (0, 1) | (1, 2)), + ); + let index = read.page_index().unwrap(); + let expected = original.page_index().unwrap(); + for rg in 0..3 { + for col in 0..4 { + if let Some(ci) = index.column_index(rg, col) { + assert_eq!(Some(ci), expected.column_index(rg, col)); + } + if let Some(oi) = index.offset_index(rg, col) { + assert_eq!(Some(oi), expected.offset_index(rg, col)); + } + } + } + + // Column index absence is still tolerated under Required, but an absent + // selected offset index remains an error. + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip); + reader.read_page_indexes(&file).unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required); + let error = reader.read_page_indexes(&file).unwrap_err(); + assert_eq!(error.to_string(), "Parquet error: missing offset index"); +} + #[test] fn test_repeated_page_index_reads_replace() { // verify that repeated calls to read_page_indexes replace existing indexes From e4ba3e0813e671b10bf76504f2a963bc191c99f3 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 4 Nov 2026 12:27:15 -0800 Subject: [PATCH 3/9] deprecate the put_ functions that silently fail --- parquet/benches/arrow_statistics.rs | 2 +- parquet/src/arrow/push_decoder/mod.rs | 4 +- .../arrow/push_decoder/reader_builder/mod.rs | 42 ++++++++------- .../src/arrow/push_decoder/scan_plan/tests.rs | 4 +- parquet/src/file/metadata/mod.rs | 16 +++--- parquet/src/file/metadata/page_index.rs | 51 +++++++++++++------ 6 files changed, 76 insertions(+), 43 deletions(-) diff --git a/parquet/benches/arrow_statistics.rs b/parquet/benches/arrow_statistics.rs index af201656d77a..a869c5dc75e3 100644 --- a/parquet/benches/arrow_statistics.rs +++ b/parquet/benches/arrow_statistics.rs @@ -359,7 +359,7 @@ fn page_index_benchmark(c: &mut Criterion) { let mut builder = PageIndexBuilder::new(row_groups, num_columns); for (rg, (_, bytes)) in column_indexes.iter().enumerate() { let index = decode_column_index(bytes, physical_type).unwrap(); - builder.put_column_index(index, rg, column); + builder.try_put_column_index(index, rg, column).unwrap(); } let page_index = builder.build(); let _ = converter diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index 82dc037351f8..ba58eb4127cb 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -3141,7 +3141,9 @@ mod test { let last = offset_index.page_locations.last_mut().unwrap(); last.compressed_page_size += 1; } - page_index.put_offset_index(offset_index, row_group, column); + page_index + .try_put_offset_index(offset_index, row_group, column) + .unwrap(); } } let metadata = metadata diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 02d1f7014e38..e076a4b8a115 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -1005,8 +1005,12 @@ mod tests { .collect(), unencoded_byte_array_data_bytes: None, }; - page_index.put_offset_index(column(&[0, 4, 8]), 0, 0); - page_index.put_offset_index(column(&[0, 6, 10]), 0, 1); + page_index + .try_put_offset_index(column(&[0, 4, 8]), 0, 0) + .unwrap(); + page_index + .try_put_offset_index(column(&[0, 6, 10]), 0, 1) + .unwrap(); let page_index: Option> = Some(Arc::new(page_index.build())); let page_index = RowGroupPageIndex::new(0, page_index); let selection = RowSelection::from(vec![ @@ -1074,22 +1078,24 @@ mod tests { #[test] fn test_auto_keeps_mask_when_page_pruning_skips_pages() { let mut page_index = PageIndexBuilder::new(1, 1); - page_index.put_offset_index( - OffsetIndexMetaData { - page_locations: [0, 2, 4, 6, 8, 10] - .into_iter() - .enumerate() - .map(|(idx, first_row_index)| PageLocation { - offset: (idx * 10) as i64, - compressed_page_size: 10, - first_row_index, - }) - .collect(), - unencoded_byte_array_data_bytes: None, - }, - 0, - 0, - ); + page_index + .try_put_offset_index( + OffsetIndexMetaData { + page_locations: [0, 2, 4, 6, 8, 10] + .into_iter() + .enumerate() + .map(|(idx, first_row_index)| PageLocation { + offset: (idx * 10) as i64, + compressed_page_size: 10, + first_row_index, + }) + .collect(), + unencoded_byte_array_data_bytes: None, + }, + 0, + 0, + ) + .unwrap(); let page_index: Option> = Some(Arc::new(page_index.build())); let page_index = RowGroupPageIndex::new(0, page_index); let selection = RowSelection::from(vec![ diff --git a/parquet/src/arrow/push_decoder/scan_plan/tests.rs b/parquet/src/arrow/push_decoder/scan_plan/tests.rs index 5a81b02e68ef..301c8ad7d00e 100644 --- a/parquet/src/arrow/push_decoder/scan_plan/tests.rs +++ b/parquet/src/arrow/push_decoder/scan_plan/tests.rs @@ -730,7 +730,9 @@ fn plan_with_empty_offset_index() { if (row_group, column) == (0, 0) { offset_index.page_locations.clear(); } - page_index_builder.put_offset_index(offset_index, row_group, column); + page_index_builder + .try_put_offset_index(offset_index, row_group, column) + .unwrap(); } } let parquet_metadata = parquet_metadata diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 2cc81f325eca..824c86ea23ec 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -2189,7 +2189,7 @@ mod tests { }; } - page_index.put_column_index(column_index, 0, 0); + page_index.try_put_column_index(column_index, 0, 0).unwrap(); // Now, add in OffsetIndex let mut offset_index = OffsetIndexBuilder::new(); @@ -2200,7 +2200,7 @@ mod tests { offset_index.append_offset_and_size(2, 3); offset_index.append_unencoded_byte_array_data_bytes(Some(10)); let offset_index = offset_index.build(); - page_index.put_offset_index(offset_index, 0, 0); + page_index.try_put_offset_index(offset_index, 0, 0).unwrap(); let parquet_meta = ParquetMetaDataBuilder::new(file_metadata) .set_row_groups(row_group_meta) @@ -2323,7 +2323,7 @@ mod tests { offset_idx_builder.append_row_count(50); offset_idx_builder.append_offset_and_size(1000, 500); let offset_idx = offset_idx_builder.build(); - builder.put_offset_index(offset_idx, 0, 0); + builder.try_put_offset_index(offset_idx, 0, 0).unwrap(); let page_index = builder.build(); @@ -2372,14 +2372,16 @@ mod tests { let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32); col_idx_builder.append(false, vec![1, 0, 0, 0], vec![100, 0, 0, 0], 10, None); let col_idx = col_idx_builder.build().unwrap(); - builder.put_column_index(col_idx.clone(), 0, 0); + builder.try_put_column_index(col_idx.clone(), 0, 0).unwrap(); // Add an offset index for first column let mut offset_idx_builder = OffsetIndexBuilder::new(); offset_idx_builder.append_row_count(50); offset_idx_builder.append_offset_and_size(1000, 500); let offset_idx = offset_idx_builder.build(); - builder.put_offset_index(offset_idx.clone(), 0, 0); + builder + .try_put_offset_index(offset_idx.clone(), 0, 0) + .unwrap(); let page_index = builder.build(); @@ -2396,8 +2398,8 @@ mod tests { let mut builder = PageIndexBuilder::new_from(page_index); // Add indexes for second column - builder.put_column_index(col_idx, 0, 1); - builder.put_offset_index(offset_idx, 0, 1); + builder.try_put_column_index(col_idx, 0, 1).unwrap(); + builder.try_put_offset_index(offset_idx, 0, 1).unwrap(); let page_index = builder.build(); // now all populated diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 130446fc247a..ff5c1a0e7f31 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -578,7 +578,8 @@ impl PageIndexBuilder { /// /// The dimensions are used to validate inserted coordinates. Storage is allocated only for /// entries populated using - /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). + /// [`try_put_column_index`](Self::try_put_column_index) and + /// [`try_put_offset_index`](Self::try_put_offset_index). /// /// # Panics /// @@ -629,7 +630,7 @@ impl PageIndexBuilder { /// /// This replaces any existing column indexes with an empty sparse map having the specified /// dimensions. It can then be populated using - /// [`put_column_index`](Self::put_column_index). + /// [`try_put_column_index`](Self::try_put_column_index). /// /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). @@ -667,7 +668,7 @@ impl PageIndexBuilder { /// /// This replaces any existing offset indexes with an empty sparse map having the specified /// dimensions. It can then be populated using - /// [`put_offset_index`](Self::put_offset_index). + /// [`try_put_offset_index`](Self::try_put_offset_index). /// /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). @@ -705,6 +706,10 @@ impl PageIndexBuilder { /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), /// or the row group or column index is out of bounds, this method does nothing. + #[deprecated( + since = "60.2.0", + note = "Use `try_put_column_index` to detect invalid coordinates or unavailable index storage" + )] pub fn put_column_index( &mut self, column_index: ColumnIndexMetaData, @@ -748,6 +753,10 @@ impl PageIndexBuilder { /// /// If offset indexes were not allocated (see [`Self::allocate_offset_indexes`]), /// or the row group or column index is out of bounds, this method does nothing. + #[deprecated( + since = "60.2.0", + note = "Use `try_put_offset_index` to detect invalid coordinates or unavailable index storage" + )] pub fn put_offset_index( &mut self, offset_index: OffsetIndexMetaData, @@ -986,18 +995,30 @@ mod tests { .is_err() ); - // Legacy insertion remains a no-op for unallocated storage and invalid coordinates. - let mut legacy_unallocated = PageIndexBuilder::default(); - legacy_unallocated.put_column_index(col_index.clone(), 0, 0); - legacy_unallocated.put_offset_index(off_index.clone(), 0, 0); - let empty_page_index = legacy_unallocated.build(); - assert!(!empty_page_index.has_column_indexes()); - assert!(!empty_page_index.has_offset_indexes()); - - builder.put_column_index(col_index.clone(), 2, 0); - builder.put_column_index(col_index.clone(), 0, 3); - builder.put_offset_index(off_index.clone(), 2, 0); - builder.put_offset_index(off_index.clone(), 0, 3); + // Deprecated insertion still populates valid coordinates, and remains a no-op + // for unallocated storage and invalid coordinates. + #[expect(deprecated, reason = "verify the legacy insertion contract")] + { + let mut legacy_unallocated = PageIndexBuilder::default(); + legacy_unallocated.put_column_index(col_index.clone(), 0, 0); + legacy_unallocated.put_offset_index(off_index.clone(), 0, 0); + let empty_page_index = legacy_unallocated.build(); + assert!(!empty_page_index.has_column_indexes()); + assert!(!empty_page_index.has_offset_indexes()); + + let mut legacy_builder = PageIndexBuilder::new(2, 3); + legacy_builder.put_column_index(col_index.clone(), 0, 0); + legacy_builder.put_offset_index(off_index.clone(), 0, 0); + legacy_builder.put_column_index(col_index.clone(), 2, 0); + legacy_builder.put_column_index(col_index.clone(), 0, 3); + legacy_builder.put_offset_index(off_index.clone(), 2, 0); + legacy_builder.put_offset_index(off_index.clone(), 0, 3); + let legacy_page_index = legacy_builder.build(); + assert_eq!(legacy_page_index.column_index(0, 0), Some(&col_index)); + assert_eq!(legacy_page_index.offset_index(0, 0), Some(&off_index)); + assert_eq!(legacy_page_index.column_index(0, 1), None); + assert_eq!(legacy_page_index.offset_index(1, 0), None); + } let mut unallocated = PageIndexBuilder::default(); assert!(unallocated.try_put_column_index(col_index, 0, 0).is_err()); From 177a57ec2b1483fb6ed354e528f3d7fcfd382474 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 12 Nov 2026 09:01:04 -0500 Subject: [PATCH 4/9] docs: clean up documentatin --- parquet/examples/page_index_cache.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/parquet/examples/page_index_cache.rs b/parquet/examples/page_index_cache.rs index 34b7c340844a..79b8e020a282 100644 --- a/parquet/examples/page_index_cache.rs +++ b/parquet/examples/page_index_cache.rs @@ -20,9 +20,9 @@ //! This example keeps footer metadata and decoded page indexes in separate caches. For each //! query it: //! -//! 1. Finds the requested column chunks that are absent from the page-index cache -//! 2. Uses [`ParquetMetaDataReader::read_page_index_and_finish`] to request and decode those misses -//! 3. Moves the decoded entries into the cache and wraps them in [`Arc`] +//! 1. Finds the requested column chunks that don't yet have entries in the page-index cache +//! 2. Uses [`ParquetMetaDataReader::read_page_index_and_finish`] to fetch the missing entries +//! 3. Moves the decoded entries into the cache //! 4. Builds a query-specific [`PageIndexProvider`] from shared cache entries //! //! A production cache shared by multiple files would also include a stable file identity and @@ -47,9 +47,11 @@ use parquet::file::page_index::column_index::ColumnIndexMetaData; use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; +/// Identifier for a column chunk in a Parquet file: (row group index, column index) type ColumnChunk = (usize, usize); -/// A query-specific provider backed by handles to cached entries. +/// A query-specific [`PageIndexProvider`] backed by handles to shared entries +/// in a [`PageIndexCache`]. #[derive(Debug, Default)] struct CachedPageIndexProvider { column_indexes: HashMap>, @@ -98,8 +100,8 @@ struct PageIndexCache { } impl PageIndexCache { - /// Returns the footer metadata and a provider containing exactly the entries requested for - /// one query, loading any cache misses through `ParquetMetaDataReader` first. + /// Returns the footer metadata and a provider for the entries requested for + /// one query, loading any missing entries using [`ParquetMetaDataReader`]. fn page_index_for_query( &mut self, mut metadata: ParquetMetaData, @@ -190,7 +192,7 @@ fn covering_mask(chunks: &[ColumnChunk]) -> ColumnChunkMask { ) } -/// Reads and caches footer metadata without decoding page indexes. +/// Reads footer metadata into [`ParquetMetaData`] without decoding page indexes. fn decode_footer(file_bytes: &Bytes) -> Result { ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Skip) From 7268402561a66466eb68fa38a5f07c207fe239c0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 12 Nov 2026 09:08:53 -0500 Subject: [PATCH 5/9] Add some comments --- parquet/src/file/metadata/mod.rs | 4 +++- parquet/src/file/metadata/page_index.rs | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index bbd37e40d764..3f1241526f3e 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -107,8 +107,10 @@ pub(crate) use writer::ThriftMetadataWriter; /// The individual fields of this structure are stored in a way that make cloning /// this structure low-cost. /// -/// See the [`ParquetMetaDataBuilder`] to create and modify this structure. +/// See [`PageIndexProvider`] for details on customizing page index handling. /// +/// See the [`ParquetMetaDataBuilder`] to create and modify this structure. +/// /// [`parquet.thrift`]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift #[derive(Debug, Clone)] pub struct ParquetMetaData { diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 1937788d1b14..125dff62d4bd 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -16,6 +16,7 @@ use crate::file::page_index::{ offset_index::{OffsetIndexMetaData, PageLocation}, }; use std::sync::Arc; +use crate::file::metadata::ColumnChunkMask; /// Trait for accessing Parquet [Page Index] data for efficient page-level skipping /// @@ -125,10 +126,17 @@ use std::sync::Arc; /// } /// ``` /// +/// # Caching the Page Index +/// +/// The Page Index can be much larger than the rest of the metadata, so some +/// systems may want to incrementally fetch or assemble the relevant data. See +/// [`page_index_cache.rs`] for an example of an incremental page index cache. +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ /// [`ColumnIndex`]: crate::file::page_index::column_index::ColumnIndexMetaData /// [`OffsetIndex`]: crate::file::page_index::offset_index::OffsetIndexMetaData /// [`ColumnChunkMetaData`]: crate::file::metadata::ColumnChunkMetaData +/// [`page_index_cache.rs`]: https://github.com/apache/arrow-rs/blob/main/parquet/examples/page_index_cache.rs pub trait PageIndexProvider: Send + Sync + std::fmt::Debug { /// Returns `true` if offset index structures are available via this provider /// @@ -484,6 +492,11 @@ impl PageIndex { /// Each entry contains its `(row_group_index, column_index)` coordinate and owned index /// metadata. This can be used to move parsed indexes into a cache without cloning their /// metadata. Entries are returned in row-major order. + /// + /// For an example of using this to populate an incremental page index + /// cache, see [`page_index_cache.rs`] + /// + /// [`page_index_cache.rs`]: https://github.com/apache/arrow-rs/blob/main/parquet/examples/page_index_cache.rs #[expect(clippy::type_complexity)] pub fn into_index_entries( self, From 186b86dc5f98df1ec9de26d32b996977af331388 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 12 Nov 2026 09:15:50 -0500 Subject: [PATCH 6/9] Add a few more breadcrumbs --- parquet/src/arrow/arrow_reader/mod.rs | 6 ++++++ parquet/src/file/metadata/reader.rs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 1fc6f3be2efa..37d88eb47761 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -780,6 +780,9 @@ impl ArrowReaderOptions { /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. + /// + /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -797,6 +800,9 @@ impl ArrowReaderOptions { /// include those columns in addition to projected columns. The same loading and policy /// qualifications as [`Self::with_column_index_mask`] apply. /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. + /// + /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index 247e8533f5b2..da65f7ab0a95 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -375,6 +375,8 @@ impl ParquetMetaDataReader { /// /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. /// [`PageIndexPolicy::Required`] validates only selected column chunks. + /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self @@ -384,6 +386,8 @@ impl ParquetMetaDataReader { /// /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. /// [`PageIndexPolicy::Required`] validates only selected column chunks. + /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self From 586a0ea80c1706c726537222b4f61003a1ef56d8 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 12 Nov 2026 06:41:10 -0800 Subject: [PATCH 7/9] fix CI --- parquet/examples/page_index_cache.rs | 19 +++---------------- parquet/src/file/metadata/mod.rs | 2 +- parquet/src/file/metadata/page_index.rs | 1 - 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/parquet/examples/page_index_cache.rs b/parquet/examples/page_index_cache.rs index 79b8e020a282..bbf3fc870a77 100644 --- a/parquet/examples/page_index_cache.rs +++ b/parquet/examples/page_index_cache.rs @@ -1,19 +1,6 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! Incrementally caching page indexes with [`ParquetMetaDataReader`]. //! diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 3f1241526f3e..80cb5e4bda1a 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -110,7 +110,7 @@ pub(crate) use writer::ThriftMetadataWriter; /// See [`PageIndexProvider`] for details on customizing page index handling. /// /// See the [`ParquetMetaDataBuilder`] to create and modify this structure. -/// +/// /// [`parquet.thrift`]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift #[derive(Debug, Clone)] pub struct ParquetMetaData { diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index 125dff62d4bd..7178d6408f5a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -16,7 +16,6 @@ use crate::file::page_index::{ offset_index::{OffsetIndexMetaData, PageLocation}, }; use std::sync::Arc; -use crate::file::metadata::ColumnChunkMask; /// Trait for accessing Parquet [Page Index] data for efficient page-level skipping /// From c310eadb4dc0e04339ce391f274b6a319198e610 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 12 Nov 2026 06:55:55 -0800 Subject: [PATCH 8/9] fix another license --- parquet/src/file/metadata/column_chunk.rs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/parquet/src/file/metadata/column_chunk.rs b/parquet/src/file/metadata/column_chunk.rs index 09852135e55d..1eb48fc3df3c 100644 --- a/parquet/src/file/metadata/column_chunk.rs +++ b/parquet/src/file/metadata/column_chunk.rs @@ -1,19 +1,6 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! Sparse storage keyed by row-group and column coordinates. From ff95c1e0630e36fe765ddef712d7dd9d8318cb18 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 12 Nov 2026 07:46:01 -0800 Subject: [PATCH 9/9] Add better test documentation and explain map vs entries rationale --- parquet/src/file/metadata/column_chunk.rs | 6 +++ parquet/tests/arrow_reader/page_index.rs | 57 ++++++++++++++++++----- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/parquet/src/file/metadata/column_chunk.rs b/parquet/src/file/metadata/column_chunk.rs index 1eb48fc3df3c..f89a674036ca 100644 --- a/parquet/src/file/metadata/column_chunk.rs +++ b/parquet/src/file/metadata/column_chunk.rs @@ -3,6 +3,10 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! Sparse storage keyed by row-group and column coordinates. +//! +//! Builders use [`ColumnChunkMap`] for arbitrary insertion and replacement. Finished indexes +//! use [`ColumnChunkEntries`], a sorted boxed slice containing only populated entries. This reduces +//! retained memory at the cost of binary-search vs hash lookups. use crate::errors::{ParquetError, Result}; use crate::file::metadata::memory::HeapSize; @@ -52,6 +56,7 @@ fn validate_dimensions(num_row_groups: usize, num_columns: usize) -> Result<()> Ok(()) } +// Chunk storage backed by a HashMap to allow builders to insert to arbitrary positions. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ColumnChunkMap { num_row_groups: usize, @@ -136,6 +141,7 @@ impl HeapSize for ColumnChunkMap { } } +// Chunk storage backed by a boxed slice. Finished (immutable) indexes use this to save memory. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ColumnChunkEntries { num_row_groups: usize, diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index 692a89161f1e..480fc58aeca5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -4,7 +4,9 @@ //! Tests of selective page index population -use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, +}; use crate::custom_page_index_provider::create_test_file; @@ -74,13 +76,39 @@ fn test_parse_with_page_index_mask() { ); } +fn index_layout( + metadata: &ParquetMetaData, + has_index: impl Fn(usize, usize) -> bool, +) -> Vec { + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + (0..metadata.num_row_groups()) + .map(|rg| { + (0..num_cols) + .map(|col| if has_index(rg, col) { 'X' } else { '.' }) + .collect() + }) + .collect() +} + #[test] fn test_sparse_page_index_missing_ranges_preserve_coordinates() { + // Omitting missing ranges from compact storage must not shift subsequent + // indexes onto earlier row-group/column coordinates. + // Each string is a row group; each character is a column. X means an index + // range is present, and . means it is missing from the footer metadata. + let column_index_ranges = [".XXX", "XXXX", "XXX."]; + let offset_index_ranges = ["X.XX", "XX.X", "XXXX"]; + let file = create_test_file(); + // Keep the fully loaded indexes as a reference for the values that survive + // selection, not just whether an index exists at each coordinate. let original = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Required) .parse_and_finish(&file) .unwrap(); + // Load only the footer, then remove selected range descriptors to simulate + // missing indexes without rewriting the file. The remaining descriptors + // still point to the original bytes; no previously decoded indexes are reused. let metadata = ParquetMetaDataReader::new() .with_page_index_policy(PageIndexPolicy::Skip) .parse_and_finish(&file) @@ -97,12 +125,12 @@ fn test_sparse_page_index_missing_ranges_preserve_coordinates() { .enumerate() .map(|(col_idx, column)| { let mut column = column.clone().into_builder(); - if matches!((rg_idx, col_idx), (0, 0) | (2, 3)) { + if column_index_ranges[rg_idx].as_bytes()[col_idx] == b'.' { column = column .set_column_index_offset(None) .set_column_index_length(None); } - if matches!((rg_idx, col_idx), (0, 1) | (1, 2)) { + if offset_index_ranges[rg_idx].as_bytes()[col_idx] == b'.' { column = column .set_offset_index_offset(None) .set_offset_index_length(None); @@ -118,8 +146,8 @@ fn test_sparse_page_index_missing_ranges_preserve_coordinates() { .collect(); let metadata = builder.set_row_groups(row_groups).build(); - // Masks canonicalize unordered, duplicate, and out-of-bounds coordinates. - // Missing ranges must leave holes in coordinate space, not vector positions. + // Select column indexes in row groups 0 and 2, columns 0, 1, and 3; offset + // indexes remain unmasked. Mask axes canonicalize duplicates and ignore 99. let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) .with_page_index_policy(PageIndexPolicy::Optional) .with_column_index_mask(ColumnChunkMask::row_groups_and_columns( @@ -128,14 +156,19 @@ fn test_sparse_page_index_missing_ranges_preserve_coordinates() { )); reader.read_page_indexes(&file).unwrap(); let read = reader.finish().unwrap(); - assert_page_index_cells( - &read, - |rg, col| { - matches!(rg, 0 | 2) && matches!(col, 0 | 1 | 3) && !matches!((rg, col), (0, 0) | (2, 3)) - }, - |rg, col| !matches!((rg, col), (0, 1) | (1, 2)), - ); let index = read.page_index().unwrap(); + // Selected, available indexes must retain their coordinates despite holes + // from both the mask and missing ranges. + assert_eq!( + index_layout(&read, |rg, col| index.column_index(rg, col).is_some()), + [".X.X", "....", "XX.."] + ); + assert_eq!( + index_layout(&read, |rg, col| index.offset_index(rg, col).is_some()), + ["X.XX", "XX.X", "XXXX"] + ); + // Correct presence alone could hide values assigned to the wrong chunk. + // Compare each surviving index with the same coordinate in the original. let expected = original.page_index().unwrap(); for rg in 0..3 { for col in 0..4 {