diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index d8cbcc886a89..1cbecbb5e477 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -166,6 +166,11 @@ name = "custom_page_index" required-features = ["arrow"] path = "./examples/custom_page_index.rs" +[[example]] +name = "page_index_cache" +required-features = ["arrow"] +path = "./examples/page_index_cache.rs" + [[test]] name = "arrow_reader" required-features = ["arrow"] diff --git a/parquet/benches/arrow_statistics.rs b/parquet/benches/arrow_statistics.rs index a4a960ef439c..b2e1f72ddeb6 100644 --- a/parquet/benches/arrow_statistics.rs +++ b/parquet/benches/arrow_statistics.rs @@ -346,7 +346,7 @@ fn page_index_benchmark(c: &mut Criterion) { let mut builder = PageIndexBuilder::new(row_groups, num_columns); for (rg, (_, bytes)) in column_indexes.iter().enumerate() { let index = decode_column_index(bytes, physical_type).unwrap(); - builder.put_column_index(index, rg, column); + builder.try_put_column_index(index, rg, column).unwrap(); } let page_index = builder.build(); let _ = converter diff --git a/parquet/examples/page_index_cache.rs b/parquet/examples/page_index_cache.rs new file mode 100644 index 000000000000..bbf3fc870a77 --- /dev/null +++ b/parquet/examples/page_index_cache.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Incrementally caching page indexes with [`ParquetMetaDataReader`]. +//! +//! This example keeps footer metadata and decoded page indexes in separate caches. For each +//! query it: +//! +//! 1. Finds the requested column chunks that don't yet have entries in the page-index cache +//! 2. Uses [`ParquetMetaDataReader::read_page_index_and_finish`] to fetch the missing entries +//! 3. Moves the decoded entries into the cache +//! 4. Builds a query-specific [`PageIndexProvider`] from shared cache entries +//! +//! A production cache shared by multiple files would also include a stable file identity and +//! freshness information in its keys. This example also caches only indexes that are present; +//! it does not record negative results. Consequently, it may request a missing index again on a +//! later query. A production cache should track coordinates known not to have an index. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +use arrow::array::{Int32Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::errors::Result; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{ + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, +}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; + +/// Identifier for a column chunk in a Parquet file: (row group index, column index) +type ColumnChunk = (usize, usize); + +/// A query-specific [`PageIndexProvider`] backed by handles to shared entries +/// in a [`PageIndexCache`]. +#[derive(Debug, Default)] +struct CachedPageIndexProvider { + column_indexes: HashMap>, + offset_indexes: HashMap>, +} + +impl PageIndexProvider for CachedPageIndexProvider { + fn has_offset_indexes(&self) -> bool { + !self.offset_indexes.is_empty() + } + + fn has_column_indexes(&self) -> bool { + !self.column_indexes.is_empty() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .get(&(row_group_idx, column_idx)) + .map(Arc::as_ref) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .get(&(row_group_idx, column_idx)) + .map(Arc::as_ref) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Decoded page indexes cached independently of query-specific metadata. +#[derive(Default)] +struct PageIndexCache { + column_indexes: HashMap>, + offset_indexes: HashMap>, +} + +impl PageIndexCache { + /// Returns the footer metadata and a provider for the entries requested for + /// one query, loading any missing entries using [`ParquetMetaDataReader`]. + fn page_index_for_query( + &mut self, + mut metadata: ParquetMetaData, + file_bytes: &Bytes, + column_index_mask: &ColumnChunkMask, + offset_index_mask: &ColumnChunkMask, + ) -> Result<(ParquetMetaData, CachedPageIndexProvider)> { + let num_row_groups = metadata.num_row_groups(); + let num_columns = metadata.file_metadata().schema_descr().num_columns(); + + let missing_column_indexes = column_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter(|key| !self.column_indexes.contains_key(key)) + .collect::>(); + let missing_offset_indexes = offset_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter(|key| !self.offset_indexes.contains_key(key)) + .collect::>(); + + println!( + " cache misses: {} column indexes, {} offset indexes", + missing_column_indexes.len(), + missing_offset_indexes.len() + ); + + if !missing_column_indexes.is_empty() || !missing_offset_indexes.is_empty() { + // ColumnChunkMask describes a Cartesian product. This produces the smallest such mask + // covering the misses and may therefore decode additional entries for irregular miss + // sets. Those additional entries are useful cache population and are retained below. + let column_miss_mask = covering_mask(&missing_column_indexes); + let offset_miss_mask = covering_mask(&missing_offset_indexes); + let reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_column_index_policy(PageIndexPolicy::Optional) + .with_offset_index_policy(PageIndexPolicy::Optional) + .with_column_index_mask(column_miss_mask) + .with_offset_index_mask(offset_miss_mask); + + let (returned_metadata, page_index) = reader.read_page_index_and_finish(file_bytes)?; + metadata = returned_metadata; + if let Some(page_index) = page_index { + let (column_indexes, offset_indexes) = page_index.into_index_entries(); + self.column_indexes + .extend(column_indexes.map(|(key, index)| (key, Arc::new(index)))); + self.offset_indexes + .extend(offset_indexes.map(|(key, index)| (key, Arc::new(index)))); + } + } + + // Build an immutable provider containing only handles needed by this query. + let column_indexes = column_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter_map(|key| { + self.column_indexes + .get(&key) + .map(|index| (key, Arc::clone(index))) + }) + .collect(); + let offset_indexes = offset_index_mask + .column_chunk_indices(num_row_groups, num_columns) + .filter_map(|key| { + self.offset_indexes + .get(&key) + .map(|index| (key, Arc::clone(index))) + }) + .collect(); + + Ok(( + metadata, + CachedPageIndexProvider { + column_indexes, + offset_indexes, + }, + )) + } +} + +/// Returns the smallest Cartesian mask covering `chunks`. +fn covering_mask(chunks: &[ColumnChunk]) -> ColumnChunkMask { + if chunks.is_empty() { + return ColumnChunkMask::none(); + } + + let row_groups = chunks.iter().map(|&(row_group, _)| row_group); + let columns = chunks.iter().map(|&(_, column)| column); + ColumnChunkMask::row_groups_and_columns( + row_groups.collect::>(), + columns.collect::>(), + ) +} + +/// Reads footer metadata into [`ParquetMetaData`] without decoding page indexes. +fn decode_footer(file_bytes: &Bytes) -> Result { + ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(file_bytes) +} + +fn main() -> Result<()> { + let file_bytes = create_sample_file()?; + let cached_metadata = decode_footer(&file_bytes)?; + let mut cache = PageIndexCache::default(); + + // Query 1 filters column 0 and projects columns 0, 1, and 4 in row group 0. + let column_mask = ColumnChunkMask::row_groups_and_columns([0], [0]); + let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 1, 4]); + println!("Query 1:"); + let (cached_metadata, provider) = + cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?; + assert_query_indexes(&provider, 0, &[0], &[0, 1, 4]); + + // Attach the query-specific provider to a cheap clone of the cached footer metadata. + let query_metadata = cached_metadata + .clone() + .into_builder() + .set_page_index(Some(Arc::new(provider))) + .build(); + assert!(query_metadata.page_index().is_some()); + + // Query 2 reuses the column index and offset indexes for columns 0 and 4. Only the offset + // index for column 2 is decoded and added to the cache. + let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 2, 4]); + println!("Query 2:"); + let (_cached_metadata, provider) = + cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?; + assert_query_indexes(&provider, 0, &[0], &[0, 2, 4]); + + println!( + "Cache now contains {} column indexes and {} offset indexes", + cache.column_indexes.len(), + cache.offset_indexes.len() + ); + Ok(()) +} + +fn assert_query_indexes( + page_index: &dyn PageIndexProvider, + row_group_idx: usize, + column_indexes: &[usize], + offset_indexes: &[usize], +) { + for column_idx in 0..6 { + assert_eq!( + page_index.column_index(row_group_idx, column_idx).is_some(), + column_indexes.contains(&column_idx) + ); + assert_eq!( + page_index.offset_index(row_group_idx, column_idx).is_some(), + offset_indexes.contains(&column_idx) + ); + } +} + +fn create_sample_file() -> Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + Field::new("category", DataType::Utf8, false), + Field::new("amount", DataType::Int32, false), + ])); + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_size_limit(100) + .set_write_batch_size(10) + .build(); + let mut buffer = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut buffer, Arc::clone(&schema), Some(props))?; + + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = row_group * 50 + batch_num * 10; + let values = (offset..offset + 10).collect::>(); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(values.clone())), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 2).collect::>(), + )), + Arc::new(StringArray::from( + values + .iter() + .map(|value| format!("name{value}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 3).collect::>(), + )), + Arc::new(StringArray::from( + values + .iter() + .map(|value| if value % 2 == 0 { "even" } else { "odd" }) + .collect::>(), + )), + Arc::new(Int32Array::from( + values.iter().map(|value| value * 4).collect::>(), + )), + ], + )?; + writer.write(&batch)?; + } + writer.flush()?; + } + writer.close()?; + Ok(Bytes::from(buffer)) +} diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index 1fc6f3be2efa..37d88eb47761 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -780,6 +780,9 @@ impl ArrowReaderOptions { /// honored by loading APIs such as [`ArrowReaderMetadata::load`]; /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes. /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. + /// + /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; @@ -797,6 +800,9 @@ impl ArrowReaderOptions { /// include those columns in addition to projected columns. The same loading and policy /// qualifications as [`Self::with_column_index_mask`] apply. /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. + /// + /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs index f9a03eaa733c..d02f00d76a13 100644 --- a/parquet/src/arrow/async_reader/mod.rs +++ b/parquet/src/arrow/async_reader/mod.rs @@ -1175,7 +1175,7 @@ mod tests { .await .unwrap(); - let page_index = PageIndex::new(None, Some(vec![])); + let page_index = PageIndex::try_new(None, Some(vec![])).unwrap(); metadata.set_page_index(Some(Arc::new(page_index))); let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap(); diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index 5fcf7ecf74b0..92169632da73 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -3128,7 +3128,9 @@ mod test { let last = offset_index.page_locations.last_mut().unwrap(); last.compressed_page_size += 1; } - page_index.put_offset_index(offset_index, row_group, column); + page_index + .try_put_offset_index(offset_index, row_group, column) + .unwrap(); } } let metadata = metadata diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 3c5e7186424c..24036b7c6d87 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -992,8 +992,12 @@ mod tests { .collect(), unencoded_byte_array_data_bytes: None, }; - page_index.put_offset_index(column(&[0, 4, 8]), 0, 0); - page_index.put_offset_index(column(&[0, 6, 10]), 0, 1); + page_index + .try_put_offset_index(column(&[0, 4, 8]), 0, 0) + .unwrap(); + page_index + .try_put_offset_index(column(&[0, 6, 10]), 0, 1) + .unwrap(); let page_index: Option> = Some(Arc::new(page_index.build())); let page_index = RowGroupPageIndex::new(0, page_index); let selection = RowSelection::from(vec![ @@ -1061,22 +1065,24 @@ mod tests { #[test] fn test_auto_keeps_mask_when_page_pruning_skips_pages() { let mut page_index = PageIndexBuilder::new(1, 1); - page_index.put_offset_index( - OffsetIndexMetaData { - page_locations: [0, 2, 4, 6, 8, 10] - .into_iter() - .enumerate() - .map(|(idx, first_row_index)| PageLocation { - offset: (idx * 10) as i64, - compressed_page_size: 10, - first_row_index, - }) - .collect(), - unencoded_byte_array_data_bytes: None, - }, - 0, - 0, - ); + page_index + .try_put_offset_index( + OffsetIndexMetaData { + page_locations: [0, 2, 4, 6, 8, 10] + .into_iter() + .enumerate() + .map(|(idx, first_row_index)| PageLocation { + offset: (idx * 10) as i64, + compressed_page_size: 10, + first_row_index, + }) + .collect(), + unencoded_byte_array_data_bytes: None, + }, + 0, + 0, + ) + .unwrap(); let page_index: Option> = Some(Arc::new(page_index.build())); let page_index = RowGroupPageIndex::new(0, page_index); let selection = RowSelection::from(vec![ diff --git a/parquet/src/arrow/push_decoder/scan_plan/tests.rs b/parquet/src/arrow/push_decoder/scan_plan/tests.rs index c4f64ce76ce1..a7406297b2fe 100644 --- a/parquet/src/arrow/push_decoder/scan_plan/tests.rs +++ b/parquet/src/arrow/push_decoder/scan_plan/tests.rs @@ -717,7 +717,9 @@ fn plan_with_empty_offset_index() { if (row_group, column) == (0, 0) { offset_index.page_locations.clear(); } - page_index_builder.put_offset_index(offset_index, row_group, column); + page_index_builder + .try_put_offset_index(offset_index, row_group, column) + .unwrap(); } } let parquet_metadata = parquet_metadata diff --git a/parquet/src/file/metadata/column_chunk.rs b/parquet/src/file/metadata/column_chunk.rs new file mode 100644 index 000000000000..f89a674036ca --- /dev/null +++ b/parquet/src/file/metadata/column_chunk.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Sparse storage keyed by row-group and column coordinates. +//! +//! Builders use [`ColumnChunkMap`] for arbitrary insertion and replacement. Finished indexes +//! use [`ColumnChunkEntries`], a sorted boxed slice containing only populated entries. This reduces +//! retained memory at the cost of binary-search vs hash lookups. + +use crate::errors::{ParquetError, Result}; +use crate::file::metadata::memory::HeapSize; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct ColumnChunkKey(u64); + +impl ColumnChunkKey { + pub(crate) fn try_new(row_group_idx: usize, column_idx: usize) -> Result { + let row_group_idx = u32::try_from(row_group_idx).map_err(|_| { + ParquetError::General(format!( + "row group index exceeds packed coordinate limit: {row_group_idx}" + )) + })?; + let column_idx = u32::try_from(column_idx).map_err(|_| { + ParquetError::General(format!( + "column index exceeds packed coordinate limit: {column_idx}" + )) + })?; + Ok(Self( + (u64::from(row_group_idx) << u32::BITS) | u64::from(column_idx), + )) + } + + pub(crate) fn coordinates(self) -> (usize, usize) { + ((self.0 >> u32::BITS) as usize, (self.0 as u32) as usize) + } +} + +impl HeapSize for ColumnChunkKey { + fn heap_size(&self) -> usize { + 0 + } +} + +fn validate_dimensions(num_row_groups: usize, num_columns: usize) -> Result<()> { + // Parquet metadata collection lengths are represented by signed 32-bit Thrift values. + i32::try_from(num_row_groups).map_err(|_| { + ParquetError::General(format!( + "row group count exceeds i32::MAX: {num_row_groups}" + )) + })?; + i32::try_from(num_columns).map_err(|_| { + ParquetError::General(format!("column count exceeds i32::MAX: {num_columns}")) + })?; + Ok(()) +} + +// Chunk storage backed by a HashMap to allow builders to insert to arbitrary positions. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnChunkMap { + num_row_groups: usize, + num_columns: usize, + entries: HashMap, +} + +impl ColumnChunkMap { + pub(crate) fn try_new(num_row_groups: usize, num_columns: usize) -> Result { + validate_dimensions(num_row_groups, num_columns)?; + Ok(Self { + num_row_groups, + num_columns, + entries: HashMap::new(), + }) + } + + pub(crate) fn num_row_groups(&self) -> usize { + self.num_row_groups + } + + pub(crate) fn num_columns(&self) -> usize { + self.num_columns + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn key(&self, row_group_idx: usize, column_idx: usize) -> Result { + if row_group_idx >= self.num_row_groups || column_idx >= self.num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {} by {}", + self.num_row_groups, self.num_columns + ))); + } + ColumnChunkKey::try_new(row_group_idx, column_idx) + } + + /// Inserts or replaces an entry. + pub(crate) fn try_insert( + &mut self, + row_group_idx: usize, + column_idx: usize, + value: T, + ) -> Result> { + let key = self.key(row_group_idx, column_idx)?; + if let Some(existing) = self.entries.get_mut(&key) { + return Ok(Some(std::mem::replace(existing, value))); + } + + self.entries.try_reserve(1).map_err(|err| { + ParquetError::General(format!( + "failed to reserve storage for column chunk at row group {row_group_idx}, \ + column {column_idx}: {err}" + )) + })?; + Ok(self.entries.insert(key, value)) + } + + pub(crate) fn try_freeze(self) -> Result> { + validate_dimensions(self.num_row_groups, self.num_columns)?; + let num_entries = self.entries.len(); + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + for entry in self.entries { + entries.push(entry); + } + entries.sort_unstable_by_key(|(key, _)| *key); + ColumnChunkEntries::try_from_sorted_entries(self.num_row_groups, self.num_columns, entries) + } +} + +impl HeapSize for ColumnChunkMap { + fn heap_size(&self) -> usize { + self.entries.heap_size() + } +} + +// Chunk storage backed by a boxed slice. Finished (immutable) indexes use this to save memory. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnChunkEntries { + num_row_groups: usize, + num_columns: usize, + entries: Box<[(ColumnChunkKey, T)]>, +} + +impl ColumnChunkEntries { + /// Constructs immutable storage from entries in strictly increasing row-major order. + /// Missing coordinates need no placeholder; dimensions only bound the stored keys. + pub(crate) fn try_from_sorted_entries( + num_row_groups: usize, + num_columns: usize, + entries: Vec<(ColumnChunkKey, T)>, + ) -> Result { + validate_dimensions(num_row_groups, num_columns)?; + let mut previous = None; + for (key, _) in &entries { + let (row_group_idx, column_idx) = key.coordinates(); + if row_group_idx >= num_row_groups || column_idx >= num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {num_row_groups} by {num_columns}" + ))); + } + if previous.is_some_and(|previous| previous >= *key) { + return Err(ParquetError::General( + "column chunk entries must be unique and in row-major order".to_string(), + )); + } + previous = Some(*key); + } + Ok(Self { + num_row_groups, + num_columns, + entries: entries.into_boxed_slice(), + }) + } + + pub(crate) fn try_from_rows(rows: Vec>>) -> Result { + let num_row_groups = rows.len(); + let num_columns = rows.iter().map(Vec::len).max().unwrap_or_default(); + validate_dimensions(num_row_groups, num_columns)?; + let num_entries = rows + .iter() + .map(|columns| columns.iter().filter(|value| value.is_some()).count()) + .try_fold(0_usize, |total, count| total.checked_add(count)) + .ok_or_else(|| { + ParquetError::General("column chunk entry count exceeds usize::MAX".to_string()) + })?; + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + for (row_group_idx, columns) in rows.into_iter().enumerate() { + for (column_idx, value) in columns.into_iter().enumerate() { + if let Some(value) = value { + entries.push((ColumnChunkKey::try_new(row_group_idx, column_idx)?, value)); + } + } + } + Self::try_from_sorted_entries(num_row_groups, num_columns, entries) + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn key(&self, row_group_idx: usize, column_idx: usize) -> Result { + if row_group_idx >= self.num_row_groups || column_idx >= self.num_columns { + return Err(ParquetError::General(format!( + "column chunk coordinate out of bounds: row group {row_group_idx}, column \ + {column_idx}; dimensions are {} by {}", + self.num_row_groups, self.num_columns + ))); + } + ColumnChunkKey::try_new(row_group_idx, column_idx) + } + + pub(crate) fn get(&self, row_group_idx: usize, column_idx: usize) -> Result> { + let key = self.key(row_group_idx, column_idx)?; + let Ok(entry_idx) = self + .entries + .binary_search_by_key(&key, |(entry_key, _)| *entry_key) + else { + return Ok(None); + }; + let Some((_, value)) = self.entries.get(entry_idx) else { + return Err(ParquetError::General(format!( + "column chunk binary search returned invalid entry index {entry_idx}" + ))); + }; + Ok(Some(value)) + } + + #[cfg_attr(not(test), expect(unused))] + pub(crate) fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(key, value)| (*key, value)) + } + + pub(crate) fn into_iter(self) -> impl Iterator { + self.entries.into_vec().into_iter() + } + + pub(crate) fn try_into_map(self) -> Result> { + let mut map = ColumnChunkMap::try_new(self.num_row_groups, self.num_columns)?; + for (key, value) in self.entries { + let (row_group_idx, column_idx) = key.coordinates(); + map.try_insert(row_group_idx, column_idx, value)?; + } + Ok(map) + } +} + +impl HeapSize for ColumnChunkEntries { + fn heap_size(&self) -> usize { + self.entries.len() * std::mem::size_of::<(ColumnChunkKey, V)>() + + self + .entries + .iter() + .map(|(_, value)| value.heap_size()) + .sum::() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sparse_storage_is_sorted_and_validates_coordinates() { + let mut map = ColumnChunkMap::try_new(2, 3).unwrap(); + map.try_insert(1, 2, 12).unwrap(); + map.try_insert(0, 1, 1).unwrap(); + let entries = map.try_freeze().unwrap(); + + let coordinates = entries + .iter() + .map(|(key, _)| key.coordinates()) + .collect::>(); + assert_eq!(coordinates, vec![(0, 1), (1, 2)]); + assert_eq!(entries.get(0, 1).unwrap(), Some(&1)); + assert_eq!(entries.get(0, 2).unwrap(), None); + assert!(entries.get(2, 0).is_err()); + assert!(entries.get(0, 3).is_err()); + } + + #[test] + fn sorted_entries_preserve_sparse_coordinates() { + let entries = ColumnChunkEntries::try_from_sorted_entries( + 100, + 1_000, + vec![ + (ColumnChunkKey::try_new(0, 3).unwrap(), 3_i32), + (ColumnChunkKey::try_new(99, 999).unwrap(), 999), + ], + ) + .unwrap(); + assert_eq!(entries.get(0, 3).unwrap(), Some(&3)); + assert_eq!(entries.get(0, 2).unwrap(), None); + assert_eq!(entries.get(99, 999).unwrap(), Some(&999)); + assert_eq!( + entries.heap_size(), + 2 * std::mem::size_of::<(ColumnChunkKey, i32)>() + ); + assert!( + ColumnChunkEntries::::try_from_sorted_entries(0, 0, vec![]) + .unwrap() + .is_empty() + ); + } + + #[test] + fn sorted_entries_reject_invalid_coordinates_and_order() { + let first = ColumnChunkKey::try_new(0, 0).unwrap(); + let second = ColumnChunkKey::try_new(1, 1).unwrap(); + for entries in [vec![(first, 1), (first, 2)], vec![(second, 1), (first, 2)]] { + assert!(ColumnChunkEntries::try_from_sorted_entries(2, 2, entries).is_err()); + } + assert!(ColumnChunkEntries::try_from_sorted_entries(1, 2, vec![(second, 1)]).is_err()); + assert!(ColumnChunkEntries::try_from_sorted_entries(2, 1, vec![(second, 1)]).is_err()); + assert!( + ColumnChunkEntries::::try_from_sorted_entries(i32::MAX as usize + 1, 1, vec![],) + .is_err() + ); + } + + #[test] + fn zero_and_invalid_dimensions() { + assert!(ColumnChunkMap::<()>::try_new(0, 0).is_ok()); + assert!(ColumnChunkMap::<()>::try_new(i32::MAX as usize + 1, 1).is_err()); + assert!(ColumnChunkMap::<()>::try_new(1, i32::MAX as usize + 1).is_err()); + } + + #[test] + fn insertion_replaces_existing_value_without_growing() { + let mut map = ColumnChunkMap::try_new(1, 1).unwrap(); + assert_eq!(map.try_insert(0, 0, "first").unwrap(), None); + let capacity = map.entries.capacity(); + + assert_eq!(map.try_insert(0, 0, "replacement").unwrap(), Some("first")); + assert_eq!(map.entries.len(), 1); + assert_eq!(map.entries.capacity(), capacity); + } + + #[test] + fn freeze_and_thaw_preserve_entries() { + let mut map = ColumnChunkMap::try_new(2, 2).unwrap(); + map.try_insert(1, 0, 3).unwrap(); + let expected_entries = map.try_freeze().unwrap(); + + let mut map = ColumnChunkMap::try_new(2, 2).unwrap(); + map.try_insert(1, 0, 3).unwrap(); + let entries = map.try_freeze().unwrap(); + let thawed = entries.try_into_map().unwrap(); + assert_eq!(thawed.try_freeze().unwrap(), expected_entries); + } + + #[test] + fn frozen_heap_size_uses_packed_key_layout() { + let entries = ColumnChunkEntries::try_from_rows(vec![vec![Some(1_i32)]]).unwrap(); + assert_eq!( + entries.heap_size(), + std::mem::size_of::<(ColumnChunkKey, i32)>() + ); + } +} diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 3d4433556243..80cb5e4bda1a 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -37,6 +37,7 @@ //! //! [`external_metadata.rs`]: https://github.com/apache/arrow-rs/tree/master/parquet/examples/external_metadata.rs //! +mod column_chunk; #[cfg(feature = "arrow")] mod dictionary; mod footer_tail; @@ -106,6 +107,8 @@ pub(crate) use writer::ThriftMetadataWriter; /// The individual fields of this structure are stored in a way that make cloning /// this structure low-cost. /// +/// See [`PageIndexProvider`] for details on customizing page index handling. +/// /// See the [`ParquetMetaDataBuilder`] to create and modify this structure. /// /// [`parquet.thrift`]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift @@ -231,14 +234,16 @@ impl ParquetMetaData { /// /// 1. Includes size of self /// - /// 2. Includes heap memory for sub fields such as [`FileMetaData`] and - /// [`RowGroupMetaData`]. + /// 2. Includes heap memory for sub fields such as [`FileMetaData`], + /// [`RowGroupMetaData`], and the built-in [`PageIndex`] provider. + /// + /// 3. Includes memory reachable through shared pointers, such as [`SchemaDescPtr`] and the + /// [`Arc`]-wrapped page index provider. Therefore, the same allocation may be counted by + /// multiple metadata values or when the provider is also retained in an external cache. /// - /// 3. Includes memory from shared pointers (e.g. [`SchemaDescPtr`]). This - /// means `memory_size` will over estimate the memory size if such pointers - /// are shared. + /// 4. Does not include heap memory used by custom [`PageIndexProvider`] implementations. /// - /// 4. Does not include any allocator overheads + /// 5. Does not include any allocator overheads. pub fn memory_size(&self) -> usize { #[cfg(feature = "encryption")] let encryption_size = self.file_decryptor.heap_size(); @@ -2251,7 +2256,7 @@ mod tests { }; } - page_index.put_column_index(column_index, 0, 0); + page_index.try_put_column_index(column_index, 0, 0).unwrap(); // Now, add in OffsetIndex let mut offset_index = OffsetIndexBuilder::new(); @@ -2262,7 +2267,7 @@ mod tests { offset_index.append_offset_and_size(2, 3); offset_index.append_unencoded_byte_array_data_bytes(Some(10)); let offset_index = offset_index.build(); - page_index.put_offset_index(offset_index, 0, 0); + page_index.try_put_offset_index(offset_index, 0, 0).unwrap(); let parquet_meta = ParquetMetaDataBuilder::new(file_metadata) .set_row_groups(row_group_meta) @@ -2271,9 +2276,9 @@ mod tests { // Size with page index (includes Arc overhead plus PageIndex heap size) #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3312; + let bigger_expected_size = 3296; #[cfg(feature = "encryption")] - let bigger_expected_size = 3480; + let bigger_expected_size = 3464; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); @@ -2385,7 +2390,7 @@ mod tests { offset_idx_builder.append_row_count(50); offset_idx_builder.append_offset_and_size(1000, 500); let offset_idx = offset_idx_builder.build(); - builder.put_offset_index(offset_idx, 0, 0); + builder.try_put_offset_index(offset_idx, 0, 0).unwrap(); let page_index = builder.build(); @@ -2434,14 +2439,16 @@ mod tests { let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32); col_idx_builder.append(false, vec![1, 0, 0, 0], vec![100, 0, 0, 0], 10, None); let col_idx = col_idx_builder.build().unwrap(); - builder.put_column_index(col_idx.clone(), 0, 0); + builder.try_put_column_index(col_idx.clone(), 0, 0).unwrap(); // Add an offset index for first column let mut offset_idx_builder = OffsetIndexBuilder::new(); offset_idx_builder.append_row_count(50); offset_idx_builder.append_offset_and_size(1000, 500); let offset_idx = offset_idx_builder.build(); - builder.put_offset_index(offset_idx.clone(), 0, 0); + builder + .try_put_offset_index(offset_idx.clone(), 0, 0) + .unwrap(); let page_index = builder.build(); @@ -2458,8 +2465,8 @@ mod tests { let mut builder = PageIndexBuilder::new_from(page_index); // Add indexes for second column - builder.put_column_index(col_idx, 0, 1); - builder.put_offset_index(offset_idx, 0, 1); + builder.try_put_column_index(col_idx, 0, 1).unwrap(); + builder.try_put_offset_index(offset_idx, 0, 1).unwrap(); let page_index = builder.build(); // now all populated diff --git a/parquet/src/file/metadata/page_index.rs b/parquet/src/file/metadata/page_index.rs index d075dfcd1cca..7178d6408f5a 100644 --- a/parquet/src/file/metadata/page_index.rs +++ b/parquet/src/file/metadata/page_index.rs @@ -3,7 +3,13 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! Page Index structures for efficient page-level skipping +//! +//! Row-group and column counts are each limited to `i32::MAX`, consistent with the +//! Parquet/Thrift metadata format. Populated entries are stored sparsely, subject to platform +//! address-space and available-memory limits. +use crate::errors::{ParquetError, Result}; +use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkMap}; use crate::file::metadata::memory::HeapSize; use crate::file::page_index::{ column_index::ColumnIndexMetaData, @@ -119,10 +125,17 @@ use std::sync::Arc; /// } /// ``` /// +/// # Caching the Page Index +/// +/// The Page Index can be much larger than the rest of the metadata, so some +/// systems may want to incrementally fetch or assemble the relevant data. See +/// [`page_index_cache.rs`] for an example of an incremental page index cache. +/// /// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ /// [`ColumnIndex`]: crate::file::page_index::column_index::ColumnIndexMetaData /// [`OffsetIndex`]: crate::file::page_index::offset_index::OffsetIndexMetaData /// [`ColumnChunkMetaData`]: crate::file::metadata::ColumnChunkMetaData +/// [`page_index_cache.rs`]: https://github.com/apache/arrow-rs/blob/main/parquet/examples/page_index_cache.rs pub trait PageIndexProvider: Send + Sync + std::fmt::Debug { /// Returns `true` if offset index structures are available via this provider /// @@ -352,10 +365,11 @@ impl RowGroupPageIndex { /// Struct to encapsulate the Parquet [Page Index] /// -/// This struct provides a dense representation of the Page Index. It is -/// used internally by this crate when assembling and writing the Page -/// Index. It is also the default implementation of the [`PageIndexProvider`] -/// contained in the [`ParquetMetaData`]. +/// This struct provides a sparse, immutable representation of the Page Index, stored in row-major +/// order and keyed by row group and column index. It owns its populated column and offset index +/// metadata. Use [`Self::into_index_entries`] to move that metadata into an external cache. +/// It is used internally by this crate when assembling and writing the Page Index, and is the +/// default implementation of the [`PageIndexProvider`] contained in the [`ParquetMetaData`]. /// /// # Example: Constructing a synthetic `PageIndex` /// @@ -411,9 +425,13 @@ impl RowGroupPageIndex { /// /// // Assemble the PageIndex (one entry per row group, each with one /// // entry per column) and attach it to the metadata -/// let mut page_index = PageIndexBuilder::new(1, 1); -/// page_index.put_column_index(column_index, 0, 0); -/// page_index.put_offset_index(offset_index, 0, 0); +/// let mut page_index = PageIndexBuilder::try_new(1, 1).unwrap(); +/// page_index +/// .try_put_column_index(column_index, 0, 0) +/// .unwrap(); +/// page_index +/// .try_put_offset_index(offset_index, 0, 0) +/// .unwrap(); /// let page_index = page_index.build(); /// let metadata = metadata /// .into_builder() @@ -428,24 +446,75 @@ impl RowGroupPageIndex { /// [`ParquetMetaData`]: crate::file::metadata::ParquetMetaData #[derive(Debug, Clone, PartialEq)] pub struct PageIndex { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndex { - pub(crate) fn new( + pub(crate) fn try_new( column_indexes: Option>>>, offset_indexes: Option>>>, + ) -> Result { + let column_indexes = column_indexes + .map(ColumnChunkEntries::try_from_rows) + .transpose()?; + let offset_indexes = offset_indexes + .map(ColumnChunkEntries::try_from_rows) + .transpose()?; + + Ok(Self::from_entries(column_indexes, offset_indexes)) + } + + pub(crate) fn from_entries( + column_indexes: Option>, + offset_indexes: Option>, ) -> Self { Self { - column_indexes, - offset_indexes, + column_indexes: column_indexes.filter(|indexes| !indexes.is_empty()), + offset_indexes: offset_indexes.filter(|indexes| !indexes.is_empty()), } } /// Convert this `PageIndex` into a [`PageIndexBuilder`] pub fn into_builder(self) -> PageIndexBuilder { - self.into() + self.try_into_builder() + .unwrap_or_else(|err| panic!("failed to thaw page index: {err}")) + } + + /// Fallibly converts this `PageIndex` into a [`PageIndexBuilder`]. + pub fn try_into_builder(self) -> Result { + PageIndexBuilder::try_new_from(self) + } + + /// Consumes this page index and returns its populated column and offset index entries. + /// + /// Each entry contains its `(row_group_index, column_index)` coordinate and owned index + /// metadata. This can be used to move parsed indexes into a cache without cloning their + /// metadata. Entries are returned in row-major order. + /// + /// For an example of using this to populate an incremental page index + /// cache, see [`page_index_cache.rs`] + /// + /// [`page_index_cache.rs`]: https://github.com/apache/arrow-rs/blob/main/parquet/examples/page_index_cache.rs + #[expect(clippy::type_complexity)] + pub fn into_index_entries( + self, + ) -> ( + impl Iterator, + impl Iterator, + ) { + let column_indexes = self + .column_indexes + .into_iter() + .flat_map(ColumnChunkEntries::into_iter) + .map(|(key, index)| (key.coordinates(), index)); + let offset_indexes = self + .offset_indexes + .into_iter() + .flat_map(ColumnChunkEntries::into_iter) + .map(|(key, index)| (key.coordinates(), index)); + + (column_indexes, offset_indexes) } } @@ -463,8 +532,10 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - let rg = self.column_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.column_indexes + .as_ref()? + .get(row_group_idx, column_idx) + .ok()? } fn offset_index( @@ -472,8 +543,10 @@ impl PageIndexProvider for PageIndex { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - let rg = self.offset_indexes.as_ref()?.get(row_group_idx)?; - rg.get(column_idx)?.as_ref() + self.offset_indexes + .as_ref()? + .get(row_group_idx, column_idx) + .ok()? } fn as_any(&self) -> &dyn std::any::Any { @@ -493,42 +566,41 @@ impl HeapSize for PageIndex { /// - Populating column indexes for predicate columns (for page filtering) /// - Populating offset indexes for projected columns (for direct I/O) /// - Automatic conversion of empty structures to `None` to save memory -#[derive(Default)] +#[derive(Default, Debug)] pub struct PageIndexBuilder { - column_indexes: Option>>>, - offset_indexes: Option>>>, + column_indexes: Option>, + offset_indexes: Option>, } impl PageIndexBuilder { - /// Creates an empty index structure with space for the specified number of row groups and columns + /// Creates a new [`PageIndexBuilder`] for the specified number of row groups and columns. /// - /// Returns `Some` containing a nested vector structure where all entries are initialized to `None`. - /// The outer vector has one entry per row group, and each inner vector has one entry per column. + /// The dimensions are used to validate inserted coordinates. Storage is allocated only for + /// entries populated using + /// [`try_put_column_index`](Self::try_put_column_index) and + /// [`try_put_offset_index`](Self::try_put_offset_index). /// - /// # Type Parameters - /// * `T` - The type of index this is to be, either `ColumnIndexMetaData` or `OffsetIndexMetaData` - fn empty_index(num_row_groups: usize, num_columns: usize) -> Option>>> { - Some( - (0..num_row_groups) - .map(|_| { - let mut idx = Vec::with_capacity(num_columns); - idx.resize_with(num_columns, || None); - idx - }) - .collect(), - ) + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_new`](Self::try_new) for a fallible alternative. + pub fn new(num_row_groups: usize, num_columns: usize) -> Self { + Self::try_new(num_row_groups, num_columns).expect("invalid page index dimensions") } - /// Creates a new [`PageIndexBuilder`] with space allocated for both column and offset indexes + /// Attempts to create a new [`PageIndexBuilder`] for the specified number of row groups and columns. /// - /// This allocates empty index structures for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using - /// [`put_column_index`](Self::put_column_index) and [`put_offset_index`](Self::put_offset_index). - pub fn new(num_row_groups: usize, num_columns: usize) -> Self { - Self { - column_indexes: Self::empty_index(num_row_groups, num_columns), - offset_indexes: Self::empty_index(num_row_groups, num_columns), - } + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_new(num_row_groups: usize, num_columns: usize) -> Result { + Ok(Self { + column_indexes: Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?), + offset_indexes: Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?), + }) } /// Creates a new [`PageIndexBuilder`] from an existing [`PageIndex`] @@ -536,80 +608,196 @@ impl PageIndexBuilder { /// This takes ownership of the index structures from the provided [`PageIndex`], /// allowing them to be modified and rebuilt. Useful for updating existing page indexes. pub(crate) fn new_from(page_index: PageIndex) -> Self { - Self { - column_indexes: page_index.column_indexes, - offset_indexes: page_index.offset_indexes, - } + Self::try_new_from(page_index) + .unwrap_or_else(|err| panic!("failed to thaw page index: {err}")) + } + + fn try_new_from(page_index: PageIndex) -> Result { + Ok(Self { + column_indexes: page_index + .column_indexes + .map(ColumnChunkEntries::try_into_map) + .transpose()?, + offset_indexes: page_index + .offset_indexes + .map(ColumnChunkEntries::try_into_map) + .transpose()?, + }) } /// Allocates space for column indexes /// - /// This allocates an empty index structure for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using - /// [`put_column_index`](Self::put_column_index). + /// This replaces any existing column indexes with an empty sparse map having the specified + /// dimensions. It can then be populated using + /// [`try_put_column_index`](Self::try_put_column_index). /// /// This can be used to add column index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without column indexes). + /// + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_allocate_column_indexes`](Self::try_allocate_column_indexes) for a + /// fallible alternative. pub fn allocate_column_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.column_indexes = Self::empty_index(num_row_groups, num_columns); + self.try_allocate_column_indexes(num_row_groups, num_columns) + .expect("invalid page index dimensions"); + } + + /// Attempts to allocate space for column indexes + /// + /// This is a fallible version of [`allocate_column_indexes`](Self::allocate_column_indexes). + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_allocate_column_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + self.column_indexes = Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?); + Ok(()) } /// Allocates space for offset indexes /// - /// This allocates an empty index structure for the specified number of row groups and columns. - /// All index entries are initialized to `None` and can be populated using - /// [`put_offset_index`](Self::put_offset_index). + /// This replaces any existing offset indexes with an empty sparse map having the specified + /// dimensions. It can then be populated using + /// [`try_put_offset_index`](Self::try_put_offset_index). /// /// This can be used to add offset index storage to a builder that lacks one /// (either a `Default` builder, or one created from a [`PageIndex`] without offset indexes). + /// + /// # Panics + /// + /// Panics if either dimension exceeds `i32::MAX`, the maximum collection size representable + /// by Thrift. + /// + /// Use [`try_allocate_offset_indexes`](Self::try_allocate_offset_indexes) for a + /// fallible alternative. pub fn allocate_offset_indexes(&mut self, num_row_groups: usize, num_columns: usize) { - self.offset_indexes = Self::empty_index(num_row_groups, num_columns); + self.try_allocate_offset_indexes(num_row_groups, num_columns) + .expect("invalid page index dimensions"); + } + + /// Attempts to allocate space for offset indexes + /// + /// This is a fallible version of [`allocate_offset_indexes`](Self::allocate_offset_indexes). + /// + /// # Errors + /// + /// Returns an error if either dimension exceeds `i32::MAX`, the maximum collection size + /// representable by Thrift. + pub fn try_allocate_offset_indexes( + &mut self, + num_row_groups: usize, + num_columns: usize, + ) -> Result<()> { + self.offset_indexes = Some(ColumnChunkMap::try_new(num_row_groups, num_columns)?); + Ok(()) } /// Sets the column index for a specific row group and column /// /// If column indexes were not allocated (see [`Self::allocate_column_indexes`]), /// or the row group or column index is out of bounds, this method does nothing. + #[deprecated( + since = "60.2.0", + note = "Use `try_put_column_index` to detect invalid coordinates or unavailable index storage" + )] pub fn put_column_index( &mut self, column_index: ColumnIndexMetaData, row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.column_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(column_index); + // preserve stated contract that this does nothing when indexes were not allocated + // or passed in indices are out of bounds + let Some(indexes) = self.column_indexes.as_ref() else { + return; + }; + if row_group_idx >= indexes.num_row_groups() || column_idx >= indexes.num_columns() { + return; } + self.try_put_column_index(column_index, row_group_idx, column_idx) + .unwrap_or_else(|err| panic!("{err}")); + } + + /// Attempts to set the column index for a specific row group and column. + /// + /// # Errors + /// + /// Returns an error, and drops `column_index`, if column indexes were not allocated or the + /// coordinate is out of bounds. + pub fn try_put_column_index( + &mut self, + column_index: ColumnIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> Result<()> { + let indexes = self + .column_indexes + .as_mut() + .ok_or_else(|| ParquetError::General("column indexes are not allocated".to_string()))?; + indexes + .try_insert(row_group_idx, column_idx, column_index) + .map(|_| ()) } /// Sets the offset index for a specific row group and column /// /// If offset indexes were not allocated (see [`Self::allocate_offset_indexes`]), /// or the row group or column index is out of bounds, this method does nothing. + #[deprecated( + since = "60.2.0", + note = "Use `try_put_offset_index` to detect invalid coordinates or unavailable index storage" + )] pub fn put_offset_index( &mut self, offset_index: OffsetIndexMetaData, row_group_idx: usize, column_idx: usize, ) { - if let Some(ref mut indexes) = self.offset_indexes - && let Some(row_group) = indexes.get_mut(row_group_idx) - && let Some(column_slot) = row_group.get_mut(column_idx) - { - *column_slot = Some(offset_index); + // preserve stated contract that this does nothing when indexes were not allocated + // or passed in indices are out of bounds + let Some(indexes) = self.offset_indexes.as_ref() else { + return; + }; + if row_group_idx >= indexes.num_row_groups() || column_idx >= indexes.num_columns() { + return; } + self.try_put_offset_index(offset_index, row_group_idx, column_idx) + .unwrap_or_else(|err| panic!("{err}")); } - /// Checks if an index structure is entirely empty (all entries are None) - fn is_empty_index(index: Option<&Vec>>>) -> bool { - match index { - None => true, - Some(row_groups) => row_groups - .iter() - .all(|columns| columns.iter().all(|entry| entry.is_none())), - } + /// Attempts to set the offset index for a specific row group and column. + /// + /// # Errors + /// + /// Returns an error, and drops `offset_index`, if offset indexes were not allocated or the + /// coordinate is out of bounds. + pub fn try_put_offset_index( + &mut self, + offset_index: OffsetIndexMetaData, + row_group_idx: usize, + column_idx: usize, + ) -> Result<()> { + let indexes = self + .offset_indexes + .as_mut() + .ok_or_else(|| ParquetError::General("offset indexes are not allocated".to_string()))?; + indexes + .try_insert(row_group_idx, column_idx, offset_index) + .map(|_| ()) + } + + /// Checks if an index structure is entirely empty. + fn is_empty_index(index: Option<&ColumnChunkMap>) -> bool { + index.is_none_or(ColumnChunkMap::is_empty) } /// Consumes the builder and returns a [`PageIndex`] @@ -620,6 +808,11 @@ impl PageIndexBuilder { /// - [`PageIndex::has_column_indexes()`] and [`PageIndex::has_offset_indexes()`] /// correctly return `false` for unpopulated indexes pub fn build(self) -> PageIndex { + self.try_build().expect("failed to build page index") + } + + /// Fallibly consumes the builder and returns a [`PageIndex`]. + pub fn try_build(self) -> Result { let column_indexes = if Self::is_empty_index(self.column_indexes.as_ref()) { None } else { @@ -632,7 +825,10 @@ impl PageIndexBuilder { self.offset_indexes }; - PageIndex::new(column_indexes, offset_indexes) + Ok(PageIndex { + column_indexes: column_indexes.map(ColumnChunkMap::try_freeze).transpose()?, + offset_indexes: offset_indexes.map(ColumnChunkMap::try_freeze).transpose()?, + }) } } @@ -641,3 +837,276 @@ impl From for PageIndexBuilder { Self::new_from(page_index) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::basic::Type as PhysicalType; + use crate::file::metadata::{ColumnIndexBuilder, OffsetIndexBuilder}; + + #[test] + fn test_dimensions_within_i32_max() { + // Valid dimensions should work + let builder = PageIndexBuilder::new(100, 50); + assert!(builder.column_indexes.is_some()); + assert!(builder.offset_indexes.is_some()); + + let result = PageIndexBuilder::try_new(100, 50); + assert!(result.is_ok()); + } + + #[test] + fn test_try_new_validates_dimensions() { + // Test row group count exceeding i32::MAX + let oversized = i32::MAX as usize + 1; + let result = PageIndexBuilder::try_new(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + // Test column count exceeding i32::MAX + let result = PageIndexBuilder::try_new(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_new_panics_on_oversized_row_groups() { + let oversized = i32::MAX as usize + 1; + PageIndexBuilder::new(oversized, 10); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_new_panics_on_oversized_columns() { + let oversized = i32::MAX as usize + 1; + PageIndexBuilder::new(10, oversized); + } + + #[test] + fn test_try_allocate_column_indexes_validates_dimensions() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + + let result = builder.try_allocate_column_indexes(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + let result = builder.try_allocate_column_indexes(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_allocate_column_indexes_panics_on_oversized() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + builder.allocate_column_indexes(oversized, 10); + } + + #[test] + fn test_try_allocate_offset_indexes_validates_dimensions() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + + let result = builder.try_allocate_offset_indexes(oversized, 10); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("row group count exceeds i32::MAX"), + "unexpected error: {err}" + ); + + let result = builder.try_allocate_offset_indexes(10, oversized); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.to_string().contains("column count exceeds i32::MAX"), + "unexpected error: {err}" + ); + } + + #[test] + #[should_panic(expected = "invalid page index dimensions")] + fn test_allocate_offset_indexes_panics_on_oversized() { + let mut builder = PageIndexBuilder::default(); + let oversized = i32::MAX as usize + 1; + builder.allocate_offset_indexes(oversized, 10); + } + + #[test] + fn test_page_index_builder_basic_operations() { + let mut builder = PageIndexBuilder::new(2, 3); + + // Create a simple column index + let mut col_index = ColumnIndexBuilder::new(PhysicalType::INT32); + col_index.append( + false, + 1i32.to_le_bytes().to_vec(), + 10i32.to_le_bytes().to_vec(), + 0, + None, + ); + let col_index = col_index.build().unwrap(); + + // Create a simple offset index + let mut off_index = OffsetIndexBuilder::new(); + off_index.append_row_count(100); + off_index.append_offset_and_size(1000, 500); + let off_index = off_index.build(); + + // Insert indexes and report coordinates that cannot be populated + assert!( + builder + .try_put_column_index(col_index.clone(), 0, 0) + .is_ok() + ); + assert!( + builder + .try_put_offset_index(off_index.clone(), 0, 0) + .is_ok() + ); + assert!( + builder + .try_put_column_index(col_index.clone(), 2, 0) + .is_err() + ); + assert!( + builder + .try_put_offset_index(off_index.clone(), 0, 3) + .is_err() + ); + + // Deprecated insertion still populates valid coordinates, and remains a no-op + // for unallocated storage and invalid coordinates. + #[expect(deprecated, reason = "verify the legacy insertion contract")] + { + let mut legacy_unallocated = PageIndexBuilder::default(); + legacy_unallocated.put_column_index(col_index.clone(), 0, 0); + legacy_unallocated.put_offset_index(off_index.clone(), 0, 0); + let empty_page_index = legacy_unallocated.build(); + assert!(!empty_page_index.has_column_indexes()); + assert!(!empty_page_index.has_offset_indexes()); + + let mut legacy_builder = PageIndexBuilder::new(2, 3); + legacy_builder.put_column_index(col_index.clone(), 0, 0); + legacy_builder.put_offset_index(off_index.clone(), 0, 0); + legacy_builder.put_column_index(col_index.clone(), 2, 0); + legacy_builder.put_column_index(col_index.clone(), 0, 3); + legacy_builder.put_offset_index(off_index.clone(), 2, 0); + legacy_builder.put_offset_index(off_index.clone(), 0, 3); + let legacy_page_index = legacy_builder.build(); + assert_eq!(legacy_page_index.column_index(0, 0), Some(&col_index)); + assert_eq!(legacy_page_index.offset_index(0, 0), Some(&off_index)); + assert_eq!(legacy_page_index.column_index(0, 1), None); + assert_eq!(legacy_page_index.offset_index(1, 0), None); + } + + let mut unallocated = PageIndexBuilder::default(); + assert!(unallocated.try_put_column_index(col_index, 0, 0).is_err()); + assert!(unallocated.try_put_offset_index(off_index, 0, 0).is_err()); + + // Build and verify + let page_index = builder.build(); + assert!(page_index.has_column_indexes()); + assert!(page_index.has_offset_indexes()); + assert!(page_index.column_index(0, 0).is_some()); + assert!(page_index.offset_index(0, 0).is_some()); + assert!(page_index.column_index(0, 1).is_none()); + assert!(page_index.offset_index(1, 0).is_none()); + } + + #[test] + fn test_into_index_entries_moves_owned_values_in_row_major_order() { + fn column_index(seed: i32, num_pages: usize) -> ColumnIndexMetaData { + let mut builder = ColumnIndexBuilder::new(PhysicalType::INT32); + for page in 0..num_pages { + let min = seed + page as i32; + builder.append( + false, + min.to_le_bytes().to_vec(), + (min + 10).to_le_bytes().to_vec(), + 0, + None, + ); + } + builder.build().unwrap() + } + + fn offset_index(seed: i64, num_pages: usize) -> OffsetIndexMetaData { + let mut builder = OffsetIndexBuilder::new(); + for page in 0..num_pages { + builder.append_row_count(10); + builder.append_offset_and_size(seed + page as i64 * 100, 100); + } + builder.build() + } + + let first_column = column_index(1, 1); + let second_column = column_index(2, 2); + let column_values = match &second_column { + ColumnIndexMetaData::INT32(index) => index.min_values().as_ptr(), + _ => unreachable!(), + }; + let offset = offset_index(10, 3); + let page_locations = offset.page_locations().as_ptr(); + + let mut builder = PageIndexBuilder::new(3, 3); + builder.try_put_column_index(second_column, 2, 0).unwrap(); + builder.try_put_offset_index(offset, 1, 1).unwrap(); + builder + .try_put_column_index(first_column.clone(), 0, 2) + .unwrap(); + + let (column_entries, offset_entries) = builder.build().into_index_entries(); + let column_entries = column_entries.collect::>(); + let offset_entries = offset_entries.collect::>(); + assert_eq!( + column_entries, + vec![((0, 2), first_column), ((2, 0), column_index(2, 2))] + ); + assert_eq!(offset_entries, vec![((1, 1), offset_index(10, 3))]); + match &column_entries[1].1 { + ColumnIndexMetaData::INT32(index) => { + assert_eq!(index.min_values().as_ptr(), column_values) + } + _ => unreachable!(), + } + assert_eq!( + offset_entries[0].1.page_locations().as_ptr(), + page_locations + ); + + let column = column_index(3, 1); + let mut builder = PageIndexBuilder::default(); + builder.try_allocate_column_indexes(1, 1).unwrap(); + builder.try_put_column_index(column.clone(), 0, 0).unwrap(); + let (column_entries, offset_entries) = builder.build().into_index_entries(); + assert_eq!(column_entries.collect::>(), vec![((0, 0), column)]); + assert_eq!(offset_entries.count(), 0); + } + + #[test] + fn test_page_index_try_new_discards_empty_indexes() { + let page_index = PageIndex::try_new(Some(vec![vec![None]]), Some(vec![])).unwrap(); + assert!(!page_index.has_column_indexes()); + assert!(!page_index.has_offset_indexes()); + } +} diff --git a/parquet/src/file/metadata/parser.rs b/parquet/src/file/metadata/parser.rs index b8f52b1d25d2..b4ff69943eb5 100644 --- a/parquet/src/file/metadata/parser.rs +++ b/parquet/src/file/metadata/parser.rs @@ -12,7 +12,8 @@ use std::ops::Range; use bytes::Bytes; use crate::errors::{ParquetError, Result}; -use crate::file::metadata::page_index::{PageIndex, PageIndexBuilder, PageIndexProvider}; +use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkKey}; +use crate::file::metadata::page_index::{PageIndex, PageIndexProvider}; use crate::file::metadata::thrift::parquet_metadata_from_bytes; use crate::file::metadata::{ ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions, @@ -245,34 +246,29 @@ pub(crate) fn parse_page_index( bytes: &Bytes, start_offset: u64, ) -> Result> { - let num_row_groups = metadata.num_row_groups(); - let num_columns = metadata.file_metadata().schema_descr().num_columns(); - let mut builder = PageIndexBuilder::default(); - - if column_index_policy != PageIndexPolicy::Skip { - builder.allocate_column_indexes(num_row_groups, num_columns); - parse_column_index( + let column_indexes = if column_index_policy == PageIndexPolicy::Skip { + None + } else { + Some(parse_column_index( metadata, - column_index_policy, column_index_mask, - &mut builder, bytes, start_offset, - )?; - } - if offset_index_policy != PageIndexPolicy::Skip { - builder.allocate_offset_indexes(num_row_groups, num_columns); - parse_offset_index( + )?) + }; + let offset_indexes = if offset_index_policy == PageIndexPolicy::Skip { + None + } else { + Some(parse_offset_index( metadata, offset_index_policy, offset_index_mask, - &mut builder, bytes, start_offset, - )?; - } + )?) + }; - let page_index = builder.build(); + let page_index = PageIndex::from_entries(column_indexes, offset_indexes); if page_index.has_column_indexes() || page_index.has_offset_indexes() { Ok(Some(page_index)) } else { @@ -280,17 +276,45 @@ pub(crate) fn parse_page_index( } } +/// Reserve only for selected chunks that actually have an index range. An extra +/// metadata-only pass avoids both vector growth for dense reads and reserving a +/// full rectangular grid for sparse reads or files with missing indexes. +fn reserve_index_entries( + metadata: &ParquetMetaData, + mask: &ColumnChunkMask, + index_range: impl Fn(&ColumnChunkMetaData) -> Option>, +) -> Result> { + let num_entries = + mask.row_group_indices(metadata.num_row_groups()) + .try_fold(0_usize, |count, rg_idx| { + let rg = metadata.row_group(rg_idx); + let present = mask + .column_indices(rg.num_columns()) + .filter(|&col_idx| index_range(rg.column(col_idx)).is_some()) + .count(); + count.checked_add(present).ok_or_else(|| { + ParquetError::General("column chunk entry count exceeds usize::MAX".to_string()) + }) + })?; + let mut entries = Vec::new(); + entries.try_reserve_exact(num_entries).map_err(|err| { + ParquetError::General(format!( + "failed to allocate storage for {num_entries} column chunk entries: {err}" + )) + })?; + Ok(entries) +} + fn parse_column_index( metadata: &ParquetMetaData, - column_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, - page_index_builder: &mut PageIndexBuilder, bytes: &Bytes, start_offset: u64, -) -> Result<()> { - if column_index_policy == PageIndexPolicy::Skip { - return Ok(()); - } +) -> Result> { + let mut entries = + reserve_index_entries(metadata, mask, ColumnChunkMetaData::column_index_range)?; + // Mask axes are sorted and unique, so present entries remain in row-major + // order even when selected chunks have no index. for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); for col_idx in mask.column_indices(rg.num_columns()) { @@ -299,25 +323,27 @@ fn parse_column_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_column_index(idx, rg_idx, col_idx); + entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx)); } } } - Ok(()) + ColumnChunkEntries::try_from_sorted_entries( + metadata.num_row_groups(), + metadata.file_metadata().schema_descr().num_columns(), + entries, + ) } fn parse_offset_index( metadata: &ParquetMetaData, offset_index_policy: PageIndexPolicy, mask: &ColumnChunkMask, - page_index_builder: &mut PageIndexBuilder, bytes: &Bytes, start_offset: u64, -) -> Result<()> { - if offset_index_policy == PageIndexPolicy::Skip { - return Ok(()); - } +) -> Result> { + let mut entries = + reserve_index_entries(metadata, mask, ColumnChunkMetaData::offset_index_range)?; for rg_idx in mask.row_group_indices(metadata.num_row_groups()) { let rg = metadata.row_group(rg_idx); for col_idx in mask.column_indices(rg.num_columns()) { @@ -326,14 +352,18 @@ fn parse_offset_index( let idx_bytes = get_index_bytes(bytes, start_offset, r)?; let idx = inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?; - page_index_builder.put_offset_index(idx, rg_idx, col_idx); + entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx)); } else if offset_index_policy == PageIndexPolicy::Required { return Err(general_err!("missing offset index")); } } } - Ok(()) + ColumnChunkEntries::try_from_sorted_entries( + metadata.num_row_groups(), + metadata.file_metadata().schema_descr().num_columns(), + entries, + ) } fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range) -> Result<&[u8]> { diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs index ee414b3621d4..da65f7ab0a95 100644 --- a/parquet/src/file/metadata/reader.rs +++ b/parquet/src/file/metadata/reader.rs @@ -256,6 +256,23 @@ impl ColumnChunkMask { Self::axis_indices(self.columns.as_deref(), num_columns) } + /// Returns an iterator over the `(row_group_index, column_index)` coordinates selected by + /// this mask. + /// + /// Coordinates are returned in row-major order and are bounded by `num_row_groups` and + /// `num_columns`. + pub fn column_chunk_indices( + &self, + num_row_groups: usize, + num_columns: usize, + ) -> impl Iterator + '_ { + self.row_group_indices(num_row_groups) + .flat_map(move |row_group_idx| { + self.column_indices(num_columns) + .map(move |column_idx| (row_group_idx, column_idx)) + }) + } + fn axis_indices(axis: Option<&[u32]>, len: usize) -> AxisIndices<'_> { match axis { None => AxisIndices::All(0..len), @@ -358,6 +375,8 @@ impl ParquetMetaDataReader { /// /// The mask applies only when the column index policy is not [`PageIndexPolicy::Skip`]. /// [`PageIndexPolicy::Required`] validates only selected column chunks. + /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.column_index_mask = mask; self @@ -367,6 +386,8 @@ impl ParquetMetaDataReader { /// /// The mask applies only when the offset index policy is not [`PageIndexPolicy::Skip`]. /// [`PageIndexPolicy::Required`] validates only selected column chunks. + /// + /// See [`PageIndexProvider`] for more details on customizing page index loading. pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self { self.offset_index_mask = mask; self @@ -1545,6 +1566,17 @@ mod tests { let sparse = ColumnChunkMask::row_groups_and_columns([0, 2, 4], [1, 3, 5]); assert_eq!(sparse.row_group_indices(3).collect::>(), [0, 2]); assert_eq!(sparse.column_indices(4).collect::>(), [1, 3]); + assert_eq!( + sparse.column_chunk_indices(3, 4).collect::>(), + [(0, 1), (0, 3), (2, 1), (2, 3)] + ); + assert_eq!( + all.column_chunk_indices(2, 3).collect::>(), + [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)] + ); + assert_eq!(none.column_chunk_indices(3, 3).count(), 0); + assert_eq!(sparse.column_chunk_indices(0, 4).count(), 0); + assert_eq!(sparse.column_chunk_indices(3, 0).count(), 0); } } diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index d293920fe5f5..7af2973adfaa 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -398,7 +398,7 @@ impl SerializedFileWriter { Some(offset_indexes) }; if column_indexes.is_some() || offset_indexes.is_some() { - let page_index = PageIndex::new(column_indexes, offset_indexes); + let page_index = PageIndex::try_new(column_indexes, offset_indexes)?; encoder = encoder.with_page_index(Arc::new(page_index)); } diff --git a/parquet/tests/arrow_reader/page_index.rs b/parquet/tests/arrow_reader/page_index.rs index e69574161752..480fc58aeca5 100644 --- a/parquet/tests/arrow_reader/page_index.rs +++ b/parquet/tests/arrow_reader/page_index.rs @@ -4,7 +4,9 @@ //! Tests of selective page index population -use parquet::file::metadata::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader}; +use parquet::file::metadata::{ + ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, +}; use crate::custom_page_index_provider::create_test_file; @@ -74,6 +76,123 @@ fn test_parse_with_page_index_mask() { ); } +fn index_layout( + metadata: &ParquetMetaData, + has_index: impl Fn(usize, usize) -> bool, +) -> Vec { + let num_cols = metadata.file_metadata().schema_descr().num_columns(); + (0..metadata.num_row_groups()) + .map(|rg| { + (0..num_cols) + .map(|col| if has_index(rg, col) { 'X' } else { '.' }) + .collect() + }) + .collect() +} + +#[test] +fn test_sparse_page_index_missing_ranges_preserve_coordinates() { + // Omitting missing ranges from compact storage must not shift subsequent + // indexes onto earlier row-group/column coordinates. + // Each string is a row group; each character is a column. X means an index + // range is present, and . means it is missing from the footer metadata. + let column_index_ranges = [".XXX", "XXXX", "XXX."]; + let offset_index_ranges = ["X.XX", "XX.X", "XXXX"]; + + let file = create_test_file(); + // Keep the fully loaded indexes as a reference for the values that survive + // selection, not just whether an index exists at each coordinate. + let original = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&file) + .unwrap(); + // Load only the footer, then remove selected range descriptors to simulate + // missing indexes without rewriting the file. The remaining descriptors + // still point to the original bytes; no previously decoded indexes are reused. + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Skip) + .parse_and_finish(&file) + .unwrap(); + let mut builder = metadata.into_builder(); + let row_groups = builder + .take_row_groups() + .into_iter() + .enumerate() + .map(|(rg_idx, rg)| { + let columns = rg + .columns() + .iter() + .enumerate() + .map(|(col_idx, column)| { + let mut column = column.clone().into_builder(); + if column_index_ranges[rg_idx].as_bytes()[col_idx] == b'.' { + column = column + .set_column_index_offset(None) + .set_column_index_length(None); + } + if offset_index_ranges[rg_idx].as_bytes()[col_idx] == b'.' { + column = column + .set_offset_index_offset(None) + .set_offset_index_length(None); + } + column.build().unwrap() + }) + .collect(); + rg.into_builder() + .set_column_metadata(columns) + .build() + .unwrap() + }) + .collect(); + let metadata = builder.set_row_groups(row_groups).build(); + + // Select column indexes in row groups 0 and 2, columns 0, 1, and 3; offset + // indexes remain unmasked. Mask axes canonicalize duplicates and ignore 99. + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_page_index_policy(PageIndexPolicy::Optional) + .with_column_index_mask(ColumnChunkMask::row_groups_and_columns( + [2, 0, 2, 99], + [3, 0, 1, 3, 99], + )); + reader.read_page_indexes(&file).unwrap(); + let read = reader.finish().unwrap(); + let index = read.page_index().unwrap(); + // Selected, available indexes must retain their coordinates despite holes + // from both the mask and missing ranges. + assert_eq!( + index_layout(&read, |rg, col| index.column_index(rg, col).is_some()), + [".X.X", "....", "XX.."] + ); + assert_eq!( + index_layout(&read, |rg, col| index.offset_index(rg, col).is_some()), + ["X.XX", "XX.X", "XXXX"] + ); + // Correct presence alone could hide values assigned to the wrong chunk. + // Compare each surviving index with the same coordinate in the original. + let expected = original.page_index().unwrap(); + for rg in 0..3 { + for col in 0..4 { + if let Some(ci) = index.column_index(rg, col) { + assert_eq!(Some(ci), expected.column_index(rg, col)); + } + if let Some(oi) = index.offset_index(rg, col) { + assert_eq!(Some(oi), expected.offset_index(rg, col)); + } + } + } + + // Column index absence is still tolerated under Required, but an absent + // selected offset index remains an error. + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata.clone()) + .with_column_index_policy(PageIndexPolicy::Required) + .with_offset_index_policy(PageIndexPolicy::Skip); + reader.read_page_indexes(&file).unwrap(); + let mut reader = ParquetMetaDataReader::new_with_metadata(metadata) + .with_page_index_policy(PageIndexPolicy::Required); + let error = reader.read_page_indexes(&file).unwrap_err(); + assert_eq!(error.to_string(), "Parquet error: missing offset index"); +} + #[test] fn test_repeated_page_index_reads_replace() { // verify that repeated calls to read_page_indexes replace existing indexes @@ -369,9 +488,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 13929); + assert_eq!(metadata.memory_size(), 13993); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 14353); + assert_eq!(metadata.memory_size(), 14417); // populate column 0 for column index and all columns for offset index let mut reader = ParquetMetaDataReader::new() @@ -384,9 +503,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12808); + assert_eq!(metadata.memory_size(), 10712); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 13232); + assert_eq!(metadata.memory_size(), 11136); // populate column 0 for column index and columns 0 & 2 for the offset index let mut reader = ParquetMetaDataReader::new() @@ -399,9 +518,9 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 12088); + assert_eq!(metadata.memory_size(), 9656); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 12512); + assert_eq!(metadata.memory_size(), 10080); // populate only row group 1, column index gets column 0, offset index gets // columns 0 and 2. @@ -415,7 +534,7 @@ fn test_page_index_sizes() { let metadata = reader.finish().unwrap(); assert!(metadata.page_index().is_some()); #[cfg(not(feature = "encryption"))] - assert_eq!(metadata.memory_size(), 11358); + assert_eq!(metadata.memory_size(), 8222); #[cfg(feature = "encryption")] - assert_eq!(metadata.memory_size(), 11782); + assert_eq!(metadata.memory_size(), 8646); }