Skip to content
Merged
5 changes: 5 additions & 0 deletions parquet/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,11 @@ name = "custom_page_index"
required-features = ["arrow"]
path = "./examples/custom_page_index.rs"

[[example]]
name = "page_index_cache"
required-features = ["arrow"]
path = "./examples/page_index_cache.rs"

[[test]]
name = "arrow_reader"
required-features = ["arrow"]
Expand Down
2 changes: 1 addition & 1 deletion parquet/benches/arrow_statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ fn page_index_benchmark(c: &mut Criterion) {
let mut builder = PageIndexBuilder::new(row_groups, num_columns);
for (rg, (_, bytes)) in column_indexes.iter().enumerate() {
let index = decode_column_index(bytes, physical_type).unwrap();
builder.put_column_index(index, rg, column);
builder.try_put_column_index(index, rg, column).unwrap();
}
let page_index = builder.build();
let _ = converter
Expand Down
298 changes: 298 additions & 0 deletions parquet/examples/page_index_cache.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,298 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Incrementally caching page indexes with [`ParquetMetaDataReader`].
//!
//! This example keeps footer metadata and decoded page indexes in separate caches. For each
//! query it:
//!
//! 1. Finds the requested column chunks that don't yet have entries in the page-index cache
//! 2. Uses [`ParquetMetaDataReader::read_page_index_and_finish`] to fetch the missing entries
//! 3. Moves the decoded entries into the cache
//! 4. Builds a query-specific [`PageIndexProvider`] from shared cache entries
//!
//! A production cache shared by multiple files would also include a stable file identity and
//! freshness information in its keys. This example also caches only indexes that are present;
//! it does not record negative results. Consequently, it may request a missing index again on a
//! later query. A production cache should track coordinates known not to have an index.

use std::collections::{BTreeSet, HashMap};
use std::sync::Arc;

use arrow::array::{Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use bytes::Bytes;
use parquet::arrow::ArrowWriter;
use parquet::errors::Result;
use parquet::file::metadata::page_index::PageIndexProvider;
use parquet::file::metadata::{
ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader,
};
use parquet::file::page_index::column_index::ColumnIndexMetaData;
use parquet::file::page_index::offset_index::OffsetIndexMetaData;
use parquet::file::properties::{EnabledStatistics, WriterProperties};

/// Identifier for a column chunk in a Parquet file: (row group index, column index)
type ColumnChunk = (usize, usize);

/// A query-specific [`PageIndexProvider`] backed by handles to shared entries
/// in a [`PageIndexCache`].
#[derive(Debug, Default)]
struct CachedPageIndexProvider {
column_indexes: HashMap<ColumnChunk, Arc<ColumnIndexMetaData>>,
offset_indexes: HashMap<ColumnChunk, Arc<OffsetIndexMetaData>>,
}

impl PageIndexProvider for CachedPageIndexProvider {
fn has_offset_indexes(&self) -> bool {
!self.offset_indexes.is_empty()
}

fn has_column_indexes(&self) -> bool {
!self.column_indexes.is_empty()
}

fn column_index(
&self,
row_group_idx: usize,
column_idx: usize,
) -> Option<&ColumnIndexMetaData> {
self.column_indexes
.get(&(row_group_idx, column_idx))
.map(Arc::as_ref)
}

fn offset_index(
&self,
row_group_idx: usize,
column_idx: usize,
) -> Option<&OffsetIndexMetaData> {
self.offset_indexes
.get(&(row_group_idx, column_idx))
.map(Arc::as_ref)
}

fn as_any(&self) -> &dyn std::any::Any {
self
}
}

/// Decoded page indexes cached independently of query-specific metadata.
#[derive(Default)]
struct PageIndexCache {
column_indexes: HashMap<ColumnChunk, Arc<ColumnIndexMetaData>>,
offset_indexes: HashMap<ColumnChunk, Arc<OffsetIndexMetaData>>,
}

impl PageIndexCache {
/// Returns the footer metadata and a provider for the entries requested for
/// one query, loading any missing entries using [`ParquetMetaDataReader`].
fn page_index_for_query(
&mut self,
mut metadata: ParquetMetaData,
file_bytes: &Bytes,
column_index_mask: &ColumnChunkMask,
offset_index_mask: &ColumnChunkMask,
) -> Result<(ParquetMetaData, CachedPageIndexProvider)> {
let num_row_groups = metadata.num_row_groups();
let num_columns = metadata.file_metadata().schema_descr().num_columns();

let missing_column_indexes = column_index_mask
.column_chunk_indices(num_row_groups, num_columns)
.filter(|key| !self.column_indexes.contains_key(key))
.collect::<Vec<_>>();
let missing_offset_indexes = offset_index_mask
.column_chunk_indices(num_row_groups, num_columns)
.filter(|key| !self.offset_indexes.contains_key(key))
.collect::<Vec<_>>();

println!(
" cache misses: {} column indexes, {} offset indexes",
missing_column_indexes.len(),
missing_offset_indexes.len()
);

if !missing_column_indexes.is_empty() || !missing_offset_indexes.is_empty() {
// ColumnChunkMask describes a Cartesian product. This produces the smallest such mask
// covering the misses and may therefore decode additional entries for irregular miss
// sets. Those additional entries are useful cache population and are retained below.
let column_miss_mask = covering_mask(&missing_column_indexes);
let offset_miss_mask = covering_mask(&missing_offset_indexes);
let reader = ParquetMetaDataReader::new_with_metadata(metadata)
.with_column_index_policy(PageIndexPolicy::Optional)
.with_offset_index_policy(PageIndexPolicy::Optional)
.with_column_index_mask(column_miss_mask)
.with_offset_index_mask(offset_miss_mask);

let (returned_metadata, page_index) = reader.read_page_index_and_finish(file_bytes)?;
metadata = returned_metadata;
if let Some(page_index) = page_index {
let (column_indexes, offset_indexes) = page_index.into_index_entries();
self.column_indexes
.extend(column_indexes.map(|(key, index)| (key, Arc::new(index))));
self.offset_indexes
.extend(offset_indexes.map(|(key, index)| (key, Arc::new(index))));
}
}

// Build an immutable provider containing only handles needed by this query.
let column_indexes = column_index_mask
.column_chunk_indices(num_row_groups, num_columns)
.filter_map(|key| {
self.column_indexes
.get(&key)
.map(|index| (key, Arc::clone(index)))
})
.collect();
let offset_indexes = offset_index_mask
.column_chunk_indices(num_row_groups, num_columns)
.filter_map(|key| {
self.offset_indexes
.get(&key)
.map(|index| (key, Arc::clone(index)))
})
.collect();

Ok((
metadata,
CachedPageIndexProvider {
column_indexes,
offset_indexes,
},
))
}
}

/// Returns the smallest Cartesian mask covering `chunks`.
fn covering_mask(chunks: &[ColumnChunk]) -> ColumnChunkMask {
if chunks.is_empty() {
return ColumnChunkMask::none();
}

let row_groups = chunks.iter().map(|&(row_group, _)| row_group);
let columns = chunks.iter().map(|&(_, column)| column);
ColumnChunkMask::row_groups_and_columns(
row_groups.collect::<BTreeSet<_>>(),
columns.collect::<BTreeSet<_>>(),
)
}

/// Reads footer metadata into [`ParquetMetaData`] without decoding page indexes.
fn decode_footer(file_bytes: &Bytes) -> Result<ParquetMetaData> {
ParquetMetaDataReader::new()
.with_page_index_policy(PageIndexPolicy::Skip)
.parse_and_finish(file_bytes)
}

fn main() -> Result<()> {
let file_bytes = create_sample_file()?;
let cached_metadata = decode_footer(&file_bytes)?;
let mut cache = PageIndexCache::default();

// Query 1 filters column 0 and projects columns 0, 1, and 4 in row group 0.
let column_mask = ColumnChunkMask::row_groups_and_columns([0], [0]);
let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 1, 4]);
println!("Query 1:");
let (cached_metadata, provider) =
cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?;
assert_query_indexes(&provider, 0, &[0], &[0, 1, 4]);

// Attach the query-specific provider to a cheap clone of the cached footer metadata.
let query_metadata = cached_metadata
.clone()
.into_builder()
.set_page_index(Some(Arc::new(provider)))
.build();
assert!(query_metadata.page_index().is_some());

// Query 2 reuses the column index and offset indexes for columns 0 and 4. Only the offset
// index for column 2 is decoded and added to the cache.
let offset_mask = ColumnChunkMask::row_groups_and_columns([0], [0, 2, 4]);
println!("Query 2:");
let (_cached_metadata, provider) =
cache.page_index_for_query(cached_metadata, &file_bytes, &column_mask, &offset_mask)?;
assert_query_indexes(&provider, 0, &[0], &[0, 2, 4]);

println!(
"Cache now contains {} column indexes and {} offset indexes",
cache.column_indexes.len(),
cache.offset_indexes.len()
);
Ok(())
}

fn assert_query_indexes(
page_index: &dyn PageIndexProvider,
row_group_idx: usize,
column_indexes: &[usize],
offset_indexes: &[usize],
) {
for column_idx in 0..6 {
assert_eq!(
page_index.column_index(row_group_idx, column_idx).is_some(),
column_indexes.contains(&column_idx)
);
assert_eq!(
page_index.offset_index(row_group_idx, column_idx).is_some(),
offset_indexes.contains(&column_idx)
);
}
}

fn create_sample_file() -> Result<Bytes> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("value", DataType::Int32, false),
Field::new("name", DataType::Utf8, false),
Field::new("score", DataType::Int32, false),
Field::new("category", DataType::Utf8, false),
Field::new("amount", DataType::Int32, false),
]));
let props = WriterProperties::builder()
.set_statistics_enabled(EnabledStatistics::Page)
.set_data_page_size_limit(100)
.set_write_batch_size(10)
.build();
let mut buffer = Vec::new();
let mut writer = ArrowWriter::try_new(&mut buffer, Arc::clone(&schema), Some(props))?;

for row_group in 0..3 {
for batch_num in 0..5 {
let offset = row_group * 50 + batch_num * 10;
let values = (offset..offset + 10).collect::<Vec<i32>>();
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(values.clone())),
Arc::new(Int32Array::from(
values.iter().map(|value| value * 2).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
values
.iter()
.map(|value| format!("name{value}"))
.collect::<Vec<_>>(),
)),
Arc::new(Int32Array::from(
values.iter().map(|value| value * 3).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
values
.iter()
.map(|value| if value % 2 == 0 { "even" } else { "odd" })
.collect::<Vec<_>>(),
)),
Arc::new(Int32Array::from(
values.iter().map(|value| value * 4).collect::<Vec<_>>(),
)),
],
)?;
writer.write(&batch)?;
}
writer.flush()?;
}
writer.close()?;
Ok(Bytes::from(buffer))
}
6 changes: 6 additions & 0 deletions parquet/src/arrow/arrow_reader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,9 @@ impl ArrowReaderOptions {
/// honored by loading APIs such as [`ArrowReaderMetadata::load`];
/// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes.
///
/// See [`PageIndexProvider`] for more details on customizing page index loading.
///
/// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider
/// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self {
self.column_index_mask = mask;
Expand All @@ -797,6 +800,9 @@ impl ArrowReaderOptions {
/// include those columns in addition to projected columns. The same loading and policy
/// qualifications as [`Self::with_column_index_mask`] apply.
///
/// See [`PageIndexProvider`] for more details on customizing page index loading.
///
/// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider
/// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self {
self.offset_index_mask = mask;
Expand Down
2 changes: 1 addition & 1 deletion parquet/src/arrow/async_reader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1175,7 +1175,7 @@ mod tests {
.await
.unwrap();

let page_index = PageIndex::new(None, Some(vec![]));
let page_index = PageIndex::try_new(None, Some(vec![])).unwrap();
metadata.set_page_index(Some(Arc::new(page_index)));
let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required);
let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
Expand Down
4 changes: 3 additions & 1 deletion parquet/src/arrow/push_decoder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3128,7 +3128,9 @@ mod test {
let last = offset_index.page_locations.last_mut().unwrap();
last.compressed_page_size += 1;
}
page_index.put_offset_index(offset_index, row_group, column);
page_index
.try_put_offset_index(offset_index, row_group, column)
.unwrap();
}
}
let metadata = metadata
Expand Down
Loading
Loading