From 0638b8715900d033907b5e745580d1afa4924069 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 9 Sep 2026 13:31:12 -0700 Subject: [PATCH 1/9] add test of using custom page index provider with row selection --- .../custom_page_index_provider.rs | 546 ++++++++++++++++++ parquet/tests/arrow_reader/mod.rs | 1 + 2 files changed, 547 insertions(+) create mode 100644 parquet/tests/arrow_reader/custom_page_index_provider.rs diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs new file mode 100644 index 000000000000..00e008a13ac6 --- /dev/null +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for custom PageIndexProvider implementation. +//! +//! This test validates reading with a custom PageIndexProvider through the +//! actual read path, ensuring that: +//! - Columns with page indexes use offset-index-driven fetch +//! - Columns without page indexes fall back to whole-column-chunk fetching +//! - Correct results are returned in both cases + +use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, + RowSelectionPolicy, +}; +use parquet::file::metadata::page_index::PageIndexProvider; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; +use parquet::file::page_index::column_index::ColumnIndexMetaData; +use parquet::file::page_index::index_reader::{decode_column_index, decode_offset_index}; +use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::sync::Arc; +use tempfile::NamedTempFile; + +/// A custom PageIndexProvider that only stores indexes for a subset of columns +/// +/// This provider mimics a real-world scenario where an application has loaded +/// page indexes selectively based on query predicates and projections. +#[derive(Debug, Clone)] +struct SelectivePageIndexProvider { + metadata: ParquetMetaData, + file_bytes: Bytes, + // indexes are accessed first by row_group index and then by column index + column_indexes: Option>>, + offset_indexes: Option>>, +} + +impl SelectivePageIndexProvider { + fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + Self { + metadata, + file_bytes, + column_indexes: None, + offset_indexes: None, + } + } + + /// Fetch and parse the column index for the given row group and column + fn fetch_column_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.column_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_column_index(&idx_bytes, column.column_type())?; + e.insert(idx); + } + } + Ok(()) + } + + /// Fetch and parse the offset index for the given row group and column + fn fetch_offset_index( + &mut self, + row_group_idx: usize, + column_idx: usize, + ) -> parquet::errors::Result<()> { + let map = self.offset_indexes.get_or_insert_with(HashMap::new); + let rg = map.entry(row_group_idx).or_default(); + if let Entry::Vacant(e) = rg.entry(column_idx) { + let column = self.metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = self + .file_bytes + .slice(range.start as usize..range.end as usize); + let idx = decode_offset_index(&idx_bytes)?; + e.insert(idx); + } + } + Ok(()) + } +} + +impl PageIndexProvider for SelectivePageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.offset_indexes.is_some() + } + + fn has_column_indexes(&self) -> bool { + self.column_indexes.is_some() + } + + fn column_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&ColumnIndexMetaData> { + self.column_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn offset_index( + &self, + row_group_idx: usize, + column_idx: usize, + ) -> Option<&OffsetIndexMetaData> { + self.offset_indexes + .as_ref()? + .get(&row_group_idx)? + .get(&column_idx) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// Create a test parquet file with multiple row groups and multiple pages per column +fn create_test_file() -> NamedTempFile { + let temp_file = tempfile::Builder::new() + .prefix("custom_page_index_test") + .suffix(".parquet") + .tempfile() + .expect("tempfile creation"); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + + // Create small pages and small row groups for testing + let props = WriterProperties::builder() + .set_statistics_enabled(EnabledStatistics::Page) + .set_data_page_row_count_limit(10) // Small pages for multiple pages per column + .set_write_batch_size(10) + .set_max_row_group_row_count(Some(50)) // Small row groups + .build(); + + let file = temp_file.reopen().unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); + + // Write 3 row groups with 50 rows each (150 rows total) + for row_group in 0..3 { + for batch_num in 0..5 { + let offset = (row_group * 50) + (batch_num * 10); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (offset..offset + 10).collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + (offset..offset + 10) + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + (offset..offset + 10).map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap(); + writer.write(&batch).unwrap(); + } + writer.flush().unwrap(); + } + + // check that file has properties we wanted (3 row groups, 5 pages per chunk) + let metadata = writer.close().unwrap(); + assert_eq!(metadata.num_row_groups(), 3); + assert!(metadata.page_index().is_some()); + let page_index = metadata.page_index().unwrap(); + for row_group_idx in 0..3 { + for column_idx in 0..4 { + assert!(page_index.column_index(row_group_idx, column_idx).is_some()); + assert!(page_index.offset_index(row_group_idx, column_idx).is_some()); + assert!( + page_index + .num_data_pages(row_group_idx, column_idx) + .is_some_and(|n| n == 5) + ); + } + } + temp_file +} + +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + provider.fetch_column_index(0, 1).unwrap(); + provider.fetch_offset_index(0, 1).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0).unwrap(); + provider.fetch_offset_index(2, 0).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); + + // Success! The test validates that: + // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping + // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data + // 3. For row group 1, no columns had indexes but all returned correct data + // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data + // 5. The reader correctly handled the mix of indexed and non-indexed columns +} + +#[test] +fn test_read_all_columns_with_partial_indexes() { + // This test focuses on verifying that columns without page indexes + // fall back to whole-column-chunk fetching and return correct results + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Load metadata without page indexes + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes ONLY for column 0 in all row groups + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + + for row_group_idx in 0..metadata.num_row_groups() { + provider.fetch_column_index(row_group_idx, 0).unwrap(); + provider.fetch_offset_index(row_group_idx, 0).unwrap(); + } + + // Install provider + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read with row selection + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Select rows 10-20 from row group 0 + let selection = RowSelection::from(vec![ + parquet::arrow::arrow_reader::RowSelector::skip(10), + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let batches: Vec = reader.collect::, _>>().unwrap(); + + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + + // Verify all columns have correct data + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + let id = ids.value(i); + assert_eq!(values.value(i), id * 2); + assert_eq!(names.value(i), format!("name{id}")); + assert_eq!(scores.value(i), id * 3); + } + } +} + +#[test] +fn test_custom_provider_with_no_row_selection() { + // Test that reading without RowSelection also works correctly + // with a custom page index provider + + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().clone(); + + // Create provider with indexes for first column only + let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); + provider.fetch_column_index(0, 0).unwrap(); + provider.fetch_offset_index(0, 0).unwrap(); + + let mut metadata_builder = metadata.as_ref().clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Create ArrowReaderMetadata with the custom page index + let arrow_metadata = + ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) + .unwrap(); + + // Read all data + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + let reader = builder.build().unwrap(); + + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Should read all 150 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 150); + + // Verify data integrity + let mut row_count: i32 = 0; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..batch.num_rows() { + assert_eq!(ids.value(i), row_count); + assert_eq!(values.value(i), row_count * 2); + assert_eq!(names.value(i), format!("name{row_count}")); + assert_eq!(scores.value(i), row_count * 3); + row_count += 1; + } + } +} diff --git a/parquet/tests/arrow_reader/mod.rs b/parquet/tests/arrow_reader/mod.rs index c7c77e0a444c..6c47a00b9b3d 100644 --- a/parquet/tests/arrow_reader/mod.rs +++ b/parquet/tests/arrow_reader/mod.rs @@ -42,6 +42,7 @@ mod bad_data; mod bloom_filter; #[cfg(feature = "crc")] mod checksum; +mod custom_page_index_provider; mod int96_stats_roundtrip; mod invalid_utf8; mod io; From da1ff75a0409616fc2d443776176e1efd04cf22c Mon Sep 17 00:00:00 2001 From: seidl Date: Fri, 18 Sep 2026 09:57:22 -0700 Subject: [PATCH 2/9] refactor and add usage stats --- .../custom_page_index_provider.rs | 582 ++++++++---------- 1 file changed, 248 insertions(+), 334 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 00e008a13ac6..2736b7bc35e2 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -41,41 +41,260 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fs::File; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use tempfile::NamedTempFile; +#[test] +fn test_read_with_custom_page_index_provider() { + // Step 1: Write a parquet file with page indexes + let temp_file = create_test_file(); + let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + + // Step 2: Load metadata WITHOUT page indexes initially + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + file, + ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), + ) + .unwrap(); + + let metadata = builder.metadata().as_ref(); + + // Verify we have multiple row groups + assert_eq!(metadata.num_row_groups(), 3); + + // Step 3: Create a custom provider and populate it selectively + // Simulate a scenario where: + // - We only populate indexes for row groups 0 and 2 (skipping row group 1) + // - For row group 0: populate column 0 (id) and column 1 (value) + // - For row group 2: populate column 0 (id) only + let mut provider = SelectivePageIndexProvider::new(file_bytes); + + // Populate indexes for row group 0, columns 0 and 1 + provider.fetch_column_index(0, 0, metadata).unwrap(); + provider.fetch_offset_index(0, 0, metadata).unwrap(); + provider.fetch_column_index(0, 1, metadata).unwrap(); + provider.fetch_offset_index(0, 1, metadata).unwrap(); + + // Populate indexes for row group 2, column 0 only + provider.fetch_column_index(2, 0, metadata).unwrap(); + provider.fetch_offset_index(2, 0, metadata).unwrap(); + + // Verify the provider has the expected indexes + assert!(provider.column_index(0, 0).is_some()); + assert!(provider.offset_index(0, 0).is_some()); + assert!(provider.column_index(0, 1).is_some()); + assert!(provider.offset_index(0, 1).is_some()); + assert!(provider.column_index(0, 2).is_none()); // Not populated + assert!(provider.column_index(2, 0).is_some()); + assert!(provider.column_index(2, 1).is_none()); // Not populated + + // Reset statistics so validation checks don't affect the final counts + provider.reset_stats(); + + // Step 4: Install the custom provider into metadata + let mut metadata_builder = metadata.clone().into_builder(); + metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_with_custom_index = Arc::new(metadata_builder.build()); + + // Step 5: Create ArrowReaderMetadata with the custom page index + let arrow_metadata = ArrowReaderMetadata::try_new( + metadata_with_custom_index.clone(), + ArrowReaderOptions::default(), + ) + .unwrap(); + + // Step 6: Read data with RowSelection that triggers page skipping + let file = File::open(temp_file.path()).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); + + // Create a RowSelection that: + // - Selects rows 20-30 (in row group 0) + // - Selects rows 60-70 (in row group 1) + // - Selects rows 120-130 (in row group 2) + let selection = RowSelection::from(vec![ + // Skip first 20 rows + parquet::arrow::arrow_reader::RowSelector::skip(20), + // Select rows 20-30 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 30-60 + parquet::arrow::arrow_reader::RowSelector::skip(30), + // Select rows 60-70 + parquet::arrow::arrow_reader::RowSelector::select(10), + // Skip rows 70-120 + parquet::arrow::arrow_reader::RowSelector::skip(50), + // Select rows 120-130 + parquet::arrow::arrow_reader::RowSelector::select(10), + ]); + + let reader = builder + .with_row_selection(selection) + // make sure we're actually using the page index and not using a mask + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + + // Collect all batches + let batches: Vec = reader.collect::, _>>().unwrap(); + + // Step 7: Check statistics + // Note: We need to get the provider reference from the metadata to check stats + let provider_ref = metadata_with_custom_index + .page_index() + .unwrap() + .as_any() + .downcast_ref::() + .expect("Expected SelectivePageIndexProvider"); + + // Check that the custom provider was used + let (hits, misses) = provider_ref.stats(); + assert!(hits > 0, "provider should have hits"); + assert!(misses > 0, "provider should have misses"); + + // Since we didn't use predicates, the column index should be untouched + let (col_hits, col_misses) = provider_ref.column_index_stats(); + assert_eq!(col_hits, 0, "column index should not be used"); + assert_eq!(col_misses, 0, "column index should not be used"); + + let (off_hits, off_misses) = provider_ref.offset_index_stats(); + assert!(off_hits > 0, "offset index should have hits"); + assert!(off_misses > 0, "offset index should have misses"); + + // Step 8: Verify correct results + // We expect 30 rows total: 10 from each selected range + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30); + + // Verify the actual data values + let mut all_ids = Vec::new(); + let mut all_values = Vec::new(); + let mut all_names = Vec::new(); + let mut all_scores = Vec::new(); + + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + + all_ids.extend(ids.iter().flatten()); + all_values.extend(values.iter().flatten()); + all_names.extend(names.iter().flatten().map(|s| s.to_owned())); + all_scores.extend(scores.iter().flatten()); + } + + // Expected values for the selected rows: + // rows 20-30: id=20..30, value=40..60, score=60..90 + // rows 60-70: id=60..70, value=120..140, score=180..210 + // rows 120-130: id=120..130, value=240..260, score=360..390 + let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); + let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); + let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); + + assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); + assert_eq!( + all_values, expected_values, + "Values don't match expected values" + ); + assert_eq!( + all_scores, expected_scores, + "Scores don't match expected values" + ); + assert_eq!( + all_names, expected_names, + "Names don't match expected values" + ); +} + /// A custom PageIndexProvider that only stores indexes for a subset of columns /// /// This provider mimics a real-world scenario where an application has loaded /// page indexes selectively based on query predicates and projections. -#[derive(Debug, Clone)] +#[derive(Debug)] struct SelectivePageIndexProvider { - metadata: ParquetMetaData, file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index column_indexes: Option>>, offset_indexes: Option>>, + // Usage statistics + column_index_hits: Arc, + column_index_misses: Arc, + offset_index_hits: Arc, + offset_index_misses: Arc, } impl SelectivePageIndexProvider { - fn new(metadata: ParquetMetaData, file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes) -> Self { Self { - metadata, file_bytes, column_indexes: None, offset_indexes: None, + column_index_hits: Arc::new(AtomicUsize::new(0)), + column_index_misses: Arc::new(AtomicUsize::new(0)), + offset_index_hits: Arc::new(AtomicUsize::new(0)), + offset_index_misses: Arc::new(AtomicUsize::new(0)), } } + /// Get statistics about column index usage + pub fn column_index_stats(&self) -> (usize, usize) { + ( + self.column_index_hits.load(Ordering::Relaxed), + self.column_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get statistics about offset index usage + pub fn offset_index_stats(&self) -> (usize, usize) { + ( + self.offset_index_hits.load(Ordering::Relaxed), + self.offset_index_misses.load(Ordering::Relaxed), + ) + } + + /// Get combined statistics as (hits, misses) for both index types + pub fn stats(&self) -> (usize, usize) { + let (col_hits, col_misses) = self.column_index_stats(); + let (off_hits, off_misses) = self.offset_index_stats(); + (col_hits + off_hits, col_misses + off_misses) + } + + /// Reset all statistics counters to zero + pub fn reset_stats(&self) { + self.column_index_hits.store(0, Ordering::Relaxed); + self.column_index_misses.store(0, Ordering::Relaxed); + self.offset_index_hits.store(0, Ordering::Relaxed); + self.offset_index_misses.store(0, Ordering::Relaxed); + } + /// Fetch and parse the column index for the given row group and column fn fetch_column_index( &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.column_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.column_index_range(); if let Some(range) = range { let idx_bytes = self @@ -93,11 +312,12 @@ impl SelectivePageIndexProvider { &mut self, row_group_idx: usize, column_idx: usize, + metadata: &ParquetMetaData, ) -> parquet::errors::Result<()> { let map = self.offset_indexes.get_or_insert_with(HashMap::new); let rg = map.entry(row_group_idx).or_default(); if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = self.metadata.row_group(row_group_idx).column(column_idx); + let column = metadata.row_group(row_group_idx).column(column_idx); let range = column.offset_index_range(); if let Some(range) = range { let idx_bytes = self @@ -125,10 +345,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&ColumnIndexMetaData> { - self.column_indexes + let result = self + .column_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.column_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.column_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn offset_index( @@ -136,10 +365,19 @@ impl PageIndexProvider for SelectivePageIndexProvider { row_group_idx: usize, column_idx: usize, ) -> Option<&OffsetIndexMetaData> { - self.offset_indexes + let result = self + .offset_indexes .as_ref()? .get(&row_group_idx)? - .get(&column_idx) + .get(&column_idx); + + if result.is_some() { + self.offset_index_hits.fetch_add(1, Ordering::Relaxed); + } else { + self.offset_index_misses.fetch_add(1, Ordering::Relaxed); + } + + result } fn as_any(&self) -> &dyn std::any::Any { @@ -220,327 +458,3 @@ fn create_test_file() -> NamedTempFile { } temp_file } - -#[test] -fn test_read_with_custom_page_index_provider() { - // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Verify we have multiple row groups - assert_eq!(metadata.num_row_groups(), 3); - - // Step 3: Create a custom provider and populate it selectively - // Simulate a scenario where: - // - We only populate indexes for row groups 0 and 2 (skipping row group 1) - // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - provider.fetch_column_index(0, 1).unwrap(); - provider.fetch_offset_index(0, 1).unwrap(); - - // Populate indexes for row group 2, column 0 only - provider.fetch_column_index(2, 0).unwrap(); - provider.fetch_offset_index(2, 0).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - - // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Step 5: Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Create a RowSelection that: - // - Selects rows 20-30 (in row group 0) - // - Selects rows 60-70 (in row group 1) - // - Selects rows 120-130 (in row group 2) - let selection = RowSelection::from(vec![ - // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), - // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), - // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), - // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), - // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Step 7: Verify correct results - // We expect 30 rows total: 10 from each selected range - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 30); - - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - - // Expected values for the selected rows: - // rows 20-30: id=20..30, value=40..60, score=60..90 - // rows 60-70: id=60..70, value=120..140, score=180..210 - // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); - assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" - ); - - // Success! The test validates that: - // 1. For row group 0, columns 0 and 1 had page indexes and could use page-level skipping - // 2. For row group 0, columns 2 and 3 had no indexes but still returned correct data - // 3. For row group 1, no columns had indexes but all returned correct data - // 4. For row group 2, column 0 had indexes and others didn't, but all returned correct data - // 5. The reader correctly handled the mix of indexed and non-indexed columns -} - -#[test] -fn test_read_all_columns_with_partial_indexes() { - // This test focuses on verifying that columns without page indexes - // fall back to whole-column-chunk fetching and return correct results - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - // Load metadata without page indexes - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes ONLY for column 0 in all row groups - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - - for row_group_idx in 0..metadata.num_row_groups() { - provider.fetch_column_index(row_group_idx, 0).unwrap(); - provider.fetch_offset_index(row_group_idx, 0).unwrap(); - } - - // Install provider - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read with row selection - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - // Select rows 10-20 from row group 0 - let selection = RowSelection::from(vec![ - parquet::arrow::arrow_reader::RowSelector::skip(10), - parquet::arrow::arrow_reader::RowSelector::select(10), - ]); - - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - let batches: Vec = reader.collect::, _>>().unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 10); - - // Verify all columns have correct data - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - let id = ids.value(i); - assert_eq!(values.value(i), id * 2); - assert_eq!(names.value(i), format!("name{id}")); - assert_eq!(scores.value(i), id * 3); - } - } -} - -#[test] -fn test_custom_provider_with_no_row_selection() { - // Test that reading without RowSelection also works correctly - // with a custom page index provider - - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); - - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, - ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), - ) - .unwrap(); - - let metadata = builder.metadata().clone(); - - // Create provider with indexes for first column only - let mut provider = SelectivePageIndexProvider::new(metadata.as_ref().clone(), file_bytes); - provider.fetch_column_index(0, 0).unwrap(); - provider.fetch_offset_index(0, 0).unwrap(); - - let mut metadata_builder = metadata.as_ref().clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); - let metadata_with_custom_index = Arc::new(metadata_builder.build()); - - // Create ArrowReaderMetadata with the custom page index - let arrow_metadata = - ArrowReaderMetadata::try_new(metadata_with_custom_index, ArrowReaderOptions::default()) - .unwrap(); - - // Read all data - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); - - let reader = builder.build().unwrap(); - - let batches: Vec = reader.collect::, _>>().unwrap(); - - // Should read all 150 rows - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 150); - - // Verify data integrity - let mut row_count: i32 = 0; - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - for i in 0..batch.num_rows() { - assert_eq!(ids.value(i), row_count); - assert_eq!(values.value(i), row_count * 2); - assert_eq!(names.value(i), format!("name{row_count}")); - assert_eq!(scores.value(i), row_count * 3); - row_count += 1; - } - } -} From b7b529de8a2f95b66110092eb567cbc3930a7fd2 Mon Sep 17 00:00:00 2001 From: seidl Date: Mon, 21 Sep 2026 09:42:51 -0700 Subject: [PATCH 3/9] more miri friendly --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 2736b7bc35e2..09b41df1155f 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -408,7 +408,7 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = temp_file.reopen().unwrap(); + let file = File::create(temp_file.path()).unwrap(); let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); // Write 3 row groups with 50 rows each (150 rows total) From 0969c2329209a7bab57a51cabb93415d7968ff92 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 23 Sep 2026 09:00:52 -0700 Subject: [PATCH 4/9] apply suggestion from review Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 09b41df1155f..f5b5bd32cd2a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -67,7 +67,7 @@ fn test_read_with_custom_page_index_provider() { // Simulate a scenario where: // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) - // - For row group 2: populate column 0 (id) only + // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap let mut provider = SelectivePageIndexProvider::new(file_bytes); // Populate indexes for row group 0, columns 0 and 1 @@ -76,9 +76,11 @@ fn test_read_with_custom_page_index_provider() { provider.fetch_column_index(0, 1, metadata).unwrap(); provider.fetch_offset_index(0, 1, metadata).unwrap(); - // Populate indexes for row group 2, column 0 only + // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) provider.fetch_column_index(2, 0, metadata).unwrap(); provider.fetch_offset_index(2, 0, metadata).unwrap(); + provider.fetch_column_index(2, 2, metadata).unwrap(); + provider.fetch_offset_index(2, 2, metadata).unwrap(); // Verify the provider has the expected indexes assert!(provider.column_index(0, 0).is_some()); From d93796e9b51004d54055af58717f7415ab717989 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 09:06:13 -0700 Subject: [PATCH 5/9] add another assert --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index f5b5bd32cd2a..ca5359f08e5b 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -90,6 +90,7 @@ fn test_read_with_custom_page_index_provider() { assert!(provider.column_index(0, 2).is_none()); // Not populated assert!(provider.column_index(2, 0).is_some()); assert!(provider.column_index(2, 1).is_none()); // Not populated + assert!(provider.column_index(2, 2).is_some()); // Reset statistics so validation checks don't affect the final counts provider.reset_stats(); From 2489741dacebede25e2344b60b27a18b48820b67 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:45:29 -0700 Subject: [PATCH 6/9] implement review suggestions --- .../custom_page_index_provider.rs | 414 +++++++++--------- 1 file changed, 214 insertions(+), 200 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index ca5359f08e5b..6c93193f91c9 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -23,14 +23,17 @@ //! - Columns without page indexes fall back to whole-column-chunk fetching //! - Correct results are returned in both cases -use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; +use arrow::compute::concat_batches; +use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use bytes::Bytes; +use parquet::DecodeResult; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelectionPolicy, }; +use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::file::metadata::page_index::PageIndexProvider; use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use parquet::file::page_index::column_index::ColumnIndexMetaData; @@ -38,22 +41,104 @@ use parquet::file::page_index::index_reader::{decode_column_index, decode_offset use parquet::file::page_index::offset_index::OffsetIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::fs::File; +use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::NamedTempFile; #[test] -fn test_read_with_custom_page_index_provider() { +fn test_read_with_custom_page_index_provider_sync() { + run_test(Reader::Sync); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_push() { + run_test(Reader::Push); +} + +// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged +#[cfg(feature = "async")] +#[test] +#[should_panic] +fn test_read_with_custom_page_index_provider_async() { + run_test(Reader::Async); +} + +#[derive(Debug, Clone, Copy)] +enum Reader { + Sync, + Push, + #[cfg(feature = "async")] + Async, +} + +/// Read `file` with `reader`, using `metadata` and `selection` +fn read( + reader: Reader, + file: Bytes, + metadata: ArrowReaderMetadata, + selection: RowSelection, + requested: &mut Vec>, +) -> Vec { + match reader { + Reader::Sync => ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .collect::>() + .unwrap(), + Reader::Push => { + let mut decoder = ParquetPushDecoderBuilder::new_with_metadata(metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap(); + let mut batches = vec![]; + loop { + match decoder.try_decode().unwrap() { + DecodeResult::NeedsData(ranges) => { + requested.extend(ranges.iter().cloned()); + let data = ranges + .iter() + .map(|r| file.slice(r.start as usize..r.end as usize)) + .collect(); + decoder.push_ranges(ranges, data).unwrap(); + } + DecodeResult::Data(batch) => batches.push(batch), + DecodeResult::Finished => return batches, + } + } + } + #[cfg(feature = "async")] + Reader::Async => { + use futures::TryStreamExt; + let input = std::io::Cursor::new(file.to_vec()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + parquet::arrow::ParquetRecordBatchStreamBuilder::new_with_metadata(input, metadata) + .with_row_selection(selection) + .with_row_selection_policy(RowSelectionPolicy::Selectors) + .build() + .unwrap() + .try_collect() + .await + .unwrap() + }) + } + } +} + +fn run_test(reader: Reader) { // Step 1: Write a parquet file with page indexes - let temp_file = create_test_file(); - let file_bytes = Bytes::from(std::fs::read(temp_file.path()).unwrap()); + let file_bytes = create_test_file(); // Step 2: Load metadata WITHOUT page indexes initially - let file = File::open(temp_file.path()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( - file, + file_bytes.clone(), ArrowReaderOptions::default().with_page_index_policy(PageIndexPolicy::Skip), ) .unwrap(); @@ -68,32 +153,11 @@ fn test_read_with_custom_page_index_provider() { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap - let mut provider = SelectivePageIndexProvider::new(file_bytes); - - // Populate indexes for row group 0, columns 0 and 1 - provider.fetch_column_index(0, 0, metadata).unwrap(); - provider.fetch_offset_index(0, 0, metadata).unwrap(); - provider.fetch_column_index(0, 1, metadata).unwrap(); - provider.fetch_offset_index(0, 1, metadata).unwrap(); - - // Populate indexes for row group 2, columns 0 and 2 (column 1 has no index) - provider.fetch_column_index(2, 0, metadata).unwrap(); - provider.fetch_offset_index(2, 0, metadata).unwrap(); - provider.fetch_column_index(2, 2, metadata).unwrap(); - provider.fetch_offset_index(2, 2, metadata).unwrap(); - - // Verify the provider has the expected indexes - assert!(provider.column_index(0, 0).is_some()); - assert!(provider.offset_index(0, 0).is_some()); - assert!(provider.column_index(0, 1).is_some()); - assert!(provider.offset_index(0, 1).is_some()); - assert!(provider.column_index(0, 2).is_none()); // Not populated - assert!(provider.column_index(2, 0).is_some()); - assert!(provider.column_index(2, 1).is_none()); // Not populated - assert!(provider.column_index(2, 2).is_some()); - - // Reset statistics so validation checks don't affect the final counts - provider.reset_stats(); + let provider = SelectivePageIndexProvider::new( + file_bytes.clone(), + metadata, + &[(0, 0), (0, 1), (2, 0), (2, 2)], + ); // Step 4: Install the custom provider into metadata let mut metadata_builder = metadata.clone().into_builder(); @@ -108,8 +172,6 @@ fn test_read_with_custom_page_index_provider() { .unwrap(); // Step 6: Read data with RowSelection that triggers page skipping - let file = File::open(temp_file.path()).unwrap(); - let builder = ParquetRecordBatchReaderBuilder::new_with_metadata(file, arrow_metadata); // Create a RowSelection that: // - Selects rows 20-30 (in row group 0) @@ -130,17 +192,33 @@ fn test_read_with_custom_page_index_provider() { parquet::arrow::arrow_reader::RowSelector::select(10), ]); - let reader = builder - .with_row_selection(selection) - // make sure we're actually using the page index and not using a mask - .with_row_selection_policy(RowSelectionPolicy::Selectors) - .build() - .unwrap(); - // Collect all batches - let batches: Vec = reader.collect::, _>>().unwrap(); + let mut requested = Vec::>::new(); + let batches = read( + reader, + file_bytes.clone(), + arrow_metadata, + selection, + &mut requested, + ); // Step 7: Check statistics + + // only the push test will populate requested + if matches!(reader, Reader::Push) { + let fetched = |rg: usize, col: usize| -> u64 { + let (start, len) = metadata.row_group(rg).column(col).byte_range(); + requested + .iter() + .filter(|r| r.start >= start && r.end <= start + len) + .map(|r| r.end - r.start) + .sum() + }; + let chunk_len = |rg: usize, col: usize| metadata.row_group(rg).column(col).byte_range().1; + assert!(fetched(0, 0) < chunk_len(0, 0)); // offset index: selected pages only + assert_eq!(fetched(0, 2), chunk_len(0, 2)); // no offset index: whole chunk + } + // Note: We need to get the provider reference from the metadata to check stats let provider_ref = metadata_with_custom_index .page_index() @@ -168,61 +246,15 @@ fn test_read_with_custom_page_index_provider() { let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 30); - // Verify the actual data values - let mut all_ids = Vec::new(); - let mut all_values = Vec::new(); - let mut all_names = Vec::new(); - let mut all_scores = Vec::new(); - - for batch in batches { - let ids = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let names = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - let scores = batch - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - - all_ids.extend(ids.iter().flatten()); - all_values.extend(values.iter().flatten()); - all_names.extend(names.iter().flatten().map(|s| s.to_owned())); - all_scores.extend(scores.iter().flatten()); - } - // Expected values for the selected rows: // rows 20-30: id=20..30, value=40..60, score=60..90 // rows 60-70: id=60..70, value=120..140, score=180..210 // rows 120-130: id=120..130, value=240..260, score=360..390 - let expected_ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); - let expected_values: Vec = expected_ids.iter().map(|x| x * 2).collect(); - let expected_scores: Vec = expected_ids.iter().map(|x| x * 3).collect(); - let expected_names: Vec = expected_ids.iter().map(|x| format!("name{x}")).collect(); - - assert_eq!(all_ids, expected_ids, "IDs don't match expected values"); - assert_eq!( - all_values, expected_values, - "Values don't match expected values" - ); + let ids: Vec = (20..30).chain(60..70).chain(120..130).collect(); + let expected = make_batch(&ids); assert_eq!( - all_scores, expected_scores, - "Scores don't match expected values" - ); - assert_eq!( - all_names, expected_names, - "Names don't match expected values" + concat_batches(&expected.schema(), &batches).unwrap(), + expected ); } @@ -232,32 +264,46 @@ fn test_read_with_custom_page_index_provider() { /// page indexes selectively based on query predicates and projections. #[derive(Debug)] struct SelectivePageIndexProvider { - file_bytes: Bytes, // indexes are accessed first by row_group index and then by column index - column_indexes: Option>>, - offset_indexes: Option>>, + column_indexes: Option>, + offset_indexes: Option>, // Usage statistics - column_index_hits: Arc, - column_index_misses: Arc, - offset_index_hits: Arc, - offset_index_misses: Arc, + column_index_hits: AtomicUsize, + column_index_misses: AtomicUsize, + offset_index_hits: AtomicUsize, + offset_index_misses: AtomicUsize, } impl SelectivePageIndexProvider { - fn new(file_bytes: Bytes) -> Self { + fn new(file_bytes: Bytes, metadata: &ParquetMetaData, chunks: &[(usize, usize)]) -> Self { + let mut column_indexes = HashMap::new(); + let mut offset_indexes = HashMap::new(); + for &(rg, col) in chunks { + column_indexes.insert( + (rg, col), + Self::fetch_column_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + offset_indexes.insert( + (rg, col), + Self::fetch_offset_index(rg, col, metadata, &file_bytes) + .unwrap() + .unwrap(), + ); + } Self { - file_bytes, - column_indexes: None, - offset_indexes: None, - column_index_hits: Arc::new(AtomicUsize::new(0)), - column_index_misses: Arc::new(AtomicUsize::new(0)), - offset_index_hits: Arc::new(AtomicUsize::new(0)), - offset_index_misses: Arc::new(AtomicUsize::new(0)), + column_indexes: Some(column_indexes), + offset_indexes: Some(offset_indexes), + column_index_hits: AtomicUsize::new(0), + column_index_misses: AtomicUsize::new(0), + offset_index_hits: AtomicUsize::new(0), + offset_index_misses: AtomicUsize::new(0), } } /// Get statistics about column index usage - pub fn column_index_stats(&self) -> (usize, usize) { + fn column_index_stats(&self) -> (usize, usize) { ( self.column_index_hits.load(Ordering::Relaxed), self.column_index_misses.load(Ordering::Relaxed), @@ -265,7 +311,7 @@ impl SelectivePageIndexProvider { } /// Get statistics about offset index usage - pub fn offset_index_stats(&self) -> (usize, usize) { + fn offset_index_stats(&self) -> (usize, usize) { ( self.offset_index_hits.load(Ordering::Relaxed), self.offset_index_misses.load(Ordering::Relaxed), @@ -273,64 +319,44 @@ impl SelectivePageIndexProvider { } /// Get combined statistics as (hits, misses) for both index types - pub fn stats(&self) -> (usize, usize) { + fn stats(&self) -> (usize, usize) { let (col_hits, col_misses) = self.column_index_stats(); let (off_hits, off_misses) = self.offset_index_stats(); (col_hits + off_hits, col_misses + off_misses) } - /// Reset all statistics counters to zero - pub fn reset_stats(&self) { - self.column_index_hits.store(0, Ordering::Relaxed); - self.column_index_misses.store(0, Ordering::Relaxed); - self.offset_index_hits.store(0, Ordering::Relaxed); - self.offset_index_misses.store(0, Ordering::Relaxed); - } - /// Fetch and parse the column index for the given row group and column fn fetch_column_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.column_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.column_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_column_index(&idx_bytes, column.column_type())?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.column_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_column_index(&idx_bytes, column.column_type())?)) + } else { + Ok(None) } - Ok(()) } /// Fetch and parse the offset index for the given row group and column fn fetch_offset_index( - &mut self, row_group_idx: usize, column_idx: usize, metadata: &ParquetMetaData, - ) -> parquet::errors::Result<()> { - let map = self.offset_indexes.get_or_insert_with(HashMap::new); - let rg = map.entry(row_group_idx).or_default(); - if let Entry::Vacant(e) = rg.entry(column_idx) { - let column = metadata.row_group(row_group_idx).column(column_idx); - let range = column.offset_index_range(); - if let Some(range) = range { - let idx_bytes = self - .file_bytes - .slice(range.start as usize..range.end as usize); - let idx = decode_offset_index(&idx_bytes)?; - e.insert(idx); - } + file_bytes: &Bytes, + ) -> parquet::errors::Result> { + let column = metadata.row_group(row_group_idx).column(column_idx); + let range = column.offset_index_range(); + if let Some(range) = range { + let idx_bytes = file_bytes.slice(range.start as usize..range.end as usize); + Ok(Some(decode_offset_index(&idx_bytes)?)) + } else { + Ok(None) } - Ok(()) } } @@ -351,8 +377,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .column_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.column_index_hits.fetch_add(1, Ordering::Relaxed); @@ -371,8 +396,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { let result = self .offset_indexes .as_ref()? - .get(&row_group_idx)? - .get(&column_idx); + .get(&(row_group_idx, column_idx)); if result.is_some() { self.offset_index_hits.fetch_add(1, Ordering::Relaxed); @@ -389,20 +413,7 @@ impl PageIndexProvider for SelectivePageIndexProvider { } /// Create a test parquet file with multiple row groups and multiple pages per column -fn create_test_file() -> NamedTempFile { - let temp_file = tempfile::Builder::new() - .prefix("custom_page_index_test") - .suffix(".parquet") - .tempfile() - .expect("tempfile creation"); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("value", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - Field::new("score", DataType::Int32, false), - ])); - +fn create_test_file() -> Bytes { // Create small pages and small row groups for testing let props = WriterProperties::builder() .set_statistics_enabled(EnabledStatistics::Page) @@ -411,37 +422,12 @@ fn create_test_file() -> NamedTempFile { .set_max_row_group_row_count(Some(50)) // Small row groups .build(); - let file = File::create(temp_file.path()).unwrap(); - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)).unwrap(); - // Write 3 row groups with 50 rows each (150 rows total) - for row_group in 0..3 { - for batch_num in 0..5 { - let offset = (row_group * 50) + (batch_num * 10); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from( - (offset..offset + 10).collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 2).collect::>(), - )), - Arc::new(StringArray::from( - (offset..offset + 10) - .map(|x| format!("name{x}")) - .collect::>(), - )), - Arc::new(Int32Array::from( - (offset..offset + 10).map(|x| x * 3).collect::>(), - )), - ], - ) - .unwrap(); - writer.write(&batch).unwrap(); - } - writer.flush().unwrap(); - } + let batch = make_batch(&(0..150).collect::>().as_slice()); + let mut buffer = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); // check that file has properties we wanted (3 row groups, 5 pages per chunk) let metadata = writer.close().unwrap(); @@ -459,5 +445,33 @@ fn create_test_file() -> NamedTempFile { ); } } - temp_file + Bytes::from(buffer) +} + +fn make_batch(ids: &[i32]) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Int32, false), + ])); + // Write 3 row groups with 50 rows each (150 rows total) + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 2).collect::>(), + )), + Arc::new(StringArray::from( + ids.iter() + .map(|x| format!("name{x}")) + .collect::>(), + )), + Arc::new(Int32Array::from( + ids.iter().map(|x| x * 3).collect::>(), + )), + ], + ) + .unwrap() } From 835170329bae2e3948ac900d676888e8484752af Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 14:56:14 -0700 Subject: [PATCH 7/9] clippy --- .../tests/arrow_reader/custom_page_index_provider.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 6c93193f91c9..8447146418f1 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -52,7 +52,9 @@ fn test_read_with_custom_page_index_provider_sync() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } @@ -60,7 +62,9 @@ fn test_read_with_custom_page_index_provider_push() { // this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic] +#[should_panic( + expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# +)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } @@ -423,7 +427,7 @@ fn create_test_file() -> Bytes { .build(); // Write 3 row groups with 50 rows each (150 rows total) - let batch = make_batch(&(0..150).collect::>().as_slice()); + let batch = make_batch((0..150).collect::>().as_slice()); let mut buffer = Vec::with_capacity(1024); let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema().clone(), Some(props)).unwrap(); From 57d70307f880f932cdae1f8f6ddafd43e09a2fa8 Mon Sep 17 00:00:00 2001 From: seidl Date: Wed, 23 Sep 2026 15:23:34 -0700 Subject: [PATCH 8/9] tests now pass --- parquet/tests/arrow_reader/custom_page_index_provider.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index 8447146418f1..b6348ecd221a 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -50,21 +50,13 @@ fn test_read_with_custom_page_index_provider_sync() { run_test(Reader::Sync); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_push() { run_test(Reader::Push); } -// this will fail until https://github.com/apache/arrow-rs/pull/11182 is merged #[cfg(feature = "async")] #[test] -#[should_panic( - expected = r#"called `Result::unwrap()` on an `Err` value: General("Invalid column index 2, column was not fetched")"# -)] fn test_read_with_custom_page_index_provider_async() { run_test(Reader::Async); } From ea42bb1a0ca7305f0ef278fa6b162d06b47743a0 Mon Sep 17 00:00:00 2001 From: seidl Date: Thu, 24 Sep 2026 12:55:28 -0700 Subject: [PATCH 9/9] apply changes requested in review --- .../custom_page_index_provider.rs | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/parquet/tests/arrow_reader/custom_page_index_provider.rs b/parquet/tests/arrow_reader/custom_page_index_provider.rs index b6348ecd221a..147c9a7336c0 100644 --- a/parquet/tests/arrow_reader/custom_page_index_provider.rs +++ b/parquet/tests/arrow_reader/custom_page_index_provider.rs @@ -31,7 +31,7 @@ use parquet::DecodeResult; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, - RowSelectionPolicy, + RowSelectionPolicy, RowSelector, }; use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; use parquet::file::metadata::page_index::PageIndexProvider; @@ -149,15 +149,18 @@ fn run_test(reader: Reader) { // - We only populate indexes for row groups 0 and 2 (skipping row group 1) // - For row group 0: populate column 0 (id) and column 1 (value) // - For row group 2: populate columns 0 (id) and 2 (name), so column 1 is a gap - let provider = SelectivePageIndexProvider::new( + let provider = Arc::new(SelectivePageIndexProvider::new( file_bytes.clone(), metadata, + // values are (row_group index, column index) &[(0, 0), (0, 1), (2, 0), (2, 2)], - ); + )); // Step 4: Install the custom provider into metadata - let mut metadata_builder = metadata.clone().into_builder(); - metadata_builder = metadata_builder.set_page_index(Some(Arc::new(provider))); + let metadata_builder = metadata + .clone() + .into_builder() + .set_page_index(Some(provider.clone())); let metadata_with_custom_index = Arc::new(metadata_builder.build()); // Step 5: Create ArrowReaderMetadata with the custom page index @@ -175,17 +178,17 @@ fn run_test(reader: Reader) { // - Selects rows 120-130 (in row group 2) let selection = RowSelection::from(vec![ // Skip first 20 rows - parquet::arrow::arrow_reader::RowSelector::skip(20), + RowSelector::skip(20), // Select rows 20-30 - parquet::arrow::arrow_reader::RowSelector::select(10), + RowSelector::select(10), // Skip rows 30-60 - parquet::arrow::arrow_reader::RowSelector::skip(30), + RowSelector::skip(30), // Select rows 60-70 - parquet::arrow::arrow_reader::RowSelector::select(10), + RowSelector::select(10), // Skip rows 70-120 - parquet::arrow::arrow_reader::RowSelector::skip(50), + RowSelector::skip(50), // Select rows 120-130 - parquet::arrow::arrow_reader::RowSelector::select(10), + RowSelector::select(10), ]); // Collect all batches @@ -215,25 +218,17 @@ fn run_test(reader: Reader) { assert_eq!(fetched(0, 2), chunk_len(0, 2)); // no offset index: whole chunk } - // Note: We need to get the provider reference from the metadata to check stats - let provider_ref = metadata_with_custom_index - .page_index() - .unwrap() - .as_any() - .downcast_ref::() - .expect("Expected SelectivePageIndexProvider"); - // Check that the custom provider was used - let (hits, misses) = provider_ref.stats(); + let (hits, misses) = provider.stats(); assert!(hits > 0, "provider should have hits"); assert!(misses > 0, "provider should have misses"); // Since we didn't use predicates, the column index should be untouched - let (col_hits, col_misses) = provider_ref.column_index_stats(); + let (col_hits, col_misses) = provider.column_index_stats(); assert_eq!(col_hits, 0, "column index should not be used"); assert_eq!(col_misses, 0, "column index should not be used"); - let (off_hits, off_misses) = provider_ref.offset_index_stats(); + let (off_hits, off_misses) = provider.offset_index_stats(); assert!(off_hits > 0, "offset index should have hits"); assert!(off_misses > 0, "offset index should have misses");