From 0be54f9076c096e13e3737a6016ab277c495222d Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 27 Oct 2026 09:16:14 -0700 Subject: [PATCH 1/4] fix(reader): keep matching rows when filtering a promoted column Predicates are bound to the table schema, but the row filter evaluated them on batches in the file's physical types. For a file written before an int -> long, float -> double, or decimal precision promotion, try_cast_literal cast the literal down to the file type, so an out-of-range literal became null and dropped every row, and a double literal was rounded to f32. Convert the filter columns to the table schema with a RecordBatchTransformer, as the reader does for its output, and make try_cast_literal fail instead of returning null. Page-index pruning is left to #3507 and #3521. Part of #3122. --- crates/iceberg/src/arrow/reader/pipeline.rs | 1 + .../src/arrow/reader/predicate_visitor.rs | 63 +++++++-- crates/iceberg/src/arrow/reader/row_filter.rs | 123 ++++++++++++++++-- 3 files changed, 165 insertions(+), 22 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 09fcad9cb2..5e2bcb8b20 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -540,6 +540,7 @@ impl FileScanTaskReader { record_batch_stream_builder.parquet_schema(), &iceberg_field_ids, &field_id_map, + task.schema_ref(), )?; record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..6641c730c8 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -27,7 +27,7 @@ use arrow_array::cast::AsArray; use arrow_array::types::{Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; use arrow_buffer::BooleanBuffer; -use arrow_cast::cast::cast; +use arrow_cast::cast::{CastOptions, cast_with_options}; use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; use arrow_schema::{ArrowError, DataType}; use arrow_string::like::starts_with; @@ -638,12 +638,10 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { } } -/// The Arrow type of an array that the Parquet reader reads may not match the exact Arrow type -/// that Iceberg uses for literals - but they are effectively the same logical type, -/// i.e. LargeUtf8 and Utf8 or Utf8View and Utf8 or Utf8View and LargeUtf8. -/// -/// The Arrow compute kernels that we use must match the type exactly, so first cast the literal -/// into the type of the batch we read from Parquet before sending it to the compute kernel. +/// The row filter evaluates batches converted to the table schema, but a literal can still use +/// another Arrow representation of the same type (e.g. `Binary` for a `LargeBinary` column). +/// The compute kernels need the exact type, so cast the literal, failing rather than letting a +/// lossy cast turn it into null. fn try_cast_literal( literal: &Arc, column_type: &DataType, @@ -655,7 +653,11 @@ fn try_cast_literal( return Ok(Arc::clone(literal)); } - let literal_array = cast(literal_array, column_type)?; + let options = CastOptions { + safe: false, + ..Default::default() + }; + let literal_array = cast_with_options(literal_array, column_type, &options)?; Ok(Arc::new(Scalar::new(literal_array))) } @@ -664,15 +666,15 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use arrow_array::{Array, BooleanArray, RecordBatch}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch}; + use arrow_schema::{ArrowError, DataType, Field, Schema as ArrowSchema}; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -836,4 +838,43 @@ mod tests { ); assert!(!result.is_null(2)); } + + /// A literal that does not fit the column type fails the predicate instead of becoming null. + #[test] + fn test_predicate_converter_rejects_lossy_literal_cast() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let predicate = Reference::new("x") + .less_than(Datum::long(i64::from(i32::MAX) + 1)) + .bind(schema, true) + .unwrap(); + let parquet_type = parse_message_type("message schema { required int32 x = 1; }").unwrap(); + let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); + let column_map = HashMap::from([(1, 0)]); + let column_indices = vec![0]; + let mut converter = PredicateConverter { + parquet_schema: &parquet_schema, + column_map: &column_map, + column_indices: &column_indices, + }; + let mut predicate_fn = visit(&mut converter, &predicate).unwrap(); + + // An `int` batch that was not converted to the table's `long`. + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "x", + DataType::Int32, + false, + )])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + assert!(matches!(predicate_fn(batch), Err(ArrowError::CastError(_)))); + } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 331e7f9656..f7abcd578f 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -23,18 +23,20 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use arrow_schema::ArrowError; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::{ArrowPredicateFn, RowFilter, RowSelection}; use parquet::file::metadata::ParquetMetaData; use parquet::schema::types::SchemaDescriptor; use super::{ArrowReader, PredicateConverter}; +use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::error::Result; use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::Schema; +use crate::spec::{Schema, SchemaRef}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -43,14 +45,20 @@ impl ArrowReader { parquet_schema: &SchemaDescriptor, iceberg_field_ids: &HashSet, field_id_map: &HashMap, + snapshot_schema: SchemaRef, ) -> Result { - // Collect Parquet column indices from field ids. + // Collect Parquet column indices from field ids, keeping each column's field id. // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. - let mut column_indices = iceberg_field_ids - .iter() - .filter_map(|field_id| field_id_map.get(field_id).cloned()) - .collect::>(); - column_indices.sort(); + let mut columns = Vec::with_capacity(iceberg_field_ids.len()); + columns.extend( + iceberg_field_ids + .iter() + .filter_map(|field_id| field_id_map.get(field_id).map(|&idx| (idx, *field_id))), + ); + // Leaf order is the filter batch's column order, so the transformer below passes the + // batch through unchanged when no column needs converting. + columns.sort_unstable(); + let (column_indices, field_ids): (Vec<_>, Vec<_>) = columns.into_iter().unzip(); // The converter that converts `BoundPredicates` to `ArrowPredicates` let mut converter = PredicateConverter { @@ -62,8 +70,18 @@ impl ArrowReader { // After collecting required leaf column indices used in the predicate, // creates the projection mask for the Arrow predicates. let projection_mask = ProjectionMask::leaves(parquet_schema, column_indices.clone()); - let predicate_func = visit(&mut converter, predicates)?; - let arrow_predicate = ArrowPredicateFn::new(projection_mask, predicate_func); + let mut predicate_func = visit(&mut converter, predicates)?; + // Predicates are bound to the table schema, so evaluate them on the filter columns + // converted to it, as the reader's output is (e.g. a file written before an `int` -> + // `long` promotion). + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &field_ids).build(); + let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { + let batch = transformer + .process_record_batch(batch) + .map_err(|err| ArrowError::ExternalError(Box::new(err)))?; + predicate_func(batch) + }); Ok(RowFilter::new(vec![Box::new(arrow_predicate)])) } @@ -228,8 +246,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, - RecordBatch, StringArray, + ArrayRef, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeStringArray, RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -460,6 +478,89 @@ mod tests { } } + /// Writes `column` as field 1, reads it back as `iceberg_type` under each predicate, and + /// checks the values that survive (`None` when no row does). + async fn assert_promoted_reads( + column: ArrayRef, + iceberg_type: PrimitiveType, + cases: Vec<(Predicate, Option)>, + ) { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/column.parquet", tmp.path().to_str().unwrap()); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "v", + column.data_type().clone(), + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column]).unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], false); + + let schema = value_schema(Type::Primitive(iceberg_type)); + let file_size = std::fs::metadata(&path).unwrap().len(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + for (predicate, expected) in cases { + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) + .with_case_sensitive(true) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .clone() + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); + assert_eq!(actual, expected, "{predicate}"); + } + } + + /// A file written before a type promotion keeps the old physical type, so the row + /// filter must compare its values promoted to the table type. + #[tokio::test] + async fn test_predicate_on_promoted_column() { + let longs = |values: Vec| Some(Arc::new(Int64Array::from(values)) as ArrayRef); + let all = longs(vec![1, 2]); + let above = || Datum::long(i64::from(i32::MAX) + 1); + let below = || Datum::long(i64::from(i32::MIN) - 1); + let v = || Reference::new("v"); + assert_promoted_reads( + Arc::new(Int32Array::from(vec![1, 2])), + PrimitiveType::Long, + vec![ + (v().less_than(above()), all.clone()), + (v().less_than_or_equal_to(above()), all.clone()), + (v().greater_than_or_equal_to(below()), all.clone()), + (v().not_equal_to(above()), all), + (v().is_in([Datum::long(1), above()]), longs(vec![1])), + (v().is_not_in([Datum::long(1), above()]), longs(vec![2])), + ], + ) + .await; + + // 0.1f32 promotes to a double that is greater than 0.1, not equal to it. + let promoted = Arc::new(Float64Array::from(vec![f64::from(0.1f32)])) as ArrayRef; + assert_promoted_reads( + Arc::new(Float32Array::from(vec![0.1])), + PrimitiveType::Double, + vec![ + (v().greater_than(Datum::double(0.1)), Some(promoted)), + (v().equal_to(Datum::double(0.1)), None), + ], + ) + .await; + } + /// Verifies that file splits respect byte ranges and only read specific row groups. #[tokio::test] async fn test_file_splits_respect_byte_ranges() { From 5a6cf1d04c2654b8149402047494ba8e8da4befa Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 28 Oct 2026 09:54:42 -0700 Subject: [PATCH 2/4] test(reader): cover files without field ids and date promotion; name the strict cast Reuse read_once in assert_promoted_reads, read each file with and without a field id, and add a date -> timestamp case. Explain why try_cast_literal sets safe: false. --- .../src/arrow/reader/predicate_visitor.rs | 1 + crates/iceberg/src/arrow/reader/row_filter.rs | 74 +++++++++---------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 6641c730c8..478a1c3994 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -653,6 +653,7 @@ fn try_cast_literal( return Ok(Arc::clone(literal)); } + // `safe: false` makes arrow return an error where the default would produce null. let options = CastOptions { safe: false, ..Default::default() diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index f7abcd578f..1d9b5a193b 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -246,8 +246,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, - LargeStringArray, RecordBatch, StringArray, + ArrayRef, Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeStringArray, RecordBatch, StringArray, TimestampMicrosecondArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -478,50 +478,36 @@ mod tests { } } - /// Writes `column` as field 1, reads it back as `iceberg_type` under each predicate, and - /// checks the values that survive (`None` when no row does). + /// Writes `column` as field 1, with and without a field id, reads it back as `iceberg_type` + /// under each predicate, and checks the values that survive (`None` when no row does). async fn assert_promoted_reads( column: ArrayRef, iceberg_type: PrimitiveType, cases: Vec<(Predicate, Option)>, ) { let tmp = TempDir::new().unwrap(); - let path = format!("{}/column.parquet", tmp.path().to_str().unwrap()); - let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( - "v", - column.data_type().clone(), - 1, - )])); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column]).unwrap(); - write_row_groups(&path, arrow_schema, vec![batch], false); - let schema = value_schema(Type::Primitive(iceberg_type)); - let file_size = std::fs::metadata(&path).unwrap().len(); - let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); - for (predicate, expected) in cases { - let task = FileScanTask::builder() - .with_file_size_in_bytes(file_size) - .with_start(0) - .with_length(0) - .with_data_file_path(path.clone()) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(schema.clone()) - .with_project_field_ids(vec![1]) - .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) - .with_case_sensitive(true) - .build() - .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; - let batches: Vec = reader - .clone() - .read(tasks) - .unwrap() - .stream() - .try_collect() - .await - .unwrap(); - let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); - assert_eq!(actual, expected, "{predicate}"); + for with_field_id in [true, false] { + // Without a field id, the reader assigns one by position. + let path = format!("{}/{with_field_id}.parquet", tmp.path().to_str().unwrap()); + let field = if with_field_id { + field_with_id("v", column.data_type().clone(), 1) + } else { + Field::new("v", column.data_type().clone(), false) + }; + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column.clone()]).unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], false); + + for (predicate, expected) in &cases { + let (batches, _) = + read_once(&path, schema.clone(), vec![1], predicate.clone(), false).await; + let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); + assert_eq!( + &actual, expected, + "{predicate}, with_field_id = {with_field_id}" + ); + } } } @@ -559,6 +545,16 @@ mod tests { ], ) .await; + + // 2024-01-01 promotes to midnight, which is before noon on that day. + let noon = || Datum::timestamp_from_str("2024-01-01T12:00:00").unwrap(); + let midnight = Arc::new(TimestampMicrosecondArray::from(vec![1_704_067_200_000_000])); + assert_promoted_reads( + Arc::new(Date32Array::from(vec![19723])), + PrimitiveType::Timestamp, + vec![(v().less_than(noon()), Some(midnight as ArrayRef))], + ) + .await; } /// Verifies that file splits respect byte ranges and only read specific row groups. From 03639f1bcfad19fc02e70ed77047c858eebb5182 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 28 Oct 2026 15:20:19 -0700 Subject: [PATCH 3/4] perf(reader): build the row filter transformer from the filter fields only The transformer converts its whole schema to Arrow on each file's first batch. Give it a schema of the top-level filter fields so that setup no longer grows with the table's width. --- crates/iceberg/src/arrow/reader/pipeline.rs | 2 +- crates/iceberg/src/arrow/reader/row_filter.rs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 5e2bcb8b20..473fb51b73 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -540,7 +540,7 @@ impl FileScanTaskReader { record_batch_stream_builder.parquet_schema(), &iceberg_field_ids, &field_id_map, - task.schema_ref(), + task.schema(), )?; record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 1d9b5a193b..6139db1f43 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -36,7 +36,7 @@ use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::{Schema, SchemaRef}; +use crate::spec::Schema; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -45,7 +45,7 @@ impl ArrowReader { parquet_schema: &SchemaDescriptor, iceberg_field_ids: &HashSet, field_id_map: &HashMap, - snapshot_schema: SchemaRef, + snapshot_schema: &Schema, ) -> Result { // Collect Parquet column indices from field ids, keeping each column's field id. // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. @@ -73,9 +73,18 @@ impl ArrowReader { let mut predicate_func = visit(&mut converter, predicates)?; // Predicates are bound to the table schema, so evaluate them on the filter columns // converted to it, as the reader's output is (e.g. a file written before an `int` -> - // `long` promotion). + // `long` promotion). The transformer's schema holds only the top-level filter fields, so + // its per-file setup does not grow with the table's width. + let filter_schema = Schema::builder() + .with_schema_id(snapshot_schema.schema_id()) + .with_fields( + field_ids + .iter() + .filter_map(|id| snapshot_schema.as_struct().field_by_id(*id).cloned()), + ) + .build()?; let mut transformer = - RecordBatchTransformerBuilder::new(snapshot_schema, &field_ids).build(); + RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &field_ids).build(); let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { let batch = transformer .process_record_batch(batch) From 08d7afe70c63fbc6f72f801008a7e6722a088557 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 11:22:21 -0800 Subject: [PATCH 4/4] fix(reader): build the row filter schema from each leaf's top-level field A filter leaf nested in a struct has no top-level field of its own, so the filter schema missed it and the transformer failed with `field not found` once nested predicates reach the row filter (#2906). Take each leaf's top-level field, prune it to the filter leaves, and pass the top-level ids in leaf order. --- crates/iceberg/src/arrow/reader/row_filter.rs | 79 ++++++++++++++++--- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 6139db1f43..895962c9ca 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -36,7 +36,7 @@ use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::Schema; +use crate::spec::{NestedFieldRef, Schema, Type, prune_columns}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -73,18 +73,10 @@ impl ArrowReader { let mut predicate_func = visit(&mut converter, predicates)?; // Predicates are bound to the table schema, so evaluate them on the filter columns // converted to it, as the reader's output is (e.g. a file written before an `int` -> - // `long` promotion). The transformer's schema holds only the top-level filter fields, so - // its per-file setup does not grow with the table's width. - let filter_schema = Schema::builder() - .with_schema_id(snapshot_schema.schema_id()) - .with_fields( - field_ids - .iter() - .filter_map(|id| snapshot_schema.as_struct().field_by_id(*id).cloned()), - ) - .build()?; + // `long` promotion). + let (filter_schema, root_ids) = filter_schema(snapshot_schema, &field_ids)?; let mut transformer = - RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &field_ids).build(); + RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &root_ids).build(); let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { let batch = transformer .process_record_batch(batch) @@ -247,6 +239,33 @@ impl ArrowReader { } } +/// The top-level fields holding `leaf_ids`, pruned to those leaves, and their ids in leaf order +/// (the filter batch's column order). Only these fields are converted, so the transformer's +/// per-file setup does not grow with the table's width. +fn filter_schema(snapshot_schema: &Schema, leaf_ids: &[i32]) -> Result<(Schema, Vec)> { + let mut roots: Vec = Vec::with_capacity(leaf_ids.len()); + for accessor in leaf_ids + .iter() + .filter_map(|id| snapshot_schema.accessor_by_field_id(*id)) + { + let root = &snapshot_schema.as_struct().fields()[accessor.position()]; + if !roots.iter().any(|r| r.id == root.id) { + roots.push(Arc::clone(root)); + } + } + let root_ids = roots.iter().map(|r| r.id).collect(); + let mut schema = Schema::builder().with_schema_id(snapshot_schema.schema_id()); + // `prune_columns` panics when it selects nothing, e.g. when no filter column is in the file. + if !roots.is_empty() { + let roots = Schema::builder().with_fields(roots).build()?; + let Type::Struct(pruned) = prune_columns(&roots, leaf_ids.iter().copied(), false)? else { + unreachable!("prune_columns returns a struct"); + }; + schema = schema.with_fields(pruned.fields().iter().cloned()); + } + Ok((schema.build()?, root_ids)) +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -268,13 +287,15 @@ mod tests { use parquet::schema::types::SchemaDescriptor; use tempfile::TempDir; + use super::filter_schema; use crate::Runtime; use crate::arrow::{ArrowReader, ArrowReaderBuilder}; use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, NestedField, NestedFieldRef, PrimitiveType, Schema, + SchemaRef, StructType, Type, }; async fn test_perform_read( @@ -520,6 +541,38 @@ mod tests { } } + /// The filter schema holds each filter leaf's top-level field, pruned to the filter leaves, + /// with root ids in leaf order. + #[test] + fn test_filter_schema_prunes_nested_roots() { + let person = |fields: Vec| { + NestedField::required(2, "person", Type::Struct(StructType::new(fields))).into() + }; + let n = || NestedField::required(3, "n", Type::Primitive(PrimitiveType::Long)).into(); + let m = || NestedField::optional(4, "m", Type::Primitive(PrimitiveType::Int)).into(); + let id = || NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(); + let schema = Schema::builder() + .with_fields(vec![id(), person(vec![n(), m()])]) + .build() + .unwrap(); + + for (leaf_ids, expected_fields, expected_root_ids) in [ + (vec![3], vec![person(vec![n()])], vec![2]), + (vec![4, 1, 3], vec![person(vec![n(), m()]), id()], vec![ + 2, 1, + ]), + (vec![], vec![], vec![]), + ] { + let (filter_schema, root_ids) = filter_schema(&schema, &leaf_ids).unwrap(); + assert_eq!( + filter_schema.as_struct().fields(), + expected_fields, + "{leaf_ids:?}" + ); + assert_eq!(root_ids, expected_root_ids, "{leaf_ids:?}"); + } + } + /// A file written before a type promotion keeps the old physical type, so the row /// filter must compare its values promoted to the table type. #[tokio::test]