From 9057b8a46f20ca972698623cdcea315cb92e7219 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 29 Sep 2026 17:02:29 -0400 Subject: [PATCH 1/4] fix(arrow): evaluate filters on columns missing from a data file against their identity partition value or initial-default --- crates/iceberg/src/arrow/reader/pipeline.rs | 9 + .../src/arrow/reader/predicate_visitor.rs | 501 +++++++++++++++++- crates/iceberg/src/arrow/reader/row_filter.rs | 233 +++++++- .../src/arrow/record_batch_transformer.rs | 2 +- crates/iceberg/src/expr/predicate.rs | 2 +- .../src/expr/visitors/expression_evaluator.rs | 4 +- crates/iceberg/src/scan/mod.rs | 175 +++++- 7 files changed, 910 insertions(+), 16 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index ee027c5e72..b941bf44b1 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -32,6 +32,7 @@ use parquet::arrow::{ }; use parquet::encryption::decrypt::FileDecryptionProperties; +use super::predicate_visitor::residual_for_missing_fields; use super::row_lineage::synthesize_row_id_column; use super::{ ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, @@ -603,6 +604,14 @@ impl FileScanTaskReader { &predicate, use_position_fallback, )?; + let predicate = residual_for_missing_fields( + predicate, + &iceberg_field_ids, + &field_id_map, + task.schema(), + task.partition_spec().map(Arc::as_ref), + task.partition(), + )?; let row_filter = ArrowReader::get_row_filter( &predicate, diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..1567ee3983 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -35,10 +35,12 @@ use fnv::FnvHashSet; use parquet::schema::types::SchemaDescriptor; use crate::arrow::get_arrow_datum; +use crate::arrow::record_batch_transformer::constants_map; use crate::error::{Result, invalid_data}; -use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; -use crate::expr::{BoundPredicate, BoundReference}; -use crate::spec::Datum; +use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; +use crate::expr::visitors::expression_evaluator::ExpressionEvaluatorVisitor; +use crate::expr::{BoundPredicate, BoundReference, LogicalExpression}; +use crate::spec::{Datum, Literal, PartitionSpec, Schema, Struct}; /// A visitor to collect field ids from bound predicates. pub(super) struct CollectFieldIdVisitor { @@ -195,6 +197,238 @@ impl BoundPredicateVisitor for CollectFieldIdVisitor { } } +/// Returns the residual of `predicate` for one data file: each leaf on a top-level field that is +/// missing from the file becomes `AlwaysTrue` or `AlwaysFalse`, based on the value projection +/// returns for that field. That value is the identity partition value, otherwise the field's +/// `initial-default`, and every row of the file holds it. Leaves on missing fields with neither +/// keep the null handling of the row filter and the page index evaluator. +/// +/// Leaves on nested fields are kept, because a nested field also reads as null in any row where +/// an ancestor struct is null. +pub(super) fn residual_for_missing_fields( + predicate: BoundPredicate, + predicate_field_ids: &HashSet, + field_id_map: &HashMap, + schema: &Schema, + partition_spec: Option<&PartitionSpec>, + partition: Option<&Struct>, +) -> Result { + if predicate_field_ids + .iter() + .all(|id| field_id_map.contains_key(id)) + { + return Ok(predicate); + } + + let partition_constants = match (partition_spec, partition) { + (Some(spec), Some(data)) => constants_map(spec, data, schema)?, + _ => HashMap::new(), + }; + + let mut field_ids = HashSet::new(); + let row: Struct = schema + .as_struct() + .fields() + .iter() + .map(|field| { + if !predicate_field_ids.contains(&field.id) || field_id_map.contains_key(&field.id) { + return None; + } + let value = match partition_constants.get(&field.id) { + Some(datum) => Some(Literal::Primitive(datum.literal().clone())), + None => field.initial_default.clone(), + }; + if value.is_some() { + field_ids.insert(field.id); + } + value + }) + .collect(); + + if field_ids.is_empty() { + return Ok(predicate); + } + visit( + &mut MissingFieldResidualVisitor { + row: &row, + field_ids: &field_ids, + }, + &predicate, + ) +} + +/// Replaces leaves on the fields in `field_ids` with their result on `row`, which holds each +/// top-level field's value at its position in the schema. +struct MissingFieldResidualVisitor<'a> { + row: &'a Struct, + field_ids: &'a HashSet, +} + +impl MissingFieldResidualVisitor<'_> { + fn residual( + &self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + if !self.field_ids.contains(&reference.field().id) { + return Ok(predicate.clone()); + } + if visit(&mut ExpressionEvaluatorVisitor::new(self.row), predicate)? { + Ok(BoundPredicate::AlwaysTrue) + } else { + Ok(BoundPredicate::AlwaysFalse) + } + } +} + +impl BoundPredicateVisitor for MissingFieldResidualVisitor<'_> { + type T = BoundPredicate; + + fn always_true(&mut self) -> Result { + Ok(BoundPredicate::AlwaysTrue) + } + + fn always_false(&mut self) -> Result { + Ok(BoundPredicate::AlwaysFalse) + } + + fn and(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { + Ok(lhs.and(rhs)) + } + + fn or(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { + Ok(lhs.or(rhs)) + } + + fn not(&mut self, inner: BoundPredicate) -> Result { + Ok(BoundPredicate::Not(LogicalExpression::new([Box::new( + inner, + )]))) + } + + fn is_null( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_null( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn is_nan( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_nan( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn less_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn less_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn greater_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn greater_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn r#in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } +} + /// A visitor to convert Iceberg bound predicates to Arrow predicates. pub(super) struct PredicateConverter<'a> { /// The Parquet schema descriptor. @@ -669,10 +903,15 @@ mod tests { use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; - use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; + use super::{ + CollectFieldIdVisitor, PredicateConverter, constant_bool_array, residual_for_missing_fields, + }; use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::expr::{Bind, BoundPredicate, LogicalExpression, Predicate, Reference}; + use crate::spec::{ + Datum, Literal, NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, + StructType, Transform, Type, + }; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -836,4 +1075,254 @@ mod tests { ); assert!(!result.is_null(2)); } + + /// Returns the residual of `predicate` for a data file that stores only field 1. + fn residual_for_file_with_field_1( + schema: &SchemaRef, + predicate: Predicate, + partition: Option<(&PartitionSpec, &Struct)>, + ) -> BoundPredicate { + let predicate = predicate.bind(schema.clone(), true).unwrap(); + let mut collector = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut collector, &predicate).unwrap(); + residual_for_missing_fields( + predicate, + &collector.field_ids(), + &HashMap::from([(1, 0)]), + schema, + partition.map(|(spec, _)| spec), + partition.map(|(_, data)| data), + ) + .unwrap() + } + + /// Asserts that each predicate in `always_true` has the residual `AlwaysTrue` and each in + /// `always_false` has `AlwaysFalse`, for a file that doesn't store the field `field`. + fn assert_residuals_for_missing_field_with_initial_default( + field: NestedField, + always_true: Vec, + always_false: Vec, + ) { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + field.into(), + ]) + .build() + .unwrap(), + ); + + let cases: Vec<_> = always_true + .into_iter() + .map(|predicate| (predicate, BoundPredicate::AlwaysTrue)) + .chain( + always_false + .into_iter() + .map(|predicate| (predicate, BoundPredicate::AlwaysFalse)), + ) + .collect(); + let actual: Vec<_> = cases + .iter() + .map(|(predicate, _)| { + ( + predicate.to_string(), + residual_for_file_with_field_1(&schema, predicate.clone(), None), + ) + }) + .collect(); + let expected: Vec<_> = cases + .into_iter() + .map(|(predicate, residual)| (predicate.to_string(), residual)) + .collect(); + assert_eq!(actual, expected); + } + + /// Mirrors Java's `TestMetricsRowGroupFilter.testColumnNotInFileWithInitialDefault`. + #[test] + fn test_residual_for_missing_string_field_with_initial_default() { + let country = || Reference::new("country"); + let string = Datum::string; + assert_residuals_for_missing_field_with_initial_default( + NestedField::optional(99, "country", Type::Primitive(PrimitiveType::String)) + .with_initial_default(Literal::string("US")), + vec![ + country().equal_to(string("US")), + country().is_not_null(), + country().not_equal_to(string("CA")), + country().is_in([string("US"), string("MX")]), + country().is_not_in([string("CA"), string("MX")]), + country().greater_than_or_equal_to(string("US")), + country().less_than(string("ZW")), + country().starts_with(string("U")), + country().not_starts_with(string("X")), + ], + vec![ + country().equal_to(string("CA")), + country().is_null(), + country().not_equal_to(string("US")), + country().is_in([string("CA"), string("MX")]), + country().is_not_in([string("US"), string("MX")]), + country().less_than(string("AD")), + country().greater_than(string("US")), + country().starts_with(string("X")), + country().not_starts_with(string("U")), + ], + ); + } + + /// Mirrors Java's `TestMetricsRowGroupFilter.testDateColumnNotInFileWithInitialDefault`. + #[test] + fn test_residual_for_missing_date_field_with_initial_default() { + let event_date = || Reference::new("event_date"); + assert_residuals_for_missing_field_with_initial_default( + NestedField::optional(100, "event_date", Type::Primitive(PrimitiveType::Date)) + .with_initial_default(Literal::date(42)), + vec![ + event_date().equal_to(Datum::date(42)), + event_date().less_than(Datum::date(43)), + event_date().greater_than(Datum::date(41)), + ], + vec![ + event_date().equal_to(Datum::date(41)), + event_date().less_than(Datum::date(42)), + event_date().greater_than(Datum::date(42)), + ], + ); + } + + /// Mirrors Java's `TestMetricsRowGroupFilter.testDoubleColumnNotInFileWithInitialDefault`. + #[test] + fn test_residual_for_missing_double_field_with_initial_default() { + assert_residuals_for_missing_field_with_initial_default( + NestedField::optional(101, "measurement", Type::Primitive(PrimitiveType::Double)) + .with_initial_default(Literal::double(12.5)), + vec![Reference::new("measurement").is_not_nan()], + vec![Reference::new("measurement").is_nan()], + ); + } + + /// Covers which value a leaf is evaluated against, and which leaves are kept. + #[test] + fn test_residual_for_missing_fields() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(5)) + .into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional(4, "d", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(5, "p", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional(6, "q", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional( + 7, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(8, "x", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .add_partition_field("p", "p", Transform::Identity) + .unwrap() + .add_partition_field("q", "q_bucket", Transform::Bucket(4)) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter([ + Some(Literal::long(100)), + Some(Literal::long(9)), + Some(Literal::int(2)), + ]); + + let bind = |predicate: Predicate| predicate.bind(schema.clone(), true).unwrap(); + let cases = [ + ( + "p = 9 uses the identity partition value over the initial-default", + Reference::new("p").equal_to(Datum::long(9)), + BoundPredicate::AlwaysTrue, + ), + ( + "p = 7", + Reference::new("p").equal_to(Datum::long(7)), + BoundPredicate::AlwaysFalse, + ), + ( + "q = 7 ignores the bucket partition value", + Reference::new("q").equal_to(Datum::long(7)), + BoundPredicate::AlwaysTrue, + ), + ( + "a = 3 reads the file despite a partition value and initial-default", + Reference::new("a").equal_to(Datum::long(3)), + bind(Reference::new("a").equal_to(Datum::long(3))), + ), + ( + "d IS NULL has no value to apply", + Reference::new("d").is_null(), + bind(Reference::new("d").is_null()), + ), + ( + "s.x = 7 is nested", + Reference::new("s.x").equal_to(Datum::long(7)), + bind(Reference::new("s.x").equal_to(Datum::long(7))), + ), + ( + "NOT (b = 7)", + !Reference::new("b").equal_to(Datum::long(7)), + BoundPredicate::Not(LogicalExpression::new([Box::new( + BoundPredicate::AlwaysTrue, + )])), + ), + ( + "a > 0 AND b = 8", + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("b").equal_to(Datum::long(8))), + bind(Reference::new("a").greater_than(Datum::long(0))) + .and(BoundPredicate::AlwaysFalse), + ), + ( + "d = 1 OR b = 7", + Reference::new("d") + .equal_to(Datum::long(1)) + .or(Reference::new("b").equal_to(Datum::long(7))), + bind(Reference::new("d").equal_to(Datum::long(1))).or(BoundPredicate::AlwaysTrue), + ), + ]; + + let actual: Vec<_> = cases + .iter() + .map(|(name, predicate, _)| { + let residual = residual_for_file_with_field_1( + &schema, + predicate.clone(), + Some((&partition_spec, &partition)), + ); + (*name, residual) + }) + .collect(); + let expected: Vec<_> = cases + .into_iter() + .map(|(name, _, residual)| (name, residual)) + .collect(); + assert_eq!(actual, expected); + } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 34b7391134..a7a1b04664 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -211,32 +211,38 @@ impl ArrowReader { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use std::fs::File; use std::sync::Arc; use arrow_array::cast::AsArray; + use arrow_array::types::Int64Type; use arrow_array::{ ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; + use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::Compression; - use parquet::file::metadata::{FileMetaData, ParquetMetaData, ParquetMetaDataBuilder}; + use parquet::file::metadata::{ + FileMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataBuilder, + }; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; use tempfile::TempDir; use crate::Runtime; + use crate::arrow::reader::predicate_visitor::residual_for_missing_fields; use crate::arrow::{ArrowReader, ArrowReaderBuilder}; use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, Literal, NestedField, PartitionSpec, PrimitiveType, + Schema, SchemaRef, Struct, Transform, Type, }; async fn test_perform_read( @@ -1757,4 +1763,225 @@ mod tests { .await; assert_eq!(rows(&on), 1); } + + /// A file written before column `b` was added, read with a schema in which `b` + /// has `initial-default` 7. Every row reads back as `b = 7`, so each predicate + /// must keep or drop all three rows as if `b` were stored as 7. + fn setup_absent_column_with_initial_default() -> (Arc, String, TempDir) { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "a", + DataType::Int64, + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( + vec![1, 2, 3], + ))]) + .unwrap(); + write_row_groups(&file_path, arrow_schema, vec![batch], false); + + (schema, file_path, tmp_dir) + } + + #[tokio::test] + async fn test_predicate_on_absent_column_uses_initial_default() { + let (schema, file_path, _tmp_dir) = setup_absent_column_with_initial_default(); + + // Projection applies the default. + let (batches, _) = read_once( + &file_path, + schema.clone(), + vec![1, 2], + Predicate::AlwaysTrue, + false, + ) + .await; + assert_eq!( + batches[0].column(1).as_primitive::().values(), + &[7, 7, 7] + ); + + let cases = [ + ("b = 7", Reference::new("b").equal_to(Datum::long(7)), 3), + ("b IS NOT NULL", Reference::new("b").is_not_null(), 3), + ( + "b IN (7, 8)", + Reference::new("b").is_in([Datum::long(7), Datum::long(8)]), + 3, + ), + ("b > 5", Reference::new("b").greater_than(Datum::long(5)), 3), + ("b = 8", Reference::new("b").equal_to(Datum::long(8)), 0), + ("b IS NULL", Reference::new("b").is_null(), 0), + ]; + + let mut actual = Vec::new(); + for (name, predicate, _) in &cases { + let (batches, _) = read_once( + &file_path, + schema.clone(), + vec![1, 2], + predicate.clone(), + false, + ) + .await; + actual.push(( + *name, + batches.iter().map(RecordBatch::num_rows).sum::(), + )); + } + let expected: Vec<_> = cases.iter().map(|(name, _, rows)| (*name, *rows)).collect(); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn test_page_index_on_absent_column_uses_initial_default() { + let (schema, file_path, _tmp_dir) = setup_absent_column_with_initial_default(); + let file = File::open(&file_path).unwrap(); + let metadata = ArrowReaderMetadata::load( + &file, + ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required), + ) + .unwrap() + .metadata() + .clone(); + let field_id_map = HashMap::from([(1, 0)]); + + let cases = [ + ("b = 7", Reference::new("b").equal_to(Datum::long(7))), + ("b IS NOT NULL", Reference::new("b").is_not_null()), + ( + "b IN (7, 8)", + Reference::new("b").is_in([Datum::long(7), Datum::long(8)]), + ), + ]; + + // The reader applies the residual before page index pruning. + let mut actual = Vec::new(); + for (name, predicate) in &cases { + let predicate = residual_for_missing_fields( + predicate.clone().bind(schema.clone(), true).unwrap(), + &HashSet::from([2]), + &field_id_map, + &schema, + None, + None, + ) + .unwrap(); + let selection = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &None, + &field_id_map, + &schema, + ) + .unwrap() + .expect("the file has a page index"); + actual.push((*name, selection.row_count())); + } + let expected: Vec<_> = cases.iter().map(|(name, _)| (*name, 3)).collect(); + assert_eq!(actual, expected); + } + + /// A file that doesn't store its identity partition column `p`, as after a Hive + /// migration or `add_files`. Projection reads `p` from the partition value 7, so each + /// predicate must keep or drop all three rows as if `p` were stored as 7. + #[tokio::test] + async fn test_predicate_on_absent_identity_partition_column_uses_partition_value() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "p", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("p", "p", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let partition = Struct::from_iter([Some(Literal::long(7))]); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "a", + DataType::Int64, + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( + vec![1, 2, 3], + ))]) + .unwrap(); + write_row_groups(&file_path, arrow_schema, vec![batch], false); + + let read = async |predicate: Predicate| { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) + .with_partition_spec(Some(partition_spec.clone())) + .with_partition(Some(partition.clone())) + .with_case_sensitive(false) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .build() + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap() + }; + + // Projection applies the partition value. + let batches = read(Predicate::AlwaysTrue).await; + assert_eq!( + batches[0].column(1).as_primitive::().values(), + &[7, 7, 7] + ); + + let cases = [ + ("p = 7", Reference::new("p").equal_to(Datum::long(7)), 3), + ("p IS NOT NULL", Reference::new("p").is_not_null(), 3), + ("p = 8", Reference::new("p").equal_to(Datum::long(8)), 0), + ("p IS NULL", Reference::new("p").is_null(), 0), + ]; + + let mut actual = Vec::new(); + for (name, predicate, _) in &cases { + let batches = read(predicate.clone()).await; + actual.push(( + *name, + batches.iter().map(RecordBatch::num_rows).sum::(), + )); + } + let expected: Vec<_> = cases.iter().map(|(name, _, rows)| (*name, *rows)).collect(); + assert_eq!(actual, expected); + } } diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 19976a81c5..4a40e6b195 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -73,7 +73,7 @@ fn field_with_id( /// # References /// - Spec: https://iceberg.apache.org/spec/#column-projection /// - Java: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java:constantsMap() -fn constants_map( +pub(crate) fn constants_map( partition_spec: &PartitionSpec, partition_data: &Struct, schema: &IcebergSchema, diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index f8ea896395..0d04628aae 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -69,7 +69,7 @@ impl Debug for LogicalExpression { } impl LogicalExpression { - fn new(inputs: [Box; N]) -> Self { + pub(crate) fn new(inputs: [Box; N]) -> Self { Self { inputs } } diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 31ce94a99b..4d132d1680 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -54,14 +54,14 @@ impl ExpressionEvaluator { /// evaluation logic to different parts of a data structure, /// specifically for data file partitions. #[derive(Debug)] -struct ExpressionEvaluatorVisitor<'a> { +pub(crate) struct ExpressionEvaluatorVisitor<'a> { /// Reference to a [`DataFile`]'s partition [`Struct`]. partition: &'a Struct, } impl<'a> ExpressionEvaluatorVisitor<'a> { /// Creates a new [`ExpressionEvaluatorVisitor`]. - fn new(partition: &'a Struct) -> Self { + pub(crate) fn new(partition: &'a Struct) -> Self { Self { partition } } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 7b39189bfb..29362e8584 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -676,9 +676,10 @@ pub mod tests { use tempfile::TempDir; use uuid::Uuid; - use crate::arrow::ArrowReaderBuilder; - use crate::expr::{BoundPredicate, Reference}; + use crate::arrow::{ArrowReaderBuilder, schema_to_arrow_schema}; + use crate::expr::{BoundPredicate, Predicate, Reference}; use crate::io::{FileIO, OutputFile}; + use crate::memory::tests::new_memory_catalog; use crate::metadata_columns::{ RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, @@ -696,7 +697,15 @@ pub mod tests { }; use crate::table::Table; use crate::test_utils::test_runtime; - use crate::{ErrorKind, TableIdent}; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; + use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + use crate::{Catalog, ErrorKind, NamespaceIdent, TableCreation, TableIdent}; fn render_template(template: &str, ctx: Value) -> String { let mut env = Environment::new(); @@ -4125,4 +4134,164 @@ pub mod tests { .collect(); assert_eq!(y, (1100..1200).collect::>()); } + + /// Appends one data file holding `columns`, written with the table's current schema. + async fn append_data_file( + catalog: &impl Catalog, + table: Table, + columns: Vec, + ) -> Table { + let schema = table.metadata().current_schema().clone(); + let batch = + RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema).unwrap()), columns) + .unwrap(); + let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::default(), schema), + table.file_io().clone(), + DefaultLocationGenerator::new(table.metadata()).unwrap(), + DefaultFileNameGenerator::new( + Uuid::new_v4().to_string(), + None, + DataFileFormat::Parquet, + ), + ); + let mut writer = DataFileWriterBuilder::new(rolling_file_writer_builder) + .build(None) + .await + .unwrap(); + writer.write(batch).await.unwrap(); + let data_files = writer.close().await.unwrap(); + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files(data_files) + .apply(tx) + .unwrap(); + tx.commit(catalog).await.unwrap() + } + + /// Mirrors Java's `TestFilterPushDown.testFilterPushdownOnInitialDefaultColumnAbsentFromFile`. + /// The first file is written before `c` is added with `initial-default` 'US', so its row + /// reads `c` as 'US'. The second file stores `c`. + #[tokio::test] + async fn test_scan_filter_on_initial_default_column_absent_from_file() { + use arrow_array::types::Int64Type; + use arrow_cast::cast; + use arrow_schema::DataType; + + let catalog = new_memory_catalog().await; + let namespace = NamespaceIdent::new(format!("ns-{}", Uuid::new_v4())); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("t".to_string()) + .schema(schema) + .format_version(FormatVersion::V3) + .build(), + ) + .await + .unwrap(); + + let table = append_data_file(&catalog, table, vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(StringArray::from(vec!["Alice"])), + ]) + .await; + + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column( + AddColumn::builder() + .name("c") + .field_type(Type::Primitive(PrimitiveType::String)) + .initial_default(Literal::string("US")) + .write_default(Literal::string("US")) + .build(), + ) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let table = append_data_file(&catalog, table, vec![ + Arc::new(Int64Array::from(vec![2, 3])), + Arc::new(StringArray::from(vec!["Bob", "Eve"])), + Arc::new(StringArray::from(vec!["US", "CA"])), + ]) + .await; + + let cases = [ + ("no filter", Predicate::AlwaysTrue, vec![1, 2, 3]), + ( + "c = 'US'", + Reference::new("c").equal_to(Datum::string("US")), + vec![1, 2], + ), + ("c IS NOT NULL", Reference::new("c").is_not_null(), vec![ + 1, 2, 3, + ]), + ( + "c = 'CA'", + Reference::new("c").equal_to(Datum::string("CA")), + vec![3], + ), + ("c IS NULL", Reference::new("c").is_null(), vec![]), + ]; + + let mut actual = Vec::new(); + let mut expected = Vec::new(); + for row_selection_enabled in [false, true] { + for (name, predicate, ids) in &cases { + let batches: Vec = table + .scan() + .with_filter(predicate.clone()) + .with_row_selection_enabled(row_selection_enabled) + .build() + .unwrap() + .to_arrow() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut rows: Vec<(i64, String)> = batches + .iter() + .flat_map(|batch| { + let c = cast(batch.column_by_name("c").unwrap(), &DataType::Utf8).unwrap(); + let ids = batch + .column_by_name("id") + .unwrap() + .as_primitive::(); + (0..batch.num_rows()) + .map(|i| (ids.value(i), c.as_string::().value(i).to_string())) + .collect::>() + }) + .collect(); + rows.sort(); + actual.push((*name, row_selection_enabled, rows)); + + let all_rows = [(1, "US"), (2, "US"), (3, "CA")]; + let rows = all_rows + .iter() + .filter(|(id, _)| ids.contains(id)) + .map(|(id, c)| (*id, c.to_string())) + .collect(); + expected.push((*name, row_selection_enabled, rows)); + } + } + assert_eq!(actual, expected); + } } From 24ef2181a97d0d61341647898cbc07dddedb35fc Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 29 Sep 2026 17:12:29 -0400 Subject: [PATCH 2/4] dedupe some test code. --- crates/iceberg/src/arrow/reader/row_filter.rs | 279 ++++++++---------- crates/iceberg/src/scan/mod.rs | 51 +--- 2 files changed, 134 insertions(+), 196 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index a7a1b04664..70a19fe48d 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -1764,23 +1764,8 @@ mod tests { assert_eq!(rows(&on), 1); } - /// A file written before column `b` was added, read with a schema in which `b` - /// has `initial-default` 7. Every row reads back as `b = 7`, so each predicate - /// must keep or drop all three rows as if `b` were stored as 7. - fn setup_absent_column_with_initial_default() -> (Arc, String, TempDir) { - let schema = Arc::new( - Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), - NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) - .with_initial_default(Literal::long(7)) - .into(), - ]) - .build() - .unwrap(), - ); - + /// Writes a file that stores only field 1 (`a`), with values 1, 2, 3. + fn write_file_without_b() -> (String, TempDir) { let tmp_dir = TempDir::new().unwrap(); let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( @@ -1793,66 +1778,129 @@ mod tests { ))]) .unwrap(); write_row_groups(&file_path, arrow_schema, vec![batch], false); + (file_path, tmp_dir) + } - (schema, file_path, tmp_dir) + fn schema_with_b(b: NestedField) -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + b.into(), + ]) + .build() + .unwrap(), + ) } - #[tokio::test] - async fn test_predicate_on_absent_column_uses_initial_default() { - let (schema, file_path, _tmp_dir) = setup_absent_column_with_initial_default(); + /// Reads the file from [`write_file_without_b`], in which `b` reads as 7 on every row through + /// `schema` or `partition`, and asserts the rows each predicate keeps. Each predicate must keep + /// or drop all three rows as if `b` were stored as 7. + async fn assert_absent_b_reads_as_7( + schema: SchemaRef, + partition: Option<(Arc, Struct)>, + cases: Vec<(Predicate, usize)>, + ) { + let (file_path, _tmp_dir) = write_file_without_b(); + let read = async |predicate: Predicate| { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) + .with_partition_spec(partition.as_ref().map(|(spec, _)| spec.clone())) + .with_partition(partition.as_ref().map(|(_, data)| data.clone())) + .with_case_sensitive(false) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .build() + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap() + }; - // Projection applies the default. - let (batches, _) = read_once( - &file_path, - schema.clone(), - vec![1, 2], - Predicate::AlwaysTrue, - false, - ) - .await; + let batches = read(Predicate::AlwaysTrue).await; assert_eq!( batches[0].column(1).as_primitive::().values(), &[7, 7, 7] ); - let cases = [ - ("b = 7", Reference::new("b").equal_to(Datum::long(7)), 3), - ("b IS NOT NULL", Reference::new("b").is_not_null(), 3), - ( - "b IN (7, 8)", - Reference::new("b").is_in([Datum::long(7), Datum::long(8)]), - 3, - ), - ("b > 5", Reference::new("b").greater_than(Datum::long(5)), 3), - ("b = 8", Reference::new("b").equal_to(Datum::long(8)), 0), - ("b IS NULL", Reference::new("b").is_null(), 0), - ]; - let mut actual = Vec::new(); - for (name, predicate, _) in &cases { - let (batches, _) = read_once( - &file_path, - schema.clone(), - vec![1, 2], - predicate.clone(), - false, - ) - .await; - actual.push(( - *name, - batches.iter().map(RecordBatch::num_rows).sum::(), - )); + for (predicate, _) in &cases { + let batches = read(predicate.clone()).await; + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + actual.push((predicate.to_string(), rows)); } - let expected: Vec<_> = cases.iter().map(|(name, _, rows)| (*name, *rows)).collect(); + let expected: Vec<_> = cases + .iter() + .map(|(predicate, rows)| (predicate.to_string(), *rows)) + .collect(); assert_eq!(actual, expected); } + /// A file written before column `b` was added with `initial-default` 7. + #[tokio::test] + async fn test_predicate_on_absent_column_uses_initial_default() { + let schema = schema_with_b( + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)), + ); + let b = || Reference::new("b"); + assert_absent_b_reads_as_7(schema, None, vec![ + (b().equal_to(Datum::long(7)), 3), + (b().is_not_null(), 3), + (b().is_in([Datum::long(7), Datum::long(8)]), 3), + (b().greater_than(Datum::long(5)), 3), + (b().equal_to(Datum::long(8)), 0), + (b().is_null(), 0), + ]) + .await; + } + + /// A file that doesn't store its identity partition column `b`, as after a Hive migration or + /// `add_files`, with partition value 7. + #[tokio::test] + async fn test_predicate_on_absent_identity_partition_column_uses_partition_value() { + let schema = schema_with_b(NestedField::optional( + 2, + "b", + Type::Primitive(PrimitiveType::Long), + )); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter([Some(Literal::long(7))]); + let b = || Reference::new("b"); + assert_absent_b_reads_as_7(schema, Some((Arc::new(partition_spec), partition)), vec![ + (b().equal_to(Datum::long(7)), 3), + (b().is_not_null(), 3), + (b().equal_to(Datum::long(8)), 0), + (b().is_null(), 0), + ]) + .await; + } + #[tokio::test] async fn test_page_index_on_absent_column_uses_initial_default() { - let (schema, file_path, _tmp_dir) = setup_absent_column_with_initial_default(); - let file = File::open(&file_path).unwrap(); + let schema = schema_with_b( + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)), + ); + let (file_path, _tmp_dir) = write_file_without_b(); let metadata = ArrowReaderMetadata::load( - &file, + &File::open(&file_path).unwrap(), ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required), ) .unwrap() @@ -1860,19 +1908,17 @@ mod tests { .clone(); let field_id_map = HashMap::from([(1, 0)]); - let cases = [ - ("b = 7", Reference::new("b").equal_to(Datum::long(7))), - ("b IS NOT NULL", Reference::new("b").is_not_null()), - ( - "b IN (7, 8)", - Reference::new("b").is_in([Datum::long(7), Datum::long(8)]), - ), + let b = || Reference::new("b"); + let predicates = [ + b().equal_to(Datum::long(7)), + b().is_not_null(), + b().is_in([Datum::long(7), Datum::long(8)]), ]; // The reader applies the residual before page index pruning. let mut actual = Vec::new(); - for (name, predicate) in &cases { - let predicate = residual_for_missing_fields( + for predicate in &predicates { + let residual = residual_for_missing_fields( predicate.clone().bind(schema.clone(), true).unwrap(), &HashSet::from([2]), &field_id_map, @@ -1882,7 +1928,7 @@ mod tests { ) .unwrap(); let selection = ArrowReader::get_row_selection_for_filter_predicate( - &predicate, + &residual, &metadata, &None, &field_id_map, @@ -1890,98 +1936,9 @@ mod tests { ) .unwrap() .expect("the file has a page index"); - actual.push((*name, selection.row_count())); - } - let expected: Vec<_> = cases.iter().map(|(name, _)| (*name, 3)).collect(); - assert_eq!(actual, expected); - } - - /// A file that doesn't store its identity partition column `p`, as after a Hive - /// migration or `add_files`. Projection reads `p` from the partition value 7, so each - /// predicate must keep or drop all three rows as if `p` were stored as 7. - #[tokio::test] - async fn test_predicate_on_absent_identity_partition_column_uses_partition_value() { - let schema = Arc::new( - Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), - NestedField::optional(2, "p", Type::Primitive(PrimitiveType::Long)).into(), - ]) - .build() - .unwrap(), - ); - let partition_spec = Arc::new( - PartitionSpec::builder(schema.clone()) - .add_partition_field("p", "p", Transform::Identity) - .unwrap() - .build() - .unwrap(), - ); - let partition = Struct::from_iter([Some(Literal::long(7))]); - - let tmp_dir = TempDir::new().unwrap(); - let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); - let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( - "a", - DataType::Int64, - 1, - )])); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( - vec![1, 2, 3], - ))]) - .unwrap(); - write_row_groups(&file_path, arrow_schema, vec![batch], false); - - let read = async |predicate: Predicate| { - let task = FileScanTask::builder() - .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) - .with_start(0) - .with_length(0) - .with_data_file_path(file_path.clone()) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(schema.clone()) - .with_project_field_ids(vec![1, 2]) - .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) - .with_partition_spec(Some(partition_spec.clone())) - .with_partition(Some(partition.clone())) - .with_case_sensitive(false) - .build() - .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; - ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) - .build() - .read(tasks) - .unwrap() - .stream() - .try_collect::>() - .await - .unwrap() - }; - - // Projection applies the partition value. - let batches = read(Predicate::AlwaysTrue).await; - assert_eq!( - batches[0].column(1).as_primitive::().values(), - &[7, 7, 7] - ); - - let cases = [ - ("p = 7", Reference::new("p").equal_to(Datum::long(7)), 3), - ("p IS NOT NULL", Reference::new("p").is_not_null(), 3), - ("p = 8", Reference::new("p").equal_to(Datum::long(8)), 0), - ("p IS NULL", Reference::new("p").is_null(), 0), - ]; - - let mut actual = Vec::new(); - for (name, predicate, _) in &cases { - let batches = read(predicate.clone()).await; - actual.push(( - *name, - batches.iter().map(RecordBatch::num_rows).sum::(), - )); + actual.push((predicate.to_string(), selection.row_count())); } - let expected: Vec<_> = cases.iter().map(|(name, _, rows)| (*name, *rows)).collect(); + let expected: Vec<_> = predicates.iter().map(|p| (p.to_string(), 3)).collect(); assert_eq!(actual, expected); } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 29362e8584..a41961a094 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -4177,8 +4177,6 @@ pub mod tests { #[tokio::test] async fn test_scan_filter_on_initial_default_column_absent_from_file() { use arrow_array::types::Int64Type; - use arrow_cast::cast; - use arrow_schema::DataType; let catalog = new_memory_catalog().await; let namespace = NamespaceIdent::new(format!("ns-{}", Uuid::new_v4())); @@ -4233,28 +4231,19 @@ pub mod tests { ]) .await; + let c = || Reference::new("c"); let cases = [ - ("no filter", Predicate::AlwaysTrue, vec![1, 2, 3]), - ( - "c = 'US'", - Reference::new("c").equal_to(Datum::string("US")), - vec![1, 2], - ), - ("c IS NOT NULL", Reference::new("c").is_not_null(), vec![ - 1, 2, 3, - ]), - ( - "c = 'CA'", - Reference::new("c").equal_to(Datum::string("CA")), - vec![3], - ), - ("c IS NULL", Reference::new("c").is_null(), vec![]), + (Predicate::AlwaysTrue, vec![1, 2, 3]), + (c().equal_to(Datum::string("US")), vec![1, 2]), + (c().is_not_null(), vec![1, 2, 3]), + (c().equal_to(Datum::string("CA")), vec![3]), + (c().is_null(), vec![]), ]; let mut actual = Vec::new(); let mut expected = Vec::new(); for row_selection_enabled in [false, true] { - for (name, predicate, ids) in &cases { + for (predicate, ids) in &cases { let batches: Vec = table .scan() .with_filter(predicate.clone()) @@ -4267,29 +4256,21 @@ pub mod tests { .try_collect() .await .unwrap(); - let mut rows: Vec<(i64, String)> = batches + let mut actual_ids: Vec = batches .iter() .flat_map(|batch| { - let c = cast(batch.column_by_name("c").unwrap(), &DataType::Utf8).unwrap(); - let ids = batch + batch .column_by_name("id") .unwrap() - .as_primitive::(); - (0..batch.num_rows()) - .map(|i| (ids.value(i), c.as_string::().value(i).to_string())) - .collect::>() + .as_primitive::() + .values() + .to_vec() }) .collect(); - rows.sort(); - actual.push((*name, row_selection_enabled, rows)); - - let all_rows = [(1, "US"), (2, "US"), (3, "CA")]; - let rows = all_rows - .iter() - .filter(|(id, _)| ids.contains(id)) - .map(|(id, c)| (*id, c.to_string())) - .collect(); - expected.push((*name, row_selection_enabled, rows)); + actual_ids.sort(); + let name = format!("{predicate}, row selection {row_selection_enabled}"); + actual.push((name.clone(), actual_ids)); + expected.push((name, ids.clone())); } } assert_eq!(actual, expected); From 66a8754404b1e096871e3db26bdc7642309b265e Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 5 Oct 2026 12:20:21 -0700 Subject: [PATCH 3/4] address PR feedback. negate NOT in residual, test equality deletes on missing columns --- crates/iceberg/src/arrow/reader/pipeline.rs | 2 + .../src/arrow/reader/predicate_visitor.rs | 88 ++++--- crates/iceberg/src/arrow/reader/row_filter.rs | 214 +++++++++--------- crates/iceberg/src/expr/predicate.rs | 2 +- 4 files changed, 145 insertions(+), 161 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 1ac0eece61..06c534d5d9 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -516,6 +516,8 @@ impl FileScanTaskReader { &predicate, use_position_fallback, )?; + // Apply the residual before the row filter and every pruning layer below, so they all + // see the same predicate. let predicate = residual_for_missing_fields( predicate, &iceberg_field_ids, diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 1567ee3983..f5c3882066 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -39,7 +39,7 @@ use crate::arrow::record_batch_transformer::constants_map; use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::visitors::expression_evaluator::ExpressionEvaluatorVisitor; -use crate::expr::{BoundPredicate, BoundReference, LogicalExpression}; +use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{Datum, Literal, PartitionSpec, Schema, Struct}; /// A visitor to collect field ids from bound predicates. @@ -225,7 +225,7 @@ pub(super) fn residual_for_missing_fields( _ => HashMap::new(), }; - let mut field_ids = HashSet::new(); + let mut constant_field_ids = HashSet::new(); let row: Struct = schema .as_struct() .fields() @@ -239,29 +239,29 @@ pub(super) fn residual_for_missing_fields( None => field.initial_default.clone(), }; if value.is_some() { - field_ids.insert(field.id); + constant_field_ids.insert(field.id); } value }) .collect(); - if field_ids.is_empty() { + if constant_field_ids.is_empty() { return Ok(predicate); } visit( &mut MissingFieldResidualVisitor { row: &row, - field_ids: &field_ids, + constant_field_ids: &constant_field_ids, }, &predicate, ) } -/// Replaces leaves on the fields in `field_ids` with their result on `row`, which holds each +/// Replaces leaves on the fields in `constant_field_ids` with their result on `row`, which holds each /// top-level field's value at its position in the schema. struct MissingFieldResidualVisitor<'a> { row: &'a Struct, - field_ids: &'a HashSet, + constant_field_ids: &'a HashSet, } impl MissingFieldResidualVisitor<'_> { @@ -270,7 +270,7 @@ impl MissingFieldResidualVisitor<'_> { reference: &BoundReference, predicate: &BoundPredicate, ) -> Result { - if !self.field_ids.contains(&reference.field().id) { + if !self.constant_field_ids.contains(&reference.field().id) { return Ok(predicate.clone()); } if visit(&mut ExpressionEvaluatorVisitor::new(self.row), predicate)? { @@ -301,9 +301,7 @@ impl BoundPredicateVisitor for MissingFieldResidualVisitor<'_> { } fn not(&mut self, inner: BoundPredicate) -> Result { - Ok(BoundPredicate::Not(LogicalExpression::new([Box::new( - inner, - )]))) + Ok(inner.negate()) } fn is_null( @@ -907,7 +905,7 @@ mod tests { CollectFieldIdVisitor, PredicateConverter, constant_bool_array, residual_for_missing_fields, }; use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, BoundPredicate, LogicalExpression, Predicate, Reference}; + use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::spec::{ Datum, Literal, NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, StructType, Transform, Type, @@ -1115,29 +1113,19 @@ mod tests { .unwrap(), ); - let cases: Vec<_> = always_true + let cases = always_true .into_iter() .map(|predicate| (predicate, BoundPredicate::AlwaysTrue)) .chain( always_false .into_iter() .map(|predicate| (predicate, BoundPredicate::AlwaysFalse)), - ) - .collect(); - let actual: Vec<_> = cases - .iter() - .map(|(predicate, _)| { - ( - predicate.to_string(), - residual_for_file_with_field_1(&schema, predicate.clone(), None), - ) - }) - .collect(); - let expected: Vec<_> = cases - .into_iter() - .map(|(predicate, residual)| (predicate.to_string(), residual)) - .collect(); - assert_eq!(actual, expected); + ); + for (predicate, expected) in cases { + let message = predicate.to_string(); + let residual = residual_for_file_with_field_1(&schema, predicate, None); + assert_eq!(residual, expected, "{message}"); + } } /// Mirrors Java's `TestMetricsRowGroupFilter.testColumnNotInFileWithInitialDefault`. @@ -1287,9 +1275,7 @@ mod tests { ( "NOT (b = 7)", !Reference::new("b").equal_to(Datum::long(7)), - BoundPredicate::Not(LogicalExpression::new([Box::new( - BoundPredicate::AlwaysTrue, - )])), + BoundPredicate::AlwaysFalse, ), ( "a > 0 AND b = 8", @@ -1306,23 +1292,29 @@ mod tests { .or(Reference::new("b").equal_to(Datum::long(7))), bind(Reference::new("d").equal_to(Datum::long(1))).or(BoundPredicate::AlwaysTrue), ), + ( + "the keep predicate of an equality delete on b = 7 drops every row", + Reference::new("b") + .is_null() + .or(Reference::new("b").not_equal_to(Datum::long(7))), + BoundPredicate::AlwaysFalse.or(BoundPredicate::AlwaysFalse), + ), + ( + "the keep predicate of an equality delete on b = 5 keeps every row", + Reference::new("b") + .is_null() + .or(Reference::new("b").not_equal_to(Datum::long(5))), + BoundPredicate::AlwaysFalse.or(BoundPredicate::AlwaysTrue), + ), ]; - let actual: Vec<_> = cases - .iter() - .map(|(name, predicate, _)| { - let residual = residual_for_file_with_field_1( - &schema, - predicate.clone(), - Some((&partition_spec, &partition)), - ); - (*name, residual) - }) - .collect(); - let expected: Vec<_> = cases - .into_iter() - .map(|(name, _, residual)| (name, residual)) - .collect(); - assert_eq!(actual, expected); + for (name, predicate, expected) in cases { + let residual = residual_for_file_with_field_1( + &schema, + predicate, + Some((&partition_spec, &partition)), + ); + assert_eq!(residual, expected, "{name}"); + } } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 70a19fe48d..3d08665aaa 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -211,7 +211,7 @@ impl ArrowReader { #[cfg(test)] mod tests { - use std::collections::{HashMap, HashSet}; + use std::collections::HashMap; use std::fs::File; use std::sync::Arc; @@ -223,19 +223,15 @@ mod tests { }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; - use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::Compression; - use parquet::file::metadata::{ - FileMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataBuilder, - }; + use parquet::file::metadata::{FileMetaData, ParquetMetaData, ParquetMetaDataBuilder}; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; use tempfile::TempDir; use crate::Runtime; - use crate::arrow::reader::predicate_visitor::residual_for_missing_fields; use crate::arrow::{ArrowReader, ArrowReaderBuilder}; use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::io::FileIO; @@ -1338,20 +1334,15 @@ mod tests { } async fn read_once( + reader: ArrowReader, file_path: &str, schema: Arc, project_field_ids: Vec, predicate: Predicate, - bloom_enabled: bool, + partition: Option<(Arc, Struct)>, + deletes: Vec, ) -> (Vec, u64) { - let file_io = FileIO::new_with_fs(); - let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) - .with_bloom_filter_enabled(bloom_enabled) - // Keep the fixed footer prefetch small so the byte measurement reflects - // row group I/O rather than a constant metadata read. - .with_metadata_size_hint(8 * 1024) - .build(); - + let (partition_spec, partition) = partition.unzip(); let task = FileScanTask::builder() .with_file_size_in_bytes(std::fs::metadata(file_path).unwrap().len()) .with_start(0) @@ -1361,6 +1352,9 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(project_field_ids) .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .with_deletes(deletes) + .with_partition(partition) + .with_partition_spec(partition_spec) .with_case_sensitive(false) .build() .unwrap(); @@ -1389,15 +1383,25 @@ mod tests { project_field_ids: Vec, predicate: Predicate, ) -> (Option, u64, u64) { - let (off, bytes_off) = read_once( - file_path, - schema.clone(), - project_field_ids.clone(), - predicate.clone(), - false, - ) - .await; - let (on, bytes_on) = read_once(file_path, schema, project_field_ids, predicate, true).await; + let read = |bloom_enabled| { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_bloom_filter_enabled(bloom_enabled) + // Keep the fixed footer prefetch small so the byte measurement reflects + // row group I/O rather than a constant metadata read. + .with_metadata_size_hint(8 * 1024) + .build(); + read_once( + reader, + file_path, + schema.clone(), + project_field_ids.clone(), + predicate.clone(), + None, + vec![], + ) + }; + let (off, bytes_off) = read(false).await; + let (on, bytes_on) = read(true).await; let off = collapse(&off); let on = collapse(&on); @@ -1795,57 +1799,93 @@ mod tests { } /// Reads the file from [`write_file_without_b`], in which `b` reads as 7 on every row through - /// `schema` or `partition`, and asserts the rows each predicate keeps. Each predicate must keep - /// or drop all three rows as if `b` were stored as 7. + /// `schema` or `partition`, and asserts the rows each predicate keeps. Each predicate, and each + /// equality delete on `b`, must keep or drop all three rows as if `b` were stored as 7. async fn assert_absent_b_reads_as_7( schema: SchemaRef, partition: Option<(Arc, Struct)>, cases: Vec<(Predicate, usize)>, ) { - let (file_path, _tmp_dir) = write_file_without_b(); - let read = async |predicate: Predicate| { - let task = FileScanTask::builder() - .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) - .with_start(0) - .with_length(0) - .with_data_file_path(file_path.clone()) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(schema.clone()) - .with_project_field_ids(vec![1, 2]) - .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) - .with_partition_spec(partition.as_ref().map(|(spec, _)| spec.clone())) - .with_partition(partition.as_ref().map(|(_, data)| data.clone())) - .with_case_sensitive(false) - .build() - .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; - ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) - .build() - .read(tasks) - .unwrap() - .stream() - .try_collect::>() - .await - .unwrap() + let (file_path, tmp_dir) = write_file_without_b(); + let read = async |row_selection_enabled, predicate, deletes| { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_selection_enabled(row_selection_enabled) + .build(); + let (batches, _) = read_once( + reader, + &file_path, + schema.clone(), + vec![1, 2], + predicate, + partition.clone(), + deletes, + ) + .await; + batches }; - let batches = read(Predicate::AlwaysTrue).await; + let batches = read(false, Predicate::AlwaysTrue, vec![]).await; assert_eq!( batches[0].column(1).as_primitive::().values(), &[7, 7, 7] ); - let mut actual = Vec::new(); - for (predicate, _) in &cases { - let batches = read(predicate.clone()).await; - let rows = batches.iter().map(RecordBatch::num_rows).sum::(); - actual.push((predicate.to_string(), rows)); - } - let expected: Vec<_> = cases - .iter() - .map(|(predicate, rows)| (predicate.to_string(), *rows)) + // The keep predicate for a deleted value `v` is `b IS NULL OR b != v`, and `b IS NOT NULL` + // for a deleted null. + let delete_field = Field::new("b", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])); + let delete_schema = Arc::new(ArrowSchema::new(vec![delete_field])); + let deletes: Vec<_> = [(Some(7), 0), (Some(5), 3), (None, 3)] + .into_iter() + .enumerate() + .map(|(i, (deleted_b, rows))| { + let delete_path = + format!("{}/eq-del-{i}.parquet", tmp_dir.path().to_str().unwrap()); + let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new( + Int64Array::from(vec![deleted_b]), + )]) + .unwrap(); + write_row_groups(&delete_path, delete_schema.clone(), vec![batch], false); + let delete = FileScanTaskDeleteFile { + file_size_in_bytes: std::fs::metadata(&delete_path).unwrap().len(), + file_path: delete_path, + file_type: DataContentType::EqualityDeletes, + file_format: DataFileFormat::Parquet, + partition_spec_id: 0, + equality_ids: Some(vec![2]), + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + record_count: None, + key_metadata: None, + }; + (deleted_b, delete, rows) + }) .collect(); - assert_eq!(actual, expected); + + for row_selection_enabled in [false, true] { + for (predicate, expected) in &cases { + let batches = read(row_selection_enabled, predicate.clone(), vec![]).await; + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + assert_eq!( + rows, *expected, + "{predicate}, row_selection_enabled = {row_selection_enabled}" + ); + } + for (deleted_b, delete, expected) in &deletes { + let batches = read(row_selection_enabled, Predicate::AlwaysTrue, vec![ + delete.clone(), + ]) + .await; + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + assert_eq!( + rows, *expected, + "equality delete on b = {deleted_b:?}, row_selection_enabled = {row_selection_enabled}" + ); + } + } } /// A file written before column `b` was added with `initial-default` 7. @@ -1891,54 +1931,4 @@ mod tests { ]) .await; } - - #[tokio::test] - async fn test_page_index_on_absent_column_uses_initial_default() { - let schema = schema_with_b( - NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) - .with_initial_default(Literal::long(7)), - ); - let (file_path, _tmp_dir) = write_file_without_b(); - let metadata = ArrowReaderMetadata::load( - &File::open(&file_path).unwrap(), - ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required), - ) - .unwrap() - .metadata() - .clone(); - let field_id_map = HashMap::from([(1, 0)]); - - let b = || Reference::new("b"); - let predicates = [ - b().equal_to(Datum::long(7)), - b().is_not_null(), - b().is_in([Datum::long(7), Datum::long(8)]), - ]; - - // The reader applies the residual before page index pruning. - let mut actual = Vec::new(); - for predicate in &predicates { - let residual = residual_for_missing_fields( - predicate.clone().bind(schema.clone(), true).unwrap(), - &HashSet::from([2]), - &field_id_map, - &schema, - None, - None, - ) - .unwrap(); - let selection = ArrowReader::get_row_selection_for_filter_predicate( - &residual, - &metadata, - &None, - &field_id_map, - &schema, - ) - .unwrap() - .expect("the file has a page index"); - actual.push((predicate.to_string(), selection.row_count())); - } - let expected: Vec<_> = predicates.iter().map(|p| (p.to_string(), 3)).collect(); - assert_eq!(actual, expected); - } } diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 0d04628aae..f8ea896395 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -69,7 +69,7 @@ impl Debug for LogicalExpression { } impl LogicalExpression { - pub(crate) fn new(inputs: [Box; N]) -> Self { + fn new(inputs: [Box; N]) -> Self { Self { inputs } } From 006d4b9a648b49e7ffbed59a2d2da9c8f425bd70 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 16 Oct 2026 14:24:16 -0400 Subject: [PATCH 4/4] address PR feedback --- .../src/arrow/reader/predicate_visitor.rs | 80 ++++++++++++------- .../src/expr/visitors/expression_evaluator.rs | 40 +++++----- crates/iceberg/src/scan/mod.rs | 77 ++++++++---------- 3 files changed, 104 insertions(+), 93 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index f5c3882066..fca0e90541 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -201,7 +201,8 @@ impl BoundPredicateVisitor for CollectFieldIdVisitor { /// missing from the file becomes `AlwaysTrue` or `AlwaysFalse`, based on the value projection /// returns for that field. That value is the identity partition value, otherwise the field's /// `initial-default`, and every row of the file holds it. Leaves on missing fields with neither -/// keep the null handling of the row filter and the page index evaluator. +/// keep the null handling of the row filter and the page index evaluator. The resulting constants +/// are folded out of `AND` and `OR`. /// /// Leaves on nested fields are kept, because a nested field also reads as null in any row where /// an ancestor struct is null. @@ -292,12 +293,26 @@ impl BoundPredicateVisitor for MissingFieldResidualVisitor<'_> { Ok(BoundPredicate::AlwaysFalse) } + // Fold constants so the row filter doesn't evaluate them per batch. The keep predicate of an + // equality delete file has one `OR` per deleted row, and each can fold to a constant. fn and(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { - Ok(lhs.and(rhs)) + Ok(match (lhs, rhs) { + (BoundPredicate::AlwaysFalse, _) | (_, BoundPredicate::AlwaysFalse) => { + BoundPredicate::AlwaysFalse + } + (BoundPredicate::AlwaysTrue, other) | (other, BoundPredicate::AlwaysTrue) => other, + (lhs, rhs) => lhs.and(rhs), + }) } fn or(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { - Ok(lhs.or(rhs)) + Ok(match (lhs, rhs) { + (BoundPredicate::AlwaysTrue, _) | (_, BoundPredicate::AlwaysTrue) => { + BoundPredicate::AlwaysTrue + } + (BoundPredicate::AlwaysFalse, other) | (other, BoundPredicate::AlwaysFalse) => other, + (lhs, rhs) => lhs.or(rhs), + }) } fn not(&mut self, inner: BoundPredicate) -> Result { @@ -1161,26 +1176,6 @@ mod tests { ); } - /// Mirrors Java's `TestMetricsRowGroupFilter.testDateColumnNotInFileWithInitialDefault`. - #[test] - fn test_residual_for_missing_date_field_with_initial_default() { - let event_date = || Reference::new("event_date"); - assert_residuals_for_missing_field_with_initial_default( - NestedField::optional(100, "event_date", Type::Primitive(PrimitiveType::Date)) - .with_initial_default(Literal::date(42)), - vec![ - event_date().equal_to(Datum::date(42)), - event_date().less_than(Datum::date(43)), - event_date().greater_than(Datum::date(41)), - ], - vec![ - event_date().equal_to(Datum::date(41)), - event_date().less_than(Datum::date(42)), - event_date().greater_than(Datum::date(42)), - ], - ); - } - /// Mirrors Java's `TestMetricsRowGroupFilter.testDoubleColumnNotInFileWithInitialDefault`. #[test] fn test_residual_for_missing_double_field_with_initial_default() { @@ -1282,29 +1277,54 @@ mod tests { Reference::new("a") .greater_than(Datum::long(0)) .and(Reference::new("b").equal_to(Datum::long(8))), - bind(Reference::new("a").greater_than(Datum::long(0))) - .and(BoundPredicate::AlwaysFalse), + BoundPredicate::AlwaysFalse, ), ( - "d = 1 OR b = 7", + "b = 7 AND a > 0", + Reference::new("b") + .equal_to(Datum::long(7)) + .and(Reference::new("a").greater_than(Datum::long(0))), + bind(Reference::new("a").greater_than(Datum::long(0))), + ), + ( + "b = 7 OR d = 1", + Reference::new("b") + .equal_to(Datum::long(7)) + .or(Reference::new("d").equal_to(Datum::long(1))), + BoundPredicate::AlwaysTrue, + ), + ( + "d = 1 OR b = 8", Reference::new("d") .equal_to(Datum::long(1)) - .or(Reference::new("b").equal_to(Datum::long(7))), - bind(Reference::new("d").equal_to(Datum::long(1))).or(BoundPredicate::AlwaysTrue), + .or(Reference::new("b").equal_to(Datum::long(8))), + bind(Reference::new("d").equal_to(Datum::long(1))), + ), + ( + "(a > 0 AND d IS NULL) OR b = 8 keeps the AND", + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("d").is_null()) + .or(Reference::new("b").equal_to(Datum::long(8))), + bind( + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("d").is_null()), + ), ), ( "the keep predicate of an equality delete on b = 7 drops every row", Reference::new("b") .is_null() .or(Reference::new("b").not_equal_to(Datum::long(7))), - BoundPredicate::AlwaysFalse.or(BoundPredicate::AlwaysFalse), + BoundPredicate::AlwaysFalse, ), ( "the keep predicate of an equality delete on b = 5 keeps every row", Reference::new("b") .is_null() .or(Reference::new("b").not_equal_to(Datum::long(5))), - BoundPredicate::AlwaysFalse.or(BoundPredicate::AlwaysTrue), + BoundPredicate::AlwaysTrue, ), ]; diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 95a96f21e3..c2c6d39826 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -50,19 +50,19 @@ impl ExpressionEvaluator { } } -/// Acts as a visitor for [`ExpressionEvaluator`] to apply -/// evaluation logic to different parts of a data structure, -/// specifically for data file partitions. +/// Evaluates a [`BoundPredicate`] against one row of values, such as a +/// [`DataFile`]'s partition [`Struct`]. #[derive(Debug)] pub(crate) struct ExpressionEvaluatorVisitor<'a> { - /// Reference to a [`DataFile`]'s partition [`Struct`]. - partition: &'a Struct, + /// Laid out like the struct the predicate was bound to, because each reference's accessor + /// reads its value by position. + row: &'a Struct, } impl<'a> ExpressionEvaluatorVisitor<'a> { /// Creates a new [`ExpressionEvaluatorVisitor`]. - pub(crate) fn new(partition: &'a Struct) -> Self { - Self { partition } + pub(crate) fn new(row: &'a Struct) -> Self { + Self { row } } } @@ -93,7 +93,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { } fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(_) => Ok(false), None => Ok(true), } @@ -104,21 +104,21 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { reference: &BoundReference, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(_) => Ok(true), None => Ok(false), } } fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(datum.is_nan()), None => Ok(false), } } fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(!datum.is_nan()), None => Ok(true), } @@ -130,7 +130,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum < literal), None => Ok(false), } @@ -142,7 +142,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum <= literal), None => Ok(false), } @@ -154,7 +154,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum > literal), None => Ok(false), } @@ -166,7 +166,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum >= literal), None => Ok(false), } @@ -178,7 +178,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum == literal), None => Ok(false), } @@ -190,7 +190,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum != literal), None => Ok(true), } @@ -202,7 +202,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - let Some(datum) = reference.accessor().get(self.partition)? else { + let Some(datum) = reference.accessor().get(self.row)? else { return Ok(false); }; @@ -227,7 +227,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literals: &FnvHashSet, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(literals.contains(&datum)), None => Ok(false), } @@ -239,7 +239,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literals: &FnvHashSet, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(!literals.contains(&datum)), None => Ok(true), } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index e4b99084b4..c507815d42 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -706,7 +706,7 @@ mod tests { use super::resolve_field_id; use crate::arrow::{ArrowReaderBuilder, schema_to_arrow_schema}; - use crate::expr::{BoundPredicate, Predicate, Reference}; + use crate::expr::{BoundPredicate, Reference}; use crate::io::FileIO; use crate::memory::tests::new_memory_catalog; use crate::metadata_columns::{ @@ -3157,9 +3157,12 @@ mod tests { tx.commit(catalog).await.unwrap() } - /// Mirrors Java's `TestFilterPushDown.testFilterPushdownOnInitialDefaultColumnAbsentFromFile`. - /// The first file is written before `c` is added with `initial-default` 'US', so its row - /// reads `c` as 'US'. The second file stores `c`. + /// Checks the residual end to end, through `update_schema` and `TableScan`, which the reader + /// tests in `arrow/reader/row_filter.rs` bypass. Those tests cover the other predicates and + /// row selection. Follows Java's + /// `TestFilterPushDown.testFilterPushdownOnInitialDefaultColumnAbsentFromFile`: the first file + /// is written before `c` is added with `initial-default` 'US', so its row reads `c` as 'US'. + /// The second file stores `c`. #[tokio::test] async fn test_scan_filter_on_initial_default_column_absent_from_file() { use arrow_array::types::Int64Type; @@ -3218,47 +3221,35 @@ mod tests { .await; let c = || Reference::new("c"); - let cases = [ - (Predicate::AlwaysTrue, vec![1, 2, 3]), + for (predicate, expected) in [ (c().equal_to(Datum::string("US")), vec![1, 2]), - (c().is_not_null(), vec![1, 2, 3]), - (c().equal_to(Datum::string("CA")), vec![3]), (c().is_null(), vec![]), - ]; - - let mut actual = Vec::new(); - let mut expected = Vec::new(); - for row_selection_enabled in [false, true] { - for (predicate, ids) in &cases { - let batches: Vec = table - .scan() - .with_filter(predicate.clone()) - .with_row_selection_enabled(row_selection_enabled) - .build() - .unwrap() - .to_arrow() - .await - .unwrap() - .try_collect() - .await - .unwrap(); - let mut actual_ids: Vec = batches - .iter() - .flat_map(|batch| { - batch - .column_by_name("id") - .unwrap() - .as_primitive::() - .values() - .to_vec() - }) - .collect(); - actual_ids.sort(); - let name = format!("{predicate}, row selection {row_selection_enabled}"); - actual.push((name.clone(), actual_ids)); - expected.push((name, ids.clone())); - } + ] { + let message = predicate.to_string(); + let batches: Vec = table + .scan() + .with_filter(predicate) + .build() + .unwrap() + .to_arrow() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut actual: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + actual.sort(); + assert_eq!(actual, expected, "{message}"); } - assert_eq!(actual, expected); } }