diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 4e9a0756e4..132ad98ce7 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -33,6 +33,7 @@ use parquet::arrow::{ }; use parquet::encryption::decrypt::FileDecryptionProperties; +use super::predicate_visitor::residual_for_missing_fields; use super::row_lineage::synthesize_row_id_column; use super::{ ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, @@ -563,6 +564,16 @@ impl FileScanTaskReader { &predicate, use_position_fallback, )?; + // Apply the residual before the row filter and every pruning layer below, so they all + // see the same predicate. + let predicate = residual_for_missing_fields( + predicate, + &iceberg_field_ids, + &field_id_map, + task.schema(), + task.partition_spec().map(Arc::as_ref), + task.partition(), + )?; let row_filter = ArrowReader::get_row_filter( &predicate, diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..fca0e90541 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -35,10 +35,12 @@ use fnv::FnvHashSet; use parquet::schema::types::SchemaDescriptor; use crate::arrow::get_arrow_datum; +use crate::arrow::record_batch_transformer::constants_map; use crate::error::{Result, invalid_data}; -use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; +use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; +use crate::expr::visitors::expression_evaluator::ExpressionEvaluatorVisitor; use crate::expr::{BoundPredicate, BoundReference}; -use crate::spec::Datum; +use crate::spec::{Datum, Literal, PartitionSpec, Schema, Struct}; /// A visitor to collect field ids from bound predicates. pub(super) struct CollectFieldIdVisitor { @@ -195,6 +197,251 @@ impl BoundPredicateVisitor for CollectFieldIdVisitor { } } +/// Returns the residual of `predicate` for one data file: each leaf on a top-level field that is +/// missing from the file becomes `AlwaysTrue` or `AlwaysFalse`, based on the value projection +/// returns for that field. That value is the identity partition value, otherwise the field's +/// `initial-default`, and every row of the file holds it. Leaves on missing fields with neither +/// keep the null handling of the row filter and the page index evaluator. The resulting constants +/// are folded out of `AND` and `OR`. +/// +/// Leaves on nested fields are kept, because a nested field also reads as null in any row where +/// an ancestor struct is null. +pub(super) fn residual_for_missing_fields( + predicate: BoundPredicate, + predicate_field_ids: &HashSet, + field_id_map: &HashMap, + schema: &Schema, + partition_spec: Option<&PartitionSpec>, + partition: Option<&Struct>, +) -> Result { + if predicate_field_ids + .iter() + .all(|id| field_id_map.contains_key(id)) + { + return Ok(predicate); + } + + let partition_constants = match (partition_spec, partition) { + (Some(spec), Some(data)) => constants_map(spec, data, schema)?, + _ => HashMap::new(), + }; + + let mut constant_field_ids = HashSet::new(); + let row: Struct = schema + .as_struct() + .fields() + .iter() + .map(|field| { + if !predicate_field_ids.contains(&field.id) || field_id_map.contains_key(&field.id) { + return None; + } + let value = match partition_constants.get(&field.id) { + Some(datum) => Some(Literal::Primitive(datum.literal().clone())), + None => field.initial_default.clone(), + }; + if value.is_some() { + constant_field_ids.insert(field.id); + } + value + }) + .collect(); + + if constant_field_ids.is_empty() { + return Ok(predicate); + } + visit( + &mut MissingFieldResidualVisitor { + row: &row, + constant_field_ids: &constant_field_ids, + }, + &predicate, + ) +} + +/// Replaces leaves on the fields in `constant_field_ids` with their result on `row`, which holds each +/// top-level field's value at its position in the schema. +struct MissingFieldResidualVisitor<'a> { + row: &'a Struct, + constant_field_ids: &'a HashSet, +} + +impl MissingFieldResidualVisitor<'_> { + fn residual( + &self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + if !self.constant_field_ids.contains(&reference.field().id) { + return Ok(predicate.clone()); + } + if visit(&mut ExpressionEvaluatorVisitor::new(self.row), predicate)? { + Ok(BoundPredicate::AlwaysTrue) + } else { + Ok(BoundPredicate::AlwaysFalse) + } + } +} + +impl BoundPredicateVisitor for MissingFieldResidualVisitor<'_> { + type T = BoundPredicate; + + fn always_true(&mut self) -> Result { + Ok(BoundPredicate::AlwaysTrue) + } + + fn always_false(&mut self) -> Result { + Ok(BoundPredicate::AlwaysFalse) + } + + // Fold constants so the row filter doesn't evaluate them per batch. The keep predicate of an + // equality delete file has one `OR` per deleted row, and each can fold to a constant. + fn and(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { + Ok(match (lhs, rhs) { + (BoundPredicate::AlwaysFalse, _) | (_, BoundPredicate::AlwaysFalse) => { + BoundPredicate::AlwaysFalse + } + (BoundPredicate::AlwaysTrue, other) | (other, BoundPredicate::AlwaysTrue) => other, + (lhs, rhs) => lhs.and(rhs), + }) + } + + fn or(&mut self, lhs: BoundPredicate, rhs: BoundPredicate) -> Result { + Ok(match (lhs, rhs) { + (BoundPredicate::AlwaysTrue, _) | (_, BoundPredicate::AlwaysTrue) => { + BoundPredicate::AlwaysTrue + } + (BoundPredicate::AlwaysFalse, other) | (other, BoundPredicate::AlwaysFalse) => other, + (lhs, rhs) => lhs.or(rhs), + }) + } + + fn not(&mut self, inner: BoundPredicate) -> Result { + Ok(inner.negate()) + } + + fn is_null( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_null( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn is_nan( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_nan( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn less_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn less_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn greater_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn greater_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn r#in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } + + fn not_in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + predicate: &BoundPredicate, + ) -> Result { + self.residual(reference, predicate) + } +} + /// A visitor to convert Iceberg bound predicates to Arrow predicates. pub(super) struct PredicateConverter<'a> { /// The Parquet schema descriptor. @@ -669,10 +916,15 @@ mod tests { use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; - use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; + use super::{ + CollectFieldIdVisitor, PredicateConverter, constant_bool_array, residual_for_missing_fields, + }; use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; + use crate::spec::{ + Datum, Literal, NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, + StructType, Transform, Type, + }; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -836,4 +1088,253 @@ mod tests { ); assert!(!result.is_null(2)); } + + /// Returns the residual of `predicate` for a data file that stores only field 1. + fn residual_for_file_with_field_1( + schema: &SchemaRef, + predicate: Predicate, + partition: Option<(&PartitionSpec, &Struct)>, + ) -> BoundPredicate { + let predicate = predicate.bind(schema.clone(), true).unwrap(); + let mut collector = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut collector, &predicate).unwrap(); + residual_for_missing_fields( + predicate, + &collector.field_ids(), + &HashMap::from([(1, 0)]), + schema, + partition.map(|(spec, _)| spec), + partition.map(|(_, data)| data), + ) + .unwrap() + } + + /// Asserts that each predicate in `always_true` has the residual `AlwaysTrue` and each in + /// `always_false` has `AlwaysFalse`, for a file that doesn't store the field `field`. + fn assert_residuals_for_missing_field_with_initial_default( + field: NestedField, + always_true: Vec, + always_false: Vec, + ) { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + field.into(), + ]) + .build() + .unwrap(), + ); + + let cases = always_true + .into_iter() + .map(|predicate| (predicate, BoundPredicate::AlwaysTrue)) + .chain( + always_false + .into_iter() + .map(|predicate| (predicate, BoundPredicate::AlwaysFalse)), + ); + for (predicate, expected) in cases { + let message = predicate.to_string(); + let residual = residual_for_file_with_field_1(&schema, predicate, None); + assert_eq!(residual, expected, "{message}"); + } + } + + /// Mirrors Java's `TestMetricsRowGroupFilter.testColumnNotInFileWithInitialDefault`. + #[test] + fn test_residual_for_missing_string_field_with_initial_default() { + let country = || Reference::new("country"); + let string = Datum::string; + assert_residuals_for_missing_field_with_initial_default( + NestedField::optional(99, "country", Type::Primitive(PrimitiveType::String)) + .with_initial_default(Literal::string("US")), + vec![ + country().equal_to(string("US")), + country().is_not_null(), + country().not_equal_to(string("CA")), + country().is_in([string("US"), string("MX")]), + country().is_not_in([string("CA"), string("MX")]), + country().greater_than_or_equal_to(string("US")), + country().less_than(string("ZW")), + country().starts_with(string("U")), + country().not_starts_with(string("X")), + ], + vec![ + country().equal_to(string("CA")), + country().is_null(), + country().not_equal_to(string("US")), + country().is_in([string("CA"), string("MX")]), + country().is_not_in([string("US"), string("MX")]), + country().less_than(string("AD")), + country().greater_than(string("US")), + country().starts_with(string("X")), + country().not_starts_with(string("U")), + ], + ); + } + + /// Mirrors Java's `TestMetricsRowGroupFilter.testDoubleColumnNotInFileWithInitialDefault`. + #[test] + fn test_residual_for_missing_double_field_with_initial_default() { + assert_residuals_for_missing_field_with_initial_default( + NestedField::optional(101, "measurement", Type::Primitive(PrimitiveType::Double)) + .with_initial_default(Literal::double(12.5)), + vec![Reference::new("measurement").is_not_nan()], + vec![Reference::new("measurement").is_nan()], + ); + } + + /// Covers which value a leaf is evaluated against, and which leaves are kept. + #[test] + fn test_residual_for_missing_fields() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(5)) + .into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional(4, "d", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(5, "p", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional(6, "q", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + NestedField::optional( + 7, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(8, "x", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .add_partition_field("p", "p", Transform::Identity) + .unwrap() + .add_partition_field("q", "q_bucket", Transform::Bucket(4)) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter([ + Some(Literal::long(100)), + Some(Literal::long(9)), + Some(Literal::int(2)), + ]); + + let bind = |predicate: Predicate| predicate.bind(schema.clone(), true).unwrap(); + let cases = [ + ( + "p = 9 uses the identity partition value over the initial-default", + Reference::new("p").equal_to(Datum::long(9)), + BoundPredicate::AlwaysTrue, + ), + ( + "p = 7", + Reference::new("p").equal_to(Datum::long(7)), + BoundPredicate::AlwaysFalse, + ), + ( + "q = 7 ignores the bucket partition value", + Reference::new("q").equal_to(Datum::long(7)), + BoundPredicate::AlwaysTrue, + ), + ( + "a = 3 reads the file despite a partition value and initial-default", + Reference::new("a").equal_to(Datum::long(3)), + bind(Reference::new("a").equal_to(Datum::long(3))), + ), + ( + "d IS NULL has no value to apply", + Reference::new("d").is_null(), + bind(Reference::new("d").is_null()), + ), + ( + "s.x = 7 is nested", + Reference::new("s.x").equal_to(Datum::long(7)), + bind(Reference::new("s.x").equal_to(Datum::long(7))), + ), + ( + "NOT (b = 7)", + !Reference::new("b").equal_to(Datum::long(7)), + BoundPredicate::AlwaysFalse, + ), + ( + "a > 0 AND b = 8", + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("b").equal_to(Datum::long(8))), + BoundPredicate::AlwaysFalse, + ), + ( + "b = 7 AND a > 0", + Reference::new("b") + .equal_to(Datum::long(7)) + .and(Reference::new("a").greater_than(Datum::long(0))), + bind(Reference::new("a").greater_than(Datum::long(0))), + ), + ( + "b = 7 OR d = 1", + Reference::new("b") + .equal_to(Datum::long(7)) + .or(Reference::new("d").equal_to(Datum::long(1))), + BoundPredicate::AlwaysTrue, + ), + ( + "d = 1 OR b = 8", + Reference::new("d") + .equal_to(Datum::long(1)) + .or(Reference::new("b").equal_to(Datum::long(8))), + bind(Reference::new("d").equal_to(Datum::long(1))), + ), + ( + "(a > 0 AND d IS NULL) OR b = 8 keeps the AND", + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("d").is_null()) + .or(Reference::new("b").equal_to(Datum::long(8))), + bind( + Reference::new("a") + .greater_than(Datum::long(0)) + .and(Reference::new("d").is_null()), + ), + ), + ( + "the keep predicate of an equality delete on b = 7 drops every row", + Reference::new("b") + .is_null() + .or(Reference::new("b").not_equal_to(Datum::long(7))), + BoundPredicate::AlwaysFalse, + ), + ( + "the keep predicate of an equality delete on b = 5 keeps every row", + Reference::new("b") + .is_null() + .or(Reference::new("b").not_equal_to(Datum::long(5))), + BoundPredicate::AlwaysTrue, + ), + ]; + + for (name, predicate, expected) in cases { + let residual = residual_for_file_with_field_1( + &schema, + predicate, + Some((&partition_spec, &partition)), + ); + assert_eq!(residual, expected, "{name}"); + } + } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 5f1fe5efed..1bbdf9f60c 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -225,6 +225,7 @@ mod tests { use std::sync::Arc; use arrow_array::cast::AsArray; + use arrow_array::types::Int64Type; use arrow_array::{ ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray, @@ -248,7 +249,8 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, Literal, NestedField, PartitionSpec, PrimitiveType, + Schema, SchemaRef, Struct, Transform, Type, }; async fn test_perform_read( @@ -1634,28 +1636,15 @@ mod tests { } async fn read_once( + reader: ArrowReader, file_path: &str, schema: Arc, project_field_ids: Vec, predicate: Predicate, - bloom_enabled: bool, + partition: Option<(Arc, Struct)>, + deletes: Vec, ) -> (Vec, u64) { - let file_io = FileIO::new_with_fs(); - let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) - .with_options( - ReaderOptions::builder() - .with_bloom_filter_enabled(bloom_enabled) - .build(), - ) - // Keep the fixed footer prefetch small so the byte measurement reflects - // row group I/O rather than a constant metadata read. - .with_parquet_options( - ParquetReadOptions::builder() - .with_metadata_size_hint(8 * 1024) - .build(), - ) - .build(); - + let (partition_spec, partition) = partition.unzip(); let task = FileScanTask::builder() .with_file_size_in_bytes(std::fs::metadata(file_path).unwrap().len()) .with_start(0) @@ -1665,6 +1654,9 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(project_field_ids) .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .with_deletes(deletes) + .with_partition(partition) + .with_partition_spec(partition_spec) .with_case_sensitive(false) .build() .unwrap(); @@ -1693,15 +1685,33 @@ mod tests { project_field_ids: Vec, predicate: Predicate, ) -> (Option, u64, u64) { - let (off, bytes_off) = read_once( - file_path, - schema.clone(), - project_field_ids.clone(), - predicate.clone(), - false, - ) - .await; - let (on, bytes_on) = read_once(file_path, schema, project_field_ids, predicate, true).await; + let read = |bloom_enabled| { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_bloom_filter_enabled(bloom_enabled) + .build(), + ) + // Keep the fixed footer prefetch small so the byte measurement reflects + // row group I/O rather than a constant metadata read. + .with_parquet_options( + ParquetReadOptions::builder() + .with_metadata_size_hint(8 * 1024) + .build(), + ) + .build(); + read_once( + reader, + file_path, + schema.clone(), + project_field_ids.clone(), + predicate.clone(), + None, + vec![], + ) + }; + let (off, bytes_off) = read(false).await; + let (on, bytes_on) = read(true).await; let off = collapse(&off); let on = collapse(&on); @@ -2067,4 +2077,168 @@ mod tests { .await; assert_eq!(rows(&on), 1); } + + /// Writes a file that stores only field 1 (`a`), with values 1, 2, 3. + fn write_file_without_b() -> (String, TempDir) { + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "a", + DataType::Int64, + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( + vec![1, 2, 3], + ))]) + .unwrap(); + write_row_groups(&file_path, arrow_schema, vec![batch], false); + (file_path, tmp_dir) + } + + fn schema_with_b(b: NestedField) -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + b.into(), + ]) + .build() + .unwrap(), + ) + } + + /// Reads the file from [`write_file_without_b`], in which `b` reads as 7 on every row through + /// `schema` or `partition`, and asserts the rows each predicate keeps. Each predicate, and each + /// equality delete on `b`, must keep or drop all three rows as if `b` were stored as 7. + async fn assert_absent_b_reads_as_7( + schema: SchemaRef, + partition: Option<(Arc, Struct)>, + cases: Vec<(Predicate, usize)>, + ) { + let (file_path, tmp_dir) = write_file_without_b(); + let read = async |row_selection_enabled, predicate, deletes| { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(row_selection_enabled) + .build(), + ) + .build(); + let (batches, _) = read_once( + reader, + &file_path, + schema.clone(), + vec![1, 2], + predicate, + partition.clone(), + deletes, + ) + .await; + batches + }; + + let batches = read(false, Predicate::AlwaysTrue, vec![]).await; + assert_eq!( + batches[0].column(1).as_primitive::().values(), + &[7, 7, 7] + ); + + // The keep predicate for a deleted value `v` is `b IS NULL OR b != v`, and `b IS NOT NULL` + // for a deleted null. + let delete_field = Field::new("b", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])); + let delete_schema = Arc::new(ArrowSchema::new(vec![delete_field])); + let deletes: Vec<_> = [(Some(7), 0), (Some(5), 3), (None, 3)] + .into_iter() + .enumerate() + .map(|(i, (deleted_b, rows))| { + let delete_path = + format!("{}/eq-del-{i}.parquet", tmp_dir.path().to_str().unwrap()); + let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new( + Int64Array::from(vec![deleted_b]), + )]) + .unwrap(); + write_row_groups(&delete_path, delete_schema.clone(), vec![batch], false); + let delete = FileScanTaskDeleteFile::builder() + .with_file_size_in_bytes(std::fs::metadata(&delete_path).unwrap().len()) + .with_file_path(delete_path) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![2])) + .build() + .unwrap(); + (deleted_b, delete, rows) + }) + .collect(); + + for row_selection_enabled in [false, true] { + for (predicate, expected) in &cases { + let batches = read(row_selection_enabled, predicate.clone(), vec![]).await; + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + assert_eq!( + rows, *expected, + "{predicate}, row_selection_enabled = {row_selection_enabled}" + ); + } + for (deleted_b, delete, expected) in &deletes { + let batches = read(row_selection_enabled, Predicate::AlwaysTrue, vec![ + delete.clone(), + ]) + .await; + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + assert_eq!( + rows, *expected, + "equality delete on b = {deleted_b:?}, row_selection_enabled = {row_selection_enabled}" + ); + } + } + } + + /// A file written before column `b` was added with `initial-default` 7. + #[tokio::test] + async fn test_predicate_on_absent_column_uses_initial_default() { + let schema = schema_with_b( + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)), + ); + let b = || Reference::new("b"); + assert_absent_b_reads_as_7(schema, None, vec![ + (b().equal_to(Datum::long(7)), 3), + (b().is_not_null(), 3), + (b().is_in([Datum::long(7), Datum::long(8)]), 3), + (b().greater_than(Datum::long(5)), 3), + (b().equal_to(Datum::long(8)), 0), + (b().is_null(), 0), + ]) + .await; + } + + /// A file that doesn't store its identity partition column `b`, as after a Hive migration or + /// `add_files`, with partition value 7. + #[tokio::test] + async fn test_predicate_on_absent_identity_partition_column_uses_partition_value() { + let schema = schema_with_b(NestedField::optional( + 2, + "b", + Type::Primitive(PrimitiveType::Long), + )); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter([Some(Literal::long(7))]); + let b = || Reference::new("b"); + assert_absent_b_reads_as_7(schema, Some((Arc::new(partition_spec), partition)), vec![ + (b().equal_to(Datum::long(7)), 3), + (b().is_not_null(), 3), + (b().equal_to(Datum::long(8)), 0), + (b().is_null(), 0), + ]) + .await; + } } diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 9cd626547e..618ad9e4b2 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -78,7 +78,7 @@ fn field_with_id( /// # References /// - Spec: https://iceberg.apache.org/spec/#column-projection /// - Java: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java:constantsMap() -fn constants_map( +pub(crate) fn constants_map( partition_spec: &PartitionSpec, partition_data: &Struct, schema: &IcebergSchema, diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index d9c2e1b571..c2c6d39826 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -50,19 +50,19 @@ impl ExpressionEvaluator { } } -/// Acts as a visitor for [`ExpressionEvaluator`] to apply -/// evaluation logic to different parts of a data structure, -/// specifically for data file partitions. +/// Evaluates a [`BoundPredicate`] against one row of values, such as a +/// [`DataFile`]'s partition [`Struct`]. #[derive(Debug)] -struct ExpressionEvaluatorVisitor<'a> { - /// Reference to a [`DataFile`]'s partition [`Struct`]. - partition: &'a Struct, +pub(crate) struct ExpressionEvaluatorVisitor<'a> { + /// Laid out like the struct the predicate was bound to, because each reference's accessor + /// reads its value by position. + row: &'a Struct, } impl<'a> ExpressionEvaluatorVisitor<'a> { /// Creates a new [`ExpressionEvaluatorVisitor`]. - fn new(partition: &'a Struct) -> Self { - Self { partition } + pub(crate) fn new(row: &'a Struct) -> Self { + Self { row } } } @@ -93,7 +93,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { } fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(_) => Ok(false), None => Ok(true), } @@ -104,21 +104,21 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { reference: &BoundReference, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(_) => Ok(true), None => Ok(false), } } fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(datum.is_nan()), None => Ok(false), } } fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(!datum.is_nan()), None => Ok(true), } @@ -130,7 +130,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum < literal), None => Ok(false), } @@ -142,7 +142,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum <= literal), None => Ok(false), } @@ -154,7 +154,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum > literal), None => Ok(false), } @@ -166,7 +166,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum >= literal), None => Ok(false), } @@ -178,7 +178,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum == literal), None => Ok(false), } @@ -190,7 +190,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(&datum != literal), None => Ok(true), } @@ -202,7 +202,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literal: &Datum, _predicate: &BoundPredicate, ) -> Result { - let Some(datum) = reference.accessor().get(self.partition)? else { + let Some(datum) = reference.accessor().get(self.row)? else { return Ok(false); }; @@ -227,7 +227,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literals: &FnvHashSet, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(literals.contains(&datum)), None => Ok(false), } @@ -239,7 +239,7 @@ impl BoundPredicateVisitor for ExpressionEvaluatorVisitor<'_> { literals: &FnvHashSet, _predicate: &BoundPredicate, ) -> Result { - match reference.accessor().get(self.partition)? { + match reference.accessor().get(self.row)? { Some(datum) => Ok(!literals.contains(&datum)), None => Ok(true), } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 1f488796c9..a3fe059f54 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -763,22 +763,26 @@ mod tests { }; use futures::{TryStreamExt, stream}; use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; use uuid::Uuid; use super::{resolve_column_stats_columns, resolve_field_id}; - use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions}; + use crate::arrow::{ + ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions, schema_to_arrow_schema, + }; use crate::expr::{BoundPredicate, Reference}; use crate::io::FileIO; + use crate::memory::tests::new_memory_catalog; use crate::metadata_columns::{ RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_POS, }; use crate::scan::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile, TableScan}; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MAIN_BRANCH, - ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, MappedField, - NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, - Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary, + DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, ListType, Literal, + MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, + MappedField, NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, + Schema, Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary, TableMetadataBuilder, TableProperties, Transform, Type, UnboundPartitionSpec, }; use crate::table::Table; @@ -786,8 +790,16 @@ mod tests { ROWS_PER_FIXTURE_FILE, TableTestFixture, assert_last_updated_seq_all, }; use crate::test_utils::test_runtime; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; use crate::util::snapshot::snapshot_id_as_of_time; - use crate::{ErrorKind, Runtime, TableIdent}; + use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + use crate::{Catalog, ErrorKind, NamespaceIdent, Runtime, TableCreation, TableIdent}; #[tokio::test] async fn test_table_scan_columns() { @@ -1296,7 +1308,7 @@ mod tests { .snapshot_by_id(entry.snapshot_id) .unwrap(); let schema = snapshot.schema(fixture.table.metadata()).unwrap(); - let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap()); + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); // Write distinct historical data: x = 100, versus x = 1 in the current files. let path = format!("{}/historical.parquet", fixture.table_location); @@ -4238,4 +4250,136 @@ mod tests { .collect(); assert_eq!(y, (1100..1200).collect::>()); } + + /// Appends one data file holding `columns`, written with the table's current schema. + async fn append_data_file( + catalog: &impl Catalog, + table: Table, + columns: Vec, + ) -> Table { + let schema = table.metadata().current_schema().clone(); + let batch = + RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema).unwrap()), columns) + .unwrap(); + let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::default(), schema), + table.file_io().clone(), + DefaultLocationGenerator::new(table.metadata()).unwrap(), + DefaultFileNameGenerator::new( + Uuid::new_v4().to_string(), + None, + DataFileFormat::Parquet, + ), + ); + let mut writer = DataFileWriterBuilder::new(rolling_file_writer_builder) + .build(None) + .await + .unwrap(); + writer.write(batch).await.unwrap(); + let data_files = writer.close().await.unwrap(); + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files(data_files) + .apply(tx) + .unwrap(); + tx.commit(catalog).await.unwrap() + } + + /// Checks the residual end to end, through `update_schema` and `TableScan`, which the reader + /// tests in `arrow/reader/row_filter.rs` bypass. Those tests cover the other predicates and + /// row selection. Follows Java's + /// `TestFilterPushDown.testFilterPushdownOnInitialDefaultColumnAbsentFromFile`: the first file + /// is written before `c` is added with `initial-default` 'US', so its row reads `c` as 'US'. + /// The second file stores `c`. + #[tokio::test] + async fn test_scan_filter_on_initial_default_column_absent_from_file() { + use arrow_array::types::Int64Type; + + let catalog = new_memory_catalog().await; + let namespace = NamespaceIdent::new(format!("ns-{}", Uuid::new_v4())); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("t".to_string()) + .schema(schema) + .format_version(FormatVersion::V3) + .build(), + ) + .await + .unwrap(); + + let table = append_data_file(&catalog, table, vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(StringArray::from(vec!["Alice"])), + ]) + .await; + + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column( + AddColumn::builder() + .name("c") + .field_type(Type::Primitive(PrimitiveType::String)) + .initial_default(Literal::string("US")) + .write_default(Literal::string("US")) + .build(), + ) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let table = append_data_file(&catalog, table, vec![ + Arc::new(Int64Array::from(vec![2, 3])), + Arc::new(StringArray::from(vec!["Bob", "Eve"])), + Arc::new(StringArray::from(vec!["US", "CA"])), + ]) + .await; + + let c = || Reference::new("c"); + for (predicate, expected) in [ + (c().equal_to(Datum::string("US")), vec![1, 2]), + (c().is_null(), vec![]), + ] { + let message = predicate.to_string(); + let batches: Vec = table + .scan() + .with_filter(predicate) + .build() + .unwrap() + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut actual: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + actual.sort(); + assert_eq!(actual, expected, "{message}"); + } + } }