From 506145ec4aad2cdd34ad5a2c51683620d5eda41e Mon Sep 17 00:00:00 2001 From: Matt Faltyn Date: Thu, 22 Oct 2026 13:55:56 +0200 Subject: [PATCH 1/2] fix(arrow): filter absent nullable columns in ordered comparisons --- crates/iceberg/src/arrow/reader/pipeline.rs | 3 + .../src/arrow/reader/predicate_visitor.rs | 158 +++++++++++++++++- crates/iceberg/src/arrow/reader/row_filter.rs | 99 ++++++++++- 3 files changed, 255 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 09fcad9cb2..b8a91359c6 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -540,6 +540,9 @@ impl FileScanTaskReader { record_batch_stream_builder.parquet_schema(), &iceberg_field_ids, &field_id_map, + task.partition_spec() + .zip(task.partition()) + .map(|(spec, data)| (spec.as_ref(), data)), )?; record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..90614210f1 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -38,7 +38,7 @@ use crate::arrow::get_arrow_datum; use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; use crate::expr::{BoundPredicate, BoundReference}; -use crate::spec::Datum; +use crate::spec::{Datum, PartitionSpec, Struct, Transform}; /// A visitor to collect field ids from bound predicates. pub(super) struct CollectFieldIdVisitor { @@ -203,6 +203,8 @@ pub(super) struct PredicateConverter<'a> { pub(super) column_map: &'a HashMap, /// The required column indices in Parquet schema for the predicates. pub(super) column_indices: &'a Vec, + /// Partition metadata used to distinguish a missing NULL from a constant value. + pub(super) partition: Option<(&'a PartitionSpec, &'a Struct)>, } impl PredicateConverter<'_> { @@ -249,6 +251,20 @@ impl PredicateConverter<'_> { Ok(constant_bool_array(false, batch.num_rows())) })) } + + fn missing_field_is_null(&self, reference: &BoundReference) -> bool { + if reference.field().required || reference.field().initial_default.is_some() { + return false; + } + + !self.partition.is_some_and(|(spec, data)| { + spec.fields().iter().zip(data.iter()).any(|(field, value)| { + field.source_id == reference.field().id + && field.transform == Transform::Identity + && value.is_some() + }) + }) + } } /// Builds a non-null `BooleanArray` of `len` elements all set to `value`. @@ -433,7 +449,11 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { })) } else { // A missing column, treating it as null. - self.build_always_true() + if self.missing_field_is_null(reference) { + self.build_always_false() + } else { + self.build_always_true() + } } } @@ -453,7 +473,11 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { })) } else { // A missing column, treating it as null. - self.build_always_true() + if self.missing_field_is_null(reference) { + self.build_always_false() + } else { + self.build_always_true() + } } } @@ -766,6 +790,15 @@ mod tests { predicate: Predicate, schema: SchemaRef, batch: RecordBatch, + ) -> BooleanArray { + apply_predicate_with_partition(predicate, schema, batch, None) + } + + fn apply_predicate_with_partition( + predicate: Predicate, + schema: SchemaRef, + batch: RecordBatch, + partition: Option<(&crate::spec::PartitionSpec, &crate::spec::Struct)>, ) -> BooleanArray { let bound = predicate.bind(schema, true).unwrap(); @@ -785,12 +818,131 @@ mod tests { parquet_schema: &parquet_schema, column_map: &column_map, column_indices: &column_indices, + partition, }; let mut predicate_fn = visit(&mut converter, &bound).unwrap(); predicate_fn(batch).unwrap() } + #[test] + fn test_missing_nullable_field_comparisons_match_null_semantics() { + use arrow_array::{Int64Array, RecordBatch}; + use arrow_schema::{Field, Schema as ArrowSchema}; + + use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "a", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int64Array::from(vec![ + 1, 2, 3, + ]))]) + .unwrap(); + + for predicate in [ + Reference::new("b").less_than(Datum::long(5)), + Reference::new("b").less_than_or_equal_to(Datum::long(5)), + ] { + let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone()); + assert_eq!(result, constant_bool_array(false, 3)); + } + } + + #[test] + fn test_missing_field_comparisons_preserve_known_values() { + use arrow_array::{Int64Array, RecordBatch}; + use arrow_schema::{Field, Schema as ArrowSchema}; + + use crate::spec::{ + Datum, Literal, NestedField, PartitionSpec, PrimitiveType, Schema, Struct, Transform, + Type, + }; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "a", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int64Array::from(vec![ + 1, 2, 3, + ]))]) + .unwrap(); + let spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + for predicate in [ + Reference::new("b").less_than(Datum::long(10)), + Reference::new("b").less_than_or_equal_to(Datum::long(10)), + ] { + let identity_value = Struct::from_iter([Some(Literal::long(7))]); + let result = apply_predicate_with_partition( + predicate.clone(), + schema.clone(), + batch.clone(), + Some((&spec, &identity_value)), + ); + assert_eq!(result, constant_bool_array(true, 3)); + + let identity_null = Struct::from_iter([None]); + let result = apply_predicate_with_partition( + predicate, + schema.clone(), + batch.clone(), + Some((&spec, &identity_null)), + ); + assert_eq!(result, constant_bool_array(false, 3)); + } + + for nested_field in [ + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::long(7)), + NestedField::required(2, "b", Type::Primitive(PrimitiveType::Long)), + ] { + let known_value_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + nested_field.into(), + ]) + .build() + .unwrap(), + ); + for predicate in [ + Reference::new("b").less_than(Datum::long(10)), + Reference::new("b").less_than_or_equal_to(Datum::long(10)), + ] { + let result = + apply_predicate_to_batch(predicate, known_value_schema.clone(), batch.clone()); + assert_eq!(result, constant_bool_array(true, 3)); + } + } + } + #[test] fn test_predicate_converter_nan() { use arrow_array::Float32Array; diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 331e7f9656..b05b63594e 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -34,7 +34,7 @@ use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::Schema; +use crate::spec::{PartitionSpec, Schema, Struct}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -43,6 +43,7 @@ impl ArrowReader { parquet_schema: &SchemaDescriptor, iceberg_field_ids: &HashSet, field_id_map: &HashMap, + partition: Option<(&PartitionSpec, &Struct)>, ) -> Result { // Collect Parquet column indices from field ids. // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. @@ -57,6 +58,7 @@ impl ArrowReader { parquet_schema, column_map: field_id_map, column_indices: &column_indices, + partition, }; // After collecting required leaf column indices used in the predicate, @@ -247,7 +249,8 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, Literal, NestedField, PartitionSpec, PrimitiveType, + Schema, SchemaRef, Struct, Transform, Type, }; async fn test_perform_read( @@ -1609,6 +1612,98 @@ mod tests { (batches, metrics.bytes_read()) } + #[tokio::test] + async fn test_missing_nullable_column_ordered_comparisons_return_no_rows() { + use arrow_array::Int64Array; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "a", + DataType::Int64, + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( + vec![1, 2, 3], + ))]) + .unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("missing-b.parquet"); + write_row_groups(path.to_str().unwrap(), arrow_schema, vec![batch], false); + let spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let identity_value = Struct::from_iter([Some(Literal::long(7))]); + + for (predicate, project_field_ids, has_identity_partition, expected_rows) in [ + ( + Reference::new("b").less_than(Datum::long(5)), + vec![1], + false, + 0, + ), + ( + Reference::new("b").less_than_or_equal_to(Datum::long(5)), + vec![1, 2], + false, + 0, + ), + ( + Reference::new("b").less_than(Datum::long(10)), + vec![1], + true, + 3, + ), + ( + Reference::new("b").less_than_or_equal_to(Datum::long(10)), + vec![1], + true, + 3, + ), + ] { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path.to_string_lossy().into_owned()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) + .with_partition_spec(has_identity_partition.then(|| Arc::new(spec.clone()))) + .with_partition(has_identity_partition.then(|| identity_value.clone())) + .with_case_sensitive(false) + .build() + .unwrap(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_group_filtering_enabled(false) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected_rows + ); + } + } + /// Batch boundaries differ when fewer row groups are read, so collapse to a /// single batch before comparing. `None` means no rows at all. fn collapse(batches: &[RecordBatch]) -> Option { From 90649d66c83e7639dbef5823257733bdeb79ae24 Mon Sep 17 00:00:00 2001 From: Matt Faltyn Date: Wed, 4 Nov 2026 08:37:45 +0100 Subject: [PATCH 2/2] fix(arrow): handle missing nulls in negated predicates --- .../src/arrow/reader/predicate_visitor.rs | 6 +- crates/iceberg/src/arrow/reader/row_filter.rs | 124 ++++++++++-------- 2 files changed, 75 insertions(+), 55 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 90614210f1..656c0aaea5 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -596,8 +596,9 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { // update here if arrow ever adds a native not_starts_with not(&starts_with(&left, literal.as_ref())?) })) + } else if self.missing_field_is_null(reference) { + self.build_always_false() } else { - // A missing column, treating it as null. self.build_always_true() } } @@ -655,8 +656,9 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(acc) })) + } else if !literals.is_empty() && self.missing_field_is_null(reference) { + self.build_always_false() } else { - // A missing column, treating it as null. self.build_always_true() } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index b05b63594e..e84914c2b2 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -1613,14 +1613,14 @@ mod tests { } #[tokio::test] - async fn test_missing_nullable_column_ordered_comparisons_return_no_rows() { - use arrow_array::Int64Array; + async fn test_predicates_on_missing_nullable_column() { + use arrow_array::{Int64Array, StringArray}; let schema = Arc::new( Schema::builder() .with_fields(vec![ NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), - NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(), @@ -1635,72 +1635,90 @@ mod tests { ))]) .unwrap(); let tmp_dir = TempDir::new().unwrap(); - let path = tmp_dir.path().join("missing-b.parquet"); - write_row_groups(path.to_str().unwrap(), arrow_schema, vec![batch], false); + let missing_path = tmp_dir.path().join("missing-b.parquet"); + let null_path = tmp_dir.path().join("null-b.parquet"); + let null_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("a", DataType::Int64, 1), + field_with_id("b", DataType::Utf8, 2).with_nullable(true), + ])); + let null_batch = RecordBatch::try_new(null_schema.clone(), vec![ + batch.column(0).clone(), + Arc::new(StringArray::new_null(3)), + ]) + .unwrap(); + write_row_groups( + missing_path.to_str().unwrap(), + arrow_schema, + vec![batch], + false, + ); + write_row_groups( + null_path.to_str().unwrap(), + null_schema, + vec![null_batch], + false, + ); let spec = PartitionSpec::builder(schema.clone()) .with_spec_id(7) .add_partition_field("b", "b", Transform::Identity) .unwrap() .build() .unwrap(); - let identity_value = Struct::from_iter([Some(Literal::long(7))]); + let identity_value = Struct::from_iter([Some(Literal::string("m"))]); - for (predicate, project_field_ids, has_identity_partition, expected_rows) in [ - ( - Reference::new("b").less_than(Datum::long(5)), - vec![1], - false, - 0, - ), + for (predicate, project_field_ids) in [ + (Reference::new("b").less_than(Datum::string("z")), vec![1]), ( - Reference::new("b").less_than_or_equal_to(Datum::long(5)), + Reference::new("b").less_than_or_equal_to(Datum::string("z")), vec![1, 2], - false, - 0, ), ( - Reference::new("b").less_than(Datum::long(10)), + Reference::new("b").is_not_in([Datum::string("x"), Datum::string("y")]), vec![1], - true, - 3, ), ( - Reference::new("b").less_than_or_equal_to(Datum::long(10)), - vec![1], - true, - 3, + Reference::new("b").not_starts_with(Datum::string("x")), + vec![1, 2], ), ] { - let task = FileScanTask::builder() - .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) - .with_start(0) - .with_length(0) - .with_data_file_path(path.to_string_lossy().into_owned()) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(schema.clone()) - .with_project_field_ids(project_field_ids) - .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) - .with_partition_spec(has_identity_partition.then(|| Arc::new(spec.clone()))) - .with_partition(has_identity_partition.then(|| identity_value.clone())) - .with_case_sensitive(false) - .build() - .unwrap(); - let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) - .with_row_group_filtering_enabled(false) - .build(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; - let batches: Vec = reader - .read(tasks) - .unwrap() - .stream() - .try_collect() - .await - .unwrap(); - - assert_eq!( - batches.iter().map(RecordBatch::num_rows).sum::(), - expected_rows - ); + for (path, has_identity_partition, expected_rows) in [ + (&null_path, false, 0), + (&missing_path, false, 0), + (&missing_path, true, 3), + ] { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path.to_string_lossy().into_owned()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids.clone()) + .with_predicate(Some(predicate.clone().bind(schema.clone(), true).unwrap())) + .with_partition_spec(has_identity_partition.then(|| Arc::new(spec.clone()))) + .with_partition(has_identity_partition.then(|| identity_value.clone())) + .with_case_sensitive(false) + .build() + .unwrap(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_group_filtering_enabled(false) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected_rows, + "{predicate} on {}", + path.display() + ); + } } }