Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/iceberg/src/arrow/reader/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,9 @@ impl FileScanTaskReader {
record_batch_stream_builder.parquet_schema(),
&iceberg_field_ids,
&field_id_map,
task.partition_spec()
.zip(task.partition())
.map(|(spec, data)| (spec.as_ref(), data)),
)?;
record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter);

Expand Down
164 changes: 159 additions & 5 deletions crates/iceberg/src/arrow/reader/predicate_visitor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ use crate::arrow::get_arrow_datum;
use crate::error::{Result, invalid_data};
use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor;
use crate::expr::{BoundPredicate, BoundReference};
use crate::spec::Datum;
use crate::spec::{Datum, PartitionSpec, Struct, Transform};

/// A visitor to collect field ids from bound predicates.
pub(super) struct CollectFieldIdVisitor {
Expand Down Expand Up @@ -203,6 +203,8 @@ pub(super) struct PredicateConverter<'a> {
pub(super) column_map: &'a HashMap<i32, usize>,
/// The required column indices in Parquet schema for the predicates.
pub(super) column_indices: &'a Vec<usize>,
/// Partition metadata used to distinguish a missing NULL from a constant value.
pub(super) partition: Option<(&'a PartitionSpec, &'a Struct)>,
}

impl PredicateConverter<'_> {
Expand Down Expand Up @@ -249,6 +251,20 @@ impl PredicateConverter<'_> {
Ok(constant_bool_array(false, batch.num_rows()))
}))
}

fn missing_field_is_null(&self, reference: &BoundReference) -> bool {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not_in and not_starts_with still return build_always_true() for a missing nullable column. On this branch b NOT IN ('x', 'y') and b NOT STARTS WITH 'x' give 3 rows when b is missing from the file and 0 when it's stored as NULL. Same mismatch as <, so maybe worth covering here, or a follow-up?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for catching this! Both predicates now treat an absent nullable column like a stored NULL. I added Parquet regression coverage, and the local Rust and Python suites passed.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re-ran my probe on 90649d6. NOT IN and NOT STARTS WITH now give 0 rows for the missing column, same as the stored-NULL file. NOT (b < 'm'), IS NULL and IS NOT NULL match too. LGTM.

if reference.field().required || reference.field().initial_default.is_some() {
return false;
}

!self.partition.is_some_and(|(spec, data)| {
spec.fields().iter().zip(data.iter()).any(|(field, value)| {
field.source_id == reference.field().id
&& field.transform == Transform::Identity
&& value.is_some()
})
})
}
}

/// Builds a non-null `BooleanArray` of `len` elements all set to `value`.
Expand Down Expand Up @@ -433,7 +449,11 @@ impl BoundPredicateVisitor for PredicateConverter<'_> {
}))
} else {
// A missing column, treating it as null.
self.build_always_true()
if self.missing_field_is_null(reference) {
self.build_always_false()
} else {
self.build_always_true()
}
}
}

Expand All @@ -453,7 +473,11 @@ impl BoundPredicateVisitor for PredicateConverter<'_> {
}))
} else {
// A missing column, treating it as null.
self.build_always_true()
if self.missing_field_is_null(reference) {
self.build_always_false()
} else {
self.build_always_true()
}
}
}

Expand Down Expand Up @@ -572,8 +596,9 @@ impl BoundPredicateVisitor for PredicateConverter<'_> {
// update here if arrow ever adds a native not_starts_with
not(&starts_with(&left, literal.as_ref())?)
}))
} else if self.missing_field_is_null(reference) {
self.build_always_false()
} else {
// A missing column, treating it as null.
self.build_always_true()
}
}
Expand Down Expand Up @@ -631,8 +656,9 @@ impl BoundPredicateVisitor for PredicateConverter<'_> {

Ok(acc)
}))
} else if !literals.is_empty() && self.missing_field_is_null(reference) {
self.build_always_false()
} else {
// A missing column, treating it as null.
self.build_always_true()
}
}
Expand Down Expand Up @@ -766,6 +792,15 @@ mod tests {
predicate: Predicate,
schema: SchemaRef,
batch: RecordBatch,
) -> BooleanArray {
apply_predicate_with_partition(predicate, schema, batch, None)
}

fn apply_predicate_with_partition(
predicate: Predicate,
schema: SchemaRef,
batch: RecordBatch,
partition: Option<(&crate::spec::PartitionSpec, &crate::spec::Struct)>,
) -> BooleanArray {
let bound = predicate.bind(schema, true).unwrap();

Expand All @@ -785,12 +820,131 @@ mod tests {
parquet_schema: &parquet_schema,
column_map: &column_map,
column_indices: &column_indices,
partition,
};

let mut predicate_fn = visit(&mut converter, &bound).unwrap();
predicate_fn(batch).unwrap()
}

#[test]
fn test_missing_nullable_field_comparisons_match_null_semantics() {
use arrow_array::{Int64Array, RecordBatch};
use arrow_schema::{Field, Schema as ArrowSchema};

use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type};

let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(),
])
.build()
.unwrap(),
);
let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
"a",
DataType::Int64,
false,
)]));
let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int64Array::from(vec![
1, 2, 3,
]))])
.unwrap();

for predicate in [
Reference::new("b").less_than(Datum::long(5)),
Reference::new("b").less_than_or_equal_to(Datum::long(5)),
] {
let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone());
assert_eq!(result, constant_bool_array(false, 3));
}
}

#[test]
fn test_missing_field_comparisons_preserve_known_values() {
use arrow_array::{Int64Array, RecordBatch};
use arrow_schema::{Field, Schema as ArrowSchema};

use crate::spec::{
Datum, Literal, NestedField, PartitionSpec, PrimitiveType, Schema, Struct, Transform,
Type,
};

let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long)).into(),
])
.build()
.unwrap(),
);
let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
"a",
DataType::Int64,
false,
)]));
let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int64Array::from(vec![
1, 2, 3,
]))])
.unwrap();
let spec = PartitionSpec::builder(schema.clone())
.with_spec_id(7)
.add_partition_field("b", "b", Transform::Identity)
.unwrap()
.build()
.unwrap();

for predicate in [
Reference::new("b").less_than(Datum::long(10)),
Reference::new("b").less_than_or_equal_to(Datum::long(10)),
] {
let identity_value = Struct::from_iter([Some(Literal::long(7))]);
let result = apply_predicate_with_partition(
predicate.clone(),
schema.clone(),
batch.clone(),
Some((&spec, &identity_value)),
);
assert_eq!(result, constant_bool_array(true, 3));

let identity_null = Struct::from_iter([None]);
let result = apply_predicate_with_partition(
predicate,
schema.clone(),
batch.clone(),
Some((&spec, &identity_null)),
);
assert_eq!(result, constant_bool_array(false, 3));
}

for nested_field in [
NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Long))
.with_initial_default(Literal::long(7)),
NestedField::required(2, "b", Type::Primitive(PrimitiveType::Long)),
] {
let known_value_schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
nested_field.into(),
])
.build()
.unwrap(),
);
for predicate in [
Reference::new("b").less_than(Datum::long(10)),
Reference::new("b").less_than_or_equal_to(Datum::long(10)),
] {
let result =
apply_predicate_to_batch(predicate, known_value_schema.clone(), batch.clone());
assert_eq!(result, constant_bool_array(true, 3));
}
}
}

#[test]
fn test_predicate_converter_nan() {
use arrow_array::Float32Array;
Expand Down
117 changes: 115 additions & 2 deletions crates/iceberg/src/arrow/reader/row_filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use crate::expr::BoundPredicate;
use crate::expr::visitors::bound_predicate_visitor::visit;
use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator;
use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator;
use crate::spec::Schema;
use crate::spec::{PartitionSpec, Schema, Struct};
use crate::{Error, ErrorKind};

impl ArrowReader {
Expand All @@ -43,6 +43,7 @@ impl ArrowReader {
parquet_schema: &SchemaDescriptor,
iceberg_field_ids: &HashSet<i32>,
field_id_map: &HashMap<i32, usize>,
partition: Option<(&PartitionSpec, &Struct)>,
) -> Result<RowFilter> {
// Collect Parquet column indices from field ids.
// If the field id is not found in Parquet schema, it will be ignored due to schema evolution.
Expand All @@ -57,6 +58,7 @@ impl ArrowReader {
parquet_schema,
column_map: field_id_map,
column_indices: &column_indices,
partition,
};

// After collecting required leaf column indices used in the predicate,
Expand Down Expand Up @@ -248,7 +250,8 @@ mod tests {
use crate::io::FileIO;
use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream};
use crate::spec::{
DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type,
DataContentType, DataFileFormat, Datum, Literal, NestedField, PartitionSpec, PrimitiveType,
Schema, SchemaRef, Struct, Transform, Type,
};

async fn test_perform_read(
Expand Down Expand Up @@ -1677,6 +1680,116 @@ mod tests {
(batches, metrics.bytes_read())
}

#[tokio::test]
async fn test_predicates_on_missing_nullable_column() {
use arrow_array::{Int64Array, StringArray};

let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
NestedField::optional(2, "b", Type::Primitive(PrimitiveType::String)).into(),
])
.build()
.unwrap(),
);
let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id(
"a",
DataType::Int64,
1,
)]));
let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from(
vec![1, 2, 3],
))])
.unwrap();
let tmp_dir = TempDir::new().unwrap();
let missing_path = tmp_dir.path().join("missing-b.parquet");
let null_path = tmp_dir.path().join("null-b.parquet");
let null_schema = Arc::new(ArrowSchema::new(vec![
field_with_id("a", DataType::Int64, 1),
field_with_id("b", DataType::Utf8, 2).with_nullable(true),
]));
let null_batch = RecordBatch::try_new(null_schema.clone(), vec![
batch.column(0).clone(),
Arc::new(StringArray::new_null(3)),
])
.unwrap();
write_row_groups(
missing_path.to_str().unwrap(),
arrow_schema,
vec![batch],
false,
);
write_row_groups(
null_path.to_str().unwrap(),
null_schema,
vec![null_batch],
false,
);
let spec = PartitionSpec::builder(schema.clone())
.with_spec_id(7)
.add_partition_field("b", "b", Transform::Identity)
.unwrap()
.build()
.unwrap();
let identity_value = Struct::from_iter([Some(Literal::string("m"))]);

for (predicate, project_field_ids) in [
(Reference::new("b").less_than(Datum::string("z")), vec![1]),
(
Reference::new("b").less_than_or_equal_to(Datum::string("z")),
vec![1, 2],
),
(
Reference::new("b").is_not_in([Datum::string("x"), Datum::string("y")]),
vec![1],
),
(
Reference::new("b").not_starts_with(Datum::string("x")),
vec![1, 2],
),
] {
for (path, has_identity_partition, expected_rows) in [
(&null_path, false, 0),
(&missing_path, false, 0),
(&missing_path, true, 3),
] {
let task = FileScanTask::builder()
.with_file_size_in_bytes(std::fs::metadata(path).unwrap().len())
.with_start(0)
.with_length(0)
.with_data_file_path(path.to_string_lossy().into_owned())
.with_data_file_format(DataFileFormat::Parquet)
.with_schema(schema.clone())
.with_project_field_ids(project_field_ids.clone())
.with_predicate(Some(predicate.clone().bind(schema.clone(), true).unwrap()))
.with_partition_spec(has_identity_partition.then(|| Arc::new(spec.clone())))
.with_partition(has_identity_partition.then(|| identity_value.clone()))
.with_case_sensitive(false)
.build()
.unwrap();
let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
.with_row_group_filtering_enabled(false)
.build();
let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
let batches: Vec<RecordBatch> = reader
.read(tasks)
.unwrap()
.stream()
.try_collect()
.await
.unwrap();

assert_eq!(
batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
expected_rows,
"{predicate} on {}",
path.display()
);
}
}
}

/// Batch boundaries differ when fewer row groups are read, so collapse to a
/// single batch before comparing. `None` means no rows at all.
fn collapse(batches: &[RecordBatch]) -> Option<RecordBatch> {
Expand Down
Loading