From 0dd80fb31cb66ca276475019dcc60227bc55955c Mon Sep 17 00:00:00 2001 From: JeonDaehong Date: Wed, 14 Oct 2026 21:22:36 +0900 Subject: [PATCH 1/5] fix(arrow): load equality deletes on fields nested in structs Equality ids may name a field nested in a struct, which the spec allows and EqualityDeleteWriter writes. Loading such a delete file failed because schema evolution only maps top-level field ids, so the delete file is now read by the top-level fields that hold the equality ids. The delete predicate also used the leaf name (`f`), which binds to a top-level column of the same name or fails; it now uses the full name (`s.f`). --- .../src/arrow/caching_delete_file_loader.rs | 104 ++++++++++++++++-- .../iceberg/src/arrow/delete_file_loader.rs | 28 ++++- 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 5582e4397f..529d90fed7 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -24,7 +24,7 @@ use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; use super::delete_filter::{DeleteFilter, PosDelLoadAction}; -use crate::arrow::delete_file_loader::BasicDeleteFileLoader; +use crate::arrow::delete_file_loader::{BasicDeleteFileLoader, equality_delete_projection}; use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; @@ -302,8 +302,9 @@ impl CachingDeleteFileLoader { del_filter.insert_equality_delete(task.file_path(), receiver); // Per the Iceberg spec, evolve schema for equality deletes but only for the - // equality_ids columns, not all table columns. + // columns holding the equality_ids, not all table columns. let equality_ids_vec = task.equality_ids().unwrap().to_vec(); + let projection = equality_delete_projection(&schema, &equality_ids_vec); let evolved_stream = BasicDeleteFileLoader::evolve_schema( basic_delete_file_loader .parquet_to_batch_stream( @@ -313,7 +314,7 @@ impl CachingDeleteFileLoader { ) .await?, schema, - &equality_ids_vec, + &projection, ) .await?; @@ -604,7 +605,7 @@ impl CachingDeleteFileLoader { let root_array: ArrayRef = Arc::new(StructArray::from(record_batch)); - let mut processor = EqDelColumnProcessor::new(&equality_ids); + let mut processor = EqDelColumnProcessor::new(&equality_ids, schema); visit_schema_with_partner(schema, &root_array, &mut processor, &accessor)?; let mut datum_columns_with_names = processor.finish()?; @@ -661,13 +662,15 @@ impl CachingDeleteFileLoader { struct EqDelColumnProcessor<'a> { equality_ids: &'a HashSet, + schema: &'a Schema, collected_columns: Vec<(ArrayRef, String, Type)>, } impl<'a> EqDelColumnProcessor<'a> { - fn new(equality_ids: &'a HashSet) -> Self { + fn new(equality_ids: &'a HashSet, schema: &'a Schema) -> Self { Self { equality_ids, + schema, collected_columns: Vec::with_capacity(equality_ids.len()), } } @@ -723,9 +726,17 @@ impl SchemaWithPartnerVisitor for EqDelColumnProcessor<'_> { fn field(&mut self, field: &NestedFieldRef, partner: &ArrayRef, _value: ()) -> Result<()> { if self.equality_ids.contains(&field.id) && field.field_type.as_primitive_type().is_some() { + // Use the full name so a nested field (`s.f`) binds to itself, not to a top-level + // column with the same leaf name. + let name = self.schema.name_by_field_id(field.id).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("equality delete field {} not found in schema", field.id), + ) + })?; self.collected_columns.push(( partner.clone(), - field.name.clone(), + name.to_string(), field.field_type.as_ref().clone(), )); } @@ -888,7 +899,7 @@ mod tests { .await .expect("error parsing batch stream"); - let expected = "((((((y IS NULL) OR (y != 1)) OR ((z IS NULL) OR (z != 100))) OR ((a IS NULL) OR (a != \"HELP\"))) OR ((sa IS NULL) OR (sa != 4))) OR ((b IS NULL) OR (b != 62696E6172795F64617461))) AND ((((((y IS NULL) OR (y != 2)) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR ((sa IS NULL) OR (sa != 5))) OR (b IS NOT NULL))".to_string(); + let expected = "((((((y IS NULL) OR (y != 1)) OR ((z IS NULL) OR (z != 100))) OR ((a IS NULL) OR (a != \"HELP\"))) OR ((s.sa IS NULL) OR (s.sa != 4))) OR ((b IS NULL) OR (b != 62696E6172795F64617461))) AND ((((((y IS NULL) OR (y != 2)) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR ((s.sa IS NULL) OR (s.sa != 5))) OR (b IS NOT NULL))".to_string(); assert_eq!(parsed_eq_delete.to_string(), expected); } @@ -1791,4 +1802,83 @@ mod tests { assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains("expected 2 from record_count")); } + + /// An equality delete on a field nested in a struct loads, and its predicate uses the full + /// field name. The table also has a top-level column with the same leaf name, which the + /// predicate must not refer to. + #[tokio::test] + async fn test_nested_field_equality_delete_predicate() { + use crate::arrow::delete_file_loader::equality_delete_projection; + use crate::expr::Bind; + use crate::spec::{Datum, StructType}; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap(); + + let table_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "f", Type::Primitive(PrimitiveType::Long)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + // Delete file with `s.f = 7`. + let f_field = Arc::new(simple_field("f", DataType::Int64, true, "3")); + let s_field = simple_field( + "s", + DataType::Struct(Fields::from(vec![f_field.clone()])), + true, + "2", + ); + let delete_schema = Arc::new(arrow_schema::Schema::new(vec![s_field])); + let s = StructArray::from(vec![( + f_field, + Arc::new(Int64Array::from(vec![7])) as ArrayRef, + )]); + let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new(s)]).unwrap(); + let path = format!("{table_location}/nested-eq-delete.parquet"); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), delete_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // Only the top-level struct holding field 3 is read. + assert_eq!(equality_delete_projection(&table_schema, &[3]), vec![2]); + + let loader = BasicDeleteFileLoader::new(FileIO::new_with_fs(), ScanMetrics::new()); + let stream = loader + .parquet_to_batch_stream(&path, std::fs::metadata(&path).unwrap().len(), None) + .await + .unwrap(); + let evolved = BasicDeleteFileLoader::evolve_schema(stream, table_schema.clone(), &[2]) + .await + .unwrap(); + let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + evolved, + HashSet::from([3]), + ) + .await + .unwrap(); + + let s_f = Reference::new("s.f"); + assert_eq!( + predicate, + s_f.clone().is_null().or(s_f.not_equal_to(Datum::long(7))) + ); + // It binds to the nested field, not to the top-level `f`. + let bound = predicate.bind(table_schema, true).unwrap(); + assert!(format!("{bound:?}").contains("id: 3")); + assert!(!format!("{bound:?}").contains("id: 1,")); + } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index ec61710a42..8d628887f7 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -115,6 +115,30 @@ impl BasicDeleteFileLoader { } } +/// Returns the top-level field ids to read for an equality delete file. +/// +/// Equality ids may name fields nested in structs, but schema evolution of a delete file works on +/// top-level fields, so each id is replaced by the top-level field that contains it. +pub(crate) fn equality_delete_projection(schema: &Schema, equality_ids: &[i32]) -> Vec { + use crate::spec::{NestedField, Type}; + + fn contains(field: &NestedField, id: i32) -> bool { + field.id == id + || match field.field_type.as_ref() { + Type::Struct(s) => s.fields().iter().any(|child| contains(child, id)), + _ => false, + } + } + + schema + .as_struct() + .fields() + .iter() + .filter(|field| equality_ids.iter().any(|&id| contains(field, id))) + .map(|field| field.id) + .collect() +} + #[async_trait::async_trait] impl DeleteFileLoader for BasicDeleteFileLoader { async fn read_delete_file( @@ -130,10 +154,10 @@ impl DeleteFileLoader for BasicDeleteFileLoader { ) .await?; - // For equality deletes, only evolve the equality_ids columns. + // For equality deletes, only evolve the columns holding the equality_ids. // For positional deletes (equality_ids is None), use all field IDs. let field_ids = match task.equality_ids() { - Some(ids) => ids.to_vec(), + Some(ids) => equality_delete_projection(&schema, ids), None => schema.field_id_to_name_map().keys().cloned().collect(), }; From 43d31692ce07a5064de999fbe525e0969247c750 Mon Sep 17 00:00:00 2001 From: JeonDaehong Date: Wed, 28 Oct 2026 20:14:45 +0900 Subject: [PATCH 2/5] fix(arrow): read equality deletes through the schema pruned to their fields Equality ids missing from the scan schema now fail instead of being dropped. The delete file is read as the scan schema pruned to the equality fields, like Java's DeleteFilter, and a nested field that EqualityDeleteWriter wrote as a top-level column is put back under its parent struct first, so both layouts load. --- .../src/arrow/caching_delete_file_loader.rs | 217 ++++++++++++++---- .../iceberg/src/arrow/delete_file_loader.rs | 157 +++++++++++-- 2 files changed, 307 insertions(+), 67 deletions(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 529d90fed7..800e86c115 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -24,7 +24,7 @@ use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; use super::delete_filter::{DeleteFilter, PosDelLoadAction}; -use crate::arrow::delete_file_loader::{BasicDeleteFileLoader, equality_delete_projection}; +use crate::arrow::delete_file_loader::{BasicDeleteFileLoader, evolve_equality_delete_stream}; use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; @@ -304,8 +304,7 @@ impl CachingDeleteFileLoader { // Per the Iceberg spec, evolve schema for equality deletes but only for the // columns holding the equality_ids, not all table columns. let equality_ids_vec = task.equality_ids().unwrap().to_vec(); - let projection = equality_delete_projection(&schema, &equality_ids_vec); - let evolved_stream = BasicDeleteFileLoader::evolve_schema( + let evolved_stream = evolve_equality_delete_stream( basic_delete_file_loader .parquet_to_batch_stream( task.file_path(), @@ -313,10 +312,9 @@ impl CachingDeleteFileLoader { task.key_metadata(), ) .await?, - schema, - &projection, - ) - .await?; + &schema, + &equality_ids_vec, + )?; Ok(DeleteFileContext::FreshEqDel { batch_stream: evolved_stream, @@ -1803,19 +1801,10 @@ mod tests { assert!(err.message().contains("expected 2 from record_count")); } - /// An equality delete on a field nested in a struct loads, and its predicate uses the full - /// field name. The table also has a top-level column with the same leaf name, which the - /// predicate must not refer to. - #[tokio::test] - async fn test_nested_field_equality_delete_predicate() { - use crate::arrow::delete_file_loader::equality_delete_projection; - use crate::expr::Bind; - use crate::spec::{Datum, StructType}; - - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().to_str().unwrap(); - - let table_schema = Arc::new( + /// Top-level `f` (id 1) and `s.f` (id 3, in struct `s` with id 2), so a predicate that + /// binds by leaf name would pick the wrong column. + fn nested_eq_delete_table_schema() -> SchemaRef { + Arc::new( Schema::builder() .with_fields(vec![ NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), @@ -1825,15 +1814,99 @@ mod tests { Type::Struct(StructType::new(vec![ NestedField::optional(3, "f", Type::Primitive(PrimitiveType::Long)) .into(), + NestedField::required(4, "g", Type::Primitive(PrimitiveType::Long)) + .into(), ])), ) .into(), ]) .build() .unwrap(), - ); + ) + } + + /// Loads one equality delete file with `equality_ids` through `load_deletes` and returns the + /// predicate the delete filter builds for a scan of `schema`. + async fn load_eq_delete_predicate( + path: &str, + equality_ids: Vec, + schema: SchemaRef, + ) -> Result> { + use crate::scan::FileScanTask; + + let eq_del = FileScanTaskDeleteFile::builder() + .with_file_path(path.to_string()) + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(equality_ids)) + .build() + .unwrap(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{path}.data")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_deletes(vec![eq_del]) + .with_case_sensitive(true) + .build() + .unwrap(); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + let filter = loader + .load_deletes(task.deletes(), task.schema_ref()) + .await + .unwrap()?; + filter.build_equality_delete_predicate(&task).await + } + + /// Field ids of the references in a bound predicate. + fn bound_field_ids(predicate: &crate::expr::BoundPredicate, ids: &mut Vec) { + use crate::expr::BoundPredicate; + match predicate { + BoundPredicate::And(expr) | BoundPredicate::Or(expr) => { + for input in expr.inputs() { + bound_field_ids(input, ids); + } + } + BoundPredicate::Not(expr) => bound_field_ids(expr.inputs()[0], ids), + BoundPredicate::Unary(expr) => ids.push(expr.term().field().id), + BoundPredicate::Binary(expr) => ids.push(expr.term().field().id), + BoundPredicate::Set(expr) => ids.push(expr.term().field().id), + BoundPredicate::AlwaysTrue | BoundPredicate::AlwaysFalse => {} + } + } + + /// The predicate for deleting `s.f = 7`, bound to field 3 and not to the top-level `f`. + fn assert_keeps_rows_unless_nested_f_is_7( + predicate: Option, + schema: SchemaRef, + ) { + use crate::expr::Bind; + + let predicate = predicate.expect("an equality delete predicate"); + let s_f = Reference::new("s.f"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, expected); + let mut ids = Vec::new(); + bound_field_ids(&predicate, &mut ids); + assert_eq!(ids, vec![3, 3]); + } + + /// A delete file in Java's layout: `s.f` under its parent struct `s`. + #[tokio::test] + async fn test_nested_field_equality_delete_in_struct_layout() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); - // Delete file with `s.f = 7`. let f_field = Arc::new(simple_field("f", DataType::Int64, true, "3")); let s_field = simple_field( "s", @@ -1847,38 +1920,94 @@ mod tests { Arc::new(Int64Array::from(vec![7])) as ArrayRef, )]); let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new(s)]).unwrap(); - let path = format!("{table_location}/nested-eq-delete.parquet"); + let path = format!( + "{}/nested-eq-delete.parquet", + tmp_dir.path().to_str().unwrap() + ); let mut writer = ArrowWriter::try_new(File::create(&path).unwrap(), delete_schema, None).unwrap(); writer.write(&batch).unwrap(); writer.close().unwrap(); - // Only the top-level struct holding field 3 is read. - assert_eq!(equality_delete_projection(&table_schema, &[3]), vec![2]); - - let loader = BasicDeleteFileLoader::new(FileIO::new_with_fs(), ScanMetrics::new()); - let stream = loader - .parquet_to_batch_stream(&path, std::fs::metadata(&path).unwrap().len(), None) + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) .await .unwrap(); - let evolved = BasicDeleteFileLoader::evolve_schema(stream, table_schema.clone(), &[2]) + assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + } + + /// A delete file from `EqualityDeleteWriter`, which writes `s.f` as a top-level column. + #[tokio::test] + async fn test_nested_field_equality_delete_from_equality_delete_writer() { + use crate::arrow::schema_to_arrow_schema; + use crate::writer::base_writer::equality_delete_writer::{ + EqualityDeleteFileWriterBuilder, EqualityDeleteWriterConfig, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + + let tmp_dir = TempDir::new().unwrap(); + let file_io = FileIO::new_with_fs(); + let schema = nested_eq_delete_table_schema(); + + let config = EqualityDeleteWriterConfig::new(vec![3], schema.clone()).unwrap(); + let delete_schema = arrow_schema_to_schema(config.projected_arrow_schema_ref()).unwrap(); + let rolling = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)), + file_io.clone(), + DefaultLocationGenerator::with_data_location( + tmp_dir.path().to_str().unwrap().to_string(), + ), + DefaultFileNameGenerator::new("eq-del".to_string(), None, DataFileFormat::Parquet), + ); + let mut writer = EqualityDeleteFileWriterBuilder::new(rolling, config) + .build(None) .await .unwrap(); - let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( - evolved, - HashSet::from([3]), - ) - .await - .unwrap(); - let s_f = Reference::new("s.f"); - assert_eq!( - predicate, - s_f.clone().is_null().or(s_f.not_equal_to(Datum::long(7))) + // Rows of the table: f = 1, s = {f: 7, g: 0}. + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); + let DataType::Struct(s_fields) = arrow_schema.field(1).data_type().clone() else { + unreachable!() + }; + let s = StructArray::new( + s_fields, + vec![ + Arc::new(Int64Array::from(vec![7])) as ArrayRef, + Arc::new(Int64Array::from(vec![0])) as ArrayRef, + ], + None, ); - // It binds to the nested field, not to the top-level `f`. - let bound = predicate.bind(table_schema, true).unwrap(); - assert!(format!("{bound:?}").contains("id: 3")); - assert!(!format!("{bound:?}").contains("id: 1,")); + let rows = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + Arc::new(s) as ArrayRef, + ]) + .unwrap(); + writer.write(rows).await.unwrap(); + let data_file = writer.close().await.unwrap().pop().unwrap(); + + let predicate = load_eq_delete_predicate(data_file.file_path(), vec![3], schema.clone()) + .await + .unwrap(); + assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + } + + /// An equality id the scan schema does not have is an error, not a column to skip. + #[tokio::test] + async fn test_equality_delete_with_id_missing_from_schema_fails() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap(); + // Columns y (id 2) and z (id 3). + let path = setup_write_equality_delete_file_1(table_location); + let schema = nested_eq_delete_table_schema(); + + let err = load_eq_delete_predicate(&path, vec![1, 9], schema) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("field id 9"), "{err}"); } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 8d628887f7..f0cadce4d2 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -15,18 +15,22 @@ // specific language governing permissions and limitations // under the License. +use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StructArray}; +use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema}; use futures::{StreamExt, TryStreamExt}; -use parquet::arrow::ParquetRecordBatchStreamBuilder; +use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder}; use crate::arrow::ArrowReader; use crate::arrow::reader::ParquetReadOptions; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::ScanMetrics; +use crate::arrow::schema::try_get_field_id_from_metadata; use crate::io::FileIO; use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile}; -use crate::spec::{Schema, SchemaRef}; +use crate::spec::{Schema, SchemaRef, prune_columns}; use crate::{Error, ErrorKind, Result}; /// Delete File Loader @@ -115,28 +119,134 @@ impl BasicDeleteFileLoader { } } -/// Returns the top-level field ids to read for an equality delete file. +/// Reads an equality delete file as `schema` pruned to its equality fields, the way Java's +/// `DeleteFilter` selects them, so a field nested in a struct keeps its parent struct. /// -/// Equality ids may name fields nested in structs, but schema evolution of a delete file works on -/// top-level fields, so each id is replaced by the top-level field that contains it. -pub(crate) fn equality_delete_projection(schema: &Schema, equality_ids: &[i32]) -> Vec { - use crate::spec::{NestedField, Type}; - - fn contains(field: &NestedField, id: i32) -> bool { - field.id == id - || match field.field_type.as_ref() { - Type::Struct(s) => s.fields().iter().any(|child| contains(child, id)), - _ => false, - } +/// Java writes a nested equality field under its parent structs, while `EqualityDeleteWriter` +/// writes it as a top-level column. Such a column is put back under its parents before schema +/// evolution, so both layouts read the same way. +pub(crate) fn evolve_equality_delete_stream( + record_batch_stream: ArrowRecordBatchStream, + schema: &Schema, + equality_ids: &[i32], +) -> Result { + if let Some(id) = equality_ids + .iter() + .find(|&&id| schema.field_by_id(id).is_none()) + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Equality delete field id {id} is not in the scan schema"), + )); } + let crate::spec::Type::Struct(pruned) = + prune_columns(schema, equality_ids.iter().copied(), false)? + else { + return Err(Error::new( + ErrorKind::Unexpected, + "pruning a schema did not return a struct", + )); + }; + let projection: Vec = pruned.fields().iter().map(|field| field.id).collect(); + let pruned = Arc::new( + Schema::builder() + .with_schema_id(schema.schema_id()) + .with_fields(pruned.fields().to_vec()) + .build()?, + ); + let mut transformer = RecordBatchTransformerBuilder::new(pruned.clone(), &projection).build(); + Ok(Box::pin(record_batch_stream.map(move |batch| { + batch + .and_then(|batch| nest_top_level_equality_columns(batch, &pruned)) + .and_then(|batch| transformer.process_record_batch(batch)) + })) as ArrowRecordBatchStream) +} - schema +/// Moves top-level columns whose field ids belong to fields nested in `schema` back under their +/// parent structs. A batch with no such column is returned as is. +fn nest_top_level_equality_columns(batch: RecordBatch, schema: &Schema) -> Result { + let top_level_ids: HashSet = schema .as_struct() .fields() .iter() - .filter(|field| equality_ids.iter().any(|&id| contains(field, id))) .map(|field| field.id) - .collect() + .collect(); + let mut flat = HashMap::new(); + for (index, field) in batch.schema().fields().iter().enumerate() { + if let Some(id) = try_get_field_id_from_metadata(field)? + && !top_level_ids.contains(&id) + && schema.field_by_id(id).is_some() + { + flat.insert(id, index); + } + } + if flat.is_empty() { + return Ok(batch); + } + + fn nest( + field: &crate::spec::NestedField, + batch: &RecordBatch, + flat: &HashMap, + ) -> Option<(FieldRef, ArrayRef)> { + if let Some(&index) = flat.get(&field.id) { + return Some(( + batch.schema().field(index).clone().into(), + batch.column(index).clone(), + )); + } + let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else { + return None; + }; + let (fields, columns): (Vec, Vec) = struct_type + .fields() + .iter() + .filter_map(|child| nest(child, batch, flat)) + .unzip(); + if fields.is_empty() { + return None; + } + let fields = Fields::from(fields); + let array = StructArray::new(fields.clone(), columns, None); + let arrow_field = + arrow_schema::Field::new(&field.name, arrow_schema::DataType::Struct(fields), true) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field.id.to_string(), + )])); + Some((Arc::new(arrow_field), Arc::new(array))) + } + + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (index, field) in batch.schema().fields().iter().enumerate() { + if !flat.values().any(|&flat_index| flat_index == index) { + fields.push(field.clone()); + columns.push(batch.column(index).clone()); + } + } + let present: HashSet = fields + .iter() + .map(|field| try_get_field_id_from_metadata(field)) + .collect::>>()? + .into_iter() + .flatten() + .collect(); + for field in schema.as_struct().fields() { + if present.contains(&field.id) { + continue; + } + if let Some((arrow_field, column)) = nest(field, &batch, &flat) { + fields.push(arrow_field); + columns.push(column); + } + } + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + Ok(RecordBatch::try_new_with_options( + Arc::new(ArrowSchema::new(fields)), + columns, + &options, + )?) } #[async_trait::async_trait] @@ -156,12 +266,13 @@ impl DeleteFileLoader for BasicDeleteFileLoader { // For equality deletes, only evolve the columns holding the equality_ids. // For positional deletes (equality_ids is None), use all field IDs. - let field_ids = match task.equality_ids() { - Some(ids) => equality_delete_projection(&schema, ids), - None => schema.field_id_to_name_map().keys().cloned().collect(), - }; - - Self::evolve_schema(raw_batch_stream, schema, &field_ids).await + match task.equality_ids() { + Some(ids) => evolve_equality_delete_stream(raw_batch_stream, &schema, ids), + None => { + let field_ids: Vec = schema.field_id_to_name_map().keys().cloned().collect(); + Self::evolve_schema(raw_batch_stream, schema, &field_ids).await + } + } } } From 37583f1b4dc2583bf57fa1c4a1e3b946fbde26cc Mon Sep 17 00:00:00 2001 From: JeonDaehong Date: Tue, 10 Nov 2026 00:07:41 +0000 Subject: [PATCH 3/5] fix(arrow): check equality ids before claiming the delete file load --- .../src/arrow/caching_delete_file_loader.rs | 338 ++++++++++++++---- .../iceberg/src/arrow/delete_file_loader.rs | 233 +++++++----- 2 files changed, 408 insertions(+), 163 deletions(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index f5b468b59d..6bb6d31e91 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -24,7 +24,9 @@ use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; use super::delete_filter::{DeleteFilter, PosDelLoadAction, PosDelLoadGuard}; -use crate::arrow::delete_file_loader::{BasicDeleteFileLoader, evolve_equality_delete_stream}; +use crate::arrow::delete_file_loader::{ + BasicDeleteFileLoader, evolve_equality_delete_stream, validate_equality_ids, +}; use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; @@ -306,6 +308,11 @@ impl CachingDeleteFileLoader { } DataContentType::EqualityDeletes => { + let equality_ids_vec = task.equality_ids().unwrap().to_vec(); + // Check the ids before claiming the load, so this error does not leave the file + // `Loading` for later reads of it. + validate_equality_ids(&schema, &equality_ids_vec)?; + let Some(notify) = del_filter.try_start_eq_del_load(task.file_path()) else { return Ok(DeleteFileContext::ExistingEqDel); }; @@ -315,7 +322,6 @@ impl CachingDeleteFileLoader { // Per the Iceberg spec, evolve schema for equality deletes but only for the // columns holding the equality_ids, not all table columns. - let equality_ids_vec = task.equality_ids().unwrap().to_vec(); let evolved_stream = evolve_equality_delete_stream( basic_delete_file_loader .parquet_to_batch_stream( @@ -2222,15 +2228,7 @@ mod tests { ) -> Result> { use crate::scan::FileScanTask; - let eq_del = FileScanTaskDeleteFile::builder() - .with_file_path(path.to_string()) - .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) - .with_file_type(DataContentType::EqualityDeletes) - .with_file_format(DataFileFormat::Parquet) - .with_partition_spec_id(0) - .with_equality_ids(Some(equality_ids)) - .build() - .unwrap(); + let eq_del = eq_delete_task(path, equality_ids); let task = FileScanTask::builder() .with_file_size_in_bytes(0) .with_start(0) @@ -2289,44 +2287,86 @@ mod tests { assert_eq!(ids, vec![3, 3]); } - /// A delete file in Java's layout: `s.f` under its parent struct `s`. - #[tokio::test] - async fn test_nested_field_equality_delete_in_struct_layout() { - let tmp_dir = TempDir::new().unwrap(); - let schema = nested_eq_delete_table_schema(); + /// An equality delete task for the file at `path` on `equality_ids`. + fn eq_delete_task(path: &str, equality_ids: Vec) -> FileScanTaskDeleteFile { + FileScanTaskDeleteFile::builder() + .with_file_path(path.to_string()) + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(equality_ids)) + .build() + .unwrap() + } - let f_field = Arc::new(simple_field("f", DataType::Int64, true, "3")); - let s_field = simple_field( - "s", - DataType::Struct(Fields::from(vec![f_field.clone()])), - true, - "2", - ); + /// Writes an equality delete file with `s.` (field `id`) under its parent struct `s`, + /// the way Java writes a nested equality field. `s_valid` marks the rows whose `s` is not + /// null. + fn write_struct_layout_eq_delete( + path: &str, + name: &str, + id: &str, + nullable: bool, + values: Vec, + s_valid: Option>, + ) { + let child = Arc::new(simple_field(name, DataType::Int64, nullable, id)); + let fields = Fields::from(vec![child]); + let s_field = simple_field("s", DataType::Struct(fields.clone()), true, "2"); let delete_schema = Arc::new(arrow_schema::Schema::new(vec![s_field])); - let s = StructArray::from(vec![( - f_field, - Arc::new(Int64Array::from(vec![7])) as ArrayRef, - )]); - let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new(s)]).unwrap(); - let path = format!( - "{}/nested-eq-delete.parquet", - tmp_dir.path().to_str().unwrap() + let s = StructArray::new( + fields, + vec![Arc::new(Int64Array::from(values)) as ArrayRef], + s_valid.map(Into::into), ); + let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new(s)]).unwrap(); let mut writer = - ArrowWriter::try_new(File::create(&path).unwrap(), delete_schema, None).unwrap(); + ArrowWriter::try_new(File::create(path).unwrap(), delete_schema, None).unwrap(); writer.write(&batch).unwrap(); writer.close().unwrap(); + } - let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) - .await - .unwrap(); - assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + /// A row of `nested_eq_delete_table_schema`: `f`, then `s.f` and `s.g`, or a null `s`. + type NestedRow = (Option, Option<(Option, i64)>); + + fn nested_rows_batch(schema: &Schema, rows: &[NestedRow]) -> RecordBatch { + let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(schema).unwrap()); + let DataType::Struct(s_fields) = arrow_schema.field(1).data_type().clone() else { + unreachable!() + }; + let s = StructArray::new( + s_fields, + vec![ + Arc::new(Int64Array::from_iter( + rows.iter().map(|(_, s)| s.and_then(|s| s.0)), + )) as ArrayRef, + Arc::new(Int64Array::from_iter_values( + rows.iter().map(|(_, s)| s.map_or(0, |s| s.1)), + )) as ArrayRef, + ], + Some( + rows.iter() + .map(|(_, s)| s.is_some()) + .collect::>() + .into(), + ), + ); + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from_iter(rows.iter().map(|(f, _)| *f))) as ArrayRef, + Arc::new(s) as ArrayRef, + ]) + .unwrap() } - /// A delete file from `EqualityDeleteWriter`, which writes `s.f` as a top-level column. - #[tokio::test] - async fn test_nested_field_equality_delete_from_equality_delete_writer() { - use crate::arrow::schema_to_arrow_schema; + /// Writes `rows` as an equality delete file on `equality_ids` with `EqualityDeleteWriter`, + /// which writes a nested field as a top-level column, and returns its path. + async fn write_eq_delete_with_writer( + dir: &TempDir, + schema: SchemaRef, + equality_ids: Vec, + rows: RecordBatch, + ) -> String { use crate::writer::base_writer::equality_delete_writer::{ EqualityDeleteFileWriterBuilder, EqualityDeleteWriterConfig, }; @@ -2337,65 +2377,211 @@ mod tests { use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use crate::writer::{IcebergWriter, IcebergWriterBuilder}; - let tmp_dir = TempDir::new().unwrap(); - let file_io = FileIO::new_with_fs(); - let schema = nested_eq_delete_table_schema(); - - let config = EqualityDeleteWriterConfig::new(vec![3], schema.clone()).unwrap(); + let config = EqualityDeleteWriterConfig::new(equality_ids, schema).unwrap(); let delete_schema = arrow_schema_to_schema(config.projected_arrow_schema_ref()).unwrap(); let rolling = RollingFileWriterBuilder::new_with_default_file_size( ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)), - file_io.clone(), - DefaultLocationGenerator::with_data_location( - tmp_dir.path().to_str().unwrap().to_string(), - ), + FileIO::new_with_fs(), + DefaultLocationGenerator::with_data_location(dir.path().to_str().unwrap().to_string()), DefaultFileNameGenerator::new("eq-del".to_string(), None, DataFileFormat::Parquet), ); let mut writer = EqualityDeleteFileWriterBuilder::new(rolling, config) .build(None) .await .unwrap(); - - // Rows of the table: f = 1, s = {f: 7, g: 0}. - let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); - let DataType::Struct(s_fields) = arrow_schema.field(1).data_type().clone() else { - unreachable!() - }; - let s = StructArray::new( - s_fields, - vec![ - Arc::new(Int64Array::from(vec![7])) as ArrayRef, - Arc::new(Int64Array::from(vec![0])) as ArrayRef, - ], - None, - ); - let rows = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int64Array::from(vec![1])) as ArrayRef, - Arc::new(s) as ArrayRef, - ]) - .unwrap(); writer.write(rows).await.unwrap(); let data_file = writer.close().await.unwrap().pop().unwrap(); + data_file.file_path().to_string() + } - let predicate = load_eq_delete_predicate(data_file.file_path(), vec![3], schema.clone()) + /// A delete file in Java's layout: `s.f` under its parent struct `s`. + #[tokio::test] + async fn test_nested_field_equality_delete_in_struct_layout() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "f", "3", true, vec![7], None); + + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) + .await + .unwrap(); + assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + } + + /// A delete file from `EqualityDeleteWriter`, which writes `s.f` as a top-level column. + #[tokio::test] + async fn test_nested_field_equality_delete_from_equality_delete_writer() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &[(Some(1), Some((Some(7), 0)))]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) .await .unwrap(); assert_keeps_rows_unless_nested_f_is_7(predicate, schema); } - /// An equality id the scan schema does not have is an error, not a column to skip. + /// The nesting worked out for the first batch of a file is reused for the next ones. + #[tokio::test] + async fn test_equality_delete_stream_nests_every_batch() { + let schema = nested_eq_delete_table_schema(); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let batch = |value: i64| { + RecordBatch::try_new(flat_schema.clone(), vec![ + Arc::new(Int64Array::from(vec![value])) as ArrayRef, + ]) + .unwrap() + }; + let input = Box::pin(futures::stream::iter(vec![Ok(batch(7)), Ok(batch(8))])) + as ArrowRecordBatchStream; + + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + let values: Vec = batches + .iter() + .map(|batch| { + batch + .column(0) + .as_struct() + .column(0) + .as_primitive::() + .value(0) + }) + .collect(); + assert_eq!(values, vec![7, 8]); + } + + /// An equality id the scan schema does not have is an error, not a column to skip. The check + /// runs before the load of the file is claimed, so loading it again through the same loader + /// fails the same way instead of waiting on the first load. #[tokio::test] async fn test_equality_delete_with_id_missing_from_schema_fails() { let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().to_str().unwrap(); // Columns y (id 2) and z (id 3). - let path = setup_write_equality_delete_file_1(table_location); + let path = setup_write_equality_delete_file_1(tmp_dir.path().to_str().unwrap()); let schema = nested_eq_delete_table_schema(); + let eq_del = eq_delete_task(&path, vec![1, 9]); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); - let err = load_eq_delete_predicate(&path, vec![1, 9], schema) + for _ in 0..2 { + let loaded = tokio::time::timeout( + std::time::Duration::from_secs(5), + loader.load_deletes(std::slice::from_ref(&eq_del), schema.clone()), + ) .await - .unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("field id 9"), "{err}"); + .expect("the load waited on an earlier failed load") + .unwrap(); + let Err(err) = loaded else { + panic!("expected an error for field id 9"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("field id 9"), "{err}"); + } + } + + /// `s.f = 7` matches only the first row: the second has a top-level `f` of 7 and the third + /// a null `s`. + const NESTED_ROWS: &[NestedRow] = &[ + (Some(1), Some((Some(7), 0))), + (Some(7), Some((Some(1), 1))), + (Some(2), None), + (Some(3), Some((Some(8), 2))), + ]; + + /// Scans a data file of `nested_eq_delete_table_schema` holding `NESTED_ROWS` with `deletes` + /// applied, and returns the top-level `f` of the rows left. + async fn scan_nested_rows( + dir: &TempDir, + deletes: Vec, + ) -> Vec> { + use crate::arrow::ArrowReaderBuilder; + use crate::scan::{FileScanTask, FileScanTaskStream}; + + let schema = nested_eq_delete_table_schema(); + let batch = nested_rows_batch(&schema, NESTED_ROWS); + let path = format!("{}/data.parquet", dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, 2]) + .with_deletes(deletes) + .with_case_sensitive(true) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .build() + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .iter() + .collect::>() + }) + .collect() + } + + #[tokio::test] + #[ignore = "applying a nested equality delete on scan needs #2906"] + async fn test_scan_applies_nested_equality_delete_in_struct_layout() { + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "f", "3", true, vec![7], None); + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![3])]).await; + assert_eq!(left, vec![Some(7), Some(2), Some(3)]); + } + + #[tokio::test] + #[ignore = "applying a nested equality delete on scan needs #2906"] + async fn test_scan_applies_nested_equality_delete_from_equality_delete_writer() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &NESTED_ROWS[..1]); + let path = write_eq_delete_with_writer(&tmp_dir, schema, vec![3], rows).await; + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![3])]).await; + assert_eq!(left, vec![Some(7), Some(2), Some(3)]); + } + + /// A delete row with a null `s` on `s.g`, which is required inside the optional `s`, deletes + /// the data rows whose `s` is null. Until #3306, the `s.g IS NOT NULL` it loads as binds to + /// `AlwaysTrue` (#3292) and the scan keeps those rows. + #[tokio::test] + #[ignore = "needs #2906 to apply nested equality deletes and #3306 to bind IS NULL on s.g"] + async fn test_scan_applies_null_parent_equality_delete_on_required_nested_field() { + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "g", "4", false, vec![0], Some(vec![false])); + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![4])]).await; + assert_eq!(left, vec![Some(1), Some(7), Some(3)]); } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index f0cadce4d2..965146ddf1 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StructArray}; -use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema}; +use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; use futures::{StreamExt, TryStreamExt}; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder}; @@ -119,6 +119,21 @@ impl BasicDeleteFileLoader { } } +/// Checks that every equality id is a field of the scan schema. Java's `DeleteFilter` fails the +/// same way rather than matching rows on fewer columns. +pub(crate) fn validate_equality_ids(schema: &Schema, equality_ids: &[i32]) -> Result<()> { + match equality_ids + .iter() + .find(|&&id| schema.field_by_id(id).is_none()) + { + Some(id) => Err(Error::new( + ErrorKind::DataInvalid, + format!("Equality delete field id {id} is not in the scan schema"), + )), + None => Ok(()), + } +} + /// Reads an equality delete file as `schema` pruned to its equality fields, the way Java's /// `DeleteFilter` selects them, so a field nested in a struct keeps its parent struct. /// @@ -130,15 +145,7 @@ pub(crate) fn evolve_equality_delete_stream( schema: &Schema, equality_ids: &[i32], ) -> Result { - if let Some(id) = equality_ids - .iter() - .find(|&&id| schema.field_by_id(id).is_none()) - { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Equality delete field id {id} is not in the scan schema"), - )); - } + validate_equality_ids(schema, equality_ids)?; let crate::spec::Type::Struct(pruned) = prune_columns(schema, equality_ids.iter().copied(), false)? else { @@ -155,98 +162,150 @@ pub(crate) fn evolve_equality_delete_stream( .build()?, ); let mut transformer = RecordBatchTransformerBuilder::new(pruned.clone(), &projection).build(); + // The batches of a file share its schema, so the nesting is worked out once and only + // rebuilt if a batch comes with a different schema. + let mut nesting: Option = None; Ok(Box::pin(record_batch_stream.map(move |batch| { - batch - .and_then(|batch| nest_top_level_equality_columns(batch, &pruned)) - .and_then(|batch| transformer.process_record_batch(batch)) + let batch = batch?; + let nesting = match &mut nesting { + Some(nesting) if nesting.input_schema == batch.schema() => nesting, + slot => slot.insert(EqualityColumnNesting::try_new(batch.schema(), &pruned)?), + }; + transformer.process_record_batch(nesting.apply(batch)?) })) as ArrowRecordBatchStream) } -/// Moves top-level columns whose field ids belong to fields nested in `schema` back under their -/// parent structs. A batch with no such column is returned as is. -fn nest_top_level_equality_columns(batch: RecordBatch, schema: &Schema) -> Result { - let top_level_ids: HashSet = schema - .as_struct() - .fields() - .iter() - .map(|field| field.id) - .collect(); - let mut flat = HashMap::new(); - for (index, field) in batch.schema().fields().iter().enumerate() { - if let Some(id) = try_get_field_id_from_metadata(field)? - && !top_level_ids.contains(&id) - && schema.field_by_id(id).is_some() - { - flat.insert(id, index); +/// How to build the columns of an equality delete batch once the top-level columns of fields +/// nested in the pruned schema are moved back under their parent structs. +struct EqualityColumnNesting { + input_schema: ArrowSchemaRef, + /// The output schema and columns, or `None` when no column has to move. + output: Option<(ArrowSchemaRef, Vec)>, +} + +/// An output column: a column of the input batch, or a struct built from such columns. +enum NestedColumn { + Input(usize), + Struct(Fields, Vec), +} + +impl NestedColumn { + fn build(&self, batch: &RecordBatch) -> ArrayRef { + match self { + NestedColumn::Input(index) => batch.column(*index).clone(), + NestedColumn::Struct(fields, children) => Arc::new(StructArray::new( + fields.clone(), + children.iter().map(|child| child.build(batch)).collect(), + None, + )), } } - if flat.is_empty() { - return Ok(batch); - } +} - fn nest( - field: &crate::spec::NestedField, - batch: &RecordBatch, - flat: &HashMap, - ) -> Option<(FieldRef, ArrayRef)> { - if let Some(&index) = flat.get(&field.id) { - return Some(( - batch.schema().field(index).clone().into(), - batch.column(index).clone(), - )); - } - let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else { - return None; - }; - let (fields, columns): (Vec, Vec) = struct_type +impl EqualityColumnNesting { + fn try_new(input_schema: ArrowSchemaRef, schema: &Schema) -> Result { + let top_level_ids: HashSet = schema + .as_struct() .fields() .iter() - .filter_map(|child| nest(child, batch, flat)) - .unzip(); - if fields.is_empty() { - return None; + .map(|field| field.id) + .collect(); + let mut flat = HashMap::new(); + for (index, field) in input_schema.fields().iter().enumerate() { + if let Some(id) = try_get_field_id_from_metadata(field)? + && !top_level_ids.contains(&id) + && schema.field_by_id(id).is_some() + { + flat.insert(id, index); + } + } + if flat.is_empty() { + return Ok(Self { + input_schema, + output: None, + }); } - let fields = Fields::from(fields); - let array = StructArray::new(fields.clone(), columns, None); - let arrow_field = - arrow_schema::Field::new(&field.name, arrow_schema::DataType::Struct(fields), true) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - field.id.to_string(), - )])); - Some((Arc::new(arrow_field), Arc::new(array))) - } - let mut fields: Vec = Vec::new(); - let mut columns: Vec = Vec::new(); - for (index, field) in batch.schema().fields().iter().enumerate() { - if !flat.values().any(|&flat_index| flat_index == index) { - fields.push(field.clone()); - columns.push(batch.column(index).clone()); + fn nest( + field: &crate::spec::NestedField, + input_schema: &ArrowSchema, + flat: &HashMap, + ) -> Option<(FieldRef, NestedColumn)> { + if let Some(&index) = flat.get(&field.id) { + return Some(( + input_schema.fields()[index].clone(), + NestedColumn::Input(index), + )); + } + let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else { + return None; + }; + let (fields, children): (Vec, Vec) = struct_type + .fields() + .iter() + .filter_map(|child| nest(child, input_schema, flat)) + .unzip(); + if fields.is_empty() { + return None; + } + let fields = Fields::from(fields); + let arrow_field = arrow_schema::Field::new( + &field.name, + arrow_schema::DataType::Struct(fields.clone()), + true, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field.id.to_string(), + )])); + Some(( + Arc::new(arrow_field), + NestedColumn::Struct(fields, children), + )) } - } - let present: HashSet = fields - .iter() - .map(|field| try_get_field_id_from_metadata(field)) - .collect::>>()? - .into_iter() - .flatten() - .collect(); - for field in schema.as_struct().fields() { - if present.contains(&field.id) { - continue; + + let moved: HashSet = flat.values().copied().collect(); + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (index, field) in input_schema.fields().iter().enumerate() { + if !moved.contains(&index) { + fields.push(field.clone()); + columns.push(NestedColumn::Input(index)); + } } - if let Some((arrow_field, column)) = nest(field, &batch, &flat) { - fields.push(arrow_field); - columns.push(column); + let present: HashSet = fields + .iter() + .map(|field| try_get_field_id_from_metadata(field)) + .collect::>>()? + .into_iter() + .flatten() + .collect(); + for field in schema.as_struct().fields() { + if present.contains(&field.id) { + continue; + } + if let Some((arrow_field, column)) = nest(field, &input_schema, &flat) { + fields.push(arrow_field); + columns.push(column); + } } + Ok(Self { + input_schema, + output: Some((Arc::new(ArrowSchema::new(fields)), columns)), + }) + } + + fn apply(&self, batch: RecordBatch) -> Result { + let Some((schema, columns)) = &self.output else { + return Ok(batch); + }; + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + Ok(RecordBatch::try_new_with_options( + schema.clone(), + columns.iter().map(|column| column.build(&batch)).collect(), + &options, + )?) } - let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); - Ok(RecordBatch::try_new_with_options( - Arc::new(ArrowSchema::new(fields)), - columns, - &options, - )?) } #[async_trait::async_trait] From 4050928545273ca16cc7e3af832da52d1fd4e576 Mon Sep 17 00:00:00 2001 From: JeonDaehong Date: Tue, 10 Nov 2026 00:11:43 +0000 Subject: [PATCH 4/5] fix(arrow): read a field under a null struct in an equality delete as null --- .../src/arrow/caching_delete_file_loader.rs | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 6bb6d31e91..e2a52bf35c 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -624,7 +624,8 @@ impl CachingDeleteFileLoader { } }; - let root_array: ArrayRef = Arc::new(StructArray::from(record_batch)); + let root_array: ArrayRef = + push_struct_nulls_down(&(Arc::new(StructArray::from(record_batch)) as ArrayRef))?; let mut processor = EqDelColumnProcessor::new(&equality_ids, schema); visit_schema_with_partner(schema, &root_array, &mut processor, &accessor)?; @@ -681,6 +682,38 @@ impl CachingDeleteFileLoader { } } +/// Copies the nulls of every struct in `array` down to its fields, so a field under a null struct +/// reads as null. The children of a struct array keep whatever was stored under a null parent: a +/// required `s.g` under a null `s` comes back as 0, not null. +fn push_struct_nulls_down(array: &ArrayRef) -> Result { + let Some(struct_array) = array.as_any().downcast_ref::() else { + return Ok(array.clone()); + }; + let nulls = struct_array.nulls(); + let columns = struct_array + .columns() + .iter() + .map(|column| { + let column = match nulls { + None => column.clone(), + Some(_) => arrow_array::make_array( + column + .to_data() + .into_builder() + .nulls(arrow_buffer::NullBuffer::union(nulls, column.nulls())) + .build()?, + ), + }; + push_struct_nulls_down(&column) + }) + .collect::>>()?; + Ok(Arc::new(StructArray::try_new( + struct_array.fields().clone(), + columns, + nulls.cloned(), + )?) as ArrayRef) +} + struct EqDelColumnProcessor<'a> { equality_ids: &'a HashSet, schema: &'a Schema, @@ -2422,6 +2455,27 @@ mod tests { assert_keeps_rows_unless_nested_f_is_7(predicate, schema); } + /// A delete row with a null `s` on the required `s.g`. The file stores 0 for `g` under the + /// null `s`, which loads as `s.g IS NOT NULL`, not as `s.g != 0`. + #[tokio::test] + async fn test_required_nested_field_under_null_struct_loads_as_null() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "g", "4", false, vec![0], Some(vec![false])); + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let expected = Reference::new("s.g") + .is_not_null() + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + /// The nesting worked out for the first batch of a file is reused for the next ones. #[tokio::test] async fn test_equality_delete_stream_nests_every_batch() { From 65632b682e2a32be47578a812d495e715c526ddf Mon Sep 17 00:00:00 2001 From: JeonDaehong Date: Wed, 11 Nov 2026 11:38:03 +0000 Subject: [PATCH 5/5] test(arrow): cover nested equality deletes under a shared, a two-level and a required parent The parent struct put back over a flat nested column now takes its nullability from the schema. --- .../src/arrow/caching_delete_file_loader.rs | 269 ++++++++++++++++++ .../iceberg/src/arrow/delete_file_loader.rs | 2 +- 2 files changed, 270 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index e2a52bf35c..4c827f6546 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -2514,6 +2514,275 @@ mod tests { assert_eq!(values, vec![7, 8]); } + /// A batch with a different schema gets its own nesting: a flat `s.f`, then `s.f` under `s`, + /// then a flat one again. + #[tokio::test] + async fn test_equality_delete_stream_renests_on_schema_change() { + let schema = nested_eq_delete_table_schema(); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let flat = |value: i64| { + RecordBatch::try_new(flat_schema.clone(), vec![ + Arc::new(Int64Array::from(vec![value])) as ArrayRef, + ]) + .unwrap() + }; + let s_fields = Fields::from(vec![simple_field("f", DataType::Int64, true, "3")]); + let struct_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "s", + DataType::Struct(s_fields.clone()), + true, + "2", + )])); + let nested = |value: i64| { + let s = StructArray::new( + s_fields.clone(), + vec![Arc::new(Int64Array::from(vec![value])) as ArrayRef], + None, + ); + RecordBatch::try_new(struct_schema.clone(), vec![Arc::new(s) as ArrayRef]).unwrap() + }; + let input = Box::pin(futures::stream::iter(vec![ + Ok(flat(7)), + Ok(nested(8)), + Ok(flat(9)), + ])) as ArrowRecordBatchStream; + + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + let values: Vec = batches + .iter() + .map(|batch| { + batch + .column(0) + .as_struct() + .column(0) + .as_primitive::() + .value(0) + }) + .collect(); + assert_eq!(values, vec![7, 8, 9]); + } + + /// `s.f` and `s.g` from `EqualityDeleteWriter` are two top-level columns that go back under + /// one `s`. + #[tokio::test] + async fn test_two_equality_fields_under_one_parent_from_equality_delete_writer() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &[(Some(1), Some((Some(7), 5)))]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3, 4], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![3, 4], schema.clone()) + .await + .unwrap(); + let s_f = Reference::new("s.f"); + let s_g = Reference::new("s.g"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .or(s_g.clone().is_null().or(s_g.not_equal_to(Datum::long(5)))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `f` (id 1) and the required `a.b.c` (id 4, in `b` with id 3, in `a` with id 2). + fn two_level_eq_delete_table_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional( + 2, + "a", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 3, + "b", + Type::Struct(StructType::new(vec![ + NestedField::required( + 4, + "c", + Type::Primitive(PrimitiveType::Long), + ) + .into(), + ])), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ) + } + + /// A batch of `two_level_eq_delete_table_schema` holding `c`, with `a` and `b` valid where + /// `a_valid` and `b_valid` say so. + fn two_level_rows_batch(c: Vec, a_valid: Vec, b_valid: Vec) -> RecordBatch { + let schema = two_level_eq_delete_table_schema(); + let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap()); + let DataType::Struct(a_fields) = arrow_schema.field(1).data_type().clone() else { + unreachable!() + }; + let DataType::Struct(b_fields) = a_fields[0].data_type().clone() else { + unreachable!() + }; + let rows = c.len(); + let b = StructArray::new( + b_fields, + vec![Arc::new(Int64Array::from(c)) as ArrayRef], + Some(b_valid.into()), + ); + let a = StructArray::new( + a_fields, + vec![Arc::new(b) as ArrayRef], + Some(a_valid.into()), + ); + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from(vec![0; rows])) as ArrayRef, + Arc::new(a) as ArrayRef, + ]) + .unwrap() + } + + fn keep_unless_a_b_c_is(value: i64) -> Predicate { + let c = Reference::new("a.b.c"); + c.clone().is_null().or(c.not_equal_to(Datum::long(value))) + } + + /// `a.b.c` from `EqualityDeleteWriter`, a top-level column that goes back under `b` and `a`. + #[tokio::test] + async fn test_two_level_nested_equality_delete_from_equality_delete_writer() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = two_level_eq_delete_table_schema(); + let rows = two_level_rows_batch(vec![7], vec![true], vec![true]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![4], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let expected = keep_unless_a_b_c_is(7).bind(schema, true).unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `a.b.c` under its parents, the way Java writes it. The required `c` reads back as a + /// value under a null `a` and under a null `b`, and loads as null there, so those rows give + /// `a.b.c IS NOT NULL`. + #[tokio::test] + async fn test_two_level_nested_equality_delete_in_struct_layout_with_null_parents() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = two_level_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + let rows = two_level_rows_batch(vec![5, 6, 7], vec![false, true, true], vec![ + true, false, true, + ]); + let batch = rows.project(&[1]).unwrap(); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let c_is_not_null = Reference::new("a.b.c").is_not_null(); + let expected = c_is_not_null + .clone() + .and(c_is_not_null) + .and(keep_unless_a_b_c_is(7)) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `s.f` under a required `s`: the `s` put back over a flat `s.f` is required too, and the + /// delete loads the same as under an optional `s`. + #[tokio::test] + async fn test_nested_equality_delete_under_required_parent() { + use crate::expr::Bind; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "f", Type::Primitive(PrimitiveType::Long)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let flat = RecordBatch::try_new(flat_schema, vec![ + Arc::new(Int64Array::from(vec![7])) as ArrayRef + ]) + .unwrap(); + let input = Box::pin(futures::stream::iter(vec![Ok(flat)])) as ArrowRecordBatchStream; + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + assert!(!batches[0].schema().field(0).is_nullable()); + + let tmp_dir = TempDir::new().unwrap(); + let s_fields = Fields::from(vec![simple_field("f", DataType::Int64, true, "3")]); + let s = StructArray::new( + s_fields.clone(), + vec![Arc::new(Int64Array::from(vec![7])) as ArrayRef], + None, + ); + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + simple_field("f", DataType::Int64, true, "1"), + simple_field("s", DataType::Struct(s_fields), false, "2"), + ])); + let rows = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + Arc::new(s) as ArrayRef, + ]) + .unwrap(); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3], rows).await; + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) + .await + .unwrap(); + let s_f = Reference::new("s.f"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + /// An equality id the scan schema does not have is an error, not a column to skip. The check /// runs before the load of the file is claimed, so loading it again through the same loader /// fails the same way instead of waiting on the first load. diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 965146ddf1..f052a7d25e 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -252,7 +252,7 @@ impl EqualityColumnNesting { let arrow_field = arrow_schema::Field::new( &field.name, arrow_schema::DataType::Struct(fields.clone()), - true, + !field.required, ) .with_metadata(HashMap::from([( PARQUET_FIELD_ID_META_KEY.to_string(),