diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 0711a58b35..4c827f6546 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -24,7 +24,9 @@ use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; use super::delete_filter::{DeleteFilter, PosDelLoadAction, PosDelLoadGuard}; -use crate::arrow::delete_file_loader::BasicDeleteFileLoader; +use crate::arrow::delete_file_loader::{ + BasicDeleteFileLoader, evolve_equality_delete_stream, validate_equality_ids, +}; use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; @@ -306,6 +308,11 @@ impl CachingDeleteFileLoader { } DataContentType::EqualityDeletes => { + let equality_ids_vec = task.equality_ids().unwrap().to_vec(); + // Check the ids before claiming the load, so this error does not leave the file + // `Loading` for later reads of it. + validate_equality_ids(&schema, &equality_ids_vec)?; + let Some(notify) = del_filter.try_start_eq_del_load(task.file_path()) else { return Ok(DeleteFileContext::ExistingEqDel); }; @@ -314,9 +321,8 @@ impl CachingDeleteFileLoader { del_filter.insert_equality_delete(task.file_path(), receiver); // Per the Iceberg spec, evolve schema for equality deletes but only for the - // equality_ids columns, not all table columns. - let equality_ids_vec = task.equality_ids().unwrap().to_vec(); - let evolved_stream = BasicDeleteFileLoader::evolve_schema( + // columns holding the equality_ids, not all table columns. + let evolved_stream = evolve_equality_delete_stream( basic_delete_file_loader .parquet_to_batch_stream( task.file_path(), @@ -324,10 +330,9 @@ impl CachingDeleteFileLoader { task.key_metadata(), ) .await?, - schema, + &schema, &equality_ids_vec, - ) - .await?; + )?; Ok(DeleteFileContext::FreshEqDel { batch_stream: evolved_stream, @@ -619,9 +624,10 @@ impl CachingDeleteFileLoader { } }; - let root_array: ArrayRef = Arc::new(StructArray::from(record_batch)); + let root_array: ArrayRef = + push_struct_nulls_down(&(Arc::new(StructArray::from(record_batch)) as ArrayRef))?; - let mut processor = EqDelColumnProcessor::new(&equality_ids); + let mut processor = EqDelColumnProcessor::new(&equality_ids, schema); visit_schema_with_partner(schema, &root_array, &mut processor, &accessor)?; let mut datum_columns_with_names = processor.finish()?; @@ -676,15 +682,49 @@ impl CachingDeleteFileLoader { } } +/// Copies the nulls of every struct in `array` down to its fields, so a field under a null struct +/// reads as null. The children of a struct array keep whatever was stored under a null parent: a +/// required `s.g` under a null `s` comes back as 0, not null. +fn push_struct_nulls_down(array: &ArrayRef) -> Result { + let Some(struct_array) = array.as_any().downcast_ref::() else { + return Ok(array.clone()); + }; + let nulls = struct_array.nulls(); + let columns = struct_array + .columns() + .iter() + .map(|column| { + let column = match nulls { + None => column.clone(), + Some(_) => arrow_array::make_array( + column + .to_data() + .into_builder() + .nulls(arrow_buffer::NullBuffer::union(nulls, column.nulls())) + .build()?, + ), + }; + push_struct_nulls_down(&column) + }) + .collect::>>()?; + Ok(Arc::new(StructArray::try_new( + struct_array.fields().clone(), + columns, + nulls.cloned(), + )?) as ArrayRef) +} + struct EqDelColumnProcessor<'a> { equality_ids: &'a HashSet, + schema: &'a Schema, collected_columns: Vec<(ArrayRef, String, Type)>, } impl<'a> EqDelColumnProcessor<'a> { - fn new(equality_ids: &'a HashSet) -> Self { + fn new(equality_ids: &'a HashSet, schema: &'a Schema) -> Self { Self { equality_ids, + schema, collected_columns: Vec::with_capacity(equality_ids.len()), } } @@ -740,9 +780,17 @@ impl SchemaWithPartnerVisitor for EqDelColumnProcessor<'_> { fn field(&mut self, field: &NestedFieldRef, partner: &ArrayRef, _value: ()) -> Result<()> { if self.equality_ids.contains(&field.id) && field.field_type.as_primitive_type().is_some() { + // Use the full name so a nested field (`s.f`) binds to itself, not to a top-level + // column with the same leaf name. + let name = self.schema.name_by_field_id(field.id).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("equality delete field {} not found in schema", field.id), + ) + })?; self.collected_columns.push(( partner.clone(), - field.name.clone(), + name.to_string(), field.field_type.as_ref().clone(), )); } @@ -905,7 +953,7 @@ mod tests { .await .expect("error parsing batch stream"); - let expected = "((((((y IS NULL) OR (y != 1)) OR ((z IS NULL) OR (z != 100))) OR ((a IS NULL) OR (a != \"HELP\"))) OR ((sa IS NULL) OR (sa != 4))) OR ((b IS NULL) OR (b != 62696E6172795F64617461))) AND ((((((y IS NULL) OR (y != 2)) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR ((sa IS NULL) OR (sa != 5))) OR (b IS NOT NULL))".to_string(); + let expected = "((((((y IS NULL) OR (y != 1)) OR ((z IS NULL) OR (z != 100))) OR ((a IS NULL) OR (a != \"HELP\"))) OR ((s.sa IS NULL) OR (s.sa != 4))) OR ((b IS NULL) OR (b != 62696E6172795F64617461))) AND ((((((y IS NULL) OR (y != 2)) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR ((s.sa IS NULL) OR (s.sa != 5))) OR (b IS NOT NULL))".to_string(); assert_eq!(parsed_eq_delete.to_string(), expected); } @@ -2179,4 +2227,684 @@ mod tests { .collect(); assert_eq!(positions, vec![1, 3]); } + + /// Top-level `f` (id 1) and `s.f` (id 3, in struct `s` with id 2), so a predicate that + /// binds by leaf name would pick the wrong column. + fn nested_eq_delete_table_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "f", Type::Primitive(PrimitiveType::Long)) + .into(), + NestedField::required(4, "g", Type::Primitive(PrimitiveType::Long)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ) + } + + /// Loads one equality delete file with `equality_ids` through `load_deletes` and returns the + /// predicate the delete filter builds for a scan of `schema`. + async fn load_eq_delete_predicate( + path: &str, + equality_ids: Vec, + schema: SchemaRef, + ) -> Result> { + use crate::scan::FileScanTask; + + let eq_del = eq_delete_task(path, equality_ids); + let task = FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{path}.data")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_deletes(vec![eq_del]) + .with_case_sensitive(true) + .build() + .unwrap(); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + let filter = loader + .load_deletes(task.deletes(), task.schema_ref()) + .await + .unwrap()?; + filter.build_equality_delete_predicate(&task).await + } + + /// Field ids of the references in a bound predicate. + fn bound_field_ids(predicate: &crate::expr::BoundPredicate, ids: &mut Vec) { + use crate::expr::BoundPredicate; + match predicate { + BoundPredicate::And(expr) | BoundPredicate::Or(expr) => { + for input in expr.inputs() { + bound_field_ids(input, ids); + } + } + BoundPredicate::Not(expr) => bound_field_ids(expr.inputs()[0], ids), + BoundPredicate::Unary(expr) => ids.push(expr.term().field().id), + BoundPredicate::Binary(expr) => ids.push(expr.term().field().id), + BoundPredicate::Set(expr) => ids.push(expr.term().field().id), + BoundPredicate::AlwaysTrue | BoundPredicate::AlwaysFalse => {} + } + } + + /// The predicate for deleting `s.f = 7`, bound to field 3 and not to the top-level `f`. + fn assert_keeps_rows_unless_nested_f_is_7( + predicate: Option, + schema: SchemaRef, + ) { + use crate::expr::Bind; + + let predicate = predicate.expect("an equality delete predicate"); + let s_f = Reference::new("s.f"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, expected); + let mut ids = Vec::new(); + bound_field_ids(&predicate, &mut ids); + assert_eq!(ids, vec![3, 3]); + } + + /// An equality delete task for the file at `path` on `equality_ids`. + fn eq_delete_task(path: &str, equality_ids: Vec) -> FileScanTaskDeleteFile { + FileScanTaskDeleteFile::builder() + .with_file_path(path.to_string()) + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(equality_ids)) + .build() + .unwrap() + } + + /// Writes an equality delete file with `s.` (field `id`) under its parent struct `s`, + /// the way Java writes a nested equality field. `s_valid` marks the rows whose `s` is not + /// null. + fn write_struct_layout_eq_delete( + path: &str, + name: &str, + id: &str, + nullable: bool, + values: Vec, + s_valid: Option>, + ) { + let child = Arc::new(simple_field(name, DataType::Int64, nullable, id)); + let fields = Fields::from(vec![child]); + let s_field = simple_field("s", DataType::Struct(fields.clone()), true, "2"); + let delete_schema = Arc::new(arrow_schema::Schema::new(vec![s_field])); + let s = StructArray::new( + fields, + vec![Arc::new(Int64Array::from(values)) as ArrayRef], + s_valid.map(Into::into), + ); + let batch = RecordBatch::try_new(delete_schema.clone(), vec![Arc::new(s)]).unwrap(); + let mut writer = + ArrowWriter::try_new(File::create(path).unwrap(), delete_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + + /// A row of `nested_eq_delete_table_schema`: `f`, then `s.f` and `s.g`, or a null `s`. + type NestedRow = (Option, Option<(Option, i64)>); + + fn nested_rows_batch(schema: &Schema, rows: &[NestedRow]) -> RecordBatch { + let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(schema).unwrap()); + let DataType::Struct(s_fields) = arrow_schema.field(1).data_type().clone() else { + unreachable!() + }; + let s = StructArray::new( + s_fields, + vec![ + Arc::new(Int64Array::from_iter( + rows.iter().map(|(_, s)| s.and_then(|s| s.0)), + )) as ArrayRef, + Arc::new(Int64Array::from_iter_values( + rows.iter().map(|(_, s)| s.map_or(0, |s| s.1)), + )) as ArrayRef, + ], + Some( + rows.iter() + .map(|(_, s)| s.is_some()) + .collect::>() + .into(), + ), + ); + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from_iter(rows.iter().map(|(f, _)| *f))) as ArrayRef, + Arc::new(s) as ArrayRef, + ]) + .unwrap() + } + + /// Writes `rows` as an equality delete file on `equality_ids` with `EqualityDeleteWriter`, + /// which writes a nested field as a top-level column, and returns its path. + async fn write_eq_delete_with_writer( + dir: &TempDir, + schema: SchemaRef, + equality_ids: Vec, + rows: RecordBatch, + ) -> String { + use crate::writer::base_writer::equality_delete_writer::{ + EqualityDeleteFileWriterBuilder, EqualityDeleteWriterConfig, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + + let config = EqualityDeleteWriterConfig::new(equality_ids, schema).unwrap(); + let delete_schema = arrow_schema_to_schema(config.projected_arrow_schema_ref()).unwrap(); + let rolling = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)), + FileIO::new_with_fs(), + DefaultLocationGenerator::with_data_location(dir.path().to_str().unwrap().to_string()), + DefaultFileNameGenerator::new("eq-del".to_string(), None, DataFileFormat::Parquet), + ); + let mut writer = EqualityDeleteFileWriterBuilder::new(rolling, config) + .build(None) + .await + .unwrap(); + writer.write(rows).await.unwrap(); + let data_file = writer.close().await.unwrap().pop().unwrap(); + data_file.file_path().to_string() + } + + /// A delete file in Java's layout: `s.f` under its parent struct `s`. + #[tokio::test] + async fn test_nested_field_equality_delete_in_struct_layout() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "f", "3", true, vec![7], None); + + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) + .await + .unwrap(); + assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + } + + /// A delete file from `EqualityDeleteWriter`, which writes `s.f` as a top-level column. + #[tokio::test] + async fn test_nested_field_equality_delete_from_equality_delete_writer() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &[(Some(1), Some((Some(7), 0)))]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) + .await + .unwrap(); + assert_keeps_rows_unless_nested_f_is_7(predicate, schema); + } + + /// A delete row with a null `s` on the required `s.g`. The file stores 0 for `g` under the + /// null `s`, which loads as `s.g IS NOT NULL`, not as `s.g != 0`. + #[tokio::test] + async fn test_required_nested_field_under_null_struct_loads_as_null() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "g", "4", false, vec![0], Some(vec![false])); + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let expected = Reference::new("s.g") + .is_not_null() + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// The nesting worked out for the first batch of a file is reused for the next ones. + #[tokio::test] + async fn test_equality_delete_stream_nests_every_batch() { + let schema = nested_eq_delete_table_schema(); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let batch = |value: i64| { + RecordBatch::try_new(flat_schema.clone(), vec![ + Arc::new(Int64Array::from(vec![value])) as ArrayRef, + ]) + .unwrap() + }; + let input = Box::pin(futures::stream::iter(vec![Ok(batch(7)), Ok(batch(8))])) + as ArrowRecordBatchStream; + + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + let values: Vec = batches + .iter() + .map(|batch| { + batch + .column(0) + .as_struct() + .column(0) + .as_primitive::() + .value(0) + }) + .collect(); + assert_eq!(values, vec![7, 8]); + } + + /// A batch with a different schema gets its own nesting: a flat `s.f`, then `s.f` under `s`, + /// then a flat one again. + #[tokio::test] + async fn test_equality_delete_stream_renests_on_schema_change() { + let schema = nested_eq_delete_table_schema(); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let flat = |value: i64| { + RecordBatch::try_new(flat_schema.clone(), vec![ + Arc::new(Int64Array::from(vec![value])) as ArrayRef, + ]) + .unwrap() + }; + let s_fields = Fields::from(vec![simple_field("f", DataType::Int64, true, "3")]); + let struct_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "s", + DataType::Struct(s_fields.clone()), + true, + "2", + )])); + let nested = |value: i64| { + let s = StructArray::new( + s_fields.clone(), + vec![Arc::new(Int64Array::from(vec![value])) as ArrayRef], + None, + ); + RecordBatch::try_new(struct_schema.clone(), vec![Arc::new(s) as ArrayRef]).unwrap() + }; + let input = Box::pin(futures::stream::iter(vec![ + Ok(flat(7)), + Ok(nested(8)), + Ok(flat(9)), + ])) as ArrowRecordBatchStream; + + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + let values: Vec = batches + .iter() + .map(|batch| { + batch + .column(0) + .as_struct() + .column(0) + .as_primitive::() + .value(0) + }) + .collect(); + assert_eq!(values, vec![7, 8, 9]); + } + + /// `s.f` and `s.g` from `EqualityDeleteWriter` are two top-level columns that go back under + /// one `s`. + #[tokio::test] + async fn test_two_equality_fields_under_one_parent_from_equality_delete_writer() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &[(Some(1), Some((Some(7), 5)))]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3, 4], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![3, 4], schema.clone()) + .await + .unwrap(); + let s_f = Reference::new("s.f"); + let s_g = Reference::new("s.g"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .or(s_g.clone().is_null().or(s_g.not_equal_to(Datum::long(5)))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `f` (id 1) and the required `a.b.c` (id 4, in `b` with id 3, in `a` with id 2). + fn two_level_eq_delete_table_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional( + 2, + "a", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 3, + "b", + Type::Struct(StructType::new(vec![ + NestedField::required( + 4, + "c", + Type::Primitive(PrimitiveType::Long), + ) + .into(), + ])), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ) + } + + /// A batch of `two_level_eq_delete_table_schema` holding `c`, with `a` and `b` valid where + /// `a_valid` and `b_valid` say so. + fn two_level_rows_batch(c: Vec, a_valid: Vec, b_valid: Vec) -> RecordBatch { + let schema = two_level_eq_delete_table_schema(); + let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap()); + let DataType::Struct(a_fields) = arrow_schema.field(1).data_type().clone() else { + unreachable!() + }; + let DataType::Struct(b_fields) = a_fields[0].data_type().clone() else { + unreachable!() + }; + let rows = c.len(); + let b = StructArray::new( + b_fields, + vec![Arc::new(Int64Array::from(c)) as ArrayRef], + Some(b_valid.into()), + ); + let a = StructArray::new( + a_fields, + vec![Arc::new(b) as ArrayRef], + Some(a_valid.into()), + ); + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from(vec![0; rows])) as ArrayRef, + Arc::new(a) as ArrayRef, + ]) + .unwrap() + } + + fn keep_unless_a_b_c_is(value: i64) -> Predicate { + let c = Reference::new("a.b.c"); + c.clone().is_null().or(c.not_equal_to(Datum::long(value))) + } + + /// `a.b.c` from `EqualityDeleteWriter`, a top-level column that goes back under `b` and `a`. + #[tokio::test] + async fn test_two_level_nested_equality_delete_from_equality_delete_writer() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = two_level_eq_delete_table_schema(); + let rows = two_level_rows_batch(vec![7], vec![true], vec![true]); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![4], rows).await; + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let expected = keep_unless_a_b_c_is(7).bind(schema, true).unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `a.b.c` under its parents, the way Java writes it. The required `c` reads back as a + /// value under a null `a` and under a null `b`, and loads as null there, so those rows give + /// `a.b.c IS NOT NULL`. + #[tokio::test] + async fn test_two_level_nested_equality_delete_in_struct_layout_with_null_parents() { + use crate::expr::Bind; + + let tmp_dir = TempDir::new().unwrap(); + let schema = two_level_eq_delete_table_schema(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + let rows = two_level_rows_batch(vec![5, 6, 7], vec![false, true, true], vec![ + true, false, true, + ]); + let batch = rows.project(&[1]).unwrap(); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let predicate = load_eq_delete_predicate(&path, vec![4], schema.clone()) + .await + .unwrap(); + let c_is_not_null = Reference::new("a.b.c").is_not_null(); + let expected = c_is_not_null + .clone() + .and(c_is_not_null) + .and(keep_unless_a_b_c_is(7)) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// `s.f` under a required `s`: the `s` put back over a flat `s.f` is required too, and the + /// delete loads the same as under an optional `s`. + #[tokio::test] + async fn test_nested_equality_delete_under_required_parent() { + use crate::expr::Bind; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "f", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "f", Type::Primitive(PrimitiveType::Long)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let flat_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "f", + DataType::Int64, + true, + "3", + )])); + let flat = RecordBatch::try_new(flat_schema, vec![ + Arc::new(Int64Array::from(vec![7])) as ArrayRef + ]) + .unwrap(); + let input = Box::pin(futures::stream::iter(vec![Ok(flat)])) as ArrowRecordBatchStream; + let batches: Vec = evolve_equality_delete_stream(input, &schema, &[3]) + .unwrap() + .try_collect() + .await + .unwrap(); + assert!(!batches[0].schema().field(0).is_nullable()); + + let tmp_dir = TempDir::new().unwrap(); + let s_fields = Fields::from(vec![simple_field("f", DataType::Int64, true, "3")]); + let s = StructArray::new( + s_fields.clone(), + vec![Arc::new(Int64Array::from(vec![7])) as ArrayRef], + None, + ); + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + simple_field("f", DataType::Int64, true, "1"), + simple_field("s", DataType::Struct(s_fields), false, "2"), + ])); + let rows = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + Arc::new(s) as ArrayRef, + ]) + .unwrap(); + let path = write_eq_delete_with_writer(&tmp_dir, schema.clone(), vec![3], rows).await; + let predicate = load_eq_delete_predicate(&path, vec![3], schema.clone()) + .await + .unwrap(); + let s_f = Reference::new("s.f"); + let expected = s_f + .clone() + .is_null() + .or(s_f.not_equal_to(Datum::long(7))) + .bind(schema, true) + .unwrap(); + assert_eq!(predicate, Some(expected)); + } + + /// An equality id the scan schema does not have is an error, not a column to skip. The check + /// runs before the load of the file is claimed, so loading it again through the same loader + /// fails the same way instead of waiting on the first load. + #[tokio::test] + async fn test_equality_delete_with_id_missing_from_schema_fails() { + let tmp_dir = TempDir::new().unwrap(); + // Columns y (id 2) and z (id 3). + let path = setup_write_equality_delete_file_1(tmp_dir.path().to_str().unwrap()); + let schema = nested_eq_delete_table_schema(); + let eq_del = eq_delete_task(&path, vec![1, 9]); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + + for _ in 0..2 { + let loaded = tokio::time::timeout( + std::time::Duration::from_secs(5), + loader.load_deletes(std::slice::from_ref(&eq_del), schema.clone()), + ) + .await + .expect("the load waited on an earlier failed load") + .unwrap(); + let Err(err) = loaded else { + panic!("expected an error for field id 9"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("field id 9"), "{err}"); + } + } + + /// `s.f = 7` matches only the first row: the second has a top-level `f` of 7 and the third + /// a null `s`. + const NESTED_ROWS: &[NestedRow] = &[ + (Some(1), Some((Some(7), 0))), + (Some(7), Some((Some(1), 1))), + (Some(2), None), + (Some(3), Some((Some(8), 2))), + ]; + + /// Scans a data file of `nested_eq_delete_table_schema` holding `NESTED_ROWS` with `deletes` + /// applied, and returns the top-level `f` of the rows left. + async fn scan_nested_rows( + dir: &TempDir, + deletes: Vec, + ) -> Vec> { + use crate::arrow::ArrowReaderBuilder; + use crate::scan::{FileScanTask, FileScanTaskStream}; + + let schema = nested_eq_delete_table_schema(); + let batch = nested_rows_batch(&schema, NESTED_ROWS); + let path = format!("{}/data.parquet", dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, 2]) + .with_deletes(deletes) + .with_case_sensitive(true) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .build() + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .iter() + .collect::>() + }) + .collect() + } + + #[tokio::test] + #[ignore = "applying a nested equality delete on scan needs #2906"] + async fn test_scan_applies_nested_equality_delete_in_struct_layout() { + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "f", "3", true, vec![7], None); + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![3])]).await; + assert_eq!(left, vec![Some(7), Some(2), Some(3)]); + } + + #[tokio::test] + #[ignore = "applying a nested equality delete on scan needs #2906"] + async fn test_scan_applies_nested_equality_delete_from_equality_delete_writer() { + let tmp_dir = TempDir::new().unwrap(); + let schema = nested_eq_delete_table_schema(); + let rows = nested_rows_batch(&schema, &NESTED_ROWS[..1]); + let path = write_eq_delete_with_writer(&tmp_dir, schema, vec![3], rows).await; + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![3])]).await; + assert_eq!(left, vec![Some(7), Some(2), Some(3)]); + } + + /// A delete row with a null `s` on `s.g`, which is required inside the optional `s`, deletes + /// the data rows whose `s` is null. Until #3306, the `s.g IS NOT NULL` it loads as binds to + /// `AlwaysTrue` (#3292) and the scan keeps those rows. + #[tokio::test] + #[ignore = "needs #2906 to apply nested equality deletes and #3306 to bind IS NULL on s.g"] + async fn test_scan_applies_null_parent_equality_delete_on_required_nested_field() { + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/eq-delete.parquet", tmp_dir.path().to_str().unwrap()); + write_struct_layout_eq_delete(&path, "g", "4", false, vec![0], Some(vec![false])); + + let left = scan_nested_rows(&tmp_dir, vec![eq_delete_task(&path, vec![4])]).await; + assert_eq!(left, vec![Some(1), Some(7), Some(3)]); + } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index ec61710a42..f052a7d25e 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -15,18 +15,22 @@ // specific language governing permissions and limitations // under the License. +use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StructArray}; +use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; use futures::{StreamExt, TryStreamExt}; -use parquet::arrow::ParquetRecordBatchStreamBuilder; +use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder}; use crate::arrow::ArrowReader; use crate::arrow::reader::ParquetReadOptions; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::ScanMetrics; +use crate::arrow::schema::try_get_field_id_from_metadata; use crate::io::FileIO; use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile}; -use crate::spec::{Schema, SchemaRef}; +use crate::spec::{Schema, SchemaRef, prune_columns}; use crate::{Error, ErrorKind, Result}; /// Delete File Loader @@ -115,6 +119,195 @@ impl BasicDeleteFileLoader { } } +/// Checks that every equality id is a field of the scan schema. Java's `DeleteFilter` fails the +/// same way rather than matching rows on fewer columns. +pub(crate) fn validate_equality_ids(schema: &Schema, equality_ids: &[i32]) -> Result<()> { + match equality_ids + .iter() + .find(|&&id| schema.field_by_id(id).is_none()) + { + Some(id) => Err(Error::new( + ErrorKind::DataInvalid, + format!("Equality delete field id {id} is not in the scan schema"), + )), + None => Ok(()), + } +} + +/// Reads an equality delete file as `schema` pruned to its equality fields, the way Java's +/// `DeleteFilter` selects them, so a field nested in a struct keeps its parent struct. +/// +/// Java writes a nested equality field under its parent structs, while `EqualityDeleteWriter` +/// writes it as a top-level column. Such a column is put back under its parents before schema +/// evolution, so both layouts read the same way. +pub(crate) fn evolve_equality_delete_stream( + record_batch_stream: ArrowRecordBatchStream, + schema: &Schema, + equality_ids: &[i32], +) -> Result { + validate_equality_ids(schema, equality_ids)?; + let crate::spec::Type::Struct(pruned) = + prune_columns(schema, equality_ids.iter().copied(), false)? + else { + return Err(Error::new( + ErrorKind::Unexpected, + "pruning a schema did not return a struct", + )); + }; + let projection: Vec = pruned.fields().iter().map(|field| field.id).collect(); + let pruned = Arc::new( + Schema::builder() + .with_schema_id(schema.schema_id()) + .with_fields(pruned.fields().to_vec()) + .build()?, + ); + let mut transformer = RecordBatchTransformerBuilder::new(pruned.clone(), &projection).build(); + // The batches of a file share its schema, so the nesting is worked out once and only + // rebuilt if a batch comes with a different schema. + let mut nesting: Option = None; + Ok(Box::pin(record_batch_stream.map(move |batch| { + let batch = batch?; + let nesting = match &mut nesting { + Some(nesting) if nesting.input_schema == batch.schema() => nesting, + slot => slot.insert(EqualityColumnNesting::try_new(batch.schema(), &pruned)?), + }; + transformer.process_record_batch(nesting.apply(batch)?) + })) as ArrowRecordBatchStream) +} + +/// How to build the columns of an equality delete batch once the top-level columns of fields +/// nested in the pruned schema are moved back under their parent structs. +struct EqualityColumnNesting { + input_schema: ArrowSchemaRef, + /// The output schema and columns, or `None` when no column has to move. + output: Option<(ArrowSchemaRef, Vec)>, +} + +/// An output column: a column of the input batch, or a struct built from such columns. +enum NestedColumn { + Input(usize), + Struct(Fields, Vec), +} + +impl NestedColumn { + fn build(&self, batch: &RecordBatch) -> ArrayRef { + match self { + NestedColumn::Input(index) => batch.column(*index).clone(), + NestedColumn::Struct(fields, children) => Arc::new(StructArray::new( + fields.clone(), + children.iter().map(|child| child.build(batch)).collect(), + None, + )), + } + } +} + +impl EqualityColumnNesting { + fn try_new(input_schema: ArrowSchemaRef, schema: &Schema) -> Result { + let top_level_ids: HashSet = schema + .as_struct() + .fields() + .iter() + .map(|field| field.id) + .collect(); + let mut flat = HashMap::new(); + for (index, field) in input_schema.fields().iter().enumerate() { + if let Some(id) = try_get_field_id_from_metadata(field)? + && !top_level_ids.contains(&id) + && schema.field_by_id(id).is_some() + { + flat.insert(id, index); + } + } + if flat.is_empty() { + return Ok(Self { + input_schema, + output: None, + }); + } + + fn nest( + field: &crate::spec::NestedField, + input_schema: &ArrowSchema, + flat: &HashMap, + ) -> Option<(FieldRef, NestedColumn)> { + if let Some(&index) = flat.get(&field.id) { + return Some(( + input_schema.fields()[index].clone(), + NestedColumn::Input(index), + )); + } + let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else { + return None; + }; + let (fields, children): (Vec, Vec) = struct_type + .fields() + .iter() + .filter_map(|child| nest(child, input_schema, flat)) + .unzip(); + if fields.is_empty() { + return None; + } + let fields = Fields::from(fields); + let arrow_field = arrow_schema::Field::new( + &field.name, + arrow_schema::DataType::Struct(fields.clone()), + !field.required, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field.id.to_string(), + )])); + Some(( + Arc::new(arrow_field), + NestedColumn::Struct(fields, children), + )) + } + + let moved: HashSet = flat.values().copied().collect(); + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (index, field) in input_schema.fields().iter().enumerate() { + if !moved.contains(&index) { + fields.push(field.clone()); + columns.push(NestedColumn::Input(index)); + } + } + let present: HashSet = fields + .iter() + .map(|field| try_get_field_id_from_metadata(field)) + .collect::>>()? + .into_iter() + .flatten() + .collect(); + for field in schema.as_struct().fields() { + if present.contains(&field.id) { + continue; + } + if let Some((arrow_field, column)) = nest(field, &input_schema, &flat) { + fields.push(arrow_field); + columns.push(column); + } + } + Ok(Self { + input_schema, + output: Some((Arc::new(ArrowSchema::new(fields)), columns)), + }) + } + + fn apply(&self, batch: RecordBatch) -> Result { + let Some((schema, columns)) = &self.output else { + return Ok(batch); + }; + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + Ok(RecordBatch::try_new_with_options( + schema.clone(), + columns.iter().map(|column| column.build(&batch)).collect(), + &options, + )?) + } +} + #[async_trait::async_trait] impl DeleteFileLoader for BasicDeleteFileLoader { async fn read_delete_file( @@ -130,14 +323,15 @@ impl DeleteFileLoader for BasicDeleteFileLoader { ) .await?; - // For equality deletes, only evolve the equality_ids columns. + // For equality deletes, only evolve the columns holding the equality_ids. // For positional deletes (equality_ids is None), use all field IDs. - let field_ids = match task.equality_ids() { - Some(ids) => ids.to_vec(), - None => schema.field_id_to_name_map().keys().cloned().collect(), - }; - - Self::evolve_schema(raw_batch_stream, schema, &field_ids).await + match task.equality_ids() { + Some(ids) => evolve_equality_delete_stream(raw_batch_stream, &schema, ids), + None => { + let field_ids: Vec = schema.field_id_to_name_map().keys().cloned().collect(); + Self::evolve_schema(raw_batch_stream, schema, &field_ids).await + } + } } }