Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
289 changes: 229 additions & 60 deletions crates/iceberg/src/arrow/reader/positional_deletes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@ impl ArrowReader {
let mut results: Vec<RowSelector> = Vec::new();
let mut selected_row_groups_idx = 0;
let mut current_row_group_base_idx: u64 = 0;
let mut delete_vector_iter = positional_deletes.iter();
let mut next_deleted_row_idx_opt = delete_vector_iter.next();
// Deleted positions as half-open ranges, so a run of deletes costs one step rather than
// one per row.
let mut deleted_ranges = positional_deletes.ranges();
let mut next_deleted_range = deleted_ranges.next();

for (idx, row_group_metadata) in row_group_metadata_list.iter().enumerate() {
let row_group_num_rows = row_group_metadata.num_rows() as u64;
Expand All @@ -60,16 +62,17 @@ impl ArrowReader {
// selected row group
selected_row_groups_idx += 1;
} else {
// Advance iterator past all deletes in the skipped row group.
// advance_to() positions the iterator to the first delete >= next_row_group_base_idx.
// However, if our cached next_deleted_row_idx_opt is in the skipped range,
// we need to call next() to update the cache with the newly positioned value.
delete_vector_iter.advance_to(next_row_group_base_idx);
// Only update the cache if the cached value is stale (in the skipped range)
if let Some(cached_idx) = next_deleted_row_idx_opt
&& cached_idx < next_row_group_base_idx
{
next_deleted_row_idx_opt = delete_vector_iter.next();
// Skip the deletes in this row group without walking them. A range that
// continues into the next row group is cut to start there.
match next_deleted_range {
Some((_, end)) if end <= next_row_group_base_idx => {
deleted_ranges.skip_to(next_row_group_base_idx);
next_deleted_range = deleted_ranges.next();
}
Some((start, end)) if start < next_row_group_base_idx => {
next_deleted_range = Some((next_row_group_base_idx, end));
}
_ => {}
}

// still increment the current page base index but then skip to the next row group
Expand All @@ -79,59 +82,29 @@ impl ArrowReader {
}
}

let mut next_deleted_row_idx = match next_deleted_row_idx_opt {
Some(next_deleted_row_idx) => {
// if the index of the next deleted row is beyond this row group, add a selection for
// the remainder of this row group and skip to the next row group
if next_deleted_row_idx >= next_row_group_base_idx {
results.push(RowSelector::select(row_group_num_rows as usize));
current_row_group_base_idx += row_group_num_rows;
continue;
}

next_deleted_row_idx
let mut current_idx = current_row_group_base_idx;
while let Some((start, end)) = next_deleted_range {
if start >= next_row_group_base_idx {
break;
}

// If there are no more pos deletes, add a selector for the entirety of this row group.
_ => {
results.push(RowSelector::select(row_group_num_rows as usize));
current_row_group_base_idx += row_group_num_rows;
continue;
// `select` the rows before the deleted range, then `skip` the part of the range
// that lies in this row group
let skip_start = start.max(current_idx);
let skip_end = end.min(next_row_group_base_idx);
if skip_start > current_idx {
results.push(RowSelector::select((skip_start - current_idx) as usize));
}
};

let mut current_idx = current_row_group_base_idx;
'chunks: while next_deleted_row_idx < next_row_group_base_idx {
// `select` all rows that precede the next delete index
if current_idx < next_deleted_row_idx {
let run_length = next_deleted_row_idx - current_idx;
results.push(RowSelector::select(run_length as usize));
current_idx += run_length;
if skip_end > skip_start {
results.push(RowSelector::skip((skip_end - skip_start) as usize));
}
current_idx = current_idx.max(skip_end);

// `skip` all consecutive deleted rows in the current row group
let mut run_length = 0;
while next_deleted_row_idx == current_idx
&& next_deleted_row_idx < next_row_group_base_idx
{
run_length += 1;
current_idx += 1;

next_deleted_row_idx_opt = delete_vector_iter.next();
next_deleted_row_idx = match next_deleted_row_idx_opt {
Some(next_deleted_row_idx) => next_deleted_row_idx,
_ => {
// We've processed the final positional delete.
// Conclude the skip and then break so that we select the remaining
// rows in the row group and move on to the next row group
results.push(RowSelector::skip(run_length));
break 'chunks;
}
};
}
if run_length > 0 {
results.push(RowSelector::skip(run_length));
if end > next_row_group_base_idx {
// the range continues into the next row group
break;
}
next_deleted_range = deleted_ranges.next();
}

if current_idx < next_row_group_base_idx {
Expand Down Expand Up @@ -168,7 +141,7 @@ mod tests {

use crate::Runtime;
use crate::arrow::{ArrowReader, ArrowReaderBuilder};
use crate::delete_vector::DeleteVector;
use crate::delete_vector::{DeleteVector, mixed_delete_vectors};
use crate::io::FileIO;
use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream};
use crate::spec::{DataContentType, DataFileFormat, NestedField, PrimitiveType, Schema, Type};
Expand Down Expand Up @@ -211,6 +184,202 @@ mod tests {
Arc::new(SchemaDescriptor::new(Arc::new(schema)))
}

/// Row groups and deleted runs that cross the 2^32 position boundary, with and without row
/// group selection.
#[test]
fn test_build_deletes_row_selection_across_2_pow_32() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd add a randomized test for build_deletes_row_selection next to this one: random row-group sizes, a random selected_row_groups subset and the DVs from mixed_delete_vectors, compared against a per-row oracle that run-length-encodes the selected groups.

The iterator gets a model test, but this loop was rewritten wholesale and only has hand-picked cases. The branch I'd least trust to stay correct is a range carried from a selected row group into a skipped one. mixed_delete_vectors would need to be pub(crate) under cfg(test) for this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added test_build_deletes_row_selection_matches_model: random row-group cuts (half placed next to a deleted position so they land inside runs), a random selected_row_groups subset or None, over every DV from mixed_delete_vectors (now pub(crate) under cfg(test)), compared with a selection built one deleted position at a time. Dropping the "range continues into the next row group" break, or skipping one row too far in a skipped group, makes it fail.

let schema_descr = get_test_schema_descr();
let columns: Vec<_> = schema_descr
.columns()
.iter()
.map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap())
.collect();
let row_groups = |sizes: &[i64]| -> Vec<RowGroupMetaData> {
sizes
.iter()
.enumerate()
.map(|(i, &n)| {
build_test_row_group_meta(schema_descr.clone(), columns.clone(), n, i as i16)
})
.collect()
};
let select = RowSelector::select;
let skip = RowSelector::skip;
let high = 1u64 << 32;
let big = high as usize;

// Row group 1 starts at 2^32 - 2. The run 2^32 - 3 ..= 2^32 crosses both boundaries.
let meta = row_groups(&[(1 << 32) - 2, 8]);
let deletes = DeleteVector::new(RoaringTreemap::from_iter([
1,
high - 3,
high - 2,
high - 1,
high,
high + 3,
]));
let build = |selected: Option<Vec<usize>>| {
ArrowReader::build_deletes_row_selection(&meta, &selected, &deletes).unwrap()
};
assert_eq!(
build(None),
RowSelection::from(vec![
select(1),
skip(1),
select(big - 5),
skip(4),
select(2),
skip(1),
select(2),
])
);
assert_eq!(
build(Some(vec![0])),
RowSelection::from(vec![select(1), skip(1), select(big - 5), skip(1)])
);
assert_eq!(
build(Some(vec![1])),
RowSelection::from(vec![skip(3), select(2), skip(1), select(2)])
);

// Skipping row group 0 lands past 2^32, in the next bitmap.
let meta = row_groups(&[(1 << 32) + 1, 4]);
let deletes = DeleteVector::new(RoaringTreemap::from_iter([5, high - 1, high, high + 2]));
assert_eq!(
ArrowReader::build_deletes_row_selection(&meta, &Some(vec![1]), &deletes).unwrap(),
RowSelection::from(vec![select(1), skip(1), select(2)])
);
}

/// The selection for `deleted` built one deleted position at a time.
fn expected_deletes_row_selection(
row_group_sizes: &[u64],
selected_row_groups: &Option<Vec<usize>>,
deleted: &[u64],
) -> RowSelection {
let mut selectors = Vec::new();
let mut base = 0;
for (idx, &num_rows) in row_group_sizes.iter().enumerate() {
let end = base + num_rows;
if selected_row_groups
.as_ref()
.is_none_or(|selected| selected.contains(&idx))
{
let mut row = base;
let first = deleted.partition_point(|&p| p < base);
for &pos in deleted[first..].iter().take_while(|&&p| p < end) {
selectors.push(RowSelector::select((pos - row) as usize));
selectors.push(RowSelector::skip(1));
row = pos + 1;
}
selectors.push(RowSelector::select((end - row) as usize));
}
base = end;
}
selectors.into()
}

/// An empty row group inside a deleted run, once selected and once skipped.
#[test]
fn test_build_deletes_row_selection_empty_row_group_inside_run() {
let schema_descr = get_test_schema_descr();
let columns: Vec<_> = schema_descr
.columns()
.iter()
.map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap())
.collect();
let row_groups: Vec<RowGroupMetaData> = [3, 0, 3]
.iter()
.enumerate()
.map(|(i, &n)| {
build_test_row_group_meta(schema_descr.clone(), columns.clone(), n, i as i16)
})
.collect();
// Rows 1..5 are deleted: two in row group 0 and two in row group 2.
let deletes = DeleteVector::new(RoaringTreemap::from_iter(1..5));
let select = RowSelector::select;
let skip = RowSelector::skip;
for (selected, expected) in [
(None, vec![select(1), skip(4), select(1)]),
(Some(vec![0, 1, 2]), vec![select(1), skip(4), select(1)]),
(Some(vec![0, 2]), vec![select(1), skip(4), select(1)]),
(Some(vec![1, 2]), vec![skip(2), select(1)]),
] {
assert_eq!(
ArrowReader::build_deletes_row_selection(&row_groups, &selected, &deletes).unwrap(),
RowSelection::from(expected),
"selected {selected:?}"
);
}
}

/// Random row group layouts and selections over the delete vectors the iterator tests use,
/// compared with a selection built one deleted position at a time.
#[test]
fn test_build_deletes_row_selection_matches_model() {
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};

let mut rng = StdRng::seed_from_u64(3);
let schema_descr = get_test_schema_descr();
let columns: Vec<_> = schema_descr
.columns()
.iter()
.map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap())
.collect();

for deletes in mixed_delete_vectors() {
let deleted: Vec<u64> = deletes.iter().collect();
for _ in 0..8 {
// Half the time the file ends at or before a deleted position, so the deletes
// past its end must be ignored.
let num_rows = match deleted.len() {
n if n > 0 && rng.random_bool(0.5) => {
deleted[rng.random_range(0..n)].saturating_add(rng.random_range(0..2))
}
_ => deleted.last().map_or(0, |&last| last + 1) + rng.random_range(0..100),
}
.max(1);
// Row group boundaries, half of them next to a deleted position so that they
// fall inside runs. Repeated cuts leave empty row groups.
let mut cuts: Vec<u64> = (0..rng.random_range(0..8))
.map(|_| match deleted.len() {
n if n > 0 && rng.random_bool(0.5) => (deleted[rng.random_range(0..n)]
+ rng.random_range(0..3))
.saturating_sub(1),
_ => rng.random_range(0..num_rows),
})
.filter(|&cut| cut > 0 && cut < num_rows)
.collect();
cuts.extend([0, num_rows]);
cuts.sort_unstable();
let sizes: Vec<u64> = cuts.windows(2).map(|w| w[1] - w[0]).collect();
let row_groups: Vec<RowGroupMetaData> = sizes
.iter()
.enumerate()
.map(|(i, &n)| {
build_test_row_group_meta(
schema_descr.clone(),
columns.clone(),
n as i64,
i as i16,
)
})
.collect();
let selected = rng
.random_bool(0.75)
.then(|| (0..sizes.len()).filter(|_| rng.random_bool(0.5)).collect());

assert_eq!(
ArrowReader::build_deletes_row_selection(&row_groups, &selected, &deletes)
.unwrap(),
expected_deletes_row_selection(&sizes, &selected, &deleted),
"row groups {sizes:?}, selected {selected:?}"
);
}
}
}

#[test]
fn test_build_deletes_row_selection() {
let schema_descr = get_test_schema_descr();
Expand Down
Loading
Loading