Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,28 @@ impl core::marker::Copy for iceberg::cow_rewrite::CowRewriteStats
impl core::marker::StructuralPartialEq for iceberg::cow_rewrite::CowRewriteStats
pub trait iceberg::cow_rewrite::CowBatchRewriter: core::marker::Send + core::marker::Sync
pub fn iceberg::cow_rewrite::CowBatchRewriter::rewrite_batch(&self, batch: arrow_array::record_batch::RecordBatch) -> iceberg::Result<iceberg::cow_rewrite::CowBatchRewrite>
pub mod iceberg::delete_vector
pub struct iceberg::delete_vector::DeleteVector
impl iceberg::delete_vector::DeleteVector
pub fn iceberg::delete_vector::DeleteVector::deserialize(blob: &[u8]) -> iceberg::Result<Self>
pub fn iceberg::delete_vector::DeleteVector::insert(&mut self, pos: u64) -> bool
pub fn iceberg::delete_vector::DeleteVector::insert_positions(&mut self, positions: &[u64]) -> iceberg::Result<usize>
pub fn iceberg::delete_vector::DeleteVector::is_empty(&self) -> bool
pub fn iceberg::delete_vector::DeleteVector::iter(&self) -> iceberg::delete_vector::DeleteVectorIterator<'_>
pub fn iceberg::delete_vector::DeleteVector::len(&self) -> u64
pub fn iceberg::delete_vector::DeleteVector::new(roaring_treemap: roaring::treemap::RoaringTreemap) -> iceberg::delete_vector::DeleteVector
impl core::default::Default for iceberg::delete_vector::DeleteVector
pub fn iceberg::delete_vector::DeleteVector::default() -> iceberg::delete_vector::DeleteVector
impl core::fmt::Debug for iceberg::delete_vector::DeleteVector
pub fn iceberg::delete_vector::DeleteVector::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl core::ops::bit::BitOrAssign for iceberg::delete_vector::DeleteVector
pub fn iceberg::delete_vector::DeleteVector::bitor_assign(&mut self, other: Self)
pub struct iceberg::delete_vector::DeleteVectorIterator<'a>
impl iceberg::delete_vector::DeleteVectorIterator<'_>
pub fn iceberg::delete_vector::DeleteVectorIterator<'_>::advance_to(&mut self, pos: u64)
impl core::iter::traits::iterator::Iterator for iceberg::delete_vector::DeleteVectorIterator<'_>
pub type iceberg::delete_vector::DeleteVectorIterator<'_>::Item = u64
pub fn iceberg::delete_vector::DeleteVectorIterator<'_>::next(&mut self) -> core::option::Option<Self::Item>
pub mod iceberg::encryption
pub mod iceberg::encryption::kms
pub struct iceberg::encryption::kms::GeneratedKey
Expand Down
20 changes: 19 additions & 1 deletion crates/iceberg/src/delete_vector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
// specific language governing permissions and limitations
// under the License.

//! Deletion vectors: the deleted row positions of one data file, as a Puffin
//! `deletion-vector-v1` blob decodes to. Public so an engine that reads
//! Parquet with a reader of its own can still decode the vectors Iceberg
//! writes.

use std::ops::BitOrAssign;

use bytes::Buf;
Expand All @@ -33,24 +38,28 @@ const DV_MAGIC_BYTES: usize = 4;
const DV_CRC_BYTES: usize = 4;
const DV_MIN_BLOB_BYTES: usize = DV_LENGTH_PREFIX_BYTES + DV_MAGIC_BYTES + DV_CRC_BYTES;

/// The deleted row positions of one data file: a set of 64-bit positions,
/// kept as a [`RoaringTreemap`].
#[derive(Debug, Default)]
pub struct DeleteVector {
inner: RoaringTreemap,
}

impl DeleteVector {
#[allow(unused)]
/// Wraps `roaring_treemap`, whose members are the deleted positions.
pub fn new(roaring_treemap: RoaringTreemap) -> DeleteVector {
DeleteVector {
inner: roaring_treemap,
}
}

/// Iterates the deleted positions in ascending order.
pub fn iter(&self) -> DeleteVectorIterator<'_> {
let outer = self.inner.bitmaps();
DeleteVectorIterator { outer, inner: None }
}

/// Marks `pos` as deleted, returning whether it was not already.
pub fn insert(&mut self, pos: u64) -> bool {
self.inner.insert(pos)
}
Expand All @@ -73,10 +82,16 @@ impl DeleteVector {
Ok(positions.len())
}

/// How many positions are deleted.
pub fn len(&self) -> u64 {
self.inner.len()
}

/// Whether no position is deleted.
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}

/// Parses a `deletion-vector-v1` Puffin blob into a `DeleteVector`.
///
/// The layout, defined by the Iceberg Puffin spec and matching Iceberg-Java's
Expand Down Expand Up @@ -216,6 +231,7 @@ fn decode_roaring_directory(mut reader: &[u8]) -> Result<RoaringTreemap> {
// There is a PR open on roaring to add this (https://github.com/RoaringBitmap/roaring-rs/pull/314)
// and if that gets merged then we can simplify `DeleteVectorIterator` here, refactoring `advance_to`
// to just a wrapper around the underlying iterator's method.
/// Iterates the deleted positions of a [`DeleteVector`] in ascending order.
pub struct DeleteVectorIterator<'a> {
// NB: `BitMapIter` was only exposed publicly in https://github.com/RoaringBitmap/roaring-rs/pull/316
// which is not yet released. As a consequence our Cargo.toml temporarily uses a git reference for
Expand Down Expand Up @@ -253,6 +269,8 @@ impl Iterator for DeleteVectorIterator<'_> {
}

impl DeleteVectorIterator<'_> {
/// Skips the positions below `pos`, so the next one yielded is the first
/// at or after it. Does nothing before the first position is yielded.
pub fn advance_to(&mut self, pos: u64) {
let hi = (pos >> 32) as u32;
let lo = pos as u32;
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ pub mod encryption;
mod test_utils;
pub mod writer;

mod delete_vector;
pub mod delete_vector;
pub mod metadata_columns;
pub mod partitioning;
pub mod puffin;
Expand Down
Loading