Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 11 additions & 43 deletions src/iceberg/deletes/dv_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@

#include "iceberg/deletes/dv_util_internal.h"
#include "iceberg/deletes/position_delete_index.h"
#include "iceberg/deletes/roaring_position_bitmap.h"
#include "iceberg/file_format.h"
#include "iceberg/file_io.h" // IWYU pragma: keep
#include "iceberg/manifest/manifest_entry.h"
#include "iceberg/partition_spec.h"
Expand All @@ -56,6 +54,7 @@ class DVWriter::Impl {
PositionDeleteIndex positions;
std::shared_ptr<PartitionSpec> spec;
PartitionValues partition;
puffin::BlobMetadata blob;
};

Deletes& DeletesFor(std::string_view referenced_data_file,
Expand All @@ -79,9 +78,7 @@ class DVWriter::Impl {
ICEBERG_PRECHECK(!referenced_data_file.empty(),
"Deletion vector requires a non-empty referenced data file");
ICEBERG_PRECHECK(spec != nullptr, "Deletion vector requires a partition spec");
ICEBERG_PRECHECK(pos >= 0 && pos <= RoaringPositionBitmap::kMaxPosition,
"Deletion vector position out of range [0, {}]: {}",
RoaringPositionBitmap::kMaxPosition, pos);
ICEBERG_PRECHECK(pos >= 0, "Deletion vector position must be non-negative: {}", pos);
DeletesFor(referenced_data_file, spec, partition).positions.Delete(pos);
return {};
}
Expand Down Expand Up @@ -110,26 +107,31 @@ class DVWriter::Impl {

for (auto& [path, deletes] : deletes_by_path_) {
ICEBERG_RETURN_UNEXPECTED(LoadPreviousDeletes(path, deletes));
ICEBERG_RETURN_UNEXPECTED(deletes.positions.PrepareForSerialization());
}

ICEBERG_ASSIGN_OR_RAISE(auto output_file, options_.io->NewOutputFile(options_.path));
const std::string output_path(options_.path);
ICEBERG_ASSIGN_OR_RAISE(
auto writer, puffin::PuffinWriter::Make(
std::move(output_file),
{{std::string(puffin::StandardPuffinProperties::kCreatedBy),
ICEBERG_FULL_VERSION_STRING}}));

for (auto& [path, deletes] : deletes_by_path_) {
ICEBERG_RETURN_UNEXPECTED(Write(*writer, path, deletes));
ICEBERG_ASSIGN_OR_RAISE(deletes.blob,
DVUtil::WriteDVBlob(*writer, path, deletes.positions));
}

ICEBERG_RETURN_UNEXPECTED(writer->Finish());
ICEBERG_ASSIGN_OR_RAISE(const int64_t file_size, writer->FileSize());

for (const auto& [path, _] : deletes_by_path_) {
for (const auto& [path, deletes] : deletes_by_path_) {
result_.referenced_data_files.push_back(path);
ICEBERG_ASSIGN_OR_RAISE(auto data_file, CreateDV(output_path, file_size, path));
ICEBERG_ASSIGN_OR_RAISE(
auto data_file,
DVUtil::MakeDVDataFile(options_.path, file_size, path, deletes.partition,
deletes.positions.Cardinality(), deletes.spec,
deletes.blob));
result_.data_files.push_back(std::move(data_file));
}

Expand Down Expand Up @@ -160,42 +162,8 @@ class DVWriter::Impl {
return {};
}

Status Write(puffin::PuffinWriter& writer, std::string_view path, Deletes& deletes) {
ICEBERG_ASSIGN_OR_RAISE(auto blob_metadata,
DVUtil::WriteDVBlob(writer, path, deletes.positions));
blobs_by_path_.insert_or_assign(std::string(path), std::move(blob_metadata));
return {};
}

Result<std::shared_ptr<DataFile>> CreateDV(const std::string& path, int64_t size,
std::string_view referenced_data_file) {
auto deletes = deletes_by_path_.find(referenced_data_file);
ICEBERG_CHECK(deletes != deletes_by_path_.end(), "Missing deletion vector for {}",
referenced_data_file);
auto blob_metadata = blobs_by_path_.find(referenced_data_file);
ICEBERG_CHECK(blob_metadata != blobs_by_path_.end(),
"Missing deletion vector blob for {}", referenced_data_file);

return std::make_shared<DataFile>(DataFile{
.content = DataFile::Content::kPositionDeletes,
.file_path = path,
.file_format = FileFormatType::kPuffin,
.partition = deletes->second.partition,
.record_count = deletes->second.positions.Cardinality(),
.file_size_in_bytes = size,
// TODO(gangwu): support encryption key metadata
.referenced_data_file = std::string(referenced_data_file),
.content_offset = blob_metadata->second.offset,
.content_size_in_bytes = blob_metadata->second.length,
.partition_spec_id = deletes->second.spec
? std::make_optional(deletes->second.spec->spec_id())
: std::nullopt,
});
}

DVWriterOptions options_;
std::map<std::string, Deletes, StringLess> deletes_by_path_;
std::map<std::string, puffin::BlobMetadata, StringLess> blobs_by_path_;
DeleteWriteResult result_;
bool closed_ = false;
};
Expand Down
25 changes: 17 additions & 8 deletions src/iceberg/deletes/position_delete_index.cc
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ constexpr std::array<uint8_t, 4> kMagic = {0xD1, 0xD3, 0x39, 0x64};
constexpr int32_t kLengthPrefixBytes = 4;
constexpr int32_t kMagicBytes = 4;
constexpr int32_t kCrcBytes = 4;
constexpr size_t kMaxSerializedLength = std::numeric_limits<int32_t>::max();

uint32_t ComputeCrc32(std::span<const uint8_t> bytes) {
uLong crc = crc32(0L, Z_NULL, 0);
Expand Down Expand Up @@ -142,16 +143,24 @@ void PositionDeleteIndex::Merge(const PositionDeleteIndex& other) {
other.delete_files_.end());
}

Result<std::vector<uint8_t>> PositionDeleteIndex::Serialize() {
Result<size_t> PositionDeleteIndex::PrepareForSerialization() {
bitmap_.Optimize(); // run-length encode before serializing
std::vector<uint8_t> blob(kLengthPrefixBytes);
blob.insert(blob.end(), kMagic.begin(), kMagic.end());
ICEBERG_ASSIGN_OR_RAISE(const auto vector_size, bitmap_.SerializeTo(blob));

const size_t vector_size = bitmap_.SerializedSizeInBytes();
const size_t magic_and_vector_size = kMagicBytes + vector_size;
ICEBERG_PRECHECK(magic_and_vector_size <= std::numeric_limits<int32_t>::max(),
"Deletion vector is too large to serialize: {} bytes",
magic_and_vector_size);
const size_t blob_size = kLengthPrefixBytes + magic_and_vector_size + kCrcBytes;
ICEBERG_PRECHECK(blob_size <= kMaxSerializedLength,
"Deletion vector is too large to serialize: {} bytes", blob_size);
return magic_and_vector_size;
}

Result<std::vector<uint8_t>> PositionDeleteIndex::Serialize() {
ICEBERG_ASSIGN_OR_RAISE(const auto magic_and_vector_size, PrepareForSerialization());

std::vector<uint8_t> blob;
blob.reserve(kLengthPrefixBytes + magic_and_vector_size + kCrcBytes);
blob.resize(kLengthPrefixBytes);
blob.insert(blob.end(), kMagic.begin(), kMagic.end());
ICEBERG_RETURN_UNEXPECTED(bitmap_.SerializeTo(blob));

WriteBigEndian(static_cast<int32_t>(magic_and_vector_size), blob.data());
const auto crc_offset = blob.size();
Expand Down
8 changes: 8 additions & 0 deletions src/iceberg/deletes/position_delete_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
/// \file iceberg/deletes/position_delete_index.h
/// Index of deleted row positions for a data file.

#include <cstddef>
#include <cstdint>
#include <memory>
#include <span>
Expand All @@ -34,6 +35,8 @@

namespace iceberg {

class DVWriter;

/// \brief Tracks deleted row positions using a bitmap.
///
/// This class provides a domain-specific API for position deletes
Expand All @@ -53,6 +56,8 @@ class ICEBERG_EXPORT PositionDeleteIndex {
/// \brief Mark a range of positions as deleted [pos_start, pos_end).
/// \param pos_start Start position (inclusive)
/// \param pos_end End position (exclusive)
/// \note Because pos_end is an int64_t exclusive endpoint, this method cannot
/// include INT64_MAX. Call Delete(INT64_MAX) separately.
void Delete(int64_t pos_start, int64_t pos_end);

/// \brief Check if a position is deleted.
Expand Down Expand Up @@ -97,6 +102,8 @@ class ICEBERG_EXPORT PositionDeleteIndex {
private:
explicit PositionDeleteIndex(RoaringPositionBitmap bitmap);

Result<size_t> PrepareForSerialization();

// Bulk-add positions sharing high-32-bit `key`. Private hook for
// `ForEachPositionDelete`'s bulk path; keeps `Delete` the sole public
// mutation surface.
Expand All @@ -105,6 +112,7 @@ class ICEBERG_EXPORT PositionDeleteIndex {
friend void ICEBERG_EXPORT ForEachPositionDelete(std::span<const int64_t> positions,
PositionDeleteIndex& target,
std::vector<uint32_t>& scratch);
friend class DVWriter;

RoaringPositionBitmap bitmap_;
std::vector<std::shared_ptr<DataFile>> delete_files_;
Expand Down
10 changes: 5 additions & 5 deletions src/iceberg/deletes/position_delete_range_consumer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ namespace iceberg {

namespace {

bool IsValidPosition(int64_t pos) {
return pos >= 0 && pos <= RoaringPositionBitmap::kMaxPosition;
}
bool IsValidPosition(int64_t pos) { return pos >= 0; }

// Unsigned subtraction so negative or wrap-around input can't
// false-positive via signed overflow.
Expand All @@ -45,11 +43,13 @@ bool IsAdjacent(int64_t prev, int64_t next) {
// bulk path groups by this key before flushing via `BulkAddForKey`.
int32_t HighKeyFromPosition(int64_t pos) { return static_cast<int32_t>(pos >> 32); }

// Emit `[range_start, last_position]`, collapsing singletons. Callers
// pre-filter via `IsValidPosition`, so `last_position + 1` cannot overflow.
// Emit `[range_start, last_position]`, collapsing singletons.
void EmitRange(PositionDeleteIndex& target, int64_t range_start, int64_t last_position) {
if (range_start == last_position) {
target.Delete(range_start);
} else if (last_position == RoaringPositionBitmap::kMaxPosition) {
target.Delete(range_start, last_position);
target.Delete(last_position);
} else {
target.Delete(range_start, last_position + 1);
}
Expand Down
Loading