Skip to content

Commit b05e3e8

Browse files
committed
fix: enforce the deletion vector read limit on framed writes
1 parent 4a4e535 commit b05e3e8

3 files changed

Lines changed: 18 additions & 71 deletions

File tree

‎src/iceberg/deletes/dv_writer.cc‎

Lines changed: 9 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131

3232
#include "iceberg/deletes/dv_util_internal.h"
3333
#include "iceberg/deletes/position_delete_index.h"
34-
#include "iceberg/file_format.h"
3534
#include "iceberg/file_io.h" // IWYU pragma: keep
3635
#include "iceberg/manifest/manifest_entry.h"
3736
#include "iceberg/partition_spec.h"
@@ -55,6 +54,7 @@ class DVWriter::Impl {
5554
PositionDeleteIndex positions;
5655
std::shared_ptr<PartitionSpec> spec;
5756
PartitionValues partition;
57+
puffin::BlobMetadata blob;
5858
};
5959

6060
Deletes& DeletesFor(std::string_view referenced_data_file,
@@ -107,30 +107,31 @@ class DVWriter::Impl {
107107

108108
for (auto& [path, deletes] : deletes_by_path_) {
109109
ICEBERG_RETURN_UNEXPECTED(LoadPreviousDeletes(path, deletes));
110-
}
111-
112-
for (auto& [_, deletes] : deletes_by_path_) {
113110
ICEBERG_RETURN_UNEXPECTED(deletes.positions.PrepareForSerialization());
114111
}
115112

116113
ICEBERG_ASSIGN_OR_RAISE(auto output_file, options_.io->NewOutputFile(options_.path));
117-
const std::string output_path(options_.path);
118114
ICEBERG_ASSIGN_OR_RAISE(
119115
auto writer, puffin::PuffinWriter::Make(
120116
std::move(output_file),
121117
{{std::string(puffin::StandardPuffinProperties::kCreatedBy),
122118
ICEBERG_FULL_VERSION_STRING}}));
123119

124120
for (auto& [path, deletes] : deletes_by_path_) {
125-
ICEBERG_RETURN_UNEXPECTED(Write(*writer, path, deletes));
121+
ICEBERG_ASSIGN_OR_RAISE(deletes.blob,
122+
DVUtil::WriteDVBlob(*writer, path, deletes.positions));
126123
}
127124

128125
ICEBERG_RETURN_UNEXPECTED(writer->Finish());
129126
ICEBERG_ASSIGN_OR_RAISE(const int64_t file_size, writer->FileSize());
130127

131-
for (const auto& [path, _] : deletes_by_path_) {
128+
for (const auto& [path, deletes] : deletes_by_path_) {
132129
result_.referenced_data_files.push_back(path);
133-
ICEBERG_ASSIGN_OR_RAISE(auto data_file, CreateDV(output_path, file_size, path));
130+
ICEBERG_ASSIGN_OR_RAISE(
131+
auto data_file,
132+
DVUtil::MakeDVDataFile(options_.path, file_size, path, deletes.partition,
133+
deletes.positions.Cardinality(), deletes.spec,
134+
deletes.blob));
134135
result_.data_files.push_back(std::move(data_file));
135136
}
136137

@@ -161,42 +162,8 @@ class DVWriter::Impl {
161162
return {};
162163
}
163164

164-
Status Write(puffin::PuffinWriter& writer, std::string_view path, Deletes& deletes) {
165-
ICEBERG_ASSIGN_OR_RAISE(auto blob_metadata,
166-
DVUtil::WriteDVBlob(writer, path, deletes.positions));
167-
blobs_by_path_.insert_or_assign(std::string(path), std::move(blob_metadata));
168-
return {};
169-
}
170-
171-
Result<std::shared_ptr<DataFile>> CreateDV(const std::string& path, int64_t size,
172-
std::string_view referenced_data_file) {
173-
auto deletes = deletes_by_path_.find(referenced_data_file);
174-
ICEBERG_CHECK(deletes != deletes_by_path_.end(), "Missing deletion vector for {}",
175-
referenced_data_file);
176-
auto blob_metadata = blobs_by_path_.find(referenced_data_file);
177-
ICEBERG_CHECK(blob_metadata != blobs_by_path_.end(),
178-
"Missing deletion vector blob for {}", referenced_data_file);
179-
180-
return std::make_shared<DataFile>(DataFile{
181-
.content = DataFile::Content::kPositionDeletes,
182-
.file_path = path,
183-
.file_format = FileFormatType::kPuffin,
184-
.partition = deletes->second.partition,
185-
.record_count = deletes->second.positions.Cardinality(),
186-
.file_size_in_bytes = size,
187-
// TODO(gangwu): support encryption key metadata
188-
.referenced_data_file = std::string(referenced_data_file),
189-
.content_offset = blob_metadata->second.offset,
190-
.content_size_in_bytes = blob_metadata->second.length,
191-
.partition_spec_id = deletes->second.spec
192-
? std::make_optional(deletes->second.spec->spec_id())
193-
: std::nullopt,
194-
});
195-
}
196-
197165
DVWriterOptions options_;
198166
std::map<std::string, Deletes, StringLess> deletes_by_path_;
199-
std::map<std::string, puffin::BlobMetadata, StringLess> blobs_by_path_;
200167
DeleteWriteResult result_;
201168
bool closed_ = false;
202169
};

‎src/iceberg/deletes/position_delete_index.cc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,9 @@ Result<size_t> PositionDeleteIndex::PrepareForSerialization() {
147147
bitmap_.Optimize(); // run-length encode before serializing
148148
const size_t vector_size = bitmap_.SerializedSizeInBytes();
149149
const size_t magic_and_vector_size = kMagicBytes + vector_size;
150-
ICEBERG_PRECHECK(magic_and_vector_size <= kMaxSerializedLength,
151-
"Deletion vector is too large to serialize: {} bytes",
152-
magic_and_vector_size);
150+
const size_t blob_size = kLengthPrefixBytes + magic_and_vector_size + kCrcBytes;
151+
ICEBERG_PRECHECK(blob_size <= kMaxSerializedLength,
152+
"Deletion vector is too large to serialize: {} bytes", blob_size);
153153
return magic_and_vector_size;
154154
}
155155

‎src/iceberg/test/roaring_position_bitmap_test.cc‎

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ namespace iceberg {
4242
namespace {
4343

4444
constexpr size_t kBitmapCountSizeBytes = 8;
45-
constexpr size_t kBitmapKeySizeBytes = 4;
4645
constexpr int64_t kBitmapSize = 0xFFFFFFFFL;
4746
constexpr int64_t kBitmapOffset = kBitmapSize + 1L;
4847
constexpr int64_t kContainerSize = 0xFFFF; // Character.MAX_VALUE
@@ -453,31 +452,12 @@ TEST(RoaringPositionBitmapTest, TestSerializeOrdersKeysAscending) {
453452
bitmap.Add(3);
454453

455454
ICEBERG_UNWRAP_OR_FAIL(auto serialized, bitmap.Serialize());
456-
const char* cursor = serialized.data();
457-
size_t remaining = serialized.size();
458-
ASSERT_GE(remaining, kBitmapCountSizeBytes);
459-
ASSERT_EQ(ReadLittleEndian<int64_t>(cursor), 4);
460-
cursor += kBitmapCountSizeBytes;
461-
remaining -= kBitmapCountSizeBytes;
462-
463-
std::vector<int32_t> keys;
464-
const std::vector<uint64_t> expected_cardinalities = {2, 1, 1, 1};
465-
while (keys.size() < 4) {
466-
ASSERT_GE(remaining, kBitmapKeySizeBytes);
467-
keys.push_back(ReadLittleEndian<int32_t>(cursor));
468-
cursor += kBitmapKeySizeBytes;
469-
remaining -= kBitmapKeySizeBytes;
470-
471-
auto bucket = roaring::Roaring::readSafe(cursor, remaining);
472-
ASSERT_EQ(bucket.cardinality(), expected_cardinalities[keys.size() - 1]);
473-
const size_t bucket_size = bucket.getSizeInBytes(/*portable=*/true);
474-
ASSERT_LE(bucket_size, remaining);
475-
cursor += bucket_size;
476-
remaining -= bucket_size;
477-
}
478-
479-
EXPECT_EQ(keys, (std::vector<int32_t>{0, 1, 100, std::numeric_limits<int32_t>::max()}));
480-
EXPECT_EQ(remaining, 0u);
455+
EXPECT_EQ(ReadLittleEndian<int64_t>(serialized.data()), 4);
456+
// Deserialization requires strictly increasing keys.
457+
ICEBERG_UNWRAP_OR_FAIL(auto restored, RoaringPositionBitmap::Deserialize(serialized));
458+
AssertEqualContent(restored,
459+
{3, std::numeric_limits<int32_t>::max(), (int64_t{1} << 32) | 7,
460+
(int64_t{100} << 32) | 9, std::numeric_limits<int64_t>::max()});
481461
}
482462

483463
TEST(RoaringPositionBitmapTest, TestSerializeDeserializeAllContainerBitmap) {

0 commit comments

Comments
 (0)