Skip to content

Commit ecb9157

Browse files
manuzhangclaude
andcommitted
fix(scan): track deletion vector history in changelog planning
- Attach the deletion vector that was live before a snapshot, not only the one it removed, so a vector left behind with its deleted data file is still an existing delete. - Reject position and equality delete files only when they apply to a reported data file, and prune delete manifests by the scan filter. - Subtract existing deletes from DeletedRowsScanTask::estimated_row_count. - Read each data manifest once when planning deleted rows instead of once per snapshot. - Share the stats and residual handling of changelog task planning, and stop copying the scan context into the planner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent ab1e688 commit ecb9157

5 files changed

Lines changed: 715 additions & 341 deletions

File tree

‎src/iceberg/changelog_dv_planner_internal.cc‎

Lines changed: 184 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -19,210 +19,234 @@
1919

2020
#include "iceberg/changelog_dv_planner_internal.h"
2121

22-
#include <ranges>
22+
#include <algorithm>
2323
#include <utility>
2424

25-
#include "iceberg/expression/residual_evaluator.h"
25+
#include "iceberg/expression/manifest_evaluator.h"
26+
#include "iceberg/expression/projections.h"
2627
#include "iceberg/manifest/manifest_entry.h"
27-
#include "iceberg/manifest/manifest_group.h"
2828
#include "iceberg/manifest/manifest_reader.h"
29-
#include "iceberg/util/content_file_util.h"
29+
#include "iceberg/partition_spec.h"
3030
#include "iceberg/util/executor_util_internal.h"
3131
#include "iceberg/util/macros.h"
3232

3333
namespace iceberg {
3434

35-
namespace {
36-
37-
Status ValidateChangelogDeleteFile(const DataFile& file) {
38-
if (file.content == DataFile::Content::kEqualityDeletes) {
39-
return NotSupported("Equality delete files are not supported in changelog scans: {}",
40-
file.file_path);
41-
}
42-
if (!file.IsDeletionVector()) {
43-
return NotSupported("Position delete files are not supported in changelog scans: {}",
44-
file.file_path);
45-
}
46-
ICEBERG_PRECHECK(file.referenced_data_file.has_value(),
47-
"Deletion vector {} does not reference a data file", file.file_path);
48-
return {};
49-
}
50-
51-
std::vector<std::shared_ptr<DataFile>> FindDeletionVector(
52-
const std::unordered_map<std::string, std::shared_ptr<DataFile>>& dvs_by_path,
53-
const std::string& data_file_path) {
54-
auto it = dvs_by_path.find(data_file_path);
55-
if (it == dvs_by_path.end()) {
56-
return {};
57-
}
58-
return {it->second};
59-
}
60-
61-
} // namespace
62-
63-
DeletionVectorChangelogPlanner::DeletionVectorChangelogPlanner(
64-
std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema,
65-
std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id,
66-
internal::TableScanContext context, std::vector<std::string> scan_columns,
67-
std::shared_ptr<Expression> filter,
68-
std::unordered_map<int64_t, SnapshotDeletionVectors> dvs_by_snapshot)
69-
: io_(std::move(io)),
70-
schema_(std::move(schema)),
71-
specs_by_id_(std::move(specs_by_id)),
72-
context_(std::move(context)),
73-
scan_columns_(std::move(scan_columns)),
74-
filter_(std::move(filter)),
75-
dvs_by_snapshot_(std::move(dvs_by_snapshot)) {}
76-
7735
Result<std::unique_ptr<DeletionVectorChangelogPlanner>>
7836
DeletionVectorChangelogPlanner::Make(
7937
std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema,
80-
std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id,
81-
const internal::TableScanContext& context, std::vector<std::string> scan_columns,
82-
std::shared_ptr<Expression> filter, const std::vector<ManifestFile>& delete_manifests,
83-
const std::unordered_set<int64_t>& snapshot_ids) {
38+
const std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>>& specs_by_id,
39+
std::shared_ptr<Expression> filter, bool case_sensitive, OptionalExecutor executor,
40+
const std::vector<ManifestFile>& delete_manifests,
41+
const std::unordered_map<int64_t, int64_t>& sequence_numbers) {
8442
ICEBERG_ASSIGN_OR_RAISE(
8543
auto entries,
8644
ParallelCollect(
87-
context.plan_executor, delete_manifests,
45+
executor, delete_manifests,
8846
[&](const ManifestFile& manifest) -> Result<std::vector<ManifestEntry>> {
47+
auto spec_it = specs_by_id.find(manifest.partition_spec_id);
48+
ICEBERG_CHECK(spec_it != specs_by_id.end(),
49+
"Partition spec ID {} not found when reading delete manifest",
50+
manifest.partition_spec_id);
51+
std::shared_ptr<Expression> partition_filter;
52+
if (filter != nullptr) {
53+
auto projector =
54+
Projections::Inclusive(*spec_it->second, *schema, case_sensitive);
55+
ICEBERG_ASSIGN_OR_RAISE(partition_filter, projector->Project(filter));
56+
ICEBERG_ASSIGN_OR_RAISE(
57+
auto evaluator,
58+
ManifestEvaluator::MakePartitionFilter(
59+
partition_filter, spec_it->second, *schema, case_sensitive));
60+
ICEBERG_ASSIGN_OR_RAISE(bool may_match, evaluator->Evaluate(manifest));
61+
if (!may_match) {
62+
return std::vector<ManifestEntry>{};
63+
}
64+
}
65+
8966
ICEBERG_ASSIGN_OR_RAISE(
9067
auto reader, ManifestReader::Make(manifest, io, schema, specs_by_id));
68+
if (partition_filter != nullptr) {
69+
reader->FilterPartitions(partition_filter).CaseSensitive(case_sensitive);
70+
}
9171
reader->TryDropStats();
9272
ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries());
93-
auto in_range = [&snapshot_ids](const ManifestEntry& entry) {
94-
return entry.snapshot_id.has_value() &&
95-
snapshot_ids.contains(entry.snapshot_id.value());
96-
};
97-
for (const auto& entry : entries) {
98-
ICEBERG_PRECHECK(entry.data_file != nullptr,
99-
"Invalid manifest entry with missing delete file");
100-
// A delete file removed before the range no longer applies to any data
101-
// file, so it cannot affect the changelog. Every other delete file can: a
102-
// live one applies to data files whose rows the changelog reports, and one
103-
// removed by a changelog snapshot was applied to the rows that snapshot
104-
// reports as deleted. Those must be deletion vectors.
105-
if (entry.status == ManifestStatus::kDeleted && !in_range(entry)) {
106-
continue;
107-
}
108-
ICEBERG_RETURN_UNEXPECTED(ValidateChangelogDeleteFile(*entry.data_file));
109-
}
110-
std::erase_if(entries, [&in_range](const ManifestEntry& entry) {
111-
return entry.status == ManifestStatus::kExisting || !in_range(entry);
73+
// A delete file removed before the range no longer applies to any data
74+
// file. One removed by a snapshot in the range applied until then.
75+
std::erase_if(entries, [&](const ManifestEntry& entry) {
76+
return entry.status == ManifestStatus::kDeleted &&
77+
!(entry.snapshot_id.has_value() &&
78+
sequence_numbers.contains(entry.snapshot_id.value()));
11279
});
11380
return entries;
11481
}));
11582

116-
std::unordered_map<int64_t, SnapshotDeletionVectors> dvs_by_snapshot;
83+
auto planner = std::unique_ptr<DeletionVectorChangelogPlanner>(
84+
new DeletionVectorChangelogPlanner());
11785
for (auto& entry : entries) {
118-
const int64_t snapshot_id = entry.snapshot_id.value();
119-
auto& dvs = dvs_by_snapshot[snapshot_id];
120-
auto& dvs_by_path = entry.status == ManifestStatus::kAdded ? dvs.added : dvs.removed;
121-
const std::string& data_file_path = entry.data_file->referenced_data_file.value();
122-
auto [it, inserted] =
123-
dvs_by_path.try_emplace(data_file_path, std::move(entry.data_file));
86+
ICEBERG_PRECHECK(entry.data_file != nullptr,
87+
"Invalid manifest entry with missing delete file");
88+
const DataFile& file = *entry.data_file;
89+
// The file sequence number belongs to the snapshot that committed the file, unlike
90+
// the data sequence number, which a rewrite can carry over from an older file.
12491
ICEBERG_PRECHECK(
125-
inserted, "Snapshot {} {} multiple deletion vectors for {}", snapshot_id,
126-
entry.status == ManifestStatus::kAdded ? "added" : "removed", data_file_path);
92+
entry.file_sequence_number.has_value() && entry.sequence_number.has_value(),
93+
"Missing sequence number from delete file {}", file.file_path);
94+
ICEBERG_PRECHECK(file.partition_spec_id.has_value(),
95+
"Missing partition spec ID from delete file {}", file.file_path);
96+
97+
if (file.content == DataFile::Content::kEqualityDeletes) {
98+
auto spec_it = specs_by_id.find(file.partition_spec_id.value());
99+
ICEBERG_CHECK(spec_it != specs_by_id.end(),
100+
"Partition spec ID {} not found for delete file {}",
101+
file.partition_spec_id.value(), file.file_path);
102+
KeepLatest(spec_it->second->IsUnpartitioned()
103+
? planner->global_equality_deletes_
104+
: LatestFor(planner->equality_deletes_by_partition_, file),
105+
entry.sequence_number.value(), file.file_path);
106+
continue;
107+
}
108+
109+
if (!file.IsDeletionVector()) {
110+
KeepLatest(
111+
file.referenced_data_file.has_value()
112+
? planner->position_deletes_by_path_[file.referenced_data_file.value()]
113+
: LatestFor(planner->position_deletes_by_partition_, file),
114+
entry.sequence_number.value(), file.file_path);
115+
continue;
116+
}
117+
118+
ICEBERG_PRECHECK(file.referenced_data_file.has_value(),
119+
"Deletion vector {} does not reference a data file", file.file_path);
120+
// The same vector appears once per manifest that lists it, for example as added in
121+
// the manifest of the snapshot that committed it and as deleted in the manifest of
122+
// the snapshot that removed it.
123+
auto& versions = planner->dvs_by_path_[file.referenced_data_file.value()];
124+
auto version_it = std::ranges::find_if(versions, [&file](const auto& version) {
125+
return version.file->file_path == file.file_path &&
126+
version.file->content_offset == file.content_offset;
127+
});
128+
if (version_it == versions.end()) {
129+
versions.push_back({.file = entry.data_file,
130+
.added_sequence_number = entry.file_sequence_number.value()});
131+
version_it = std::prev(versions.end());
132+
}
133+
if (entry.status == ManifestStatus::kDeleted) {
134+
version_it->removed_sequence_number =
135+
sequence_numbers.at(entry.snapshot_id.value());
136+
}
137+
}
138+
139+
for (const auto& [data_file_path, versions] : planner->dvs_by_path_) {
140+
for (auto it = versions.begin(); it != versions.end(); ++it) {
141+
ICEBERG_PRECHECK(
142+
std::ranges::find(std::next(it), versions.end(), it->added_sequence_number,
143+
&DeletionVectorVersion::added_sequence_number) ==
144+
versions.end(),
145+
"Snapshot with sequence number {} added multiple deletion vectors for {}",
146+
it->added_sequence_number, data_file_path);
147+
}
127148
}
128149

129-
return std::unique_ptr<DeletionVectorChangelogPlanner>(
130-
new DeletionVectorChangelogPlanner(
131-
std::move(io), std::move(schema), std::move(specs_by_id), context,
132-
std::move(scan_columns), std::move(filter), std::move(dvs_by_snapshot)));
150+
return planner;
133151
}
134152

135-
std::vector<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::AddedDeletes(
136-
int64_t snapshot_id, const std::string& data_file_path) const {
137-
auto it = dvs_by_snapshot_.find(snapshot_id);
138-
if (it == dvs_by_snapshot_.end()) {
139-
return {};
153+
std::optional<DeletionVectorChangelogPlanner::LatestDeleteFile>&
154+
DeletionVectorChangelogPlanner::LatestFor(
155+
PartitionMap<std::optional<LatestDeleteFile>>& deletes_by_partition,
156+
const DataFile& file) {
157+
const int32_t spec_id = file.partition_spec_id.value();
158+
if (!deletes_by_partition.contains(spec_id, file.partition)) {
159+
deletes_by_partition.put(spec_id, file.partition, std::nullopt);
140160
}
141-
return FindDeletionVector(it->second.added, data_file_path);
161+
return deletes_by_partition.get(spec_id, file.partition)->get();
142162
}
143163

144-
std::vector<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::RemovedDeletes(
145-
int64_t snapshot_id, const std::string& data_file_path) const {
146-
auto it = dvs_by_snapshot_.find(snapshot_id);
147-
if (it == dvs_by_snapshot_.end()) {
148-
return {};
164+
void DeletionVectorChangelogPlanner::KeepLatest(std::optional<LatestDeleteFile>& latest,
165+
int64_t sequence_number,
166+
const std::string& file_path) {
167+
if (!latest.has_value() || latest->sequence_number < sequence_number) {
168+
latest = LatestDeleteFile{.sequence_number = sequence_number, .file_path = file_path};
149169
}
150-
return FindDeletionVector(it->second.removed, data_file_path);
151170
}
152171

153-
Result<std::vector<std::shared_ptr<ChangelogScanTask>>>
154-
DeletionVectorChangelogPlanner::PlanDeletedRows(
155-
int64_t snapshot_id, int32_t change_ordinal, std::span<ManifestFile> data_manifests,
156-
const std::unordered_set<std::string>& added_data_file_paths) const {
157-
auto dvs_it = dvs_by_snapshot_.find(snapshot_id);
158-
if (dvs_it == dvs_by_snapshot_.end()) {
159-
return std::vector<std::shared_ptr<ChangelogScanTask>>{};
172+
std::shared_ptr<DataFile> DeletionVectorChangelogPlanner::AddedDeletionVector(
173+
int64_t sequence_number, const std::string& data_file_path) const {
174+
auto it = dvs_by_path_.find(data_file_path);
175+
if (it == dvs_by_path_.end()) {
176+
return nullptr;
160177
}
161-
const SnapshotDeletionVectors& dvs = dvs_it->second;
178+
auto version_it = std::ranges::find(it->second, sequence_number,
179+
&DeletionVectorVersion::added_sequence_number);
180+
return version_it == it->second.end() ? nullptr : version_it->file;
181+
}
162182

163-
std::unordered_set<std::string> pending_paths;
164-
for (const auto& [data_file_path, dv] : dvs.added) {
165-
if (!added_data_file_paths.contains(data_file_path)) {
166-
pending_paths.insert(data_file_path);
183+
std::unordered_map<std::string, std::vector<int64_t>>
184+
DeletionVectorChangelogPlanner::AddedDeletionVectors(
185+
const std::unordered_set<int64_t>& sequence_numbers) const {
186+
std::unordered_map<std::string, std::vector<int64_t>> added;
187+
for (const auto& [data_file_path, versions] : dvs_by_path_) {
188+
for (const auto& version : versions) {
189+
if (sequence_numbers.contains(version.added_sequence_number)) {
190+
added[data_file_path].push_back(version.added_sequence_number);
191+
}
167192
}
168193
}
169-
if (pending_paths.empty()) {
170-
return std::vector<std::shared_ptr<ChangelogScanTask>>{};
171-
}
194+
return added;
195+
}
172196

173-
ICEBERG_ASSIGN_OR_RAISE(
174-
auto manifest_group,
175-
ManifestGroup::Make(
176-
io_, schema_, specs_by_id_,
177-
std::vector<ManifestFile>(data_manifests.begin(), data_manifests.end()),
178-
/*delete_manifests=*/{}));
179-
manifest_group->CaseSensitive(context_.case_sensitive)
180-
.Select(scan_columns_)
181-
.FilterData(filter_)
182-
.FilterManifestEntries([&pending_paths](const ManifestEntry& entry) {
183-
return entry.data_file != nullptr &&
184-
pending_paths.contains(entry.data_file->file_path);
185-
})
186-
.IgnoreDeleted()
187-
.ColumnsToKeepStats(context_.columns_to_keep_stats)
188-
.PlanWith(context_.plan_executor);
189-
if (context_.ignore_residuals) {
190-
manifest_group->IgnoreResiduals();
197+
Result<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::ExistingDeletionVector(
198+
int64_t sequence_number, const std::string& data_file_path) const {
199+
auto it = dvs_by_path_.find(data_file_path);
200+
if (it == dvs_by_path_.end()) {
201+
return nullptr;
191202
}
192-
193-
auto create_tasks_func =
194-
[&](std::vector<ManifestEntry>&& entries,
195-
const TaskContext& ctx) -> Result<std::vector<std::shared_ptr<ScanTask>>> {
196-
std::vector<std::shared_ptr<ScanTask>> tasks;
197-
tasks.reserve(entries.size());
198-
199-
for (auto& entry : entries) {
200-
ICEBERG_PRECHECK(entry.data_file != nullptr,
201-
"Invalid manifest entry with missing data file");
202-
203-
if (ctx.drop_stats) {
204-
ContentFileUtil::DropAllStats(*entry.data_file);
205-
} else if (!ctx.columns_to_keep_stats.empty()) {
206-
ContentFileUtil::DropUnselectedStats(*entry.data_file, ctx.columns_to_keep_stats);
207-
}
208-
209-
ICEBERG_ASSIGN_OR_RAISE(auto residual,
210-
ctx.residuals->ResidualFor(entry.data_file->partition));
211-
const std::string& data_file_path = entry.data_file->file_path;
212-
auto added_deletes = FindDeletionVector(dvs.added, data_file_path);
213-
auto existing_deletes = FindDeletionVector(dvs.removed, data_file_path);
214-
tasks.push_back(std::make_shared<DeletedRowsScanTask>(
215-
change_ordinal, snapshot_id, std::move(entry.data_file),
216-
std::move(added_deletes), std::move(existing_deletes), std::move(residual)));
203+
std::shared_ptr<DataFile> existing;
204+
for (const auto& version : it->second) {
205+
const bool live = version.added_sequence_number < sequence_number &&
206+
(!version.removed_sequence_number.has_value() ||
207+
version.removed_sequence_number.value() >= sequence_number);
208+
if (!live) {
209+
continue;
217210
}
218-
return tasks;
219-
};
211+
ICEBERG_PRECHECK(existing == nullptr,
212+
"Multiple deletion vectors apply to {} before sequence number {}",
213+
data_file_path, sequence_number);
214+
existing = version.file;
215+
}
216+
return existing;
217+
}
220218

221-
ICEBERG_ASSIGN_OR_RAISE(auto tasks, manifest_group->Plan(create_tasks_func));
222-
return tasks | std::views::transform([](const auto& task) {
223-
return std::static_pointer_cast<ChangelogScanTask>(task);
224-
}) |
225-
std::ranges::to<std::vector>();
219+
Status DeletionVectorChangelogPlanner::ValidateNoOtherDeletes(
220+
int64_t data_sequence_number, const DataFile& data_file) const {
221+
ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(),
222+
"Missing partition spec ID from data file {}", data_file.file_path);
223+
const int32_t spec_id = data_file.partition_spec_id.value();
224+
auto in_partition = [&](const PartitionMap<std::optional<LatestDeleteFile>>& deletes) {
225+
auto latest = deletes.get(spec_id, data_file.partition);
226+
return latest.has_value() ? latest->get() : std::nullopt;
227+
};
228+
auto for_path = position_deletes_by_path_.find(data_file.file_path);
229+
230+
// An equality delete file applies to data files written before it, and a position
231+
// delete file also to those written in the same commit.
232+
for (const auto& latest :
233+
{global_equality_deletes_, in_partition(equality_deletes_by_partition_)}) {
234+
if (latest.has_value() && latest->sequence_number > data_sequence_number) {
235+
return NotSupported(
236+
"Equality delete files are not supported in changelog scans: {} applies to {}",
237+
latest->file_path, data_file.file_path);
238+
}
239+
}
240+
for (const auto& latest :
241+
{in_partition(position_deletes_by_partition_),
242+
for_path == position_deletes_by_path_.end() ? std::nullopt : for_path->second}) {
243+
if (latest.has_value() && latest->sequence_number >= data_sequence_number) {
244+
return NotSupported(
245+
"Position delete files are not supported in changelog scans: {} applies to {}",
246+
latest->file_path, data_file.file_path);
247+
}
248+
}
249+
return {};
226250
}
227251

228252
} // namespace iceberg

0 commit comments

Comments
 (0)