|
19 | 19 |
|
20 | 20 | #include "iceberg/changelog_dv_planner_internal.h" |
21 | 21 |
|
22 | | -#include <ranges> |
| 22 | +#include <algorithm> |
23 | 23 | #include <utility> |
24 | 24 |
|
25 | | -#include "iceberg/expression/residual_evaluator.h" |
| 25 | +#include "iceberg/expression/manifest_evaluator.h" |
| 26 | +#include "iceberg/expression/projections.h" |
26 | 27 | #include "iceberg/manifest/manifest_entry.h" |
27 | | -#include "iceberg/manifest/manifest_group.h" |
28 | 28 | #include "iceberg/manifest/manifest_reader.h" |
29 | | -#include "iceberg/util/content_file_util.h" |
| 29 | +#include "iceberg/partition_spec.h" |
30 | 30 | #include "iceberg/util/executor_util_internal.h" |
31 | 31 | #include "iceberg/util/macros.h" |
32 | 32 |
|
33 | 33 | namespace iceberg { |
34 | 34 |
|
35 | | -namespace { |
36 | | - |
37 | | -Status ValidateChangelogDeleteFile(const DataFile& file) { |
38 | | - if (file.content == DataFile::Content::kEqualityDeletes) { |
39 | | - return NotSupported("Equality delete files are not supported in changelog scans: {}", |
40 | | - file.file_path); |
41 | | - } |
42 | | - if (!file.IsDeletionVector()) { |
43 | | - return NotSupported("Position delete files are not supported in changelog scans: {}", |
44 | | - file.file_path); |
45 | | - } |
46 | | - ICEBERG_PRECHECK(file.referenced_data_file.has_value(), |
47 | | - "Deletion vector {} does not reference a data file", file.file_path); |
48 | | - return {}; |
49 | | -} |
50 | | - |
51 | | -std::vector<std::shared_ptr<DataFile>> FindDeletionVector( |
52 | | - const std::unordered_map<std::string, std::shared_ptr<DataFile>>& dvs_by_path, |
53 | | - const std::string& data_file_path) { |
54 | | - auto it = dvs_by_path.find(data_file_path); |
55 | | - if (it == dvs_by_path.end()) { |
56 | | - return {}; |
57 | | - } |
58 | | - return {it->second}; |
59 | | -} |
60 | | - |
61 | | -} // namespace |
62 | | - |
63 | | -DeletionVectorChangelogPlanner::DeletionVectorChangelogPlanner( |
64 | | - std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema, |
65 | | - std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id, |
66 | | - internal::TableScanContext context, std::vector<std::string> scan_columns, |
67 | | - std::shared_ptr<Expression> filter, |
68 | | - std::unordered_map<int64_t, SnapshotDeletionVectors> dvs_by_snapshot) |
69 | | - : io_(std::move(io)), |
70 | | - schema_(std::move(schema)), |
71 | | - specs_by_id_(std::move(specs_by_id)), |
72 | | - context_(std::move(context)), |
73 | | - scan_columns_(std::move(scan_columns)), |
74 | | - filter_(std::move(filter)), |
75 | | - dvs_by_snapshot_(std::move(dvs_by_snapshot)) {} |
76 | | - |
77 | 35 | Result<std::unique_ptr<DeletionVectorChangelogPlanner>> |
78 | 36 | DeletionVectorChangelogPlanner::Make( |
79 | 37 | std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema, |
80 | | - std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id, |
81 | | - const internal::TableScanContext& context, std::vector<std::string> scan_columns, |
82 | | - std::shared_ptr<Expression> filter, const std::vector<ManifestFile>& delete_manifests, |
83 | | - const std::unordered_set<int64_t>& snapshot_ids) { |
| 38 | + const std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>>& specs_by_id, |
| 39 | + std::shared_ptr<Expression> filter, bool case_sensitive, OptionalExecutor executor, |
| 40 | + const std::vector<ManifestFile>& delete_manifests, |
| 41 | + const std::unordered_map<int64_t, int64_t>& sequence_numbers) { |
84 | 42 | ICEBERG_ASSIGN_OR_RAISE( |
85 | 43 | auto entries, |
86 | 44 | ParallelCollect( |
87 | | - context.plan_executor, delete_manifests, |
| 45 | + executor, delete_manifests, |
88 | 46 | [&](const ManifestFile& manifest) -> Result<std::vector<ManifestEntry>> { |
| 47 | + auto spec_it = specs_by_id.find(manifest.partition_spec_id); |
| 48 | + ICEBERG_CHECK(spec_it != specs_by_id.end(), |
| 49 | + "Partition spec ID {} not found when reading delete manifest", |
| 50 | + manifest.partition_spec_id); |
| 51 | + std::shared_ptr<Expression> partition_filter; |
| 52 | + if (filter != nullptr) { |
| 53 | + auto projector = |
| 54 | + Projections::Inclusive(*spec_it->second, *schema, case_sensitive); |
| 55 | + ICEBERG_ASSIGN_OR_RAISE(partition_filter, projector->Project(filter)); |
| 56 | + ICEBERG_ASSIGN_OR_RAISE( |
| 57 | + auto evaluator, |
| 58 | + ManifestEvaluator::MakePartitionFilter( |
| 59 | + partition_filter, spec_it->second, *schema, case_sensitive)); |
| 60 | + ICEBERG_ASSIGN_OR_RAISE(bool may_match, evaluator->Evaluate(manifest)); |
| 61 | + if (!may_match) { |
| 62 | + return std::vector<ManifestEntry>{}; |
| 63 | + } |
| 64 | + } |
| 65 | + |
89 | 66 | ICEBERG_ASSIGN_OR_RAISE( |
90 | 67 | auto reader, ManifestReader::Make(manifest, io, schema, specs_by_id)); |
| 68 | + if (partition_filter != nullptr) { |
| 69 | + reader->FilterPartitions(partition_filter).CaseSensitive(case_sensitive); |
| 70 | + } |
91 | 71 | reader->TryDropStats(); |
92 | 72 | ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); |
93 | | - auto in_range = [&snapshot_ids](const ManifestEntry& entry) { |
94 | | - return entry.snapshot_id.has_value() && |
95 | | - snapshot_ids.contains(entry.snapshot_id.value()); |
96 | | - }; |
97 | | - for (const auto& entry : entries) { |
98 | | - ICEBERG_PRECHECK(entry.data_file != nullptr, |
99 | | - "Invalid manifest entry with missing delete file"); |
100 | | - // A delete file removed before the range no longer applies to any data |
101 | | - // file, so it cannot affect the changelog. Every other delete file can: a |
102 | | - // live one applies to data files whose rows the changelog reports, and one |
103 | | - // removed by a changelog snapshot was applied to the rows that snapshot |
104 | | - // reports as deleted. Those must be deletion vectors. |
105 | | - if (entry.status == ManifestStatus::kDeleted && !in_range(entry)) { |
106 | | - continue; |
107 | | - } |
108 | | - ICEBERG_RETURN_UNEXPECTED(ValidateChangelogDeleteFile(*entry.data_file)); |
109 | | - } |
110 | | - std::erase_if(entries, [&in_range](const ManifestEntry& entry) { |
111 | | - return entry.status == ManifestStatus::kExisting || !in_range(entry); |
| 73 | + // A delete file removed before the range no longer applies to any data |
| 74 | + // file. One removed by a snapshot in the range applied until then. |
| 75 | + std::erase_if(entries, [&](const ManifestEntry& entry) { |
| 76 | + return entry.status == ManifestStatus::kDeleted && |
| 77 | + !(entry.snapshot_id.has_value() && |
| 78 | + sequence_numbers.contains(entry.snapshot_id.value())); |
112 | 79 | }); |
113 | 80 | return entries; |
114 | 81 | })); |
115 | 82 |
|
116 | | - std::unordered_map<int64_t, SnapshotDeletionVectors> dvs_by_snapshot; |
| 83 | + auto planner = std::unique_ptr<DeletionVectorChangelogPlanner>( |
| 84 | + new DeletionVectorChangelogPlanner()); |
117 | 85 | for (auto& entry : entries) { |
118 | | - const int64_t snapshot_id = entry.snapshot_id.value(); |
119 | | - auto& dvs = dvs_by_snapshot[snapshot_id]; |
120 | | - auto& dvs_by_path = entry.status == ManifestStatus::kAdded ? dvs.added : dvs.removed; |
121 | | - const std::string& data_file_path = entry.data_file->referenced_data_file.value(); |
122 | | - auto [it, inserted] = |
123 | | - dvs_by_path.try_emplace(data_file_path, std::move(entry.data_file)); |
| 86 | + ICEBERG_PRECHECK(entry.data_file != nullptr, |
| 87 | + "Invalid manifest entry with missing delete file"); |
| 88 | + const DataFile& file = *entry.data_file; |
| 89 | + // The file sequence number belongs to the snapshot that committed the file, unlike |
| 90 | + // the data sequence number, which a rewrite can carry over from an older file. |
124 | 91 | ICEBERG_PRECHECK( |
125 | | - inserted, "Snapshot {} {} multiple deletion vectors for {}", snapshot_id, |
126 | | - entry.status == ManifestStatus::kAdded ? "added" : "removed", data_file_path); |
| 92 | + entry.file_sequence_number.has_value() && entry.sequence_number.has_value(), |
| 93 | + "Missing sequence number from delete file {}", file.file_path); |
| 94 | + ICEBERG_PRECHECK(file.partition_spec_id.has_value(), |
| 95 | + "Missing partition spec ID from delete file {}", file.file_path); |
| 96 | + |
| 97 | + if (file.content == DataFile::Content::kEqualityDeletes) { |
| 98 | + auto spec_it = specs_by_id.find(file.partition_spec_id.value()); |
| 99 | + ICEBERG_CHECK(spec_it != specs_by_id.end(), |
| 100 | + "Partition spec ID {} not found for delete file {}", |
| 101 | + file.partition_spec_id.value(), file.file_path); |
| 102 | + KeepLatest(spec_it->second->IsUnpartitioned() |
| 103 | + ? planner->global_equality_deletes_ |
| 104 | + : LatestFor(planner->equality_deletes_by_partition_, file), |
| 105 | + entry.sequence_number.value(), file.file_path); |
| 106 | + continue; |
| 107 | + } |
| 108 | + |
| 109 | + if (!file.IsDeletionVector()) { |
| 110 | + KeepLatest( |
| 111 | + file.referenced_data_file.has_value() |
| 112 | + ? planner->position_deletes_by_path_[file.referenced_data_file.value()] |
| 113 | + : LatestFor(planner->position_deletes_by_partition_, file), |
| 114 | + entry.sequence_number.value(), file.file_path); |
| 115 | + continue; |
| 116 | + } |
| 117 | + |
| 118 | + ICEBERG_PRECHECK(file.referenced_data_file.has_value(), |
| 119 | + "Deletion vector {} does not reference a data file", file.file_path); |
| 120 | + // The same vector appears once per manifest that lists it, for example as added in |
| 121 | + // the manifest of the snapshot that committed it and as deleted in the manifest of |
| 122 | + // the snapshot that removed it. |
| 123 | + auto& versions = planner->dvs_by_path_[file.referenced_data_file.value()]; |
| 124 | + auto version_it = std::ranges::find_if(versions, [&file](const auto& version) { |
| 125 | + return version.file->file_path == file.file_path && |
| 126 | + version.file->content_offset == file.content_offset; |
| 127 | + }); |
| 128 | + if (version_it == versions.end()) { |
| 129 | + versions.push_back({.file = entry.data_file, |
| 130 | + .added_sequence_number = entry.file_sequence_number.value()}); |
| 131 | + version_it = std::prev(versions.end()); |
| 132 | + } |
| 133 | + if (entry.status == ManifestStatus::kDeleted) { |
| 134 | + version_it->removed_sequence_number = |
| 135 | + sequence_numbers.at(entry.snapshot_id.value()); |
| 136 | + } |
| 137 | + } |
| 138 | + |
| 139 | + for (const auto& [data_file_path, versions] : planner->dvs_by_path_) { |
| 140 | + for (auto it = versions.begin(); it != versions.end(); ++it) { |
| 141 | + ICEBERG_PRECHECK( |
| 142 | + std::ranges::find(std::next(it), versions.end(), it->added_sequence_number, |
| 143 | + &DeletionVectorVersion::added_sequence_number) == |
| 144 | + versions.end(), |
| 145 | + "Snapshot with sequence number {} added multiple deletion vectors for {}", |
| 146 | + it->added_sequence_number, data_file_path); |
| 147 | + } |
127 | 148 | } |
128 | 149 |
|
129 | | - return std::unique_ptr<DeletionVectorChangelogPlanner>( |
130 | | - new DeletionVectorChangelogPlanner( |
131 | | - std::move(io), std::move(schema), std::move(specs_by_id), context, |
132 | | - std::move(scan_columns), std::move(filter), std::move(dvs_by_snapshot))); |
| 150 | + return planner; |
133 | 151 | } |
134 | 152 |
|
135 | | -std::vector<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::AddedDeletes( |
136 | | - int64_t snapshot_id, const std::string& data_file_path) const { |
137 | | - auto it = dvs_by_snapshot_.find(snapshot_id); |
138 | | - if (it == dvs_by_snapshot_.end()) { |
139 | | - return {}; |
| 153 | +std::optional<DeletionVectorChangelogPlanner::LatestDeleteFile>& |
| 154 | +DeletionVectorChangelogPlanner::LatestFor( |
| 155 | + PartitionMap<std::optional<LatestDeleteFile>>& deletes_by_partition, |
| 156 | + const DataFile& file) { |
| 157 | + const int32_t spec_id = file.partition_spec_id.value(); |
| 158 | + if (!deletes_by_partition.contains(spec_id, file.partition)) { |
| 159 | + deletes_by_partition.put(spec_id, file.partition, std::nullopt); |
140 | 160 | } |
141 | | - return FindDeletionVector(it->second.added, data_file_path); |
| 161 | + return deletes_by_partition.get(spec_id, file.partition)->get(); |
142 | 162 | } |
143 | 163 |
|
144 | | -std::vector<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::RemovedDeletes( |
145 | | - int64_t snapshot_id, const std::string& data_file_path) const { |
146 | | - auto it = dvs_by_snapshot_.find(snapshot_id); |
147 | | - if (it == dvs_by_snapshot_.end()) { |
148 | | - return {}; |
| 164 | +void DeletionVectorChangelogPlanner::KeepLatest(std::optional<LatestDeleteFile>& latest, |
| 165 | + int64_t sequence_number, |
| 166 | + const std::string& file_path) { |
| 167 | + if (!latest.has_value() || latest->sequence_number < sequence_number) { |
| 168 | + latest = LatestDeleteFile{.sequence_number = sequence_number, .file_path = file_path}; |
149 | 169 | } |
150 | | - return FindDeletionVector(it->second.removed, data_file_path); |
151 | 170 | } |
152 | 171 |
|
153 | | -Result<std::vector<std::shared_ptr<ChangelogScanTask>>> |
154 | | -DeletionVectorChangelogPlanner::PlanDeletedRows( |
155 | | - int64_t snapshot_id, int32_t change_ordinal, std::span<ManifestFile> data_manifests, |
156 | | - const std::unordered_set<std::string>& added_data_file_paths) const { |
157 | | - auto dvs_it = dvs_by_snapshot_.find(snapshot_id); |
158 | | - if (dvs_it == dvs_by_snapshot_.end()) { |
159 | | - return std::vector<std::shared_ptr<ChangelogScanTask>>{}; |
| 172 | +std::shared_ptr<DataFile> DeletionVectorChangelogPlanner::AddedDeletionVector( |
| 173 | + int64_t sequence_number, const std::string& data_file_path) const { |
| 174 | + auto it = dvs_by_path_.find(data_file_path); |
| 175 | + if (it == dvs_by_path_.end()) { |
| 176 | + return nullptr; |
160 | 177 | } |
161 | | - const SnapshotDeletionVectors& dvs = dvs_it->second; |
| 178 | + auto version_it = std::ranges::find(it->second, sequence_number, |
| 179 | + &DeletionVectorVersion::added_sequence_number); |
| 180 | + return version_it == it->second.end() ? nullptr : version_it->file; |
| 181 | +} |
162 | 182 |
|
163 | | - std::unordered_set<std::string> pending_paths; |
164 | | - for (const auto& [data_file_path, dv] : dvs.added) { |
165 | | - if (!added_data_file_paths.contains(data_file_path)) { |
166 | | - pending_paths.insert(data_file_path); |
| 183 | +std::unordered_map<std::string, std::vector<int64_t>> |
| 184 | +DeletionVectorChangelogPlanner::AddedDeletionVectors( |
| 185 | + const std::unordered_set<int64_t>& sequence_numbers) const { |
| 186 | + std::unordered_map<std::string, std::vector<int64_t>> added; |
| 187 | + for (const auto& [data_file_path, versions] : dvs_by_path_) { |
| 188 | + for (const auto& version : versions) { |
| 189 | + if (sequence_numbers.contains(version.added_sequence_number)) { |
| 190 | + added[data_file_path].push_back(version.added_sequence_number); |
| 191 | + } |
167 | 192 | } |
168 | 193 | } |
169 | | - if (pending_paths.empty()) { |
170 | | - return std::vector<std::shared_ptr<ChangelogScanTask>>{}; |
171 | | - } |
| 194 | + return added; |
| 195 | +} |
172 | 196 |
|
173 | | - ICEBERG_ASSIGN_OR_RAISE( |
174 | | - auto manifest_group, |
175 | | - ManifestGroup::Make( |
176 | | - io_, schema_, specs_by_id_, |
177 | | - std::vector<ManifestFile>(data_manifests.begin(), data_manifests.end()), |
178 | | - /*delete_manifests=*/{})); |
179 | | - manifest_group->CaseSensitive(context_.case_sensitive) |
180 | | - .Select(scan_columns_) |
181 | | - .FilterData(filter_) |
182 | | - .FilterManifestEntries([&pending_paths](const ManifestEntry& entry) { |
183 | | - return entry.data_file != nullptr && |
184 | | - pending_paths.contains(entry.data_file->file_path); |
185 | | - }) |
186 | | - .IgnoreDeleted() |
187 | | - .ColumnsToKeepStats(context_.columns_to_keep_stats) |
188 | | - .PlanWith(context_.plan_executor); |
189 | | - if (context_.ignore_residuals) { |
190 | | - manifest_group->IgnoreResiduals(); |
| 197 | +Result<std::shared_ptr<DataFile>> DeletionVectorChangelogPlanner::ExistingDeletionVector( |
| 198 | + int64_t sequence_number, const std::string& data_file_path) const { |
| 199 | + auto it = dvs_by_path_.find(data_file_path); |
| 200 | + if (it == dvs_by_path_.end()) { |
| 201 | + return nullptr; |
191 | 202 | } |
192 | | - |
193 | | - auto create_tasks_func = |
194 | | - [&](std::vector<ManifestEntry>&& entries, |
195 | | - const TaskContext& ctx) -> Result<std::vector<std::shared_ptr<ScanTask>>> { |
196 | | - std::vector<std::shared_ptr<ScanTask>> tasks; |
197 | | - tasks.reserve(entries.size()); |
198 | | - |
199 | | - for (auto& entry : entries) { |
200 | | - ICEBERG_PRECHECK(entry.data_file != nullptr, |
201 | | - "Invalid manifest entry with missing data file"); |
202 | | - |
203 | | - if (ctx.drop_stats) { |
204 | | - ContentFileUtil::DropAllStats(*entry.data_file); |
205 | | - } else if (!ctx.columns_to_keep_stats.empty()) { |
206 | | - ContentFileUtil::DropUnselectedStats(*entry.data_file, ctx.columns_to_keep_stats); |
207 | | - } |
208 | | - |
209 | | - ICEBERG_ASSIGN_OR_RAISE(auto residual, |
210 | | - ctx.residuals->ResidualFor(entry.data_file->partition)); |
211 | | - const std::string& data_file_path = entry.data_file->file_path; |
212 | | - auto added_deletes = FindDeletionVector(dvs.added, data_file_path); |
213 | | - auto existing_deletes = FindDeletionVector(dvs.removed, data_file_path); |
214 | | - tasks.push_back(std::make_shared<DeletedRowsScanTask>( |
215 | | - change_ordinal, snapshot_id, std::move(entry.data_file), |
216 | | - std::move(added_deletes), std::move(existing_deletes), std::move(residual))); |
| 203 | + std::shared_ptr<DataFile> existing; |
| 204 | + for (const auto& version : it->second) { |
| 205 | + const bool live = version.added_sequence_number < sequence_number && |
| 206 | + (!version.removed_sequence_number.has_value() || |
| 207 | + version.removed_sequence_number.value() >= sequence_number); |
| 208 | + if (!live) { |
| 209 | + continue; |
217 | 210 | } |
218 | | - return tasks; |
219 | | - }; |
| 211 | + ICEBERG_PRECHECK(existing == nullptr, |
| 212 | + "Multiple deletion vectors apply to {} before sequence number {}", |
| 213 | + data_file_path, sequence_number); |
| 214 | + existing = version.file; |
| 215 | + } |
| 216 | + return existing; |
| 217 | +} |
220 | 218 |
|
221 | | - ICEBERG_ASSIGN_OR_RAISE(auto tasks, manifest_group->Plan(create_tasks_func)); |
222 | | - return tasks | std::views::transform([](const auto& task) { |
223 | | - return std::static_pointer_cast<ChangelogScanTask>(task); |
224 | | - }) | |
225 | | - std::ranges::to<std::vector>(); |
| 219 | +Status DeletionVectorChangelogPlanner::ValidateNoOtherDeletes( |
| 220 | + int64_t data_sequence_number, const DataFile& data_file) const { |
| 221 | + ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(), |
| 222 | + "Missing partition spec ID from data file {}", data_file.file_path); |
| 223 | + const int32_t spec_id = data_file.partition_spec_id.value(); |
| 224 | + auto in_partition = [&](const PartitionMap<std::optional<LatestDeleteFile>>& deletes) { |
| 225 | + auto latest = deletes.get(spec_id, data_file.partition); |
| 226 | + return latest.has_value() ? latest->get() : std::nullopt; |
| 227 | + }; |
| 228 | + auto for_path = position_deletes_by_path_.find(data_file.file_path); |
| 229 | + |
| 230 | + // An equality delete file applies to data files written before it, and a position |
| 231 | + // delete file also to those written in the same commit. |
| 232 | + for (const auto& latest : |
| 233 | + {global_equality_deletes_, in_partition(equality_deletes_by_partition_)}) { |
| 234 | + if (latest.has_value() && latest->sequence_number > data_sequence_number) { |
| 235 | + return NotSupported( |
| 236 | + "Equality delete files are not supported in changelog scans: {} applies to {}", |
| 237 | + latest->file_path, data_file.file_path); |
| 238 | + } |
| 239 | + } |
| 240 | + for (const auto& latest : |
| 241 | + {in_partition(position_deletes_by_partition_), |
| 242 | + for_path == position_deletes_by_path_.end() ? std::nullopt : for_path->second}) { |
| 243 | + if (latest.has_value() && latest->sequence_number >= data_sequence_number) { |
| 244 | + return NotSupported( |
| 245 | + "Position delete files are not supported in changelog scans: {} applies to {}", |
| 246 | + latest->file_path, data_file.file_path); |
| 247 | + } |
| 248 | + } |
| 249 | + return {}; |
226 | 250 | } |
227 | 251 |
|
228 | 252 | } // namespace iceberg |
0 commit comments