From 2ed4eabff80981f3d5512657dbfd3dbbe06e5cd0 Mon Sep 17 00:00:00 2001 From: Xinli shang Date: Mon, 7 Sep 2026 16:55:16 -0700 Subject: [PATCH 1/5] feat: add CherryPickOperation and wire SnapshotManager::Cherrypick Implements cherry-picking an append or dynamic partition overwrite onto the current state, with source-snapshot-id and WAP publish tracking, non-ancestor and replaced-partition validation. SnapshotManager routes fast-forwardable picks to SetSnapshot, since those produce no snapshot. Closes the CherryPickOperation item in #637. --- src/iceberg/CMakeLists.txt | 1 + src/iceberg/test/CMakeLists.txt | 1 + .../test/cherry_pick_operation_test.cc | 476 ++++++++++++++++++ src/iceberg/transaction.cc | 8 + src/iceberg/transaction.h | 7 + src/iceberg/type_fwd.h | 1 + src/iceberg/update/cherry_pick_operation.cc | 333 ++++++++++++ src/iceberg/update/cherry_pick_operation.h | 113 +++++ src/iceberg/update/set_snapshot.cc | 11 + src/iceberg/update/set_snapshot.h | 9 + src/iceberg/update/snapshot_manager.cc | 28 +- src/iceberg/update/snapshot_manager.h | 7 +- src/iceberg/util/snapshot_util.cc | 9 + src/iceberg/util/snapshot_util.h | 11 + 14 files changed, 1012 insertions(+), 3 deletions(-) create mode 100644 src/iceberg/test/cherry_pick_operation_test.cc create mode 100644 src/iceberg/update/cherry_pick_operation.cc create mode 100644 src/iceberg/update/cherry_pick_operation.h diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index b5fb4874e2..7394ce82c6 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -119,6 +119,7 @@ set(ICEBERG_SOURCES transform.cc transform_function.cc type.cc + update/cherry_pick_operation.cc update/delete_files.cc update/expire_snapshots.cc update/fast_append.cc diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 956a0cbd6b..058a9dd3cc 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -234,6 +234,7 @@ if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(table_update_test USE_BUNDLE SOURCES + cherry_pick_operation_test.cc delete_files_test.cc expire_snapshots_test.cc fast_append_test.cc diff --git a/src/iceberg/test/cherry_pick_operation_test.cc b/src/iceberg/test/cherry_pick_operation_test.cc new file mode 100644 index 0000000000..f8d22f01f3 --- /dev/null +++ b/src/iceberg/test/cherry_pick_operation_test.cc @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/update/cherry_pick_operation.h" + +#include +#include +#include +#include + +#include +#include + +#include "iceberg/avro/avro_register.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/partition_spec.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" +#include "iceberg/update/delete_files.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/overwrite_files.h" +#include "iceberg/update/replace_partitions.h" +#include "iceberg/update/snapshot_manager.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +// The base table (TableMetadataV2ValidMinimal.json) has schema {x: long (id 1), +// y: long (id 2), z: long (id 3)} and partitions by identity(x) as spec 0. +class CherryPickOperationTest : public UpdateTestBase { + protected: + static void SetUpTestSuite() { avro::RegisterAll(); } + + std::string MetadataResource() const override { + return "TableMetadataV2ValidMinimal.json"; + } + + void SetUp() override { + UpdateTestBase::SetUp(); + + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); + + file_a_ = MakeDataFile("/data/file_a.parquet", /*partition_x=*/1L); + file_b_ = MakeDataFile("/data/file_b.parquet", /*partition_x=*/2L); + replacement_a_ = MakeDataFile("/data/file_a_replacement.parquet", /*partition_x=*/1L); + conflict_a_ = MakeDataFile("/data/file_a_conflict.parquet", /*partition_x=*/1L); + } + + std::shared_ptr MakeDataFile(const std::string& path, int64_t partition_x) { + auto f = std::make_shared(); + f->content = DataFile::Content::kData; + f->file_path = table_location_ + path; + f->file_format = FileFormatType::kParquet; + f->partition = PartitionValues(std::vector{Literal::Long(partition_x)}); + f->file_size_in_bytes = 1024; + f->record_count = 100; + f->partition_spec_id = spec_->spec_id(); + return f; + } + + // Live (non-deleted) data file paths across the current snapshot's manifests. + Result> LiveDataFilePaths() { + std::vector paths; + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DataManifests(file_io_)); + for (const auto& manifest : manifests) { + ICEBERG_ASSIGN_OR_RAISE( + auto spec, table_->metadata()->PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, file_io_, schema_, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); + for (const auto& entry : entries) { + if (entry.data_file) { + paths.push_back(entry.data_file->file_path); + } + } + } + return paths; + } + + int64_t CommitAppend(const std::shared_ptr& file) { + auto fa = table_->NewFastAppend(); + EXPECT_TRUE(fa.has_value()); + fa.value()->AppendFile(file); + EXPECT_THAT(fa.value()->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + auto snap = table_->current_snapshot(); + EXPECT_TRUE(snap.has_value()); + return snap.value()->snapshot_id; + } + + // Commit a staged append and return the staged snapshot's ID. The table's + // current snapshot is unchanged. + int64_t StageAppend(const std::shared_ptr& file, + const std::string& wap_id = "") { + auto fa = table_->NewFastAppend(); + EXPECT_TRUE(fa.has_value()); + fa.value()->StageOnly(); + if (!wap_id.empty()) { + fa.value()->Set(SnapshotSummaryFields::kWAPId, wap_id); + } + fa.value()->AppendFile(file); + EXPECT_THAT(fa.value()->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + return table_->metadata()->snapshots.back()->snapshot_id; + } + + // Commit a staged dynamic partition overwrite and return its snapshot ID. + int64_t StageReplacePartitions(const std::shared_ptr& file) { + auto ctx = TransactionContext::Make(table_, TransactionKind::kUpdate); + EXPECT_TRUE(ctx.has_value()); + auto op = ReplacePartitions::Make(TableName(), std::move(ctx.value())); + EXPECT_TRUE(op.has_value()); + op.value()->StageOnly(); + op.value()->AddFile(file); + EXPECT_THAT(op.value()->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + return table_->metadata()->snapshots.back()->snapshot_id; + } + + // Commit a staged overwrite that is not a dynamic partition overwrite, and + // return its snapshot ID. + int64_t StageOverwrite(const std::shared_ptr& added, + const std::shared_ptr& removed) { + auto ctx = TransactionContext::Make(table_, TransactionKind::kUpdate); + EXPECT_TRUE(ctx.has_value()); + auto op = OverwriteFiles::Make(TableName(), std::move(ctx.value())); + EXPECT_TRUE(op.has_value()); + op.value()->StageOnly(); + op.value()->AddFile(added); + op.value()->DeleteFile(removed); + EXPECT_THAT(op.value()->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + return table_->metadata()->snapshots.back()->snapshot_id; + } + + void RollbackTo(int64_t snapshot_id) { + ICEBERG_UNWRAP_OR_FAIL(auto manager, table_->NewSnapshotManager()); + manager->RollbackTo(snapshot_id); + EXPECT_THAT(manager->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + } + + void CommitDelete(const std::string& path) { + ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles()); + delete_files->DeleteFile(path); + EXPECT_THAT(delete_files->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + } + + Status Cherrypick(int64_t snapshot_id) { + ICEBERG_ASSIGN_OR_RAISE(auto manager, table_->NewSnapshotManager()); + manager->Cherrypick(snapshot_id); + ICEBERG_RETURN_UNEXPECTED(manager->Commit()); + return table_->Refresh(); + } + + std::shared_ptr spec_; + std::shared_ptr schema_; + std::shared_ptr file_a_; + std::shared_ptr file_b_; + std::shared_ptr replacement_a_; + std::shared_ptr conflict_a_; +}; + +// A staged dynamic overwrite is re-applied onto a state that moved on, so a new +// snapshot is produced rather than a fast-forward. +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwrite) { + CommitAppend(file_a_); + int64_t staged_id = StageReplacePartitions(replacement_a_); + CommitAppend(file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path, + replacement_a_->file_path)); +} + +// The same pick works when the staged overwrite has no parent snapshot. +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteWithoutParent) { + int64_t staged_id = StageReplacePartitions(replacement_a_); + CommitAppend(file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path, + replacement_a_->file_path)); +} + +// A file added concurrently into a replaced partition blocks the pick. +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteConflict) { + CommitAppend(file_a_); + int64_t staged_id = StageReplacePartitions(replacement_a_); + int64_t last_snapshot_id = CommitAppend(conflict_a_); + + EXPECT_THAT( + Cherrypick(staged_id), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot cherry-pick replace partitions with changed partition: " + "x=1"))); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, last_snapshot_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT( + live, ::testing::UnorderedElementsAre(file_a_->file_path, conflict_a_->file_path)); +} + +// A file the staged overwrite removed must still be present at pick time. +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteDeleteConflict) { + CommitAppend(file_a_); + int64_t staged_id = StageReplacePartitions(replacement_a_); + CommitAppend(file_b_); + CommitDelete(file_a_->file_path); + ICEBERG_UNWRAP_OR_FAIL(auto before, table_->current_snapshot()); + int64_t last_snapshot_id = before->snapshot_id; + + EXPECT_THAT(Cherrypick(staged_id), IsError(ErrorKind::kValidationFailed)); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, last_snapshot_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path)); +} + +// A staged append is re-applied on top of the current state. +TEST_F(CherryPickOperationTest, CherryPickAppend) { + CommitAppend(file_a_); + int64_t staged_id = StageAppend(replacement_a_); + CommitAppend(file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kPublishedWAPId)); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path, + replacement_a_->file_path)); +} + +// When the picked snapshot's parent is the current snapshot, the pick moves the +// current snapshot instead of creating one. +TEST_F(CherryPickOperationTest, FastForwardSetsCurrentSnapshot) { + CommitAppend(file_a_); + int64_t staged_id = StageAppend(file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, staged_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); +} + +// An overwrite without "replace-partitions" is not pickable, but it is still +// fast-forwarded to when it is a child of the current snapshot. +TEST_F(CherryPickOperationTest, FastForwardOverwriteSetsCurrentSnapshot) { + CommitAppend(file_a_); + int64_t staged_id = StageOverwrite(/*added=*/replacement_a_, /*removed=*/file_a_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kOperation), + DataOperation::kOverwrite); + // A fast-forward publishes the staged snapshot itself, so it carries no + // source-snapshot-id. + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kSourceSnapshotId)); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(replacement_a_->file_path)); +} + +// A dynamic overwrite whose parent has been rolled off the current history +// cannot be picked, because the partitions it replaced cannot be checked. +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteParentNotAncestor) { + int64_t first_id = CommitAppend(file_a_); + CommitAppend(file_b_); + int64_t staged_id = StageReplacePartitions(replacement_a_); + RollbackTo(first_id); + + EXPECT_THAT(Cherrypick(staged_id), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage(std::format("Cannot cherry-pick overwrite not based on " + "an ancestor of the current state: {}", + staged_id)))); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, first_id); +} + +// A WAP id already published by an ancestor is rejected even when the staged +// snapshot is a child of the current one and would otherwise fast-forward. +TEST_F(CherryPickOperationTest, DuplicateWapPublishOnFastForwardRejected) { + CommitAppend(file_a_); + int64_t first_staged_id = StageAppend(file_b_, /*wap_id=*/"wap-123"); + + EXPECT_THAT(Cherrypick(first_staged_id), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto picked, table_->current_snapshot()); + EXPECT_EQ(picked->snapshot_id, first_staged_id); + + // Staged on top of the snapshot just published, so this is a fast-forward. + int64_t second_staged_id = StageAppend(conflict_a_, /*wap_id=*/"wap-123"); + + EXPECT_THAT( + Cherrypick(second_staged_id), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Duplicate request to cherry pick wap id that " + "was published already: wap-123"))); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, first_staged_id); +} + +// A fast-forward staged against one state must not be applied to another. The +// commit below is retried after a concurrent append, and re-applying the +// fast-forward there would discard that append. +TEST_F(CherryPickOperationTest, FastForwardInvalidatedByConcurrentCommitRejected) { + CommitAppend(file_a_); + int64_t staged_id = StageAppend(file_b_); + + // Stage the fast-forward, but hold the transaction open. + ICEBERG_UNWRAP_OR_FAIL(auto txn, Transaction::Make(table_, TransactionKind::kUpdate)); + ICEBERG_UNWRAP_OR_FAIL(auto manager, SnapshotManager::Make(txn)); + manager->Cherrypick(staged_id); + EXPECT_THAT(manager->Commit(), IsOk()); + + // A separate table handle advances the current snapshot in the catalog, so + // the staged snapshot's parent is no longer current. + ICEBERG_UNWRAP_OR_FAIL(auto other_table, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto append, other_table->NewFastAppend()); + append->AppendFile(conflict_a_); + ASSERT_THAT(append->Commit(), IsOk()); + + EXPECT_THAT(txn->Commit(), + HasErrorMessage(std::format( + "Cannot fast-forward to {}: not a child of the current table state", + staged_id))); + + // The concurrent append survives; the staged snapshot was not published. + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT( + live, ::testing::UnorderedElementsAre(file_a_->file_path, conflict_a_->file_path)); +} + +// A snapshot already in the current history cannot be picked again. +TEST_F(CherryPickOperationTest, CherryPickAncestorRejected) { + int64_t first_id = CommitAppend(file_a_); + CommitAppend(file_b_); + + EXPECT_THAT(Cherrypick(first_id), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage(std::format( + "Cannot cherrypick snapshot {}: already an ancestor", first_id)))); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); +} + +// The same WAP id cannot be picked twice. +TEST_F(CherryPickOperationTest, DuplicateWapPublishRejected) { + CommitAppend(file_a_); + int64_t first_staged_id = StageAppend(file_b_, /*wap_id=*/"wap-123"); + int64_t second_staged_id = StageAppend(conflict_a_, /*wap_id=*/"wap-123"); + + EXPECT_THAT(Cherrypick(first_staged_id), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto picked, table_->current_snapshot()); + EXPECT_EQ(picked->snapshot_id, first_staged_id); + + EXPECT_THAT( + Cherrypick(second_staged_id), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Duplicate request to cherry pick wap id that " + "was published already: wap-123"))); +} + +// A picked snapshot with no WAP id records only the source snapshot. +TEST_F(CherryPickOperationTest, NonWapCherrypick) { + CommitAppend(file_a_); + int64_t staged_id = StageAppend(replacement_a_); + CommitAppend(file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kPublishedWAPId)); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); +} + +// A snapshot that is neither an append nor a dynamic overwrite can only be +// fast-forwarded. +TEST_F(CherryPickOperationTest, NonPickableOperationRejected) { + CommitAppend(file_a_); + CommitAppend(file_b_); + + auto delete_files = table_->NewDeleteFiles(); + ASSERT_TRUE(delete_files.has_value()); + delete_files.value()->StageOnly(); + delete_files.value()->DeleteFile(file_a_->file_path); + ASSERT_THAT(delete_files.value()->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + int64_t staged_id = table_->metadata()->snapshots.back()->snapshot_id; + + CommitAppend(conflict_a_); + + EXPECT_THAT(Cherrypick(staged_id), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage(std::format( + "Cannot cherry-pick snapshot {}: not append, dynamic overwrite, " + "or fast-forward", + staged_id)))); +} + +TEST_F(CherryPickOperationTest, UnknownSnapshotRejected) { + CommitAppend(file_a_); + + EXPECT_THAT( + Cherrypick(/*snapshot_id=*/-99), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot cherry-pick unknown snapshot ID: -99"))); +} + +} // namespace iceberg diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index 391d6f6ca3..e079f8fd8e 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -35,6 +35,7 @@ #include "iceberg/table_requirement.h" #include "iceberg/table_requirements.h" #include "iceberg/table_update.h" +#include "iceberg/update/cherry_pick_operation.h" #include "iceberg/update/delete_files.h" #include "iceberg/update/expire_snapshots.h" #include "iceberg/update/fast_append.h" @@ -635,6 +636,13 @@ Result> Transaction::NewUpdateLocation() { return update_location; } +Result> Transaction::NewCherryPickOperation() { + ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr cherry_pick, + CherryPickOperation::Make(ctx_->table->name().name, ctx_)); + ICEBERG_RETURN_UNEXPECTED(AddUpdate(cherry_pick)); + return cherry_pick; +} + Result> Transaction::NewSetSnapshot() { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr set_snapshot, SetSnapshot::Make(ctx_)); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 75e3c831e6..348f517865 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -153,6 +153,13 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> NewSnapshotManager(); + /// \brief Create a new CherryPickOperation to apply the changes of an existing + /// snapshot onto the current state. + /// + /// \note Intended for use by SnapshotManager. Prefer + /// SnapshotManager::Cherrypick(), which also handles the fast-forward case. + Result> NewCherryPickOperation(); + /// \brief Create a new SetSnapshot to set the current snapshot or rollback to a /// previous snapshot and commit the changes. Result> NewSetSnapshot(); diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index f45dd0115b..84a9919097 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -250,6 +250,7 @@ class Transaction; class TransactionContext; /// \brief Update family. +class CherryPickOperation; class DeleteFiles; class ExpireSnapshots; class FastAppend; diff --git a/src/iceberg/update/cherry_pick_operation.cc b/src/iceberg/update/cherry_pick_operation.cc new file mode 100644 index 0000000000..778d82b95a --- /dev/null +++ b/src/iceberg/update/cherry_pick_operation.cc @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/update/cherry_pick_operation.h" + +#include +#include + +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/transaction.h" +#include "iceberg/util/error_collector.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/snapshot_util.h" + +namespace iceberg { + +namespace { + +Result> MakeManifestReader( + const ManifestFile& manifest, const std::shared_ptr& file_io, + const TableMetadata& metadata) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + TableMetadataCache metadata_cache(&metadata); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); + return ManifestReader::Make(manifest, file_io, std::move(schema), specs_by_id.get()); +} + +/// \brief Data files added and removed by a snapshot. +struct SnapshotChanges { + std::vector> added; + std::vector> removed; +}; + +/// \brief Read the data files that the given snapshot added and removed. +/// +/// Only manifests written by the snapshot carry its own entries, so manifests +/// inherited from earlier snapshots are skipped. +Result ReadSnapshotChanges(const Snapshot& snapshot, + const std::shared_ptr& file_io, + const TableMetadata& metadata) { + SnapshotChanges changes; + SnapshotCache cache(&snapshot); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DataManifests(file_io)); + for (const auto& manifest : manifests) { + if (manifest.added_snapshot_id != snapshot.snapshot_id) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto reader, MakeManifestReader(manifest, file_io, metadata)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); + for (const auto& entry : entries) { + if (!entry.data_file) { + continue; + } + if (entry.status == ManifestStatus::kAdded) { + changes.added.push_back(entry.data_file); + } else if (entry.status == ManifestStatus::kDeleted) { + changes.removed.push_back(entry.data_file); + } + } + } + return changes; +} + +std::string StagedWapId(const Snapshot& snapshot) { + auto it = snapshot.summary.find(SnapshotSummaryFields::kWAPId); + return it == snapshot.summary.end() ? std::string() : it->second; +} + +std::string PublishedWapId(const Snapshot& snapshot) { + auto it = snapshot.summary.find(SnapshotSummaryFields::kPublishedWAPId); + return it == snapshot.summary.end() ? std::string() : it->second; +} + +Result> CurrentAncestorIds(const TableMetadata& metadata) { + std::vector ids; + if (metadata.current_snapshot_id == kInvalidSnapshotId) { + return ids; + } + ICEBERG_ASSIGN_OR_RAISE( + auto ancestors, SnapshotUtil::AncestorsOf(metadata, metadata.current_snapshot_id)); + ids.reserve(ancestors.size()); + for (const auto& ancestor : ancestors) { + ids.push_back(ancestor->snapshot_id); + } + return ids; +} + +/// \brief Fail if the WAP id staged on the picked snapshot was already +/// published, and return that id when the snapshot has one. +Result ValidateWapPublish(const TableMetadata& metadata, + int64_t wap_snapshot_id) { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata.SnapshotById(wap_snapshot_id)); + std::string wap_id = StagedWapId(*snapshot); + if (wap_id.empty()) { + return wap_id; + } + + ICEBERG_ASSIGN_OR_RAISE(auto ancestor_ids, CurrentAncestorIds(metadata)); + for (int64_t ancestor_id : ancestor_ids) { + ICEBERG_ASSIGN_OR_RAISE(auto ancestor, metadata.SnapshotById(ancestor_id)); + if (wap_id == StagedWapId(*ancestor) || wap_id == PublishedWapId(*ancestor)) { + return CommitFailed( + "Duplicate request to cherry pick wap id that was published already: {}", + wap_id); + } + } + return wap_id; +} + +bool IsReplacePartitions(const Snapshot& snapshot) { + auto it = snapshot.summary.find(SnapshotSummaryFields::kReplacePartitions); + return it != snapshot.summary.end() && it->second == "true"; +} + +} // namespace + +Result> CherryPickOperation::Make( + std::string table_name, std::shared_ptr ctx) { + ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); + ICEBERG_PRECHECK(ctx != nullptr, "Cannot create CherryPickOperation without a context"); + return std::unique_ptr( + new CherryPickOperation(std::move(table_name), std::move(ctx))); +} + +CherryPickOperation::CherryPickOperation(std::string table_name, + std::shared_ptr ctx) + : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} + +std::string CherryPickOperation::operation() { + if (cherrypick_snapshot_ == nullptr) { + return DataOperation::kAppend; + } + auto op = cherrypick_snapshot_->Operation(); + return op.has_value() ? std::string(*op) : DataOperation::kAppend; +} + +Status CherryPickOperation::ValidateFastForward(const TableMetadata& metadata, + const Snapshot& snapshot) { + // Java runs the WAP check only for the two pickable operations; any other + // snapshot reaches a fast-forward without one. + const auto operation = snapshot.Operation(); + const bool is_pickable = + operation == DataOperation::kAppend || + (operation == DataOperation::kOverwrite && IsReplacePartitions(snapshot)); + if (!is_pickable) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE(std::ignore, + ValidateWapPublish(metadata, snapshot.snapshot_id)); + return {}; +} + +bool CherryPickOperation::IsFastForward() const { + return cherrypick_snapshot_ != nullptr && + SnapshotUtil::CanFastForward(ctx_->current(), *cherrypick_snapshot_); +} + +CherryPickOperation& CherryPickOperation::Cherrypick(int64_t snapshot_id) { + const TableMetadata& metadata = ctx_->current(); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + cherrypick_snapshot_, metadata.SnapshotById(snapshot_id), + "Cannot cherry-pick unknown snapshot ID: {}", snapshot_id); + + const auto picked_operation = cherrypick_snapshot_->Operation(); + const bool is_append = picked_operation == DataOperation::kAppend; + const bool is_dynamic_overwrite = picked_operation == DataOperation::kOverwrite && + IsReplacePartitions(*cherrypick_snapshot_); + + if (!is_append && !is_dynamic_overwrite) { + ICEBERG_BUILDER_CHECK( + IsFastForward(), + "Cannot cherry-pick snapshot {}: not append, dynamic overwrite, or fast-forward", + snapshot_id); + return *this; + } + + if (is_dynamic_overwrite) { + // The replaced partitions can only be checked against files added since the + // picked snapshot's parent, so that parent must still be in the history. + if (cherrypick_snapshot_->parent_snapshot_id.has_value()) { + ICEBERG_BUILDER_ASSIGN_OR_RETURN( + bool is_ancestor, + SnapshotUtil::IsAncestorOf(metadata, + cherrypick_snapshot_->parent_snapshot_id.value())); + ICEBERG_BUILDER_CHECK(is_ancestor, + "Cannot cherry-pick overwrite not based on an ancestor of " + "the current state: {}", + snapshot_id); + } + } + + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto wap_id, + ValidateWapPublish(metadata, snapshot_id)); + if (!wap_id.empty()) { + Set(SnapshotSummaryFields::kPublishedWAPId, wap_id); + } + Set(SnapshotSummaryFields::kSourceSnapshotId, std::to_string(snapshot_id)); + + auto io = ctx_->table->io(); + ICEBERG_BUILDER_ASSIGN_OR_RETURN( + auto changes, ReadSnapshotChanges(*cherrypick_snapshot_, io, metadata)); + + if (is_dynamic_overwrite) { + // A replace can only be re-applied if the files it removed are all present. + FailMissingDeletePaths(); + replaced_partitions_.emplace(); + } + + for (const auto& added : changes.added) { + ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(added)); + if (replaced_partitions_.has_value()) { + ICEBERG_BUILDER_CHECK(added->partition_spec_id.has_value(), + "Data file must have partition spec ID"); + replaced_partitions_->add(added->partition_spec_id.value(), added->partition); + } + } + + if (is_dynamic_overwrite) { + for (const auto& removed : changes.removed) { + ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(removed)); + } + } + + return *this; +} + +Status CherryPickOperation::ValidateNonAncestor(const TableMetadata& metadata, + int64_t snapshot_id) const { + ICEBERG_ASSIGN_OR_RAISE(bool is_ancestor, + SnapshotUtil::IsAncestorOf(metadata, snapshot_id)); + if (is_ancestor) { + return CommitFailed("Cannot cherrypick snapshot {}: already an ancestor", + snapshot_id); + } + + const std::string snapshot_id_str = std::to_string(snapshot_id); + ICEBERG_ASSIGN_OR_RAISE(auto ancestor_ids, CurrentAncestorIds(metadata)); + for (int64_t ancestor_id : ancestor_ids) { + ICEBERG_ASSIGN_OR_RAISE(auto ancestor, metadata.SnapshotById(ancestor_id)); + auto it = ancestor->summary.find(SnapshotSummaryFields::kSourceSnapshotId); + if (it != ancestor->summary.end() && it->second == snapshot_id_str) { + return CommitFailed( + "Cannot cherrypick snapshot {}: already picked to create ancestor {}", + snapshot_id, ancestor_id); + } + } + return {}; +} + +Status CherryPickOperation::ValidateReplacedPartitions( + const TableMetadata& metadata) const { + if (!replaced_partitions_.has_value() || + metadata.current_snapshot_id == kInvalidSnapshotId) { + return {}; + } + + const auto parent_id = cherrypick_snapshot_->parent_snapshot_id; + if (parent_id.has_value()) { + ICEBERG_ASSIGN_OR_RAISE(bool is_ancestor, + SnapshotUtil::IsAncestorOf(metadata, parent_id.value())); + if (!is_ancestor) { + return ValidationFailed( + "Cannot cherry-pick overwrite, based on non-ancestor of the current state: {}", + parent_id.value()); + } + } + + // Walk back from the current snapshot to the picked snapshot's parent and + // reject any file added into a partition this pick replaces. + auto io = ctx_->table->io(); + ICEBERG_ASSIGN_OR_RAISE( + auto ancestors, SnapshotUtil::AncestorsOf(metadata, metadata.current_snapshot_id)); + for (const auto& ancestor : ancestors) { + if (parent_id.has_value() && ancestor->snapshot_id == parent_id.value()) { + break; + } + ICEBERG_ASSIGN_OR_RAISE(auto changes, ReadSnapshotChanges(*ancestor, io, metadata)); + for (const auto& added : changes.added) { + if (!added->partition_spec_id.has_value() || + !replaced_partitions_->contains(added->partition_spec_id.value(), + added->partition)) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(*added->partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto partition_path, spec->PartitionPath(added->partition)); + return ValidationFailed( + "Cannot cherry-pick replace partitions with changed partition: {}", + partition_path); + } + } + return {}; +} + +Status CherryPickOperation::Validate(const TableMetadata& current_metadata, + const std::shared_ptr& snapshot) { + if (cherrypick_snapshot_ == nullptr || IsFastForward()) { + return {}; + } + + ICEBERG_RETURN_UNEXPECTED( + ValidateNonAncestor(current_metadata, cherrypick_snapshot_->snapshot_id)); + ICEBERG_RETURN_UNEXPECTED(ValidateReplacedPartitions(current_metadata)); + ICEBERG_ASSIGN_OR_RAISE( + std::ignore, + ValidateWapPublish(current_metadata, cherrypick_snapshot_->snapshot_id)); + return {}; +} + +} // namespace iceberg diff --git a/src/iceberg/update/cherry_pick_operation.h b/src/iceberg/update/cherry_pick_operation.h new file mode 100644 index 0000000000..9ddaabff85 --- /dev/null +++ b/src/iceberg/update/cherry_pick_operation.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/update/cherry_pick_operation.h + +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" +#include "iceberg/update/merging_snapshot_update.h" +#include "iceberg/util/partition_value_util.h" + +namespace iceberg { + +/// \brief Cherry-picks the changes of a snapshot onto the current state. +/// +/// This update is not exposed through the Table API. It is part of the +/// Transaction API intended for use in SnapshotManager. +/// +/// Three kinds of snapshot can be picked. An append snapshot has its added +/// data files re-applied on top of the current state. An overwrite snapshot +/// carrying "replace-partitions"="true" has both its added and its removed +/// data files re-applied, and can only be picked while the partitions it +/// replaced are unchanged. Any other snapshot can only be fast-forwarded. +/// +/// A fast-forward moves the current state to the picked snapshot without +/// producing a new one, which this operation cannot express because Apply() +/// always builds a new snapshot. Callers detect that case with +/// SnapshotUtil::CanFastForward() and set the current snapshot instead of +/// committing this operation; ValidateFastForward() applies the checks this +/// operation would otherwise have run. +/// +/// The new snapshot records the picked snapshot in "source-snapshot-id". When +/// the picked snapshot carries a "wap.id", that id is recorded in +/// "published-wap-id" and the pick fails if the id was already published. +class ICEBERG_EXPORT CherryPickOperation : public MergingSnapshotUpdate { + public: + /// \brief Create a new CherryPickOperation instance. + /// + /// \param table_name The name of the table + /// \param ctx The transaction context + /// \return A new CherryPickOperation instance + static Result> Make( + std::string table_name, std::shared_ptr ctx); + + /// \brief Apply the changes of the given snapshot to the current state. + /// + /// \param snapshot_id The ID of the snapshot whose changes to apply + /// \return Reference to this for method chaining + CherryPickOperation& Cherrypick(int64_t snapshot_id); + + /// \brief Run the checks that apply when the given snapshot is fast-forwarded + /// to rather than picked. + /// + /// A fast-forward publishes the picked snapshot itself, so the WAP id staged + /// on it must not already have been published. Mirrors Java, where this check + /// runs in cherrypick() for append and dynamic overwrite snapshots, before + /// apply() elects to fast-forward. + /// + /// \param metadata The table metadata to fast-forward + /// \param snapshot The snapshot to fast-forward to + static Status ValidateFastForward(const TableMetadata& metadata, + const Snapshot& snapshot); + + std::string operation() override; + + protected: + Status Validate(const TableMetadata& current_metadata, + const std::shared_ptr& snapshot) override; + + private: + explicit CherryPickOperation(std::string table_name, + std::shared_ptr ctx); + + /// \brief Whether the snapshot passed to Cherrypick() can be fast-forwarded. + bool IsFastForward() const; + + /// \brief Fail if the picked snapshot is already part of the current history, + /// either directly or as the source of an earlier pick. + Status ValidateNonAncestor(const TableMetadata& metadata, int64_t snapshot_id) const; + + /// \brief Fail if any partition replaced by the picked snapshot received new + /// files after the picked snapshot's parent. + Status ValidateReplacedPartitions(const TableMetadata& metadata) const; + + std::shared_ptr cherrypick_snapshot_; + // Set only when the picked snapshot is a dynamic partition overwrite. + std::optional replaced_partitions_; +}; + +} // namespace iceberg diff --git a/src/iceberg/update/set_snapshot.cc b/src/iceberg/update/set_snapshot.cc index 0054c9d329..997b859c0e 100644 --- a/src/iceberg/update/set_snapshot.cc +++ b/src/iceberg/update/set_snapshot.cc @@ -85,6 +85,11 @@ SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { return SetCurrentSnapshot(snapshot_id); } +SetSnapshot& SetSnapshot::RequireFastForward() { + require_fast_forward_ = true; + return *this; +} + Result SetSnapshot::Apply() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); @@ -102,6 +107,12 @@ Result SetSnapshot::Apply() { "Cannot roll back to unknown snapshot id: {}", target_snapshot_id_.value()); + if (require_fast_forward_) { + ICEBERG_CHECK(SnapshotUtil::CanFastForward(base_metadata, *snapshot_result.value()), + "Cannot fast-forward to {}: not a child of the current table state", + target_snapshot_id_.value()); + } + // If this is a rollback, validate that the target is still an ancestor if (is_rollback_) { ICEBERG_ASSIGN_OR_RAISE( diff --git a/src/iceberg/update/set_snapshot.h b/src/iceberg/update/set_snapshot.h index 431e636b29..b48abcf4ce 100644 --- a/src/iceberg/update/set_snapshot.h +++ b/src/iceberg/update/set_snapshot.h @@ -50,6 +50,14 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { /// \brief Rollback table's state to a specific Snapshot identified by id. SetSnapshot& RollbackTo(int64_t snapshot_id); + /// \brief Require the target snapshot to still be a fast-forward when applied. + /// + /// Without this, a commit retried against refreshed metadata would move the + /// current snapshot even if a concurrent commit has since advanced it, + /// discarding that commit. Used by SnapshotManager::Cherrypick(), which + /// expresses a cherry-pick fast-forward as a set of the current snapshot. + SetSnapshot& RequireFastForward(); + Kind kind() const final { return Kind::kSetSnapshot; } bool IsRetryable() const override { return true; } @@ -67,6 +75,7 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { std::optional target_snapshot_id_; bool is_rollback_{false}; + bool require_fast_forward_{false}; }; } // namespace iceberg diff --git a/src/iceberg/update/snapshot_manager.cc b/src/iceberg/update/snapshot_manager.cc index 405b029f8a..fe35c0f21f 100644 --- a/src/iceberg/update/snapshot_manager.cc +++ b/src/iceberg/update/snapshot_manager.cc @@ -23,10 +23,12 @@ #include "iceberg/table.h" #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" +#include "iceberg/update/cherry_pick_operation.h" #include "iceberg/update/fast_append.h" #include "iceberg/update/set_snapshot.h" #include "iceberg/update/update_snapshot_reference.h" #include "iceberg/util/macros.h" +#include "iceberg/util/snapshot_util.h" namespace iceberg { @@ -55,8 +57,30 @@ SnapshotManager::~SnapshotManager() = default; SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); - // TODO(anyone): Implement cherrypick operation - ICEBERG_BUILDER_CHECK(false, "Cherrypick operation not yet implemented"); + const TableMetadata& metadata = transaction_->current(); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + auto snapshot, metadata.SnapshotById(snapshot_id), + "Cannot cherry-pick unknown snapshot ID: {}", snapshot_id); + + // A fast-forward produces no new snapshot, so move the current snapshot + // instead of creating a cherry-pick operation. + if (SnapshotUtil::CanFastForward(metadata, *snapshot)) { + ICEBERG_BUILDER_RETURN_IF_ERROR( + CherryPickOperation::ValidateFastForward(metadata, *snapshot)); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); + // The commit may be retried against refreshed metadata, where a concurrent + // commit can have moved the current snapshot. Without this the retry would + // apply as a plain branch move and discard that commit. + set_snapshot->RequireFastForward(); + set_snapshot->SetCurrentSnapshot(snapshot_id); + ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit()); + return *this; + } + + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto cherry_pick, + transaction_->NewCherryPickOperation()); + cherry_pick->Cherrypick(snapshot_id); + ICEBERG_BUILDER_RETURN_IF_ERROR(cherry_pick->Commit()); return *this; } diff --git a/src/iceberg/update/snapshot_manager.h b/src/iceberg/update/snapshot_manager.h index fd81f83397..3dfdb25f39 100644 --- a/src/iceberg/update/snapshot_manager.h +++ b/src/iceberg/update/snapshot_manager.h @@ -46,7 +46,9 @@ namespace iceberg { /// instead of the one that was current when the audited changes were created. This class /// adds support for cherry-picking the changes from an orphan snapshot by applying them /// to the current snapshot. The output of the operation is a new snapshot with the -/// changes from cherry-picked snapshot. +/// changes from cherry-picked snapshot, except when the picked snapshot is a child of +/// the current one, in which case the current state fast-forwards to it and no snapshot +/// is created. class ICEBERG_EXPORT SnapshotManager : public ErrorCollector { public: /// \brief Create a SnapshotManager that owns its own transaction. @@ -63,6 +65,9 @@ class ICEBERG_EXPORT SnapshotManager : public ErrorCollector { /// \brief Apply supported changes in given snapshot and create a new snapshot which /// will be set as the current snapshot on commit. /// + /// If the given snapshot is a child of the current snapshot, the current state + /// fast-forwards to it on commit and no new snapshot is created. + /// /// \param snapshot_id a Snapshot ID whose changes to apply /// \return Reference to this for method chaining SnapshotManager& Cherrypick(int64_t snapshot_id); diff --git a/src/iceberg/util/snapshot_util.cc b/src/iceberg/util/snapshot_util.cc index 6aed282606..aefa1236d5 100644 --- a/src/iceberg/util/snapshot_util.cc +++ b/src/iceberg/util/snapshot_util.cc @@ -71,6 +71,15 @@ Result SnapshotUtil::IsAncestorOf(const TableMetadata& metadata, return IsAncestorOf(metadata, current->snapshot_id, ancestor_snapshot_id); } +bool SnapshotUtil::CanFastForward(const TableMetadata& metadata, + const Snapshot& snapshot) { + if (metadata.current_snapshot_id == kInvalidSnapshotId) { + return !snapshot.parent_snapshot_id.has_value(); + } + return snapshot.parent_snapshot_id.has_value() && + snapshot.parent_snapshot_id.value() == metadata.current_snapshot_id; +} + Result SnapshotUtil::IsAncestorOf(const Table& table, int64_t snapshot_id, int64_t ancestor_snapshot_id) { return IsAncestorOf(*table.metadata(), snapshot_id, ancestor_snapshot_id); diff --git a/src/iceberg/util/snapshot_util.h b/src/iceberg/util/snapshot_util.h index 5c832d65fe..2088d4eae9 100644 --- a/src/iceberg/util/snapshot_util.h +++ b/src/iceberg/util/snapshot_util.h @@ -81,6 +81,17 @@ class ICEBERG_EXPORT SnapshotUtil { static Result IsAncestorOf(const TableMetadata& metadata, int64_t ancestor_snapshot_id); + /// \brief Returns whether the given snapshot can be fast-forwarded onto the metadata's + /// current state, that is, its parent is the current snapshot or both are absent. + /// + /// A fast-forward moves the current state to an existing snapshot rather than creating + /// a new one. + /// + /// \param metadata The table metadata to fast-forward + /// \param snapshot The snapshot to fast-forward to + /// \return true if snapshot can be fast-forwarded onto the current state + static bool CanFastForward(const TableMetadata& metadata, const Snapshot& snapshot); + /// \brief Returns whether ancestor_snapshot_id is an ancestor of snapshot_id. /// /// \param table The table to check From a0fb54ceb33b059922ed89f81eb401f878fee552 Mon Sep 17 00:00:00 2001 From: alice-shang Date: Sat, 26 Sep 2026 10:12:25 -0700 Subject: [PATCH 2/5] fix: allow SnapshotUtil::IsAncestorOf to handle an empty current snapshot metadata.Snapshot() raised "No current snapshot" when the ancestor check had nothing to compare against, which surfaced as a cherry-pick failure when picking onto a branch with no history yet. Java's equivalent (SnapshotUtil.ancestorsOf(null, ...)) treats this as an empty ancestor list instead of an error, so nothing can be an ancestor of it; mirror that here. --- .../test/cherry_pick_operation_test.cc | 33 +++++++++++++++++++ src/iceberg/test/snapshot_util_test.cc | 13 ++++++++ src/iceberg/util/snapshot_util.cc | 10 ++++-- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/iceberg/test/cherry_pick_operation_test.cc b/src/iceberg/test/cherry_pick_operation_test.cc index f8d22f01f3..bd2289f434 100644 --- a/src/iceberg/test/cherry_pick_operation_test.cc +++ b/src/iceberg/test/cherry_pick_operation_test.cc @@ -114,6 +114,19 @@ class CherryPickOperationTest : public UpdateTestBase { return snap.value()->snapshot_id; } + // Commit an append onto the given branch and return its snapshot ID. Main's + // current snapshot is unchanged. + int64_t CommitAppendToBranch(const std::string& branch, + const std::shared_ptr& file) { + auto fa = table_->NewFastAppend(); + EXPECT_TRUE(fa.has_value()); + fa.value()->ToBranch(branch); + fa.value()->AppendFile(file); + EXPECT_THAT(fa.value()->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + return table_->metadata()->snapshots.back()->snapshot_id; + } + // Commit a staged append and return the staged snapshot's ID. The table's // current snapshot is unchanged. int64_t StageAppend(const std::shared_ptr& file, @@ -280,6 +293,26 @@ TEST_F(CherryPickOperationTest, CherryPickAppend) { replacement_a_->file_path)); } +// A snapshot picked from another branch of an otherwise empty table can be +// applied to main even though main has no current snapshot yet. The pick's +// parent (on the source branch) keeps it from qualifying as a fast-forward, so +// it must go through the ordinary ancestor checks with no current snapshot to +// compare against. +TEST_F(CherryPickOperationTest, CherryPickAppendOntoEmptyMain) { + CommitAppendToBranch("b1", file_a_); + int64_t staged_id = CommitAppendToBranch("b1", file_b_); + + EXPECT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path)); +} + // When the picked snapshot's parent is the current snapshot, the pick moves the // current snapshot instead of creating one. TEST_F(CherryPickOperationTest, FastForwardSetsCurrentSnapshot) { diff --git a/src/iceberg/test/snapshot_util_test.cc b/src/iceberg/test/snapshot_util_test.cc index 25aa59ab3b..46557dd067 100644 --- a/src/iceberg/test/snapshot_util_test.cc +++ b/src/iceberg/test/snapshot_util_test.cc @@ -222,6 +222,19 @@ TEST_F(SnapshotUtilTest, IsAncestorOf) { EXPECT_FALSE(result5); } +// No current snapshot means an empty ancestor history, so nothing can be an +// ancestor of it; this must return false rather than raising "No current +// snapshot", matching Java's SnapshotUtil.ancestorsOf(null, ...). +TEST_F(SnapshotUtilTest, IsAncestorOfNoCurrentSnapshot) { + TableMetadata metadata; + metadata.current_snapshot_id = kInvalidSnapshotId; + metadata.snapshots = {CreateSnapshot(base_snapshot_id_, std::nullopt, 1, base_timestamp_)}; + + ICEBERG_UNWRAP_OR_FAIL(auto result, + SnapshotUtil::IsAncestorOf(metadata, base_snapshot_id_)); + EXPECT_FALSE(result); +} + TEST_F(SnapshotUtilTest, CurrentAncestors) { ICEBERG_UNWRAP_OR_FAIL(auto ancestors, SnapshotUtil::CurrentAncestors(*table_)); auto ids = ExtractSnapshotIds(ancestors); diff --git a/src/iceberg/util/snapshot_util.cc b/src/iceberg/util/snapshot_util.cc index aefa1236d5..35d6d9d36d 100644 --- a/src/iceberg/util/snapshot_util.cc +++ b/src/iceberg/util/snapshot_util.cc @@ -66,9 +66,13 @@ Result SnapshotUtil::IsAncestorOf(const Table& table, Result SnapshotUtil::IsAncestorOf(const TableMetadata& metadata, int64_t ancestor_snapshot_id) { - ICEBERG_ASSIGN_OR_RAISE(auto current, metadata.Snapshot()); - ICEBERG_CHECK(current != nullptr, "Current snapshot is null"); - return IsAncestorOf(metadata, current->snapshot_id, ancestor_snapshot_id); + // No current snapshot means an empty ancestor history, so nothing can be an + // ancestor of it; matches Java's SnapshotUtil.ancestorsOf(null, ...), which + // returns an empty iterable instead of raising. + if (metadata.current_snapshot_id == kInvalidSnapshotId) { + return false; + } + return IsAncestorOf(metadata, metadata.current_snapshot_id, ancestor_snapshot_id); } bool SnapshotUtil::CanFastForward(const TableMetadata& metadata, From 026a97d2e7dda342fc18e309971a365e766ffa05 Mon Sep 17 00:00:00 2001 From: alice-shang Date: Fri, 9 Oct 2026 09:06:54 +0900 Subject: [PATCH 3/5] fix: use SnapshotReader after rename and apply clang-format Co-Authored-By: Claude Opus 5.5 --- src/iceberg/test/cherry_pick_operation_test.cc | 4 ++-- src/iceberg/test/snapshot_util_test.cc | 3 ++- src/iceberg/update/cherry_pick_operation.cc | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/iceberg/test/cherry_pick_operation_test.cc b/src/iceberg/test/cherry_pick_operation_test.cc index bd2289f434..a7f5b554bb 100644 --- a/src/iceberg/test/cherry_pick_operation_test.cc +++ b/src/iceberg/test/cherry_pick_operation_test.cc @@ -86,8 +86,8 @@ class CherryPickOperationTest : public UpdateTestBase { Result> LiveDataFilePaths() { std::vector paths; ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); - SnapshotCache cache(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DataManifests(file_io_)); + SnapshotReader snapshot_reader(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot_reader.DataManifests(file_io_)); for (const auto& manifest : manifests) { ICEBERG_ASSIGN_OR_RAISE( auto spec, table_->metadata()->PartitionSpecById(manifest.partition_spec_id)); diff --git a/src/iceberg/test/snapshot_util_test.cc b/src/iceberg/test/snapshot_util_test.cc index 46557dd067..bdca4a1719 100644 --- a/src/iceberg/test/snapshot_util_test.cc +++ b/src/iceberg/test/snapshot_util_test.cc @@ -228,7 +228,8 @@ TEST_F(SnapshotUtilTest, IsAncestorOf) { TEST_F(SnapshotUtilTest, IsAncestorOfNoCurrentSnapshot) { TableMetadata metadata; metadata.current_snapshot_id = kInvalidSnapshotId; - metadata.snapshots = {CreateSnapshot(base_snapshot_id_, std::nullopt, 1, base_timestamp_)}; + metadata.snapshots = { + CreateSnapshot(base_snapshot_id_, std::nullopt, 1, base_timestamp_)}; ICEBERG_UNWRAP_OR_FAIL(auto result, SnapshotUtil::IsAncestorOf(metadata, base_snapshot_id_)); diff --git a/src/iceberg/update/cherry_pick_operation.cc b/src/iceberg/update/cherry_pick_operation.cc index 778d82b95a..1e091c4ecb 100644 --- a/src/iceberg/update/cherry_pick_operation.cc +++ b/src/iceberg/update/cherry_pick_operation.cc @@ -61,8 +61,8 @@ Result ReadSnapshotChanges(const Snapshot& snapshot, const std::shared_ptr& file_io, const TableMetadata& metadata) { SnapshotChanges changes; - SnapshotCache cache(&snapshot); - ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DataManifests(file_io)); + SnapshotReader snapshot_reader(&snapshot); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot_reader.DataManifests(file_io)); for (const auto& manifest : manifests) { if (manifest.added_snapshot_id != snapshot.snapshot_id) { continue; From 338997dc309e9dbdab25fae4937434817c01026e Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Sun, 11 Oct 2026 00:02:57 +0800 Subject: [PATCH 4/5] fix: re-evaluate fast-forward on each cherry-pick attempt --- src/iceberg/cpp20_compatibility_internal.h | 1 + .../test/cherry_pick_operation_test.cc | 665 +++++++++++++----- src/iceberg/test/snapshot_util_test.cc | 3 - src/iceberg/transaction.cc | 5 +- src/iceberg/transaction.h | 3 - src/iceberg/update/cherry_pick_operation.cc | 224 +++--- src/iceberg/update/cherry_pick_operation.h | 50 +- src/iceberg/update/set_snapshot.cc | 11 - src/iceberg/update/set_snapshot.h | 9 - src/iceberg/update/snapshot_manager.cc | 21 - src/iceberg/update/snapshot_update.h | 3 +- src/iceberg/util/snapshot_util.cc | 13 +- src/iceberg/util/snapshot_util.h | 11 - 13 files changed, 625 insertions(+), 394 deletions(-) diff --git a/src/iceberg/cpp20_compatibility_internal.h b/src/iceberg/cpp20_compatibility_internal.h index 5c54b5441b..8c4a465f03 100644 --- a/src/iceberg/cpp20_compatibility_internal.h +++ b/src/iceberg/cpp20_compatibility_internal.h @@ -139,6 +139,7 @@ #include "iceberg/transform_function.h" #include "iceberg/type.h" #include "iceberg/type_fwd.h" +#include "iceberg/update/cherry_pick_operation.h" #include "iceberg/update/delete_files.h" #include "iceberg/update/expire_snapshots.h" #include "iceberg/update/fast_append.h" diff --git a/src/iceberg/test/cherry_pick_operation_test.cc b/src/iceberg/test/cherry_pick_operation_test.cc index a7f5b554bb..0819e0f0c3 100644 --- a/src/iceberg/test/cherry_pick_operation_test.cc +++ b/src/iceberg/test/cherry_pick_operation_test.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -36,6 +37,7 @@ #include "iceberg/snapshot.h" #include "iceberg/table.h" #include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" #include "iceberg/test/matchers.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transaction.h" @@ -44,22 +46,40 @@ #include "iceberg/update/overwrite_files.h" #include "iceberg/update/replace_partitions.h" #include "iceberg/update/snapshot_manager.h" +#include "iceberg/update/update_partition_spec.h" +#include "iceberg/update/update_properties.h" #include "iceberg/util/macros.h" namespace iceberg { -// The base table (TableMetadataV2ValidMinimal.json) has schema {x: long (id 1), -// y: long (id 2), z: long (id 3)} and partitions by identity(x) as spec 0. class CherryPickOperationTest : public UpdateTestBase { - protected: + public: static void SetUpTestSuite() { avro::RegisterAll(); } + protected: + /// Most cherry-pick behavior is version independent; only tests that assert + /// version-dependent metadata derive from CherryPickFormatVersionTest. + virtual int8_t format_version() const { return 2; } + std::string MetadataResource() const override { - return "TableMetadataV2ValidMinimal.json"; + return format_version() == 3 ? "TableMetadataV3ValidMinimal.json" + : "TableMetadataV2ValidMinimal.json"; } void SetUp() override { UpdateTestBase::SetUp(); + if (format_version() == 1) { + auto metadata = std::make_shared(*table_->metadata()); + metadata->format_version = 1; + metadata->last_sequence_number = 0; + const auto metadata_location = + std::format("{}/metadata/v1.metadata.json", table_location_); + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), + IsOk()); + ASSERT_THAT(catalog_->DropTable(table_ident_, /*purge=*/false), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(table_, + catalog_->RegisterTable(table_ident_, metadata_location)); + } ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); @@ -82,108 +102,106 @@ class CherryPickOperationTest : public UpdateTestBase { return f; } - // Live (non-deleted) data file paths across the current snapshot's manifests. - Result> LiveDataFilePaths() { - std::vector paths; - ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); - SnapshotReader snapshot_reader(snapshot.get()); + Result> LiveDataEntries(const Snapshot& snapshot) { + std::vector entries; + SnapshotReader snapshot_reader(&snapshot); ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot_reader.DataManifests(file_io_)); for (const auto& manifest : manifests) { ICEBERG_ASSIGN_OR_RAISE( auto spec, table_->metadata()->PartitionSpecById(manifest.partition_spec_id)); ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(manifest, file_io_, schema_, spec)); - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); - for (const auto& entry : entries) { - if (entry.data_file) { - paths.push_back(entry.data_file->file_path); - } - } + ICEBERG_ASSIGN_OR_RAISE(auto live, reader->LiveEntries()); + entries.insert(entries.end(), live.begin(), live.end()); + } + return entries; + } + + Result> LiveDataFilePaths() { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + ICEBERG_ASSIGN_OR_RAISE(auto entries, LiveDataEntries(*snapshot)); + std::vector paths; + for (const auto& entry : entries) { + paths.push_back(entry.data_file->file_path); } return paths; } - int64_t CommitAppend(const std::shared_ptr& file) { - auto fa = table_->NewFastAppend(); - EXPECT_TRUE(fa.has_value()); - fa.value()->AppendFile(file); - EXPECT_THAT(fa.value()->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); - auto snap = table_->current_snapshot(); - EXPECT_TRUE(snap.has_value()); - return snap.value()->snapshot_id; + Result CommitAppend(const std::shared_ptr& file) { + ICEBERG_ASSIGN_OR_RAISE(auto append, table_->NewFastAppend()); + append->AppendFile(file); + ICEBERG_RETURN_UNEXPECTED(append->Commit()); + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + return snapshot->snapshot_id; } - // Commit an append onto the given branch and return its snapshot ID. Main's - // current snapshot is unchanged. - int64_t CommitAppendToBranch(const std::string& branch, - const std::shared_ptr& file) { - auto fa = table_->NewFastAppend(); - EXPECT_TRUE(fa.has_value()); - fa.value()->ToBranch(branch); - fa.value()->AppendFile(file); - EXPECT_THAT(fa.value()->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); - return table_->metadata()->snapshots.back()->snapshot_id; + Result CommitAppendToBranch(const std::string& branch, + const std::shared_ptr& file) { + ICEBERG_ASSIGN_OR_RAISE(auto append, table_->NewFastAppend()); + append->ToBranch(branch).AppendFile(file); + ICEBERG_RETURN_UNEXPECTED(append->Commit()); + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); + return table_->metadata()->refs.at(branch)->snapshot_id; } - // Commit a staged append and return the staged snapshot's ID. The table's - // current snapshot is unchanged. - int64_t StageAppend(const std::shared_ptr& file, - const std::string& wap_id = "") { - auto fa = table_->NewFastAppend(); - EXPECT_TRUE(fa.has_value()); - fa.value()->StageOnly(); - if (!wap_id.empty()) { - fa.value()->Set(SnapshotSummaryFields::kWAPId, wap_id); + Result StageAppend(const std::shared_ptr& file, + const std::optional& wap_id = std::nullopt) { + ICEBERG_ASSIGN_OR_RAISE(auto append, table_->NewFastAppend()); + append->StageOnly(); + if (wap_id.has_value()) { + append->Set(SnapshotSummaryFields::kWAPId, *wap_id); } - fa.value()->AppendFile(file); - EXPECT_THAT(fa.value()->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); + append->AppendFile(file); + ICEBERG_RETURN_UNEXPECTED(append->Commit()); + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); return table_->metadata()->snapshots.back()->snapshot_id; } - // Commit a staged dynamic partition overwrite and return its snapshot ID. - int64_t StageReplacePartitions(const std::shared_ptr& file) { - auto ctx = TransactionContext::Make(table_, TransactionKind::kUpdate); - EXPECT_TRUE(ctx.has_value()); - auto op = ReplacePartitions::Make(TableName(), std::move(ctx.value())); - EXPECT_TRUE(op.has_value()); - op.value()->StageOnly(); - op.value()->AddFile(file); - EXPECT_THAT(op.value()->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); + Result StageReplacePartitions(const std::shared_ptr& file) { + ICEBERG_ASSIGN_OR_RAISE(auto transaction, table_->NewTransaction()); + ICEBERG_ASSIGN_OR_RAISE(auto replace, transaction->NewReplacePartitions()); + replace->StageOnly(); + replace->Set(SnapshotSummaryFields::kReplacePartitions, "true"); + replace->AddFile(file); + ICEBERG_RETURN_UNEXPECTED(replace->Commit()); + ICEBERG_RETURN_UNEXPECTED(transaction->Commit()); + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); return table_->metadata()->snapshots.back()->snapshot_id; } - // Commit a staged overwrite that is not a dynamic partition overwrite, and - // return its snapshot ID. - int64_t StageOverwrite(const std::shared_ptr& added, - const std::shared_ptr& removed) { - auto ctx = TransactionContext::Make(table_, TransactionKind::kUpdate); - EXPECT_TRUE(ctx.has_value()); - auto op = OverwriteFiles::Make(TableName(), std::move(ctx.value())); - EXPECT_TRUE(op.has_value()); - op.value()->StageOnly(); - op.value()->AddFile(added); - op.value()->DeleteFile(removed); - EXPECT_THAT(op.value()->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); + Result StageOverwrite(const std::shared_ptr& added, + const std::shared_ptr& removed) { + ICEBERG_ASSIGN_OR_RAISE(auto transaction, table_->NewTransaction()); + ICEBERG_ASSIGN_OR_RAISE(auto overwrite, transaction->NewOverwrite()); + overwrite->StageOnly(); + overwrite->AddFile(added).DeleteFile(removed); + ICEBERG_RETURN_UNEXPECTED(overwrite->Commit()); + ICEBERG_RETURN_UNEXPECTED(transaction->Commit()); + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); return table_->metadata()->snapshots.back()->snapshot_id; } - void RollbackTo(int64_t snapshot_id) { - ICEBERG_UNWRAP_OR_FAIL(auto manager, table_->NewSnapshotManager()); + Status RollbackTo(int64_t snapshot_id) { + ICEBERG_ASSIGN_OR_RAISE(auto manager, table_->NewSnapshotManager()); manager->RollbackTo(snapshot_id); - EXPECT_THAT(manager->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_RETURN_UNEXPECTED(manager->Commit()); + return table_->Refresh(); } - void CommitDelete(const std::string& path) { - ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles()); + Status CommitDelete(const std::string& path) { + ICEBERG_ASSIGN_OR_RAISE(auto delete_files, table_->NewDeleteFiles()); delete_files->DeleteFile(path); - EXPECT_THAT(delete_files->Commit(), IsOk()); - EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_RETURN_UNEXPECTED(delete_files->Commit()); + return table_->Refresh(); + } + + Result> StageCherrypick(int64_t snapshot_id) { + ICEBERG_ASSIGN_OR_RAISE(auto transaction, table_->NewTransaction()); + ICEBERG_ASSIGN_OR_RAISE(auto manager, transaction->NewSnapshotManager()); + manager->Cherrypick(snapshot_id); + ICEBERG_RETURN_UNEXPECTED(manager->Commit()); + return transaction; } Status Cherrypick(int64_t snapshot_id) { @@ -193,6 +211,43 @@ class CherryPickOperationTest : public UpdateTestBase { return table_->Refresh(); } + /// Commit an append through a second table handle while another transaction is still + /// open, so that transaction must rebase and replay at commit time. + Result CommitFromOtherHandle(const std::shared_ptr& file) { + ICEBERG_ASSIGN_OR_RAISE(auto other_table, catalog_->LoadTable(table_ident_)); + ICEBERG_ASSIGN_OR_RAISE(auto append, other_table->NewFastAppend()); + append->AppendFile(file); + ICEBERG_RETURN_UNEXPECTED(append->Commit()); + ICEBERG_RETURN_UNEXPECTED(other_table->Refresh()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, other_table->current_snapshot()); + return snapshot->snapshot_id; + } + + /// A fast-forward publishes the staged snapshot itself: no new snapshot, no new + /// sequence number or row ids, and one new snapshot-log entry for it. + void ExpectFastForward(const std::shared_ptr& before, + int64_t staged_id) { + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, staged_id); + EXPECT_EQ(table_->metadata()->snapshots.size(), before->snapshots.size()); + EXPECT_EQ(table_->metadata()->last_sequence_number, before->last_sequence_number); + EXPECT_EQ(table_->metadata()->next_row_id, before->next_row_id); + ASSERT_EQ(table_->metadata()->snapshot_log.size(), before->snapshot_log.size() + 1); + EXPECT_EQ(table_->metadata()->snapshot_log.back().snapshot_id, staged_id); + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kSourceSnapshotId)); + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kPublishedWAPId)); + } + + /// A pick replayed on top of a moved state writes a new snapshot that is linked to the + /// picked one and parented on the state it was applied to. + void ExpectReplayedPick(int64_t staged_id, int64_t parent_snapshot_id) { + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->parent_snapshot_id, parent_snapshot_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); + } + std::shared_ptr spec_; std::shared_ptr schema_; std::shared_ptr file_a_; @@ -201,14 +256,20 @@ class CherryPickOperationTest : public UpdateTestBase { std::shared_ptr conflict_a_; }; +class CherryPickFormatVersionTest : public CherryPickOperationTest, + public ::testing::WithParamInterface { + protected: + int8_t format_version() const override { return GetParam(); } +}; + // A staged dynamic overwrite is re-applied onto a state that moved on, so a new // snapshot is produced rather than a fast-forward. TEST_F(CherryPickOperationTest, CherryPickDynamicOverwrite) { - CommitAppend(file_a_); - int64_t staged_id = StageReplacePartitions(replacement_a_); - CommitAppend(file_b_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_NE(snapshot->snapshot_id, staged_id); @@ -222,10 +283,10 @@ TEST_F(CherryPickOperationTest, CherryPickDynamicOverwrite) { // The same pick works when the staged overwrite has no parent snapshot. TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteWithoutParent) { - int64_t staged_id = StageReplacePartitions(replacement_a_); - CommitAppend(file_b_); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_NE(snapshot->snapshot_id, staged_id); @@ -237,9 +298,9 @@ TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteWithoutParent) { // A file added concurrently into a replaced partition blocks the pick. TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteConflict) { - CommitAppend(file_a_); - int64_t staged_id = StageReplacePartitions(replacement_a_); - int64_t last_snapshot_id = CommitAppend(conflict_a_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto last_snapshot_id, CommitAppend(conflict_a_)); EXPECT_THAT( Cherrypick(staged_id), @@ -257,52 +318,86 @@ TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteConflict) { // A file the staged overwrite removed must still be present at pick time. TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteDeleteConflict) { - CommitAppend(file_a_); - int64_t staged_id = StageReplacePartitions(replacement_a_); - CommitAppend(file_b_); - CommitDelete(file_a_->file_path); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); + ASSERT_THAT(CommitDelete(file_a_->file_path), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto before, table_->current_snapshot()); int64_t last_snapshot_id = before->snapshot_id; EXPECT_THAT(Cherrypick(staged_id), IsError(ErrorKind::kValidationFailed)); - EXPECT_THAT(table_->Refresh(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_EQ(snapshot->snapshot_id, last_snapshot_id); ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path)); } -// A staged append is re-applied on top of the current state. -TEST_F(CherryPickOperationTest, CherryPickAppend) { - CommitAppend(file_a_); - int64_t staged_id = StageAppend(replacement_a_); - CommitAppend(file_b_); +TEST_P(CherryPickFormatVersionTest, CherryPickAppend) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_)); + ICEBERG_UNWRAP_OR_FAIL(auto staged, table_->SnapshotById(staged_id)); + ASSERT_THAT(CommitAppend(conflict_a_), IsOk()); + const auto before = table_->metadata(); + ICEBERG_UNWRAP_OR_FAIL(auto parent, table_->current_snapshot()); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_NE(snapshot->snapshot_id, staged_id); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), std::to_string(staged_id)); EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kPublishedWAPId)); - + EXPECT_EQ(snapshot->parent_snapshot_id, parent->snapshot_id); + const int64_t expected_sequence = + GetParam() == 1 ? 0 : before->last_sequence_number + 1; + EXPECT_EQ(snapshot->sequence_number, expected_sequence); + EXPECT_EQ(table_->metadata()->last_sequence_number, expected_sequence); + EXPECT_EQ(table_->metadata()->snapshots.size(), before->snapshots.size() + 1); + ASSERT_EQ(table_->metadata()->snapshot_log.size(), before->snapshot_log.size() + 1); + EXPECT_EQ(table_->metadata()->snapshot_log.back().snapshot_id, snapshot->snapshot_id); + ExpectBranch(std::string(SnapshotRef::kMainBranch), snapshot->snapshot_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "3"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalRecords), "300"); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, LiveDataEntries(*snapshot)); + ASSERT_EQ(entries.size(), 3); + int added_files = 0; + for (const auto& entry : entries) { + if (entry.data_file->file_path == file_b_->file_path) { + ++added_files; + EXPECT_EQ(entry.status, ManifestStatus::kAdded); + EXPECT_EQ(entry.snapshot_id, snapshot->snapshot_id); + EXPECT_EQ(entry.sequence_number, expected_sequence); + EXPECT_EQ(entry.file_sequence_number, expected_sequence); + if (GetParam() == 3) { + EXPECT_EQ(entry.data_file->first_row_id, before->next_row_id); + EXPECT_NE(entry.data_file->first_row_id, staged->first_row_id); + } else { + EXPECT_FALSE(entry.data_file->first_row_id.has_value()); + } + } + } + EXPECT_EQ(added_files, 1); ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); - EXPECT_THAT(live, - ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path, - replacement_a_->file_path)); + EXPECT_THAT(live, ::testing::UnorderedElementsAre( + file_a_->file_path, file_b_->file_path, conflict_a_->file_path)); + if (GetParam() == 3) { + EXPECT_EQ(snapshot->first_row_id, before->next_row_id); + EXPECT_EQ(snapshot->added_rows, file_b_->record_count); + EXPECT_EQ(table_->metadata()->next_row_id, + before->next_row_id + file_b_->record_count); + } } -// A snapshot picked from another branch of an otherwise empty table can be -// applied to main even though main has no current snapshot yet. The pick's -// parent (on the source branch) keeps it from qualifying as a fast-forward, so -// it must go through the ordinary ancestor checks with no current snapshot to -// compare against. +// Main is empty, but the source snapshot has a parent on another branch. TEST_F(CherryPickOperationTest, CherryPickAppendOntoEmptyMain) { - CommitAppendToBranch("b1", file_a_); - int64_t staged_id = CommitAppendToBranch("b1", file_b_); + ASSERT_THAT(CommitAppendToBranch("b1", file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, CommitAppendToBranch("b1", file_b_)); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_NE(snapshot->snapshot_id, staged_id); @@ -313,37 +408,39 @@ TEST_F(CherryPickOperationTest, CherryPickAppendOntoEmptyMain) { EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path)); } +// When the picked snapshot's parent is the current snapshot, the pick moves the +// current snapshot instead of creating one. // When the picked snapshot's parent is the current snapshot, the pick moves the // current snapshot instead of creating one. TEST_F(CherryPickOperationTest, FastForwardSetsCurrentSnapshot) { - CommitAppend(file_a_); - int64_t staged_id = StageAppend(file_b_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_)); + const auto before = table_->metadata(); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->snapshot_id, staged_id); + ExpectFastForward(before, staged_id); ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); } +// An overwrite without "replace-partitions" is not pickable, but it is still +// fast-forwarded to when it is a child of the current snapshot. // An overwrite without "replace-partitions" is not pickable, but it is still // fast-forwarded to when it is a child of the current snapshot. TEST_F(CherryPickOperationTest, FastForwardOverwriteSetsCurrentSnapshot) { - CommitAppend(file_a_); - int64_t staged_id = StageOverwrite(/*added=*/replacement_a_, /*removed=*/file_a_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, + StageOverwrite(/*added=*/replacement_a_, /*removed=*/file_a_)); + const auto before = table_->metadata(); - EXPECT_THAT(Cherrypick(staged_id), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + ExpectFastForward(before, staged_id); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->snapshot_id, staged_id); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kOperation), DataOperation::kOverwrite); - // A fast-forward publishes the staged snapshot itself, so it carries no - // source-snapshot-id. - EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kSourceSnapshotId)); - ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); EXPECT_THAT(live, ::testing::UnorderedElementsAre(replacement_a_->file_path)); } @@ -351,10 +448,10 @@ TEST_F(CherryPickOperationTest, FastForwardOverwriteSetsCurrentSnapshot) { // A dynamic overwrite whose parent has been rolled off the current history // cannot be picked, because the partitions it replaced cannot be checked. TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteParentNotAncestor) { - int64_t first_id = CommitAppend(file_a_); - CommitAppend(file_b_); - int64_t staged_id = StageReplacePartitions(replacement_a_); - RollbackTo(first_id); + ICEBERG_UNWRAP_OR_FAIL(auto first_id, CommitAppend(file_a_)); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ASSERT_THAT(RollbackTo(first_id), IsOk()); EXPECT_THAT(Cherrypick(staged_id), ::testing::AllOf( @@ -370,15 +467,17 @@ TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteParentNotAncestor) { // A WAP id already published by an ancestor is rejected even when the staged // snapshot is a child of the current one and would otherwise fast-forward. TEST_F(CherryPickOperationTest, DuplicateWapPublishOnFastForwardRejected) { - CommitAppend(file_a_); - int64_t first_staged_id = StageAppend(file_b_, /*wap_id=*/"wap-123"); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto first_staged_id, + StageAppend(file_b_, /*wap_id=*/"wap-123")); - EXPECT_THAT(Cherrypick(first_staged_id), IsOk()); + ASSERT_THAT(Cherrypick(first_staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto picked, table_->current_snapshot()); EXPECT_EQ(picked->snapshot_id, first_staged_id); // Staged on top of the snapshot just published, so this is a fast-forward. - int64_t second_staged_id = StageAppend(conflict_a_, /*wap_id=*/"wap-123"); + ICEBERG_UNWRAP_OR_FAIL(auto second_staged_id, + StageAppend(conflict_a_, /*wap_id=*/"wap-123")); EXPECT_THAT( Cherrypick(second_staged_id), @@ -386,49 +485,36 @@ TEST_F(CherryPickOperationTest, DuplicateWapPublishOnFastForwardRejected) { HasErrorMessage("Duplicate request to cherry pick wap id that " "was published already: wap-123"))); - EXPECT_THAT(table_->Refresh(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_EQ(snapshot->snapshot_id, first_staged_id); } -// A fast-forward staged against one state must not be applied to another. The -// commit below is retried after a concurrent append, and re-applying the -// fast-forward there would discard that append. -TEST_F(CherryPickOperationTest, FastForwardInvalidatedByConcurrentCommitRejected) { - CommitAppend(file_a_); - int64_t staged_id = StageAppend(file_b_); - - // Stage the fast-forward, but hold the transaction open. - ICEBERG_UNWRAP_OR_FAIL(auto txn, Transaction::Make(table_, TransactionKind::kUpdate)); - ICEBERG_UNWRAP_OR_FAIL(auto manager, SnapshotManager::Make(txn)); - manager->Cherrypick(staged_id); - EXPECT_THAT(manager->Commit(), IsOk()); - - // A separate table handle advances the current snapshot in the catalog, so - // the staged snapshot's parent is no longer current. - ICEBERG_UNWRAP_OR_FAIL(auto other_table, catalog_->LoadTable(table_ident_)); - ICEBERG_UNWRAP_OR_FAIL(auto append, other_table->NewFastAppend()); - append->AppendFile(conflict_a_); - ASSERT_THAT(append->Commit(), IsOk()); +TEST_F(CherryPickOperationTest, NonPickableFastForwardInvalidatedByConcurrentCommit) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageOverwrite(replacement_a_, file_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto concurrent_id, CommitFromOtherHandle(file_b_)); - EXPECT_THAT(txn->Commit(), - HasErrorMessage(std::format( - "Cannot fast-forward to {}: not a child of the current table state", - staged_id))); - - // The concurrent append survives; the staged snapshot was not published. - EXPECT_THAT(table_->Refresh(), IsOk()); + EXPECT_THAT(transaction->Commit(), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage(std::format( + "Cannot cherry-pick snapshot {}: not append, dynamic overwrite, " + "or fast-forward", + staged_id)))); + ASSERT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->snapshot_id, concurrent_id); ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); - EXPECT_THAT( - live, ::testing::UnorderedElementsAre(file_a_->file_path, conflict_a_->file_path)); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); } // A snapshot already in the current history cannot be picked again. TEST_F(CherryPickOperationTest, CherryPickAncestorRejected) { - int64_t first_id = CommitAppend(file_a_); - CommitAppend(file_b_); + ICEBERG_UNWRAP_OR_FAIL(auto first_id, CommitAppend(file_a_)); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); EXPECT_THAT(Cherrypick(first_id), ::testing::AllOf( @@ -443,11 +529,13 @@ TEST_F(CherryPickOperationTest, CherryPickAncestorRejected) { // The same WAP id cannot be picked twice. TEST_F(CherryPickOperationTest, DuplicateWapPublishRejected) { - CommitAppend(file_a_); - int64_t first_staged_id = StageAppend(file_b_, /*wap_id=*/"wap-123"); - int64_t second_staged_id = StageAppend(conflict_a_, /*wap_id=*/"wap-123"); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto first_staged_id, + StageAppend(file_b_, /*wap_id=*/"wap-123")); + ICEBERG_UNWRAP_OR_FAIL(auto second_staged_id, + StageAppend(conflict_a_, /*wap_id=*/"wap-123")); - EXPECT_THAT(Cherrypick(first_staged_id), IsOk()); + ASSERT_THAT(Cherrypick(first_staged_id), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto picked, table_->current_snapshot()); EXPECT_EQ(picked->snapshot_id, first_staged_id); @@ -458,25 +546,11 @@ TEST_F(CherryPickOperationTest, DuplicateWapPublishRejected) { "was published already: wap-123"))); } -// A picked snapshot with no WAP id records only the source snapshot. -TEST_F(CherryPickOperationTest, NonWapCherrypick) { - CommitAppend(file_a_); - int64_t staged_id = StageAppend(replacement_a_); - CommitAppend(file_b_); - - EXPECT_THAT(Cherrypick(staged_id), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kPublishedWAPId)); - EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), - std::to_string(staged_id)); -} - // A snapshot that is neither an append nor a dynamic overwrite can only be // fast-forwarded. TEST_F(CherryPickOperationTest, NonPickableOperationRejected) { - CommitAppend(file_a_); - CommitAppend(file_b_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); auto delete_files = table_->NewDeleteFiles(); ASSERT_TRUE(delete_files.has_value()); @@ -486,7 +560,7 @@ TEST_F(CherryPickOperationTest, NonPickableOperationRejected) { ASSERT_THAT(table_->Refresh(), IsOk()); int64_t staged_id = table_->metadata()->snapshots.back()->snapshot_id; - CommitAppend(conflict_a_); + ASSERT_THAT(CommitAppend(conflict_a_), IsOk()); EXPECT_THAT(Cherrypick(staged_id), ::testing::AllOf( @@ -498,7 +572,7 @@ TEST_F(CherryPickOperationTest, NonPickableOperationRejected) { } TEST_F(CherryPickOperationTest, UnknownSnapshotRejected) { - CommitAppend(file_a_); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); EXPECT_THAT( Cherrypick(/*snapshot_id=*/-99), @@ -506,4 +580,223 @@ TEST_F(CherryPickOperationTest, UnknownSnapshotRejected) { HasErrorMessage("Cannot cherry-pick unknown snapshot ID: -99"))); } +TEST_F(CherryPickOperationTest, FastForwardOntoEmptyMain) { + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_a_, "wap-123")); + const auto before = table_->metadata(); + + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + + ExpectFastForward(before, staged_id); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kWAPId), "wap-123"); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::ElementsAre(file_a_->file_path)); +} + +TEST_F(CherryPickOperationTest, FastForwardAppendReplayedAfterConcurrentAppend) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_, "wap-123")); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto concurrent_id, CommitFromOtherHandle(conflict_a_)); + + ASSERT_THAT(transaction->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ExpectReplayedPick(staged_id, concurrent_id); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kPublishedWAPId), "wap-123"); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre( + file_a_->file_path, file_b_->file_path, conflict_a_->file_path)); +} + +TEST_F(CherryPickOperationTest, FastForwardDynamicOverwriteReplayedWithoutConflict) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto concurrent_id, CommitFromOtherHandle(file_b_)); + + ASSERT_THAT(transaction->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ExpectReplayedPick(staged_id, concurrent_id); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->Operation(), DataOperation::kOverwrite); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre(file_b_->file_path, + replacement_a_->file_path)); +} + +TEST_F(CherryPickOperationTest, FastForwardDynamicOverwriteReplayedWithConflict) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto concurrent_id, CommitFromOtherHandle(conflict_a_)); + + EXPECT_THAT(transaction->Commit(), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot cherry-pick replace partitions with changed " + "partition"))); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, concurrent_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT( + live, ::testing::UnorderedElementsAre(file_a_->file_path, conflict_a_->file_path)); +} + +TEST_F(CherryPickOperationTest, CherryPickReplayedAsFastForwardAfterRollback) { + ICEBERG_UNWRAP_OR_FAIL(auto parent_id, CommitAppend(file_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_)); + ASSERT_THAT(CommitAppend(conflict_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto draft, transaction->current().Snapshot()); + ASSERT_NE(draft->snapshot_id, staged_id); + ASSERT_THAT(file_io_->ReadFile(draft->manifest_list, std::nullopt), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto other_table, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto rollback, other_table->NewSnapshotManager()); + rollback->RollbackTo(parent_id); + ASSERT_THAT(rollback->Commit(), IsOk()); + ASSERT_THAT(other_table->Refresh(), IsOk()); + const auto before = other_table->metadata(); + + ASSERT_THAT(transaction->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ExpectFastForward(before, staged_id); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_THAT(file_io_->ReadFile(draft->manifest_list, std::nullopt), + IsError(ErrorKind::kIOError)); + EXPECT_THAT(file_io_->ReadFile(snapshot->manifest_list, std::nullopt), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); +} + +TEST_F(CherryPickOperationTest, FastForwardSucceedsAfterCommitRetries) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_, "wap-123")); + const auto before = table_->metadata(); + FailCommits(2); + + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + + ExpectFastForward(before, staged_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); +} + +TEST_F(CherryPickOperationTest, CherryPickRecordsPublishedWapId) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_, "wap-123")); + ICEBERG_UNWRAP_OR_FAIL(auto second_staged_id, StageAppend(replacement_a_, "wap-123")); + ASSERT_THAT(CommitAppend(conflict_a_), IsOk()); + + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_NE(snapshot->snapshot_id, staged_id); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(staged_id)); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kPublishedWAPId), "wap-123"); + EXPECT_FALSE(snapshot->summary.contains(SnapshotSummaryFields::kWAPId)); + EXPECT_THAT(Cherrypick(second_staged_id), + HasErrorMessage("Duplicate request to cherry pick wap id")); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto after, table_->current_snapshot()); + EXPECT_EQ(after->snapshot_id, snapshot->snapshot_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre( + file_a_->file_path, file_b_->file_path, conflict_a_->file_path)); + + // An explicitly empty wap id is still recorded as published-wap-id. + auto extra = MakeDataFile("/data/file_extra.parquet", /*partition_x=*/4L); + auto bridge = MakeDataFile("/data/file_bridge.parquet", /*partition_x=*/5L); + ICEBERG_UNWRAP_OR_FAIL(auto empty_wap_id, StageAppend(extra, /*wap_id=*/"")); + ASSERT_THAT(CommitAppend(bridge), IsOk()); + + ASSERT_THAT(Cherrypick(empty_wap_id), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto empty_wap_snapshot, table_->current_snapshot()); + EXPECT_EQ(empty_wap_snapshot->summary.at(SnapshotSummaryFields::kPublishedWAPId), ""); + EXPECT_EQ(empty_wap_snapshot->summary.at(SnapshotSummaryFields::kSourceSnapshotId), + std::to_string(empty_wap_id)); +} + +TEST_F(CherryPickOperationTest, DuplicateSourceSnapshotRejected) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageAppend(file_b_)); + ASSERT_THAT(CommitAppend(conflict_a_), IsOk()); + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto picked, table_->current_snapshot()); + + EXPECT_THAT(Cherrypick(staged_id), + ::testing::AllOf( + IsError(ErrorKind::kValidationFailed), + HasErrorMessage(std::format( + "Cannot cherrypick snapshot {}: already picked to create ancestor " + "{}", + staged_id, picked->snapshot_id)))); + + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, picked->snapshot_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, ::testing::UnorderedElementsAre( + file_a_->file_path, file_b_->file_path, conflict_a_->file_path)); +} + +TEST_F(CherryPickOperationTest, DuplicateWapPublishedDuringReplayRejected) { + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto first_id, StageAppend(file_b_, "wap-123")); + ICEBERG_UNWRAP_OR_FAIL(auto second_id, StageAppend(replacement_a_, "wap-123")); + ICEBERG_UNWRAP_OR_FAIL(auto transaction, StageCherrypick(second_id)); + + ICEBERG_UNWRAP_OR_FAIL(auto other_table, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto publish, other_table->NewSnapshotManager()); + publish->Cherrypick(first_id); + ASSERT_THAT(publish->Commit(), IsOk()); + + EXPECT_THAT( + transaction->Commit(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Duplicate request to cherry pick wap id"))); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->snapshot_id, first_id); + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT(live, + ::testing::UnorderedElementsAre(file_a_->file_path, file_b_->file_path)); +} + +TEST_F(CherryPickOperationTest, CherryPickDynamicOverwriteFromMergedManifest) { + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kManifestMinMergeCount.key(), "1"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(CommitAppend(file_a_), IsOk()); + ASSERT_THAT(CommitAppend(file_b_), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto staged_id, StageReplacePartitions(replacement_a_)); + ICEBERG_UNWRAP_OR_FAIL(auto staged, table_->SnapshotById(staged_id)); + ICEBERG_UNWRAP_OR_FAIL(auto staged_entries, LiveDataEntries(*staged)); + ASSERT_THAT(staged_entries, ::testing::Contains(::testing::Field( + &ManifestEntry::status, ManifestStatus::kExisting))); + auto new_file = MakeDataFile("/data/another_partition.parquet", 3L); + ASSERT_THAT(CommitAppend(new_file), IsOk()); + + ASSERT_THAT(Cherrypick(staged_id), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto live, LiveDataFilePaths()); + EXPECT_THAT( + live, ::testing::UnorderedElementsAre(file_b_->file_path, replacement_a_->file_path, + new_file->file_path)); +} + +INSTANTIATE_TEST_SUITE_P(FormatVersions, CherryPickFormatVersionTest, + ::testing::Values(1, 2, 3), + [](const ::testing::TestParamInfo& info) { + return std::format("V{}", info.param); + }); + } // namespace iceberg diff --git a/src/iceberg/test/snapshot_util_test.cc b/src/iceberg/test/snapshot_util_test.cc index bdca4a1719..3b6204ffdd 100644 --- a/src/iceberg/test/snapshot_util_test.cc +++ b/src/iceberg/test/snapshot_util_test.cc @@ -222,9 +222,6 @@ TEST_F(SnapshotUtilTest, IsAncestorOf) { EXPECT_FALSE(result5); } -// No current snapshot means an empty ancestor history, so nothing can be an -// ancestor of it; this must return false rather than raising "No current -// snapshot", matching Java's SnapshotUtil.ancestorsOf(null, ...). TEST_F(SnapshotUtilTest, IsAncestorOfNoCurrentSnapshot) { TableMetadata metadata; metadata.current_snapshot_id = kInvalidSnapshotId; diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index e079f8fd8e..0eaa3c9762 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -357,11 +357,14 @@ Status Transaction::ApplyUpdateSnapshot(SnapshotUpdate& update) { const auto& base = ctx_->metadata_builder->current(); ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + if (result.snapshot == nullptr) { + return {}; + } // Create a temp builder to check if this is an empty update auto temp_update = TableMetadataBuilder::BuildFrom(&base); if (base.SnapshotById(result.snapshot->snapshot_id).has_value()) { - // This is a rollback operation + // Move the branch to an existing snapshot. temp_update->SetBranchSnapshot(result.snapshot->snapshot_id, result.target_branch); } else if (result.stage_only) { temp_update->AddSnapshot(result.snapshot); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 348f517865..47678b15e3 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -155,9 +155,6 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> NewCherryPickOperation(); /// \brief Create a new SetSnapshot to set the current snapshot or rollback to a diff --git a/src/iceberg/update/cherry_pick_operation.cc b/src/iceberg/update/cherry_pick_operation.cc index 1e091c4ecb..d0a330760f 100644 --- a/src/iceberg/update/cherry_pick_operation.cc +++ b/src/iceberg/update/cherry_pick_operation.cc @@ -30,9 +30,11 @@ #include "iceberg/table.h" #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" +#include "iceberg/util/content_file_util.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util.h" +#include "iceberg/util/string_util.h" namespace iceberg { @@ -68,29 +70,34 @@ Result ReadSnapshotChanges(const Snapshot& snapshot, continue; } ICEBERG_ASSIGN_OR_RAISE(auto reader, MakeManifestReader(manifest, file_io, metadata)); - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); - for (const auto& entry : entries) { - if (!entry.data_file) { + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->EntriesStream()); + while (true) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, entries->Next()); + if (!entry.has_value()) { + break; + } + if (!entry->data_file) { continue; } - if (entry.status == ManifestStatus::kAdded) { - changes.added.push_back(entry.data_file); - } else if (entry.status == ManifestStatus::kDeleted) { - changes.removed.push_back(entry.data_file); + if (entry->status == ManifestStatus::kAdded) { + changes.added.push_back(std::move(entry->data_file)); + } else if (entry->status == ManifestStatus::kDeleted) { + ContentFileUtil::DropAllStats(*entry->data_file); + changes.removed.push_back(std::move(entry->data_file)); } } } return changes; } -std::string StagedWapId(const Snapshot& snapshot) { +std::optional StagedWapId(const Snapshot& snapshot) { auto it = snapshot.summary.find(SnapshotSummaryFields::kWAPId); - return it == snapshot.summary.end() ? std::string() : it->second; + return it == snapshot.summary.end() ? std::nullopt : std::make_optional(it->second); } -std::string PublishedWapId(const Snapshot& snapshot) { +std::optional PublishedWapId(const Snapshot& snapshot) { auto it = snapshot.summary.find(SnapshotSummaryFields::kPublishedWAPId); - return it == snapshot.summary.end() ? std::string() : it->second; + return it == snapshot.summary.end() ? std::nullopt : std::make_optional(it->second); } Result> CurrentAncestorIds(const TableMetadata& metadata) { @@ -109,11 +116,11 @@ Result> CurrentAncestorIds(const TableMetadata& metadata) { /// \brief Fail if the WAP id staged on the picked snapshot was already /// published, and return that id when the snapshot has one. -Result ValidateWapPublish(const TableMetadata& metadata, - int64_t wap_snapshot_id) { +Result> ValidateWapPublish(const TableMetadata& metadata, + int64_t wap_snapshot_id) { ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata.SnapshotById(wap_snapshot_id)); - std::string wap_id = StagedWapId(*snapshot); - if (wap_id.empty()) { + auto wap_id = StagedWapId(*snapshot); + if (!wap_id.has_value() || wap_id->empty()) { return wap_id; } @@ -121,9 +128,9 @@ Result ValidateWapPublish(const TableMetadata& metadata, for (int64_t ancestor_id : ancestor_ids) { ICEBERG_ASSIGN_OR_RAISE(auto ancestor, metadata.SnapshotById(ancestor_id)); if (wap_id == StagedWapId(*ancestor) || wap_id == PublishedWapId(*ancestor)) { - return CommitFailed( + return ValidationFailed( "Duplicate request to cherry pick wap id that was published already: {}", - wap_id); + *wap_id); } } return wap_id; @@ -131,7 +138,8 @@ Result ValidateWapPublish(const TableMetadata& metadata, bool IsReplacePartitions(const Snapshot& snapshot) { auto it = snapshot.summary.find(SnapshotSummaryFields::kReplacePartitions); - return it != snapshot.summary.end() && it->second == "true"; + return it != snapshot.summary.end() && + StringUtils::EqualsIgnoreCase(it->second, "true"); } } // namespace @@ -149,39 +157,46 @@ CherryPickOperation::CherryPickOperation(std::string table_name, : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} std::string CherryPickOperation::operation() { - if (cherrypick_snapshot_ == nullptr) { - return DataOperation::kAppend; - } - auto op = cherrypick_snapshot_->Operation(); - return op.has_value() ? std::string(*op) : DataOperation::kAppend; + ICEBERG_DCHECK(cherrypick_snapshot_ != nullptr, "Uninitialized cherry-pick operation"); + return std::string(cherrypick_snapshot_->Operation().value()); } -Status CherryPickOperation::ValidateFastForward(const TableMetadata& metadata, - const Snapshot& snapshot) { - // Java runs the WAP check only for the two pickable operations; any other - // snapshot reaches a fast-forward without one. - const auto operation = snapshot.Operation(); - const bool is_pickable = - operation == DataOperation::kAppend || - (operation == DataOperation::kOverwrite && IsReplacePartitions(snapshot)); - if (!is_pickable) { - return {}; +bool CherryPickOperation::IsFastForward(const TableMetadata& base) const { + if (base.current_snapshot_id == kInvalidSnapshotId) { + return !cherrypick_snapshot_->parent_snapshot_id.has_value(); } - - ICEBERG_ASSIGN_OR_RAISE(std::ignore, - ValidateWapPublish(metadata, snapshot.snapshot_id)); - return {}; + return cherrypick_snapshot_->parent_snapshot_id == base.current_snapshot_id; } -bool CherryPickOperation::IsFastForward() const { - return cherrypick_snapshot_ != nullptr && - SnapshotUtil::CanFastForward(ctx_->current(), *cherrypick_snapshot_); +Result CherryPickOperation::Apply() { + ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + const auto& base = this->base(); + if (cherrypick_snapshot_ == nullptr) { + ICEBERG_ASSIGN_OR_RAISE(auto current, + SnapshotUtil::OptionalLatestSnapshot( + base, std::string(SnapshotRef::kMainBranch))); + return ApplyResult{.snapshot = std::move(current), + .target_branch = std::string(SnapshotRef::kMainBranch)}; + } + + const bool is_fast_forward = IsFastForward(base); + if (require_fast_forward_ || is_fast_forward) { + ICEBERG_CHECK( + is_fast_forward, + "Cannot cherry-pick snapshot {}: not append, dynamic overwrite, or fast-forward", + cherrypick_snapshot_->snapshot_id); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, + base.SnapshotById(cherrypick_snapshot_->snapshot_id)); + return ApplyResult{.snapshot = std::move(snapshot), + .target_branch = std::string(SnapshotRef::kMainBranch)}; + } + return SnapshotUpdate::Apply(); } CherryPickOperation& CherryPickOperation::Cherrypick(int64_t snapshot_id) { - const TableMetadata& metadata = ctx_->current(); + const TableMetadata& current = ctx_->current(); ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( - cherrypick_snapshot_, metadata.SnapshotById(snapshot_id), + cherrypick_snapshot_, current.SnapshotById(snapshot_id), "Cannot cherry-pick unknown snapshot ID: {}", snapshot_id); const auto picked_operation = cherrypick_snapshot_->Operation(); @@ -191,19 +206,21 @@ CherryPickOperation& CherryPickOperation::Cherrypick(int64_t snapshot_id) { if (!is_append && !is_dynamic_overwrite) { ICEBERG_BUILDER_CHECK( - IsFastForward(), + IsFastForward(current), "Cannot cherry-pick snapshot {}: not append, dynamic overwrite, or fast-forward", snapshot_id); + require_fast_forward_ = true; return *this; } if (is_dynamic_overwrite) { - // The replaced partitions can only be checked against files added since the - // picked snapshot's parent, so that parent must still be in the history. + // Partition conflicts are checked against files added since the parent snapshot. + // The parent must be a current ancestor, or absent if the overwrite was based on + // an empty table. if (cherrypick_snapshot_->parent_snapshot_id.has_value()) { ICEBERG_BUILDER_ASSIGN_OR_RETURN( bool is_ancestor, - SnapshotUtil::IsAncestorOf(metadata, + SnapshotUtil::IsAncestorOf(current, cherrypick_snapshot_->parent_snapshot_id.value())); ICEBERG_BUILDER_CHECK(is_ancestor, "Cannot cherry-pick overwrite not based on an ancestor of " @@ -212,57 +229,57 @@ CherryPickOperation& CherryPickOperation::Cherrypick(int64_t snapshot_id) { } } - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto wap_id, - ValidateWapPublish(metadata, snapshot_id)); - if (!wap_id.empty()) { - Set(SnapshotSummaryFields::kPublishedWAPId, wap_id); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto wap_id, ValidateWapPublish(current, snapshot_id)); + if (wap_id.has_value()) { + Set(SnapshotSummaryFields::kPublishedWAPId, *wap_id); } Set(SnapshotSummaryFields::kSourceSnapshotId, std::to_string(snapshot_id)); - auto io = ctx_->table->io(); - ICEBERG_BUILDER_ASSIGN_OR_RETURN( - auto changes, ReadSnapshotChanges(*cherrypick_snapshot_, io, metadata)); - if (is_dynamic_overwrite) { // A replace can only be re-applied if the files it removed are all present. FailMissingDeletePaths(); replaced_partitions_.emplace(); } - for (const auto& added : changes.added) { - ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(added)); + auto io = ctx_->table->io(); + ICEBERG_BUILDER_ASSIGN_OR_RETURN( + auto changes, ReadSnapshotChanges(*cherrypick_snapshot_, io, current)); + + for (const auto& added_file : changes.added) { + ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(added_file)); if (replaced_partitions_.has_value()) { - ICEBERG_BUILDER_CHECK(added->partition_spec_id.has_value(), + ICEBERG_BUILDER_CHECK(added_file->partition_spec_id.has_value(), "Data file must have partition spec ID"); - replaced_partitions_->add(added->partition_spec_id.value(), added->partition); + replaced_partitions_->add(added_file->partition_spec_id.value(), + added_file->partition); } } if (is_dynamic_overwrite) { - for (const auto& removed : changes.removed) { - ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(removed)); + for (const auto& deleted_file : changes.removed) { + ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(deleted_file)); } } return *this; } -Status CherryPickOperation::ValidateNonAncestor(const TableMetadata& metadata, +Status CherryPickOperation::ValidateNonAncestor(const TableMetadata& meta, int64_t snapshot_id) const { ICEBERG_ASSIGN_OR_RAISE(bool is_ancestor, - SnapshotUtil::IsAncestorOf(metadata, snapshot_id)); + SnapshotUtil::IsAncestorOf(meta, snapshot_id)); if (is_ancestor) { - return CommitFailed("Cannot cherrypick snapshot {}: already an ancestor", - snapshot_id); + return ValidationFailed("Cannot cherrypick snapshot {}: already an ancestor", + snapshot_id); } const std::string snapshot_id_str = std::to_string(snapshot_id); - ICEBERG_ASSIGN_OR_RAISE(auto ancestor_ids, CurrentAncestorIds(metadata)); + ICEBERG_ASSIGN_OR_RAISE(auto ancestor_ids, CurrentAncestorIds(meta)); for (int64_t ancestor_id : ancestor_ids) { - ICEBERG_ASSIGN_OR_RAISE(auto ancestor, metadata.SnapshotById(ancestor_id)); + ICEBERG_ASSIGN_OR_RAISE(auto ancestor, meta.SnapshotById(ancestor_id)); auto it = ancestor->summary.find(SnapshotSummaryFields::kSourceSnapshotId); if (it != ancestor->summary.end() && it->second == snapshot_id_str) { - return CommitFailed( + return ValidationFailed( "Cannot cherrypick snapshot {}: already picked to create ancestor {}", snapshot_id, ancestor_id); } @@ -270,17 +287,16 @@ Status CherryPickOperation::ValidateNonAncestor(const TableMetadata& metadata, return {}; } -Status CherryPickOperation::ValidateReplacedPartitions( - const TableMetadata& metadata) const { +Status CherryPickOperation::ValidateReplacedPartitions(const TableMetadata& meta) const { if (!replaced_partitions_.has_value() || - metadata.current_snapshot_id == kInvalidSnapshotId) { + meta.current_snapshot_id == kInvalidSnapshotId) { return {}; } const auto parent_id = cherrypick_snapshot_->parent_snapshot_id; if (parent_id.has_value()) { ICEBERG_ASSIGN_OR_RAISE(bool is_ancestor, - SnapshotUtil::IsAncestorOf(metadata, parent_id.value())); + SnapshotUtil::IsAncestorOf(meta, parent_id.value())); if (!is_ancestor) { return ValidationFailed( "Cannot cherry-pick overwrite, based on non-ancestor of the current state: {}", @@ -291,42 +307,56 @@ Status CherryPickOperation::ValidateReplacedPartitions( // Walk back from the current snapshot to the picked snapshot's parent and // reject any file added into a partition this pick replaces. auto io = ctx_->table->io(); - ICEBERG_ASSIGN_OR_RAISE( - auto ancestors, SnapshotUtil::AncestorsOf(metadata, metadata.current_snapshot_id)); - for (const auto& ancestor : ancestors) { - if (parent_id.has_value() && ancestor->snapshot_id == parent_id.value()) { - break; - } - ICEBERG_ASSIGN_OR_RAISE(auto changes, ReadSnapshotChanges(*ancestor, io, metadata)); - for (const auto& added : changes.added) { - if (!added->partition_spec_id.has_value() || - !replaced_partitions_->contains(added->partition_spec_id.value(), - added->partition)) { + ICEBERG_ASSIGN_OR_RAISE(auto snapshots, SnapshotUtil::AncestorsBetween( + meta, meta.current_snapshot_id, parent_id)); + // TODO: Parallelize added-file validation using the planning executor. + // Bound active readers and wait for in-flight tasks before returning. + for (const auto& snap : snapshots) { + SnapshotReader snapshot_reader(snap.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot_reader.DataManifests(io)); + for (const auto& manifest : manifests) { + if (manifest.added_snapshot_id != snap->snapshot_id) { continue; } - ICEBERG_ASSIGN_OR_RAISE(auto spec, - metadata.PartitionSpecById(*added->partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto partition_path, spec->PartitionPath(added->partition)); - return ValidationFailed( - "Cannot cherry-pick replace partitions with changed partition: {}", - partition_path); + ICEBERG_ASSIGN_OR_RAISE(auto reader, MakeManifestReader(manifest, io, meta)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->EntriesStream()); + while (true) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, entries->Next()); + if (!entry.has_value()) { + break; + } + if (entry->status != ManifestStatus::kAdded || !entry->data_file) { + continue; + } + const auto& new_file = entry->data_file; + if (!new_file->partition_spec_id.has_value() || + !replaced_partitions_->contains(new_file->partition_spec_id.value(), + new_file->partition)) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto spec, + meta.PartitionSpecById(*new_file->partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto partition_path, + spec->PartitionPath(new_file->partition)); + return ValidationFailed( + "Cannot cherry-pick replace partitions with changed partition: {}", + partition_path); + } } } return {}; } -Status CherryPickOperation::Validate(const TableMetadata& current_metadata, - const std::shared_ptr& snapshot) { - if (cherrypick_snapshot_ == nullptr || IsFastForward()) { +Status CherryPickOperation::Validate( + const TableMetadata& base, + [[maybe_unused]] const std::shared_ptr& snapshot) { + if (IsFastForward(base)) { return {}; } - ICEBERG_RETURN_UNEXPECTED( - ValidateNonAncestor(current_metadata, cherrypick_snapshot_->snapshot_id)); - ICEBERG_RETURN_UNEXPECTED(ValidateReplacedPartitions(current_metadata)); - ICEBERG_ASSIGN_OR_RAISE( - std::ignore, - ValidateWapPublish(current_metadata, cherrypick_snapshot_->snapshot_id)); + ICEBERG_RETURN_UNEXPECTED(ValidateNonAncestor(base, cherrypick_snapshot_->snapshot_id)); + ICEBERG_RETURN_UNEXPECTED(ValidateReplacedPartitions(base)); + ICEBERG_RETURN_UNEXPECTED(ValidateWapPublish(base, cherrypick_snapshot_->snapshot_id)); return {}; } diff --git a/src/iceberg/update/cherry_pick_operation.h b/src/iceberg/update/cherry_pick_operation.h index 9ddaabff85..5c3211c417 100644 --- a/src/iceberg/update/cherry_pick_operation.h +++ b/src/iceberg/update/cherry_pick_operation.h @@ -34,27 +34,10 @@ namespace iceberg { -/// \brief Cherry-picks the changes of a snapshot onto the current state. +/// \brief Cherry-picks or fast-forwards the current state to a snapshot. /// /// This update is not exposed through the Table API. It is part of the /// Transaction API intended for use in SnapshotManager. -/// -/// Three kinds of snapshot can be picked. An append snapshot has its added -/// data files re-applied on top of the current state. An overwrite snapshot -/// carrying "replace-partitions"="true" has both its added and its removed -/// data files re-applied, and can only be picked while the partitions it -/// replaced are unchanged. Any other snapshot can only be fast-forwarded. -/// -/// A fast-forward moves the current state to the picked snapshot without -/// producing a new one, which this operation cannot express because Apply() -/// always builds a new snapshot. Callers detect that case with -/// SnapshotUtil::CanFastForward() and set the current snapshot instead of -/// committing this operation; ValidateFastForward() applies the checks this -/// operation would otherwise have run. -/// -/// The new snapshot records the picked snapshot in "source-snapshot-id". When -/// the picked snapshot carries a "wap.id", that id is recorded in -/// "published-wap-id" and the pick fails if the id was already published. class ICEBERG_EXPORT CherryPickOperation : public MergingSnapshotUpdate { public: /// \brief Create a new CherryPickOperation instance. @@ -65,29 +48,18 @@ class ICEBERG_EXPORT CherryPickOperation : public MergingSnapshotUpdate { static Result> Make( std::string table_name, std::shared_ptr ctx); - /// \brief Apply the changes of the given snapshot to the current state. + /// \brief Cherry-pick the changes of the given snapshot, or fast-forward to it. /// /// \param snapshot_id The ID of the snapshot whose changes to apply /// \return Reference to this for method chaining CherryPickOperation& Cherrypick(int64_t snapshot_id); - /// \brief Run the checks that apply when the given snapshot is fast-forwarded - /// to rather than picked. - /// - /// A fast-forward publishes the picked snapshot itself, so the WAP id staged - /// on it must not already have been published. Mirrors Java, where this check - /// runs in cherrypick() for append and dynamic overwrite snapshots, before - /// apply() elects to fast-forward. - /// - /// \param metadata The table metadata to fast-forward - /// \param snapshot The snapshot to fast-forward to - static Status ValidateFastForward(const TableMetadata& metadata, - const Snapshot& snapshot); + protected: + Result Apply() override; std::string operation() override; - protected: - Status Validate(const TableMetadata& current_metadata, + Status Validate(const TableMetadata& base, const std::shared_ptr& snapshot) override; private: @@ -95,18 +67,18 @@ class ICEBERG_EXPORT CherryPickOperation : public MergingSnapshotUpdate { std::shared_ptr ctx); /// \brief Whether the snapshot passed to Cherrypick() can be fast-forwarded. - bool IsFastForward() const; + bool IsFastForward(const TableMetadata& base) const; /// \brief Fail if the picked snapshot is already part of the current history, /// either directly or as the source of an earlier pick. - Status ValidateNonAncestor(const TableMetadata& metadata, int64_t snapshot_id) const; + Status ValidateNonAncestor(const TableMetadata& meta, int64_t snapshot_id) const; - /// \brief Fail if any partition replaced by the picked snapshot received new - /// files after the picked snapshot's parent. - Status ValidateReplacedPartitions(const TableMetadata& metadata) const; + /// \brief If there is a current snapshot, require an ancestral parent (if any) + /// and reject files added to replaced partitions since that parent. + Status ValidateReplacedPartitions(const TableMetadata& meta) const; std::shared_ptr cherrypick_snapshot_; - // Set only when the picked snapshot is a dynamic partition overwrite. + bool require_fast_forward_ = false; std::optional replaced_partitions_; }; diff --git a/src/iceberg/update/set_snapshot.cc b/src/iceberg/update/set_snapshot.cc index 997b859c0e..0054c9d329 100644 --- a/src/iceberg/update/set_snapshot.cc +++ b/src/iceberg/update/set_snapshot.cc @@ -85,11 +85,6 @@ SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { return SetCurrentSnapshot(snapshot_id); } -SetSnapshot& SetSnapshot::RequireFastForward() { - require_fast_forward_ = true; - return *this; -} - Result SetSnapshot::Apply() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); @@ -107,12 +102,6 @@ Result SetSnapshot::Apply() { "Cannot roll back to unknown snapshot id: {}", target_snapshot_id_.value()); - if (require_fast_forward_) { - ICEBERG_CHECK(SnapshotUtil::CanFastForward(base_metadata, *snapshot_result.value()), - "Cannot fast-forward to {}: not a child of the current table state", - target_snapshot_id_.value()); - } - // If this is a rollback, validate that the target is still an ancestor if (is_rollback_) { ICEBERG_ASSIGN_OR_RAISE( diff --git a/src/iceberg/update/set_snapshot.h b/src/iceberg/update/set_snapshot.h index b48abcf4ce..431e636b29 100644 --- a/src/iceberg/update/set_snapshot.h +++ b/src/iceberg/update/set_snapshot.h @@ -50,14 +50,6 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { /// \brief Rollback table's state to a specific Snapshot identified by id. SetSnapshot& RollbackTo(int64_t snapshot_id); - /// \brief Require the target snapshot to still be a fast-forward when applied. - /// - /// Without this, a commit retried against refreshed metadata would move the - /// current snapshot even if a concurrent commit has since advanced it, - /// discarding that commit. Used by SnapshotManager::Cherrypick(), which - /// expresses a cherry-pick fast-forward as a set of the current snapshot. - SetSnapshot& RequireFastForward(); - Kind kind() const final { return Kind::kSetSnapshot; } bool IsRetryable() const override { return true; } @@ -75,7 +67,6 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { std::optional target_snapshot_id_; bool is_rollback_{false}; - bool require_fast_forward_{false}; }; } // namespace iceberg diff --git a/src/iceberg/update/snapshot_manager.cc b/src/iceberg/update/snapshot_manager.cc index fe35c0f21f..d825403b56 100644 --- a/src/iceberg/update/snapshot_manager.cc +++ b/src/iceberg/update/snapshot_manager.cc @@ -28,7 +28,6 @@ #include "iceberg/update/set_snapshot.h" #include "iceberg/update/update_snapshot_reference.h" #include "iceberg/util/macros.h" -#include "iceberg/util/snapshot_util.h" namespace iceberg { @@ -57,26 +56,6 @@ SnapshotManager::~SnapshotManager() = default; SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); - const TableMetadata& metadata = transaction_->current(); - ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( - auto snapshot, metadata.SnapshotById(snapshot_id), - "Cannot cherry-pick unknown snapshot ID: {}", snapshot_id); - - // A fast-forward produces no new snapshot, so move the current snapshot - // instead of creating a cherry-pick operation. - if (SnapshotUtil::CanFastForward(metadata, *snapshot)) { - ICEBERG_BUILDER_RETURN_IF_ERROR( - CherryPickOperation::ValidateFastForward(metadata, *snapshot)); - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); - // The commit may be retried against refreshed metadata, where a concurrent - // commit can have moved the current snapshot. Without this the retry would - // apply as a plain branch move and discard that commit. - set_snapshot->RequireFastForward(); - set_snapshot->SetCurrentSnapshot(snapshot_id); - ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit()); - return *this; - } - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto cherry_pick, transaction_->NewCherryPickOperation()); cherry_pick->Cherrypick(snapshot_id); diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index cfb0518868..3f06c8b3ed 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -62,6 +62,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { public: /// \brief Result of applying a snapshot update struct ApplyResult { + // Null when applying an unconfigured update to an empty table. std::shared_ptr snapshot; std::string target_branch; bool stage_only = false; @@ -200,7 +201,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { friend class Transaction; /// Build snapshot state for the transaction's current metadata. - Result Apply(); + virtual Result Apply(); Status Finalize(const TableMetadata& committed) override; Status CleanStaged() override; diff --git a/src/iceberg/util/snapshot_util.cc b/src/iceberg/util/snapshot_util.cc index 35d6d9d36d..885189c2e0 100644 --- a/src/iceberg/util/snapshot_util.cc +++ b/src/iceberg/util/snapshot_util.cc @@ -66,24 +66,13 @@ Result SnapshotUtil::IsAncestorOf(const Table& table, Result SnapshotUtil::IsAncestorOf(const TableMetadata& metadata, int64_t ancestor_snapshot_id) { - // No current snapshot means an empty ancestor history, so nothing can be an - // ancestor of it; matches Java's SnapshotUtil.ancestorsOf(null, ...), which - // returns an empty iterable instead of raising. + // An empty current state has no ancestors. if (metadata.current_snapshot_id == kInvalidSnapshotId) { return false; } return IsAncestorOf(metadata, metadata.current_snapshot_id, ancestor_snapshot_id); } -bool SnapshotUtil::CanFastForward(const TableMetadata& metadata, - const Snapshot& snapshot) { - if (metadata.current_snapshot_id == kInvalidSnapshotId) { - return !snapshot.parent_snapshot_id.has_value(); - } - return snapshot.parent_snapshot_id.has_value() && - snapshot.parent_snapshot_id.value() == metadata.current_snapshot_id; -} - Result SnapshotUtil::IsAncestorOf(const Table& table, int64_t snapshot_id, int64_t ancestor_snapshot_id) { return IsAncestorOf(*table.metadata(), snapshot_id, ancestor_snapshot_id); diff --git a/src/iceberg/util/snapshot_util.h b/src/iceberg/util/snapshot_util.h index 2088d4eae9..5c832d65fe 100644 --- a/src/iceberg/util/snapshot_util.h +++ b/src/iceberg/util/snapshot_util.h @@ -81,17 +81,6 @@ class ICEBERG_EXPORT SnapshotUtil { static Result IsAncestorOf(const TableMetadata& metadata, int64_t ancestor_snapshot_id); - /// \brief Returns whether the given snapshot can be fast-forwarded onto the metadata's - /// current state, that is, its parent is the current snapshot or both are absent. - /// - /// A fast-forward moves the current state to an existing snapshot rather than creating - /// a new one. - /// - /// \param metadata The table metadata to fast-forward - /// \param snapshot The snapshot to fast-forward to - /// \return true if snapshot can be fast-forwarded onto the current state - static bool CanFastForward(const TableMetadata& metadata, const Snapshot& snapshot); - /// \brief Returns whether ancestor_snapshot_id is an ancestor of snapshot_id. /// /// \param table The table to check From 8737c1465d8bae791287b7149022634550229bfc Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Sun, 11 Oct 2026 00:17:20 +0800 Subject: [PATCH 5/5] fix: unchain ToBranch().AppendFile() for C++20 compat builds --- src/iceberg/test/cherry_pick_operation_test.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/iceberg/test/cherry_pick_operation_test.cc b/src/iceberg/test/cherry_pick_operation_test.cc index 0819e0f0c3..cb55de9d0b 100644 --- a/src/iceberg/test/cherry_pick_operation_test.cc +++ b/src/iceberg/test/cherry_pick_operation_test.cc @@ -139,7 +139,10 @@ class CherryPickOperationTest : public UpdateTestBase { Result CommitAppendToBranch(const std::string& branch, const std::shared_ptr& file) { ICEBERG_ASSIGN_OR_RAISE(auto append, table_->NewFastAppend()); - append->ToBranch(branch).AppendFile(file); + // ToBranch() returns SnapshotUpdate& in C++20 compat builds, so keep the + // FastAppend-only call separate. + append->ToBranch(branch); + append->AppendFile(file); ICEBERG_RETURN_UNEXPECTED(append->Commit()); ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); return table_->metadata()->refs.at(branch)->snapshot_id;