diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..a08d7340d3 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1512,6 +1512,37 @@ Fast-forward `from_branch` to point at the snapshot referenced by `to_ref`. +#### Cherry-picking a snapshot + +Cherry-pick the latest snapshot of the `audit-branch` branch onto `main`: + +```python +snapshot_id = table.snapshot_by_name("audit-branch").snapshot_id +table.manage_snapshots().cherrypick(snapshot_id).commit() +``` + + + +!!! info "Cherry-pick Behavior" + + * Case 1: If the snapshot's parent is the current snapshot of `main`, `main` is fast-forwarded to it. + * Case 2: Otherwise, if the snapshot is an `append`, its data files are committed to `main` as a new + snapshot. The new snapshot's `source-snapshot-id` summary property is set to the picked snapshot. + * Case 3: Any other operation raises `ValidationException`. + * Picking a snapshot that is already an ancestor of `main`, or was already cherry-picked, raises + `CherrypickAncestorCommitException`. + + + +Cherry-picking can also publish a staged write-audit-publish snapshot. Stage it with a `wap.id`: + +```python +table.append(df, branch=None, snapshot_properties={"wap.id": "nightly-2024-01-01"}) +``` + +When the snapshot is replayed onto `main`, the new snapshot records the id as `published-wap-id`. +Publishing the same `wap.id` twice raises `DuplicateWAPCommitException`. + #### Example Use-Case: write-audit-publish (WAP) The use of branching & fast-forwarding enable the usage of the write-audit-publish (WAP) process: diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 4a601b0d13..e104f3439d 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -148,6 +148,14 @@ class ValidationException(Exception): """Raised when validation fails.""" +class CherrypickAncestorCommitException(ValidationException): + """Raised when a cherry-picked snapshot is already an ancestor of, or was already picked into, the current state.""" + + +class DuplicateWAPCommitException(ValidationException): + """Raised when a cherry-pick would publish a wap id that was already published.""" + + class NoSuchSnapshotRefError(ValueError): """Raised when a named snapshot ref (branch or tag) does not exist.""" diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 624ce486b0..f5dac8cb9d 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -48,6 +48,9 @@ DELETED_RECORDS = "deleted-records" ADDED_EQUALITY_DELETE_FILES = "added-equality-delete-files" REMOVED_DELETE_FILES = "removed-delete-files" +STAGED_WAP_ID = "wap.id" +PUBLISHED_WAP_ID = "published-wap-id" +SOURCE_SNAPSHOT_ID = "source-snapshot-id" REMOVED_EQUALITY_DELETES = "removed-equality-deletes" REMOVED_EQUALITY_DELETE_FILES = "removed-equality-delete-files" REMOVED_FILE_SIZE = "removed-files-size" diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..ba399790cc 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -29,6 +29,8 @@ from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.exceptions import ( + CherrypickAncestorCommitException, + DuplicateWAPCommitException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, @@ -59,6 +61,9 @@ from pyiceberg.schema import Schema from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( + PUBLISHED_WAP_ID, + SOURCE_SNAPSHOT_ID, + STAGED_WAP_ID, Operation, Snapshot, SnapshotSummaryCollector, @@ -721,6 +726,63 @@ def _deleted_entries(self) -> list[ManifestEntry]: return [] +class _CherrypickAppendFiles(_FastAppendFiles): + """Replay the data files added by an append snapshot as a new snapshot on main.""" + + _cherrypick_snapshot_id: int + + def __init__(self, cherrypick_snapshot_id: int, transaction: Transaction, io: FileIO, snapshot_properties: dict[str, str]): + super().__init__(operation=Operation.APPEND, transaction=transaction, io=io, snapshot_properties=snapshot_properties) + self._cherrypick_snapshot_id = cherrypick_snapshot_id + + def _validate_concurrency(self) -> None: + """Validate against the refreshed state that the snapshot was not picked or its wap id published concurrently.""" + metadata = self._transaction._table.metadata + _validate_cherrypick_non_ancestor(metadata, self._cherrypick_snapshot_id) + _validate_wap_publish(metadata, self._cherrypick_snapshot_id) + + +def _validate_cherrypick_non_ancestor(metadata: TableMetadata, snapshot_id: int) -> None: + ancestors = list(ancestors_of(metadata.current_snapshot(), metadata)) + if any(ancestor.snapshot_id == snapshot_id for ancestor in ancestors): + raise CherrypickAncestorCommitException(f"Cannot cherrypick snapshot {snapshot_id}: already an ancestor") + + for ancestor in ancestors: + if ancestor.summary is not None and ancestor.summary.get(SOURCE_SNAPSHOT_ID) == str(snapshot_id): + raise CherrypickAncestorCommitException( + f"Cannot cherrypick snapshot {snapshot_id}: already picked to create ancestor {ancestor.snapshot_id}" + ) + + +def _validate_wap_publish(metadata: TableMetadata, snapshot_id: int) -> str | None: + """Return the staged wap id of the snapshot, if any, after checking it was not already published to main.""" + snapshot = metadata.snapshot_by_id(snapshot_id) + wap_id = snapshot.summary.get(STAGED_WAP_ID) if snapshot is not None and snapshot.summary is not None else None + if not wap_id: + return None + + for ancestor in ancestors_of(metadata.current_snapshot(), metadata): + if ancestor.summary is not None and wap_id in ( + ancestor.summary.get(STAGED_WAP_ID), + ancestor.summary.get(PUBLISHED_WAP_ID), + ): + raise DuplicateWAPCommitException(f"Duplicate request to cherry pick wap id that was published already: {wap_id}") + + return wap_id + + +def _added_data_files(snapshot: Snapshot, io: FileIO) -> list[DataFile]: + added_data_files = [] + for manifest in snapshot.manifests(io=io): + if manifest.content == ManifestContent.DATA and manifest.added_snapshot_id == snapshot.snapshot_id: + for entry in manifest.fetch_manifest_entry(io=io, discard_deleted=True): + if entry.status == ManifestEntryStatus.ADDED: + # Picked files are new rows on main, so they inherit row ids from the publishing snapshot + entry.data_file.first_row_id = None + added_data_files.append(entry.data_file) + return added_data_files + + class _MergeAppendFiles(_FastAppendFiles): _target_size_bytes: int _min_count_to_merge: int @@ -1290,6 +1352,61 @@ def rollback_to_timestamp(self, timestamp_ms: int) -> ManageSnapshots: return self.set_current_snapshot(snapshot_id=snapshot.snapshot_id) + def cherrypick(self, snapshot_id: int) -> ManageSnapshots: + """Apply the changes of a snapshot to the current state of the main branch. + + An append snapshot is replayed as a new snapshot on top of main, which records the picked snapshot in + its ``source-snapshot-id`` summary property and, for a staged write-audit-publish snapshot, its + ``published-wap-id``. Any other snapshot can only be picked when its parent is the current snapshot, + in which case main is fast-forwarded to it. + + Args: + snapshot_id: The ID of the snapshot to cherry-pick. + + Returns: + This for method chaining. + + Raises: + ValidationException: If the snapshot does not exist, or is neither an append nor a fast-forward. + CherrypickAncestorCommitException: If the snapshot is already an ancestor of, or was already picked into, main. + DuplicateWAPCommitException: If the wap id of the snapshot was already published. + """ + self._commit_if_ref_updates_exist() + + metadata = self._transaction.table_metadata + snapshot = metadata.snapshot_by_id(snapshot_id) + if snapshot is None: + raise ValidationException(f"Cannot cherry-pick unknown snapshot ID: {snapshot_id}") + + is_append = snapshot.summary is not None and snapshot.summary.operation == Operation.APPEND + wap_id = _validate_wap_publish(metadata, snapshot_id) if is_append else None + + current_snapshot = metadata.current_snapshot() + current_snapshot_id = current_snapshot.snapshot_id if current_snapshot is not None else None + if snapshot.parent_snapshot_id == current_snapshot_id: + return self.set_current_snapshot(snapshot_id=snapshot_id) + + if not is_append: + raise ValidationException(f"Cannot cherry-pick snapshot {snapshot_id}: not append or fast-forward") + + _validate_cherrypick_non_ancestor(metadata, snapshot_id) + + snapshot_properties = {SOURCE_SNAPSHOT_ID: str(snapshot_id)} + if wap_id is not None: + snapshot_properties[PUBLISHED_WAP_ID] = wap_id + + io = self._transaction._table.io + producer = _CherrypickAppendFiles( + cherrypick_snapshot_id=snapshot_id, + transaction=self._transaction, + io=io, + snapshot_properties=snapshot_properties, + ) + for data_file in _added_data_files(snapshot, io): + producer.append_data_file(data_file) + producer.commit() + return self + def _is_current_ancestor(self, snapshot_id: int) -> bool: return snapshot_id in self._current_ancestors() diff --git a/tests/table/test_manage_snapshots.py b/tests/table/test_manage_snapshots.py index 4fd6c06ae3..716164a1ab 100644 --- a/tests/table/test_manage_snapshots.py +++ b/tests/table/test_manage_snapshots.py @@ -18,17 +18,23 @@ from unittest.mock import MagicMock from uuid import uuid4 +import pyarrow as pa import pytest +from pyiceberg.catalog import Catalog from pyiceberg.exceptions import ( + CherrypickAncestorCommitException, + DuplicateWAPCommitException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, + ValidationException, ) from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.table import CommitTableResponse, Table from pyiceberg.table.metadata import TableMetadataUtil from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.table.snapshots import PUBLISHED_WAP_ID, SOURCE_SNAPSHOT_ID, STAGED_WAP_ID, Operation, Snapshot from pyiceberg.table.update import RemoveSnapshotRefUpdate, SetSnapshotRefUpdate, TableRequirement, TableUpdate # The two snapshots in the table_v2 fixture: CHILD's parent is PARENT. @@ -585,3 +591,106 @@ def test_fast_forward_branch__with_operation_chaining__preserves_retention(self, assert feature_updates[1].max_ref_age_ms == 5000 assert feature_updates[1].max_snapshot_age_ms == 6000 assert feature_updates[1].min_snapshots_to_keep == 7 + + +def _cherrypick_table(catalog: Catalog) -> Table: + catalog.create_namespace("default") + table = catalog.create_table("default.cherrypick", schema=pa.schema([pa.field("x", pa.int64())])) + table.append(pa.table({"x": [1]})) + return table + + +def _stage(table: Table, rows: list[int], wap_id: str | None = None) -> Snapshot: + """Write a snapshot on top of main without moving any ref, as a write-audit-publish stage does.""" + table.append(pa.table({"x": rows}), branch=None, snapshot_properties={STAGED_WAP_ID: wap_id} if wap_id else {}) + return table.snapshots()[-1] + + +def _main_rows(table: Table) -> list[int]: + return sorted(table.scan().to_arrow()["x"].to_pylist()) + + +def test_cherrypick_replays_append_onto_diverged_main(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + main_snapshot = table.current_snapshot() + assert main_snapshot is not None + table.manage_snapshots().create_branch(main_snapshot.snapshot_id, "audit").commit() + table.append(pa.table({"x": [2]}), branch="audit") + audit_snapshot = table.snapshot_by_name("audit") + assert audit_snapshot is not None + table.append(pa.table({"x": [3]})) + head = table.current_snapshot() + assert head is not None + + table.manage_snapshots().cherrypick(audit_snapshot.snapshot_id).commit() + + picked = table.current_snapshot() + assert picked is not None + assert picked.snapshot_id not in (audit_snapshot.snapshot_id, head.snapshot_id) + assert picked.parent_snapshot_id == head.snapshot_id + assert picked.summary is not None + assert picked.summary.operation == Operation.APPEND + assert picked.summary[SOURCE_SNAPSHOT_ID] == str(audit_snapshot.snapshot_id) + assert picked.summary.get(PUBLISHED_WAP_ID) is None + assert _main_rows(table) == [1, 2, 3] + + with pytest.raises(CherrypickAncestorCommitException, match="already picked to create ancestor"): + table.manage_snapshots().cherrypick(audit_snapshot.snapshot_id).commit() + + +def test_cherrypick_fast_forwards_when_parent_is_current(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + staged = _stage(table, [2], wap_id="wap-1") + assert _main_rows(table) == [1] + + table.manage_snapshots().cherrypick(staged.snapshot_id).commit() + + current = table.current_snapshot() + assert current is not None + assert current.snapshot_id == staged.snapshot_id + assert _main_rows(table) == [1, 2] + + +def test_cherrypick_publishes_wap_id_once(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + staged = _stage(table, [2], wap_id="wap-1") + table.append(pa.table({"x": [3]})) + + table.manage_snapshots().cherrypick(staged.snapshot_id).commit() + + picked = table.current_snapshot() + assert picked is not None and picked.summary is not None + assert picked.summary[PUBLISHED_WAP_ID] == "wap-1" + assert _main_rows(table) == [1, 2, 3] + + duplicate = _stage(table, [4], wap_id="wap-1") + table.append(pa.table({"x": [5]})) + with pytest.raises(DuplicateWAPCommitException, match="wap-1"): + table.manage_snapshots().cherrypick(duplicate.snapshot_id).commit() + + +def test_cherrypick_rejects_ancestor(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + first = table.current_snapshot() + assert first is not None + table.append(pa.table({"x": [2]})) + + with pytest.raises(CherrypickAncestorCommitException, match="already an ancestor"): + table.manage_snapshots().cherrypick(first.snapshot_id).commit() + + +def test_cherrypick_rejects_unknown_snapshot(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + + with pytest.raises(ValidationException, match="Cannot cherry-pick unknown snapshot ID: 42"): + table.manage_snapshots().cherrypick(42).commit() + + +def test_cherrypick_rejects_non_append_that_is_not_fast_forward(catalog: Catalog) -> None: + table = _cherrypick_table(catalog) + table.delete("x == 1", branch=None) + staged_delete = table.snapshots()[-1] + table.append(pa.table({"x": [2]})) + + with pytest.raises(ValidationException, match="not append or fast-forward"): + table.manage_snapshots().cherrypick(staged_delete.snapshot_id).commit()