Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1512,6 +1512,37 @@ Fast-forward `from_branch` to point at the snapshot referenced by `to_ref`.

<!-- markdownlint-enable MD046 -->

#### Cherry-picking a snapshot

Cherry-pick the latest snapshot of the `audit-branch` branch onto `main`:

```python
snapshot_id = table.snapshot_by_name("audit-branch").snapshot_id
table.manage_snapshots().cherrypick(snapshot_id).commit()
```

<!-- markdownlint-disable MD046 -- Allowing indented multi-line formatting in admonition-->

!!! info "Cherry-pick Behavior"

* Case 1: If the snapshot's parent is the current snapshot of `main`, `main` is fast-forwarded to it.
* Case 2: Otherwise, if the snapshot is an `append`, its data files are committed to `main` as a new
snapshot. The new snapshot's `source-snapshot-id` summary property is set to the picked snapshot.
* Case 3: Any other operation raises `ValidationException`.
* Picking a snapshot that is already an ancestor of `main`, or was already cherry-picked, raises
`CherrypickAncestorCommitException`.

<!-- markdownlint-enable MD046 -->

Cherry-picking can also publish a staged write-audit-publish snapshot. Stage it with a `wap.id`:

```python
table.append(df, branch=None, snapshot_properties={"wap.id": "nightly-2024-01-01"})
```

When the snapshot is replayed onto `main`, the new snapshot records the id as `published-wap-id`.
Publishing the same `wap.id` twice raises `DuplicateWAPCommitException`.

#### Example Use-Case: write-audit-publish (WAP)

The use of branching & fast-forwarding enable the usage of the write-audit-publish (WAP) process:
Expand Down
8 changes: 8 additions & 0 deletions pyiceberg/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,14 @@ class ValidationException(Exception):
"""Raised when validation fails."""


class CherrypickAncestorCommitException(ValidationException):
"""Raised when a cherry-picked snapshot is already an ancestor of, or was already picked into, the current state."""


class DuplicateWAPCommitException(ValidationException):
"""Raised when a cherry-pick would publish a wap id that was already published."""


class NoSuchSnapshotRefError(ValueError):
"""Raised when a named snapshot ref (branch or tag) does not exist."""

Expand Down
3 changes: 3 additions & 0 deletions pyiceberg/table/snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@
DELETED_RECORDS = "deleted-records"
ADDED_EQUALITY_DELETE_FILES = "added-equality-delete-files"
REMOVED_DELETE_FILES = "removed-delete-files"
STAGED_WAP_ID = "wap.id"
PUBLISHED_WAP_ID = "published-wap-id"
SOURCE_SNAPSHOT_ID = "source-snapshot-id"
REMOVED_EQUALITY_DELETES = "removed-equality-deletes"
REMOVED_EQUALITY_DELETE_FILES = "removed-equality-delete-files"
REMOVED_FILE_SIZE = "removed-files-size"
Expand Down
117 changes: 117 additions & 0 deletions pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@

from pyiceberg.avro.codecs import AvroCompressionCodec
from pyiceberg.exceptions import (
CherrypickAncestorCommitException,
DuplicateWAPCommitException,
NoSuchSnapshotRefError,
NotAncestorError,
SnapshotRefTypeError,
Expand Down Expand Up @@ -59,6 +61,9 @@
from pyiceberg.schema import Schema
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
from pyiceberg.table.snapshots import (
PUBLISHED_WAP_ID,
SOURCE_SNAPSHOT_ID,
STAGED_WAP_ID,
Operation,
Snapshot,
SnapshotSummaryCollector,
Expand Down Expand Up @@ -721,6 +726,63 @@ def _deleted_entries(self) -> list[ManifestEntry]:
return []


class _CherrypickAppendFiles(_FastAppendFiles):
"""Replay the data files added by an append snapshot as a new snapshot on main."""

_cherrypick_snapshot_id: int

def __init__(self, cherrypick_snapshot_id: int, transaction: Transaction, io: FileIO, snapshot_properties: dict[str, str]):
super().__init__(operation=Operation.APPEND, transaction=transaction, io=io, snapshot_properties=snapshot_properties)
self._cherrypick_snapshot_id = cherrypick_snapshot_id

def _validate_concurrency(self) -> None:
"""Validate against the refreshed state that the snapshot was not picked or its wap id published concurrently."""
metadata = self._transaction._table.metadata
_validate_cherrypick_non_ancestor(metadata, self._cherrypick_snapshot_id)
_validate_wap_publish(metadata, self._cherrypick_snapshot_id)


def _validate_cherrypick_non_ancestor(metadata: TableMetadata, snapshot_id: int) -> None:
ancestors = list(ancestors_of(metadata.current_snapshot(), metadata))
if any(ancestor.snapshot_id == snapshot_id for ancestor in ancestors):
raise CherrypickAncestorCommitException(f"Cannot cherrypick snapshot {snapshot_id}: already an ancestor")

for ancestor in ancestors:
if ancestor.summary is not None and ancestor.summary.get(SOURCE_SNAPSHOT_ID) == str(snapshot_id):
raise CherrypickAncestorCommitException(
f"Cannot cherrypick snapshot {snapshot_id}: already picked to create ancestor {ancestor.snapshot_id}"
)


def _validate_wap_publish(metadata: TableMetadata, snapshot_id: int) -> str | None:
"""Return the staged wap id of the snapshot, if any, after checking it was not already published to main."""
snapshot = metadata.snapshot_by_id(snapshot_id)
wap_id = snapshot.summary.get(STAGED_WAP_ID) if snapshot is not None and snapshot.summary is not None else None
if not wap_id:
return None

for ancestor in ancestors_of(metadata.current_snapshot(), metadata):
if ancestor.summary is not None and wap_id in (
ancestor.summary.get(STAGED_WAP_ID),
ancestor.summary.get(PUBLISHED_WAP_ID),
):
raise DuplicateWAPCommitException(f"Duplicate request to cherry pick wap id that was published already: {wap_id}")

return wap_id


def _added_data_files(snapshot: Snapshot, io: FileIO) -> list[DataFile]:
added_data_files = []
for manifest in snapshot.manifests(io=io):
if manifest.content == ManifestContent.DATA and manifest.added_snapshot_id == snapshot.snapshot_id:
for entry in manifest.fetch_manifest_entry(io=io, discard_deleted=True):
if entry.status == ManifestEntryStatus.ADDED:
# Picked files are new rows on main, so they inherit row ids from the publishing snapshot
entry.data_file.first_row_id = None
added_data_files.append(entry.data_file)
return added_data_files


class _MergeAppendFiles(_FastAppendFiles):
_target_size_bytes: int
_min_count_to_merge: int
Expand Down Expand Up @@ -1290,6 +1352,61 @@ def rollback_to_timestamp(self, timestamp_ms: int) -> ManageSnapshots:

return self.set_current_snapshot(snapshot_id=snapshot.snapshot_id)

def cherrypick(self, snapshot_id: int) -> ManageSnapshots:
"""Apply the changes of a snapshot to the current state of the main branch.

An append snapshot is replayed as a new snapshot on top of main, which records the picked snapshot in
its ``source-snapshot-id`` summary property and, for a staged write-audit-publish snapshot, its
``published-wap-id``. Any other snapshot can only be picked when its parent is the current snapshot,
in which case main is fast-forwarded to it.

Args:
snapshot_id: The ID of the snapshot to cherry-pick.

Returns:
This for method chaining.

Raises:
ValidationException: If the snapshot does not exist, or is neither an append nor a fast-forward.
CherrypickAncestorCommitException: If the snapshot is already an ancestor of, or was already picked into, main.
DuplicateWAPCommitException: If the wap id of the snapshot was already published.
"""
self._commit_if_ref_updates_exist()

metadata = self._transaction.table_metadata
snapshot = metadata.snapshot_by_id(snapshot_id)
if snapshot is None:
raise ValidationException(f"Cannot cherry-pick unknown snapshot ID: {snapshot_id}")

is_append = snapshot.summary is not None and snapshot.summary.operation == Operation.APPEND
wap_id = _validate_wap_publish(metadata, snapshot_id) if is_append else None

current_snapshot = metadata.current_snapshot()
current_snapshot_id = current_snapshot.snapshot_id if current_snapshot is not None else None
if snapshot.parent_snapshot_id == current_snapshot_id:
return self.set_current_snapshot(snapshot_id=snapshot_id)

if not is_append:
raise ValidationException(f"Cannot cherry-pick snapshot {snapshot_id}: not append or fast-forward")

_validate_cherrypick_non_ancestor(metadata, snapshot_id)

snapshot_properties = {SOURCE_SNAPSHOT_ID: str(snapshot_id)}
if wap_id is not None:
snapshot_properties[PUBLISHED_WAP_ID] = wap_id

io = self._transaction._table.io
producer = _CherrypickAppendFiles(
cherrypick_snapshot_id=snapshot_id,
transaction=self._transaction,
io=io,
snapshot_properties=snapshot_properties,
)
for data_file in _added_data_files(snapshot, io):
producer.append_data_file(data_file)
producer.commit()
return self

def _is_current_ancestor(self, snapshot_id: int) -> bool:
return snapshot_id in self._current_ancestors()

Expand Down
109 changes: 109 additions & 0 deletions tests/table/test_manage_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,23 @@
from unittest.mock import MagicMock
from uuid import uuid4

import pyarrow as pa
import pytest

from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import (
CherrypickAncestorCommitException,
DuplicateWAPCommitException,
NoSuchSnapshotRefError,
NotAncestorError,
SnapshotRefTypeError,
ValidationException,
)
from pyiceberg.io.pyarrow import PyArrowFileIO
from pyiceberg.table import CommitTableResponse, Table
from pyiceberg.table.metadata import TableMetadataUtil
from pyiceberg.table.refs import SnapshotRef, SnapshotRefType
from pyiceberg.table.snapshots import PUBLISHED_WAP_ID, SOURCE_SNAPSHOT_ID, STAGED_WAP_ID, Operation, Snapshot
from pyiceberg.table.update import RemoveSnapshotRefUpdate, SetSnapshotRefUpdate, TableRequirement, TableUpdate

# The two snapshots in the table_v2 fixture: CHILD's parent is PARENT.
Expand Down Expand Up @@ -585,3 +591,106 @@ def test_fast_forward_branch__with_operation_chaining__preserves_retention(self,
assert feature_updates[1].max_ref_age_ms == 5000
assert feature_updates[1].max_snapshot_age_ms == 6000
assert feature_updates[1].min_snapshots_to_keep == 7


def _cherrypick_table(catalog: Catalog) -> Table:
catalog.create_namespace("default")
table = catalog.create_table("default.cherrypick", schema=pa.schema([pa.field("x", pa.int64())]))
table.append(pa.table({"x": [1]}))
return table


def _stage(table: Table, rows: list[int], wap_id: str | None = None) -> Snapshot:
"""Write a snapshot on top of main without moving any ref, as a write-audit-publish stage does."""
table.append(pa.table({"x": rows}), branch=None, snapshot_properties={STAGED_WAP_ID: wap_id} if wap_id else {})
return table.snapshots()[-1]


def _main_rows(table: Table) -> list[int]:
return sorted(table.scan().to_arrow()["x"].to_pylist())


def test_cherrypick_replays_append_onto_diverged_main(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)
main_snapshot = table.current_snapshot()
assert main_snapshot is not None
table.manage_snapshots().create_branch(main_snapshot.snapshot_id, "audit").commit()
table.append(pa.table({"x": [2]}), branch="audit")
audit_snapshot = table.snapshot_by_name("audit")
assert audit_snapshot is not None
table.append(pa.table({"x": [3]}))
head = table.current_snapshot()
assert head is not None

table.manage_snapshots().cherrypick(audit_snapshot.snapshot_id).commit()

picked = table.current_snapshot()
assert picked is not None
assert picked.snapshot_id not in (audit_snapshot.snapshot_id, head.snapshot_id)
assert picked.parent_snapshot_id == head.snapshot_id
assert picked.summary is not None
assert picked.summary.operation == Operation.APPEND
assert picked.summary[SOURCE_SNAPSHOT_ID] == str(audit_snapshot.snapshot_id)
assert picked.summary.get(PUBLISHED_WAP_ID) is None
assert _main_rows(table) == [1, 2, 3]

with pytest.raises(CherrypickAncestorCommitException, match="already picked to create ancestor"):
table.manage_snapshots().cherrypick(audit_snapshot.snapshot_id).commit()


def test_cherrypick_fast_forwards_when_parent_is_current(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)
staged = _stage(table, [2], wap_id="wap-1")
assert _main_rows(table) == [1]

table.manage_snapshots().cherrypick(staged.snapshot_id).commit()

current = table.current_snapshot()
assert current is not None
assert current.snapshot_id == staged.snapshot_id
assert _main_rows(table) == [1, 2]


def test_cherrypick_publishes_wap_id_once(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)
staged = _stage(table, [2], wap_id="wap-1")
table.append(pa.table({"x": [3]}))

table.manage_snapshots().cherrypick(staged.snapshot_id).commit()

picked = table.current_snapshot()
assert picked is not None and picked.summary is not None
assert picked.summary[PUBLISHED_WAP_ID] == "wap-1"
assert _main_rows(table) == [1, 2, 3]

duplicate = _stage(table, [4], wap_id="wap-1")
table.append(pa.table({"x": [5]}))
with pytest.raises(DuplicateWAPCommitException, match="wap-1"):
table.manage_snapshots().cherrypick(duplicate.snapshot_id).commit()


def test_cherrypick_rejects_ancestor(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)
first = table.current_snapshot()
assert first is not None
table.append(pa.table({"x": [2]}))

with pytest.raises(CherrypickAncestorCommitException, match="already an ancestor"):
table.manage_snapshots().cherrypick(first.snapshot_id).commit()


def test_cherrypick_rejects_unknown_snapshot(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)

with pytest.raises(ValidationException, match="Cannot cherry-pick unknown snapshot ID: 42"):
table.manage_snapshots().cherrypick(42).commit()


def test_cherrypick_rejects_non_append_that_is_not_fast_forward(catalog: Catalog) -> None:
table = _cherrypick_table(catalog)
table.delete("x == 1", branch=None)
staged_delete = table.snapshots()[-1]
table.append(pa.table({"x": [2]}))

with pytest.raises(ValidationException, match="not append or fast-forward"):
table.manage_snapshots().cherrypick(staged_delete.snapshot_id).commit()
Loading