Skip to content

Commit 05ae65d

Browse files
committed
Add ManageSnapshots.cherry_pick_snapshot
Publishing a Write-Audit-Publish staged write means replaying a snapshot's changes onto the current table state. pyiceberg can stage the write on a branch and validate it there, but has no way to publish it, so the last step of the loop has to happen outside Python. Add cherry_pick_snapshot(), mirroring Java's CherryPickOperation: an append snapshot is replayed as a new snapshot on top of current carrying its added data files, recording source-snapshot-id and published-wap-id. A snapshot with another operation is fast-forwarded to when its parent is already current, and rejected otherwise. Picking an existing ancestor is a no-op, and a wap.id may only be published once. Dynamic overwrite is not covered; it raises rather than silently doing nothing. Signed-off-by: 1fanwang <1fannnw@gmail.com>
1 parent 7d0f503 commit 05ae65d

4 files changed

Lines changed: 335 additions & 0 deletions

File tree

‎mkdocs/docs/api.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1483,6 +1483,38 @@ Remove an existing branch:
14831483
table.manage_snapshots().remove_branch("dev").commit()
14841484
```
14851485

1486+
### Write-Audit-Publish
1487+
1488+
Stage a write on a branch, validate it, then publish it to the table with
1489+
`cherry_pick_snapshot`. The staged data is invisible to readers of the table until it is
1490+
published.
1491+
1492+
```python
1493+
# Write: stage the changes on an audit branch
1494+
table.manage_snapshots().create_branch(
1495+
snapshot_id=table.metadata.current_snapshot_id,
1496+
branch_name="audit-2024-01-15",
1497+
).commit()
1498+
1499+
table = catalog.load_table("db.table")
1500+
table.append(new_rows, branch="audit-2024-01-15", snapshot_properties={"wap.id": "etl-2024-01-15"})
1501+
1502+
# Audit: validate the staged data without affecting readers of the table
1503+
table = catalog.load_table("db.table")
1504+
staged = table.metadata.refs["audit-2024-01-15"].snapshot_id
1505+
assert len(table.scan(snapshot_id=staged).to_arrow()) > 0
1506+
1507+
# Publish: replay the staged changes onto the current table state
1508+
table.manage_snapshots().cherry_pick_snapshot(staged).commit()
1509+
```
1510+
1511+
The published snapshot records `source-snapshot-id`, and `published-wap-id` when the staged
1512+
snapshot carried a `wap.id`. A given `wap.id` can only be published once.
1513+
1514+
If the table has not changed since the branch was cut, the branch is fast-forwarded rather than
1515+
replayed. Picking a snapshot that is already an ancestor of the current state does nothing. Only
1516+
append snapshots can be replayed; anything else raises.
1517+
14861518
## Table Maintenance
14871519

14881520
PyIceberg provides table maintenance operations through the `table.maintenance` API. This provides a clean interface for performing maintenance tasks like snapshot expiration.

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,12 @@
8080
if TYPE_CHECKING:
8181
from pyiceberg.table import Transaction
8282

83+
# Snapshot summary properties for Write-Audit-Publish, mirroring Java's SnapshotSummary:
84+
# https://github.com/apache/iceberg/blob/0d60b2bb8780d781f4ceb69032418d222b649ae5/core/src/main/java/org/apache/iceberg/SnapshotSummary.java#L60-L62
85+
_STAGED_WAP_ID_PROP = "wap.id"
86+
_PUBLISHED_WAP_ID_PROP = "published-wap-id"
87+
_SOURCE_SNAPSHOT_ID_PROP = "source-snapshot-id"
88+
8389

8490
def _new_manifest_file_name(num: int, commit_uuid: uuid.UUID) -> str:
8591
return f"{commit_uuid}-m{num}.avro"
@@ -974,6 +980,87 @@ def set_current_snapshot(self, snapshot_id: int | None = None, ref_name: str | N
974980
self._transaction._stage(update, requirement)
975981
return self
976982

983+
def cherry_pick_snapshot(self, snapshot_id: int) -> ManageSnapshots:
984+
"""Apply the changes in a snapshot to the current table state.
985+
986+
Creates a new snapshot on top of the current one carrying the data files the picked
987+
snapshot added, which is how a Write-Audit-Publish staged write is published. The picked
988+
snapshot itself is left in place.
989+
990+
Append snapshots are always replayed, matching Java's ``CherryPickOperation``. A snapshot
991+
with another operation is fast-forwarded to when its parent is already the current
992+
snapshot, and rejected otherwise. Picking a snapshot that is already an ancestor of the
993+
current state does nothing.
994+
995+
A snapshot staged with a ``wap.id`` can only be published once. The new snapshot records
996+
``source-snapshot-id``, and ``published-wap-id`` when the picked snapshot carried one.
997+
998+
Args:
999+
snapshot_id: The ID of the snapshot to cherry-pick.
1000+
1001+
Returns:
1002+
This for method chaining.
1003+
1004+
Raises:
1005+
ValueError: If the snapshot does not exist, its ``wap.id`` was already published, or
1006+
its operation cannot be cherry-picked.
1007+
"""
1008+
self._commit_if_ref_updates_exist()
1009+
1010+
metadata = self._transaction.table_metadata
1011+
picked = metadata.snapshot_by_id(snapshot_id)
1012+
if picked is None:
1013+
raise ValueError(f"Cannot cherry-pick unknown snapshot id: {snapshot_id}")
1014+
1015+
if self._is_current_ancestor(snapshot_id):
1016+
return self
1017+
1018+
wap_id = self._validate_wap_publish(picked)
1019+
operation = picked.summary.operation if picked.summary else None
1020+
1021+
if operation == Operation.APPEND:
1022+
snapshot_properties = {_SOURCE_SNAPSHOT_ID_PROP: str(snapshot_id)}
1023+
if wap_id is not None:
1024+
snapshot_properties[_PUBLISHED_WAP_ID_PROP] = wap_id
1025+
1026+
with self._transaction.update_snapshot(snapshot_properties=snapshot_properties).fast_append() as append:
1027+
for data_file in self._added_data_files(picked):
1028+
append.append_data_file(data_file)
1029+
return self
1030+
1031+
if picked.parent_snapshot_id == metadata.current_snapshot_id:
1032+
return self.set_current_snapshot(snapshot_id=snapshot_id)
1033+
1034+
raise ValueError(
1035+
f"Cannot cherry-pick snapshot {snapshot_id}: not append, dynamic overwrite, or fast-forward "
1036+
f"(operation: {operation}). Only append snapshots can be replayed."
1037+
)
1038+
1039+
def _added_data_files(self, snapshot: Snapshot) -> list[DataFile]:
1040+
"""Return the data files the given snapshot added, excluding entries carried over from its ancestors."""
1041+
io = self._transaction._table.io
1042+
return [
1043+
entry.data_file
1044+
for manifest in snapshot.manifests(io)
1045+
if manifest.added_snapshot_id == snapshot.snapshot_id
1046+
for entry in manifest.fetch_manifest_entry(io, discard_deleted=True)
1047+
if entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == snapshot.snapshot_id
1048+
]
1049+
1050+
def _validate_wap_publish(self, picked: Snapshot) -> str | None:
1051+
"""Return the picked snapshot's staged wap id, rejecting one that an ancestor already published."""
1052+
wap_id = picked.summary.additional_properties.get(_STAGED_WAP_ID_PROP) if picked.summary else None
1053+
if not wap_id:
1054+
return None
1055+
1056+
metadata = self._transaction.table_metadata
1057+
for ancestor in ancestors_of(metadata.current_snapshot(), metadata):
1058+
props = ancestor.summary.additional_properties if ancestor.summary else {}
1059+
if wap_id in (props.get(_STAGED_WAP_ID_PROP), props.get(_PUBLISHED_WAP_ID_PROP)):
1060+
raise ValueError(f"Duplicate request to cherry pick wap id that was published already: {wap_id}")
1061+
1062+
return wap_id
1063+
9771064
def rollback_to_snapshot(self, snapshot_id: int) -> ManageSnapshots:
9781065
"""Rollback the table to the given snapshot id.
9791066

‎tests/integration/test_snapshot_operations.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,3 +332,48 @@ def test_rollback_to_timestamp_chained_with_tag(table_with_snapshots: Table) ->
332332
assert table_with_snapshots.metadata.refs[tag_name] == SnapshotRef(
333333
snapshot_id=current_snapshot.snapshot_id, snapshot_ref_type="tag"
334334
)
335+
336+
337+
@pytest.mark.integration
338+
@pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")])
339+
def test_cherry_pick_snapshot_publishes_staged_write(catalog: Catalog) -> None:
340+
"""Publish a branch-staged write onto a main that has moved on since the branch was cut."""
341+
catalog.create_namespace_if_not_exists("default")
342+
identifier = f"default.test_cherry_pick_{uuid.uuid4().hex[:8]}"
343+
arrow_schema = pa.schema([pa.field("id", pa.int64(), nullable=False)])
344+
tbl = catalog.create_table(identifier=identifier, schema=arrow_schema)
345+
346+
tbl.append(pa.Table.from_pylist([{"id": 1}], schema=arrow_schema))
347+
tbl = catalog.load_table(identifier)
348+
current_snapshot_id = tbl.metadata.current_snapshot_id
349+
assert current_snapshot_id is not None
350+
tbl.manage_snapshots().create_branch(snapshot_id=current_snapshot_id, branch_name="audit").commit()
351+
352+
tbl = catalog.load_table(identifier)
353+
tbl.append(
354+
pa.Table.from_pylist([{"id": 2}, {"id": 3}], schema=arrow_schema),
355+
branch="audit",
356+
snapshot_properties={"wap.id": "etl-001"},
357+
)
358+
tbl = catalog.load_table(identifier)
359+
staged = tbl.metadata.refs["audit"].snapshot_id
360+
361+
# main advances independently, so this is a replay rather than a fast-forward
362+
tbl.append(pa.Table.from_pylist([{"id": 9}], schema=arrow_schema))
363+
tbl = catalog.load_table(identifier)
364+
assert sorted(tbl.scan().to_arrow().column("id").to_pylist()) == [1, 9]
365+
366+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
367+
368+
tbl = catalog.load_table(identifier)
369+
assert sorted(tbl.scan().to_arrow().column("id").to_pylist()) == [1, 2, 3, 9]
370+
published = tbl.current_snapshot()
371+
assert published is not None and published.summary is not None
372+
summary = published.summary.additional_properties
373+
assert summary["source-snapshot-id"] == str(staged)
374+
assert summary["published-wap-id"] == "etl-001"
375+
376+
with pytest.raises(ValueError, match="Duplicate request to cherry pick wap id"):
377+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
378+
379+
catalog.drop_table(identifier)

‎tests/table/test_manage_snapshots.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717
from unittest.mock import MagicMock
1818
from uuid import uuid4
1919

20+
import pyarrow as pa
2021
import pytest
2122

23+
from pyiceberg.catalog import Catalog
2224
from pyiceberg.table import CommitTableResponse, Table
2325
from pyiceberg.table.update import SetSnapshotRefUpdate, TableUpdate
2426

@@ -177,3 +179,172 @@ def test_set_current_snapshot_chained_with_create_tag(table_v2: Table) -> None:
177179
# The main branch should point to the same snapshot as the tag
178180
main_update = next(u for u in set_ref_updates if u.ref_name == "main")
179181
assert main_update.snapshot_id == snapshot_one
182+
183+
184+
def _staged_wap_table(
185+
catalog_with_warehouse: Catalog,
186+
namespace: str,
187+
advance_main: bool = True,
188+
wap_id: str | None = None,
189+
) -> tuple[Table, int]:
190+
"""Build the WAP shape: a staged branch write, with main optionally moved on since.
191+
192+
Returns the reloaded table and the staged snapshot id.
193+
"""
194+
catalog_with_warehouse.create_namespace(namespace)
195+
schema = pa.schema([pa.field("id", pa.int64())])
196+
tbl = catalog_with_warehouse.create_table(f"{namespace}.tbl", schema=schema)
197+
198+
tbl.append(pa.table({"id": [1]}, schema=schema))
199+
tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl")
200+
201+
tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit()
202+
tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl")
203+
204+
tbl.append(
205+
pa.table({"id": [2, 3]}, schema=schema),
206+
branch="audit",
207+
snapshot_properties={"wap.id": wap_id} if wap_id else {},
208+
)
209+
tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl")
210+
staged = tbl.metadata.refs["audit"].snapshot_id
211+
212+
if advance_main:
213+
tbl.append(pa.table({"id": [9]}, schema=schema))
214+
tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl")
215+
216+
return tbl, staged
217+
218+
219+
def _ids(tbl: Table) -> list[int]:
220+
return sorted(tbl.scan().to_arrow().column("id").to_pylist())
221+
222+
223+
def _current_summary_props(tbl: Table) -> dict[str, str]:
224+
snapshot = tbl.current_snapshot()
225+
assert snapshot is not None and snapshot.summary is not None
226+
return snapshot.summary.additional_properties
227+
228+
229+
def _current_snapshot_id(tbl: Table) -> int:
230+
snapshot_id = tbl.metadata.current_snapshot_id
231+
assert snapshot_id is not None
232+
return snapshot_id
233+
234+
235+
def test_cherry_pick_snapshot_replays_staged_append(catalog_with_warehouse: Catalog) -> None:
236+
"""The staged rows land on main even though main moved on after the branch was cut."""
237+
tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_replay")
238+
assert _ids(tbl) == [1, 9]
239+
240+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
241+
242+
tbl = catalog_with_warehouse.load_table("cp_replay.tbl")
243+
assert _ids(tbl) == [1, 2, 3, 9]
244+
assert _current_summary_props(tbl)["source-snapshot-id"] == str(staged)
245+
246+
247+
def test_cherry_pick_snapshot_records_published_wap_id(catalog_with_warehouse: Catalog) -> None:
248+
tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_wap", wap_id="etl-001")
249+
250+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
251+
252+
tbl = catalog_with_warehouse.load_table("cp_wap.tbl")
253+
assert _current_summary_props(tbl)["published-wap-id"] == "etl-001"
254+
255+
256+
def test_cherry_pick_snapshot_rejects_duplicate_wap_publish(catalog_with_warehouse: Catalog) -> None:
257+
"""A wap id may only be published once."""
258+
tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_dup", wap_id="etl-001")
259+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
260+
261+
tbl = catalog_with_warehouse.load_table("cp_dup.tbl")
262+
with pytest.raises(ValueError, match="Duplicate request to cherry pick wap id"):
263+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
264+
265+
266+
def test_cherry_pick_snapshot_replays_append_even_when_parent_is_current(
267+
catalog_with_warehouse: Catalog,
268+
) -> None:
269+
"""Appends are always replayed, so the wap trail is recorded even on an unmoved table."""
270+
tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_ff", advance_main=False, wap_id="etl-001")
271+
assert _ids(tbl) == [1]
272+
273+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
274+
275+
tbl = catalog_with_warehouse.load_table("cp_ff.tbl")
276+
assert _ids(tbl) == [1, 2, 3]
277+
assert tbl.metadata.current_snapshot_id != staged
278+
assert _current_summary_props(tbl)["published-wap-id"] == "etl-001"
279+
280+
281+
def test_cherry_pick_snapshot_fast_forwards_non_append(catalog_with_warehouse: Catalog) -> None:
282+
"""A non-append whose parent is current cannot be replayed, so the ref advances to it."""
283+
catalog_with_warehouse.create_namespace("cp_ffna")
284+
schema = pa.schema([pa.field("id", pa.int64())])
285+
tbl = catalog_with_warehouse.create_table("cp_ffna.tbl", schema=schema)
286+
tbl.append(pa.table({"id": [1, 2]}, schema=schema))
287+
tbl = catalog_with_warehouse.load_table("cp_ffna.tbl")
288+
289+
tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit()
290+
tbl = catalog_with_warehouse.load_table("cp_ffna.tbl")
291+
tbl.delete("id = 1", branch="audit")
292+
tbl = catalog_with_warehouse.load_table("cp_ffna.tbl")
293+
staged = tbl.metadata.refs["audit"].snapshot_id
294+
295+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()
296+
297+
tbl = catalog_with_warehouse.load_table("cp_ffna.tbl")
298+
assert tbl.metadata.current_snapshot_id == staged
299+
assert _ids(tbl) == [2]
300+
301+
302+
def test_cherry_pick_snapshot_is_noop_for_ancestor(catalog_with_warehouse: Catalog) -> None:
303+
"""Picking a snapshot already in main's history changes nothing."""
304+
catalog_with_warehouse.create_namespace("cp_anc")
305+
schema = pa.schema([pa.field("id", pa.int64())])
306+
tbl = catalog_with_warehouse.create_table("cp_anc.tbl", schema=schema)
307+
tbl.append(pa.table({"id": [1]}, schema=schema))
308+
tbl = catalog_with_warehouse.load_table("cp_anc.tbl")
309+
first = _current_snapshot_id(tbl)
310+
tbl.append(pa.table({"id": [2]}, schema=schema))
311+
tbl = catalog_with_warehouse.load_table("cp_anc.tbl")
312+
before = _current_snapshot_id(tbl)
313+
314+
tbl.manage_snapshots().cherry_pick_snapshot(first).commit()
315+
316+
tbl = catalog_with_warehouse.load_table("cp_anc.tbl")
317+
assert tbl.metadata.current_snapshot_id == before
318+
assert _ids(tbl) == [1, 2]
319+
320+
321+
def test_cherry_pick_snapshot_rejects_unknown_snapshot(catalog_with_warehouse: Catalog) -> None:
322+
catalog_with_warehouse.create_namespace("cp_unknown")
323+
schema = pa.schema([pa.field("id", pa.int64())])
324+
tbl = catalog_with_warehouse.create_table("cp_unknown.tbl", schema=schema)
325+
tbl.append(pa.table({"id": [1]}, schema=schema))
326+
tbl = catalog_with_warehouse.load_table("cp_unknown.tbl")
327+
328+
with pytest.raises(ValueError, match="Cannot cherry-pick unknown snapshot id"):
329+
tbl.manage_snapshots().cherry_pick_snapshot(1234567890).commit()
330+
331+
332+
def test_cherry_pick_snapshot_rejects_non_append(catalog_with_warehouse: Catalog) -> None:
333+
"""Only append snapshots can be replayed; anything else must say so rather than silently skip."""
334+
catalog_with_warehouse.create_namespace("cp_nonappend")
335+
schema = pa.schema([pa.field("id", pa.int64())])
336+
tbl = catalog_with_warehouse.create_table("cp_nonappend.tbl", schema=schema)
337+
tbl.append(pa.table({"id": [1, 2]}, schema=schema))
338+
tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl")
339+
340+
tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit()
341+
tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl")
342+
tbl.delete("id = 1", branch="audit")
343+
tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl")
344+
staged = tbl.metadata.refs["audit"].snapshot_id
345+
346+
tbl.append(pa.table({"id": [9]}, schema=schema))
347+
tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl")
348+
349+
with pytest.raises(ValueError, match="not append, dynamic overwrite, or fast-forward"):
350+
tbl.manage_snapshots().cherry_pick_snapshot(staged).commit()

0 commit comments

Comments
 (0)