Skip to content

Commit 2d9f5d5

Browse files
committed
Preserve a ref's retention policy when its snapshot moves
Branch writes and current-snapshot moves rebuilt refs without carrying their retention fields, clearing policies that create_branch had set. Preserve the existing ref's max ref age, max snapshot age, and min snapshots to keep on those move paths. Refs without retention stay unset. Signed-off-by: 1fanwang <1fannnw@gmail.com>
1 parent ebbc0ba commit 2d9f5d5

3 files changed

Lines changed: 138 additions & 0 deletions

File tree

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,7 @@ def _commit(self) -> UpdatesAndRequirements:
367367
(),
368368
)
369369
else:
370+
existing_ref = self._transaction.table_metadata.refs.get(self._target_branch)
370371
return (
371372
(
372373
add_snapshot_update,
@@ -375,6 +376,9 @@ def _commit(self) -> UpdatesAndRequirements:
375376
parent_snapshot_id=self._parent_snapshot_id,
376377
ref_name=self._target_branch,
377378
type=SnapshotRefType.BRANCH,
379+
max_ref_age_ms=existing_ref.max_ref_age_ms if existing_ref else None,
380+
max_snapshot_age_ms=existing_ref.max_snapshot_age_ms if existing_ref else None,
381+
min_snapshots_to_keep=existing_ref.min_snapshots_to_keep if existing_ref else None,
378382
),
379383
),
380384
(
@@ -1205,10 +1209,14 @@ def set_current_snapshot(self, snapshot_id: int | None = None, ref_name: str | N
12051209
if self._transaction.table_metadata.snapshot_by_id(target_snapshot_id) is None:
12061210
raise ValueError(f"Cannot set current snapshot to unknown snapshot id: {target_snapshot_id}")
12071211

1212+
existing_ref = self._transaction.table_metadata.refs.get(MAIN_BRANCH)
12081213
update, requirement = self._transaction._set_ref_snapshot(
12091214
snapshot_id=target_snapshot_id,
12101215
ref_name=MAIN_BRANCH,
12111216
type=SnapshotRefType.BRANCH,
1217+
max_ref_age_ms=existing_ref.max_ref_age_ms if existing_ref else None,
1218+
max_snapshot_age_ms=existing_ref.max_snapshot_age_ms if existing_ref else None,
1219+
min_snapshots_to_keep=existing_ref.min_snapshots_to_keep if existing_ref else None,
12121220
)
12131221
self._transaction._stage(update, requirement)
12141222
return self

‎tests/integration/test_writes/test_writes.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2235,6 +2235,32 @@ def test_append_to_existing_branch(session_catalog: Catalog, arrow_table_with_nu
22352235
assert branch_snapshot.parent_snapshot_id == main_snapshot.snapshot_id
22362236

22372237

2238+
@pytest.mark.integration
2239+
def test_write_to_branch_preserves_retention(session_catalog: Catalog, arrow_table_with_null: pa.Table) -> None:
2240+
"""Writing to a branch must not reset the retention policy it was created with."""
2241+
identifier = "default.test_branch_write_retention"
2242+
branch = "retention_branch"
2243+
tbl = _create_table(session_catalog, identifier, {"format-version": "2"}, [arrow_table_with_null])
2244+
assert tbl.metadata.current_snapshot_id is not None
2245+
2246+
tbl.manage_snapshots().create_branch(
2247+
snapshot_id=tbl.metadata.current_snapshot_id,
2248+
branch_name=branch,
2249+
max_ref_age_ms=86400000,
2250+
max_snapshot_age_ms=3600000,
2251+
min_snapshots_to_keep=5,
2252+
).commit()
2253+
2254+
tbl.append(arrow_table_with_null, branch=branch)
2255+
tbl.append(arrow_table_with_null, branch=branch)
2256+
2257+
ref = tbl.metadata.refs[branch]
2258+
assert ref.max_ref_age_ms == 86400000
2259+
assert ref.max_snapshot_age_ms == 3600000
2260+
assert ref.min_snapshots_to_keep == 5
2261+
assert tbl.metadata.refs["main"].max_ref_age_ms is None
2262+
2263+
22382264
@pytest.mark.integration
22392265
def test_delete_to_existing_branch(session_catalog: Catalog, arrow_table_with_null: pa.Table) -> None:
22402266
identifier = "default.test_existing_branch_delete"

‎tests/table/test_manage_snapshots.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
from unittest.mock import MagicMock
1919
from uuid import uuid4
2020

21+
import pyarrow as pa
2122
import pytest
2223

24+
from pyiceberg.catalog import Catalog
2325
from pyiceberg.exceptions import (
2426
NoSuchSnapshotRefError,
2527
NotAncestorError,
@@ -512,3 +514,105 @@ def test_fast_forward_branch__with_operation_chaining__preserves_retention(self,
512514
assert feature_updates[1].max_ref_age_ms == 5000
513515
assert feature_updates[1].max_snapshot_age_ms == 6000
514516
assert feature_updates[1].min_snapshots_to_keep == 7
517+
518+
519+
def test_branch_write_preserves_retention(catalog_with_warehouse: Catalog) -> None:
520+
"""Writing to a branch keeps the retention policy it was created with."""
521+
catalog_with_warehouse.create_namespace("branch_retention")
522+
schema = pa.schema([pa.field("id", pa.int64())])
523+
tbl = catalog_with_warehouse.create_table("branch_retention.tbl", schema=schema)
524+
tbl.append(pa.table({"id": [1]}, schema=schema))
525+
tbl = catalog_with_warehouse.load_table("branch_retention.tbl")
526+
527+
snapshot_id = tbl.metadata.current_snapshot_id
528+
assert snapshot_id is not None
529+
tbl.manage_snapshots().create_branch(
530+
snapshot_id=snapshot_id,
531+
branch_name="audit",
532+
max_ref_age_ms=86400000,
533+
max_snapshot_age_ms=3600000,
534+
min_snapshots_to_keep=5,
535+
).commit()
536+
537+
for i in range(3):
538+
tbl = catalog_with_warehouse.load_table("branch_retention.tbl")
539+
tbl.append(pa.table({"id": [i + 2]}, schema=schema), branch="audit")
540+
541+
ref = catalog_with_warehouse.load_table("branch_retention.tbl").metadata.refs["audit"]
542+
assert ref.max_ref_age_ms == 86400000
543+
assert ref.max_snapshot_age_ms == 3600000
544+
assert ref.min_snapshots_to_keep == 5
545+
546+
547+
def test_branch_write_without_retention_stays_unset(catalog_with_warehouse: Catalog) -> None:
548+
"""A branch created without a retention policy does not gain one from a write."""
549+
catalog_with_warehouse.create_namespace("branch_no_retention")
550+
schema = pa.schema([pa.field("id", pa.int64())])
551+
tbl = catalog_with_warehouse.create_table("branch_no_retention.tbl", schema=schema)
552+
tbl.append(pa.table({"id": [1]}, schema=schema))
553+
tbl = catalog_with_warehouse.load_table("branch_no_retention.tbl")
554+
555+
snapshot_id = tbl.metadata.current_snapshot_id
556+
assert snapshot_id is not None
557+
tbl.manage_snapshots().create_branch(snapshot_id=snapshot_id, branch_name="plain").commit()
558+
559+
tbl = catalog_with_warehouse.load_table("branch_no_retention.tbl")
560+
tbl.append(pa.table({"id": [2]}, schema=schema), branch="plain")
561+
562+
ref = catalog_with_warehouse.load_table("branch_no_retention.tbl").metadata.refs["plain"]
563+
assert ref.max_ref_age_ms is None
564+
assert ref.max_snapshot_age_ms is None
565+
assert ref.min_snapshots_to_keep is None
566+
567+
568+
def test_rollback_preserves_retention(catalog_with_warehouse: Catalog) -> None:
569+
"""Moving a ref keeps its retention policy; rollback and set_current_snapshot move main."""
570+
catalog_with_warehouse.create_namespace("rollback_retention")
571+
schema = pa.schema([pa.field("id", pa.int64())])
572+
tbl = catalog_with_warehouse.create_table("rollback_retention.tbl", schema=schema)
573+
tbl.append(pa.table({"id": [1]}, schema=schema))
574+
tbl = catalog_with_warehouse.load_table("rollback_retention.tbl")
575+
first = tbl.metadata.current_snapshot_id
576+
assert first is not None
577+
578+
tbl.append(pa.table({"id": [2]}, schema=schema))
579+
tbl = catalog_with_warehouse.load_table("rollback_retention.tbl")
580+
head = tbl.metadata.current_snapshot_id
581+
assert head is not None
582+
583+
tbl.manage_snapshots().create_branch(
584+
snapshot_id=head,
585+
branch_name="main",
586+
max_ref_age_ms=99999,
587+
max_snapshot_age_ms=1234,
588+
min_snapshots_to_keep=7,
589+
).commit()
590+
591+
tbl = catalog_with_warehouse.load_table("rollback_retention.tbl")
592+
tbl.manage_snapshots().rollback_to_snapshot(first).commit()
593+
594+
ref = catalog_with_warehouse.load_table("rollback_retention.tbl").metadata.refs["main"]
595+
assert ref.snapshot_id == first
596+
assert ref.max_ref_age_ms == 99999
597+
assert ref.max_snapshot_age_ms == 1234
598+
assert ref.min_snapshots_to_keep == 7
599+
600+
601+
def test_rollback_without_retention_stays_unset(catalog_with_warehouse: Catalog) -> None:
602+
"""A ref with no retention policy does not gain one from being moved."""
603+
catalog_with_warehouse.create_namespace("rollback_plain")
604+
schema = pa.schema([pa.field("id", pa.int64())])
605+
tbl = catalog_with_warehouse.create_table("rollback_plain.tbl", schema=schema)
606+
tbl.append(pa.table({"id": [1]}, schema=schema))
607+
tbl = catalog_with_warehouse.load_table("rollback_plain.tbl")
608+
first = tbl.metadata.current_snapshot_id
609+
assert first is not None
610+
611+
tbl.append(pa.table({"id": [2]}, schema=schema))
612+
tbl = catalog_with_warehouse.load_table("rollback_plain.tbl")
613+
tbl.manage_snapshots().rollback_to_snapshot(first).commit()
614+
615+
ref = catalog_with_warehouse.load_table("rollback_plain.tbl").metadata.refs["main"]
616+
assert ref.max_ref_age_ms is None
617+
assert ref.max_snapshot_age_ms is None
618+
assert ref.min_snapshots_to_keep is None

0 commit comments

Comments
 (0)