Skip to content

Commit 60d0a7e

Browse files
committed
Validate each branch against its own commits on retry
The retry path and the empty-output validation path resolved one CommitWindow from the first snapshot producer and reused it for every producer, so in a multi-branch transaction a delete or overwrite on a later branch was validated against the first branch's commits and could skip validation entirely. Resolve one window per target branch, based on the first producer on that branch, since later producers on the same branch start from a snapshot staged by this transaction. Closes #4316
1 parent 1787b41 commit 60d0a7e

2 files changed

Lines changed: 86 additions & 14 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1194,17 +1194,10 @@ def commit_transaction(self) -> Table:
11941194
elif self._snapshot_producers:
11951195
# An empty staged output (e.g. a delete whose plan matched nothing) skips the
11961196
# commit loop above, so run concurrency validation explicitly before reporting success.
1197-
from pyiceberg.table.update.snapshot import CommitWindow
1198-
11991197
try:
12001198
self._table.refresh()
1201-
commit_window = CommitWindow.resolve(
1202-
self._table.metadata,
1203-
self._snapshot_producers[0]._starting_snapshot_id,
1204-
self._snapshot_producers[0]._target_branch,
1205-
)
1199+
self._resolve_commit_windows()
12061200
for producer in self._snapshot_producers:
1207-
producer._commit_window = commit_window
12081201
producer._validate_concurrency()
12091202
except Exception:
12101203
for producer in self._snapshot_producers:
@@ -1224,20 +1217,30 @@ def _cleanup_uncommitted_manifests(self) -> None:
12241217
for producer in self._snapshot_producers:
12251218
producer._cleanup_uncommitted()
12261219

1220+
def _resolve_commit_windows(self) -> None:
1221+
"""Resolve each producer's commit window against the refreshed metadata of its own branch.
1222+
1223+
The window starts where the transaction first wrote to the branch. Later producers on the same
1224+
branch start from a snapshot staged by this transaction, which the catalog has never seen.
1225+
"""
1226+
from pyiceberg.table.update.snapshot import CommitWindow
1227+
1228+
windows: dict[str | None, CommitWindow] = {}
1229+
for producer in self._snapshot_producers:
1230+
branch = producer._target_branch
1231+
if branch not in windows:
1232+
windows[branch] = CommitWindow.resolve(self._table.metadata, producer._starting_snapshot_id, branch)
1233+
producer._commit_window = windows[branch]
1234+
12271235
def _rebuild_snapshot_updates(self) -> None:
12281236
"""Rebuild snapshot updates for retry by re-executing registered producers."""
12291237
from pyiceberg.table.update import AddSnapshotUpdate, AssertRefSnapshotId, SetSnapshotRefUpdate
1230-
from pyiceberg.table.update.snapshot import CommitWindow
12311238

12321239
self._updates = tuple(u for u in self._updates if not isinstance(u, (AddSnapshotUpdate, SetSnapshotRefUpdate)))
12331240
self._requirements = tuple(r for r in self._requirements if not isinstance(r, AssertRefSnapshotId))
12341241

1235-
starting_id = self._snapshot_producers[0]._starting_snapshot_id if self._snapshot_producers else None
1236-
target_branch = self._snapshot_producers[0]._target_branch if self._snapshot_producers else None
1237-
commit_window = CommitWindow.resolve(self._table.metadata, starting_id, target_branch)
1238-
1242+
self._resolve_commit_windows()
12391243
for producer in self._snapshot_producers:
1240-
producer._commit_window = commit_window
12411244
producer._refresh_for_retry()
12421245
producer._validate_concurrency()
12431246
updates, requirements = producer._commit()

‎tests/table/test_commit_retry.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1363,3 +1363,72 @@ def test_transaction_stages_one_ref_assertion_per_ref(catalog: Catalog) -> None:
13631363
ref_assertions = {r.ref: r.snapshot_id for r in tx._requirements if isinstance(r, AssertRefSnapshotId)}
13641364
assert ref_assertions == {"b": base_snapshot_id, "main": base_snapshot_id}
13651365
assert len([r for r in tx._requirements if isinstance(r, AssertRefSnapshotId)]) == 2
1366+
1367+
1368+
def _table_with_branch(catalog: Catalog, identifier: str) -> Any:
1369+
import pyarrow as pa
1370+
1371+
tbl = catalog.create_table(
1372+
identifier,
1373+
schema=_test_schema(),
1374+
properties={TableProperties.COMMIT_MIN_RETRY_WAIT_MS: "1", TableProperties.COMMIT_MAX_RETRY_WAIT_MS: "2"},
1375+
)
1376+
tbl.append(pa.table({"x": [0, 1]}))
1377+
snapshot_id = tbl.metadata.current_snapshot_id
1378+
assert snapshot_id is not None
1379+
tbl.manage_snapshots().create_branch(snapshot_id=snapshot_id, branch_name="b").commit()
1380+
return tbl
1381+
1382+
1383+
def test_retry_validates_each_branch_against_its_own_commits(catalog: Catalog) -> None:
1384+
"""A retried delete on a second branch is validated against that branch, not the first producer's branch."""
1385+
import pyarrow as pa
1386+
1387+
catalog.create_namespace("default")
1388+
tx = _table_with_branch(catalog, "default.retry_window_per_branch").transaction()
1389+
tx.append(pa.table({"x": [9]}))
1390+
tx.delete("x == 1", branch="b")
1391+
1392+
# A row matching the delete lands on b after the delete was planned; main is untouched.
1393+
catalog.load_table("default.retry_window_per_branch").append(pa.table({"x": [1]}), branch="b")
1394+
1395+
with pytest.raises(ValidationException):
1396+
tx.commit_transaction()
1397+
1398+
refreshed = catalog.load_table("default.retry_window_per_branch")
1399+
assert sorted(refreshed.scan().use_ref("b").to_arrow()["x"].to_pylist()) == [0, 1, 1]
1400+
assert sorted(refreshed.scan().to_arrow()["x"].to_pylist()) == [0, 1]
1401+
1402+
1403+
def test_empty_plan_validation_checks_each_branch_against_its_own_commits(catalog: Catalog) -> None:
1404+
"""Empty-plan deletes on two branches are each validated against their own branch."""
1405+
import pyarrow as pa
1406+
1407+
catalog.create_namespace("default")
1408+
tx = _table_with_branch(catalog, "default.empty_plan_window_per_branch").transaction()
1409+
with pytest.warns(UserWarning): # neither delete matches anything at staging time
1410+
tx.delete("x > 45")
1411+
tx.delete("x > 45", branch="b")
1412+
1413+
catalog.load_table("default.empty_plan_window_per_branch").append(pa.table({"x": [50]}), branch="b")
1414+
1415+
with pytest.raises(ValidationException):
1416+
tx.commit_transaction()
1417+
1418+
1419+
def test_retry_with_two_writes_to_one_branch_succeeds(catalog: Catalog) -> None:
1420+
"""The second write to a branch starts from a staged snapshot; its retry window must use the committed base."""
1421+
import pyarrow as pa
1422+
1423+
catalog.create_namespace("default")
1424+
tx = _table_with_branch(catalog, "default.retry_two_writes_one_branch").transaction()
1425+
tx.append(pa.table({"x": [9]}))
1426+
tx.delete("x == 0")
1427+
1428+
# Non-conflicting concurrent append to the same branch forces a retry.
1429+
catalog.load_table("default.retry_two_writes_one_branch").append(pa.table({"x": [5]}))
1430+
1431+
tx.commit_transaction()
1432+
1433+
rows = sorted(catalog.load_table("default.retry_two_writes_one_branch").scan().to_arrow()["x"].to_pylist())
1434+
assert rows == [1, 5, 9]

0 commit comments

Comments
 (0)