From 1787b41bf0a17de43a6524ea5284d53a7f5b1ebe Mon Sep 17 00:00:00 2001 From: avison9 Date: Mon, 9 Nov 2026 15:51:54 +0100 Subject: [PATCH 1/2] Assert every ref a transaction writes, not just the first Transaction._stage de-duplicated requirements by type, so a transaction writing two branches only sent AssertRefSnapshotId for the first one and a concurrent change to the second was overwritten. Key ref assertions by ref name so each ref keeps its base assertion, as Java does. Closes #4183 --- pyiceberg/table/__init__.py | 10 +++-- tests/table/test_commit_retry.py | 73 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index df7aaf824f..40118478ad 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -306,10 +306,14 @@ def _stage( self._updates += updates # For the requirements, it does not make sense to add a requirement more than once - # For example, you cannot assert that the current schema has two different IDs - existing_requirements = {type(requirement) for requirement in self._requirements} + # For example, you cannot assert that the current schema has two different IDs. + # Ref assertions are the exception: they apply per ref, so each ref keeps its own first assertion. + def _requirement_key(requirement: TableRequirement) -> tuple[type[TableRequirement], str | None]: + return type(requirement), requirement.ref if isinstance(requirement, AssertRefSnapshotId) else None + + existing_requirements = {_requirement_key(requirement) for requirement in self._requirements} for new_requirement in requirements: - if type(new_requirement) not in existing_requirements: + if _requirement_key(new_requirement) not in existing_requirements: self._requirements = self._requirements + (new_requirement,) return self diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index ab95457e23..dff1071c9c 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -1290,3 +1290,76 @@ def test_negative_wait_properties_do_not_mask_commit_failure(catalog: Catalog) - result = catalog.load_table("default.negative_wait_test").scan().to_arrow() assert len(result) == 6 + + +def test_transaction_writing_two_branches_detects_concurrent_change_to_either(catalog: Catalog) -> None: + """A transaction writing two branches must assert the base snapshot of both refs. + + Writing the branch first used to drop the assertion for main, so a concurrent change to main was + overwritten (rollback undone, or an append lost on v1) instead of being caught and retried. + """ + import pyarrow as pa + + from pyiceberg.table import Table + + def rollback_main(table: Table, rollback_to: int) -> None: + table.manage_snapshots().rollback_to_snapshot(rollback_to).commit() + + def append_to_main(table: Table, _: int) -> None: + table.append(pa.table({"x": [4]})) + + catalog.create_namespace("default") + for format_version in (1, 2): + for concurrent_change, expected_main in ((rollback_main, [1, 3]), (append_to_main, [1, 2, 3, 4])): + identifier = f"default.two_branches_v{format_version}_{concurrent_change.__name__}" + tbl = catalog.create_table( + identifier, + schema=_test_schema(), + properties={ + "format-version": str(format_version), + TableProperties.COMMIT_MIN_RETRY_WAIT_MS: "1", + TableProperties.COMMIT_MAX_RETRY_WAIT_MS: "2", + }, + ) + tbl.append(pa.table({"x": [1]})) + first_snapshot_id = tbl.metadata.current_snapshot_id + assert first_snapshot_id is not None + tbl.append(pa.table({"x": [2]})) + second_snapshot_id = tbl.metadata.current_snapshot_id + assert second_snapshot_id is not None + tbl.manage_snapshots().create_branch(snapshot_id=second_snapshot_id, branch_name="b").commit() + + # Lands after `tbl` was loaded, so the transaction below starts from a stale main. + concurrent_change(catalog.load_table(identifier), first_snapshot_id) + + with tbl.transaction() as tx: + tx.append(pa.table({"x": [3]}), branch="b") + tx.append(pa.table({"x": [3]})) + + refreshed = catalog.load_table(identifier) + assert sorted(refreshed.scan().to_arrow()["x"].to_pylist()) == expected_main, identifier + assert sorted(refreshed.scan().use_ref("b").to_arrow()["x"].to_pylist()) == [1, 2, 3], identifier + + +def test_transaction_stages_one_ref_assertion_per_ref(catalog: Catalog) -> None: + """Each ref written in a transaction keeps exactly one assertion, holding its base snapshot id.""" + import pyarrow as pa + + from pyiceberg.table.update import AssertRefSnapshotId + + catalog.create_namespace("default") + tbl = catalog.create_table("default.ref_assertions", schema=_test_schema()) + tbl.append(pa.table({"x": [1]})) + base_snapshot_id = tbl.metadata.current_snapshot_id + assert base_snapshot_id is not None + tbl.manage_snapshots().create_branch(snapshot_id=base_snapshot_id, branch_name="b").commit() + + tx = tbl.transaction() + tx.append(pa.table({"x": [2]}), branch="b") + tx.append(pa.table({"x": [3]})) + # A second write to main asserts the snapshot staged above; only the base assertion must be kept. + tx.append(pa.table({"x": [4]})) + + ref_assertions = {r.ref: r.snapshot_id for r in tx._requirements if isinstance(r, AssertRefSnapshotId)} + assert ref_assertions == {"b": base_snapshot_id, "main": base_snapshot_id} + assert len([r for r in tx._requirements if isinstance(r, AssertRefSnapshotId)]) == 2 From 60d0a7eb0f0cfeb294536c0e61cbb34f435d7e85 Mon Sep 17 00:00:00 2001 From: avison9 Date: Mon, 9 Nov 2026 16:03:26 +0100 Subject: [PATCH 2/2] Validate each branch against its own commits on retry The retry path and the empty-output validation path resolved one CommitWindow from the first snapshot producer and reused it for every producer, so in a multi-branch transaction a delete or overwrite on a later branch was validated against the first branch's commits and could skip validation entirely. Resolve one window per target branch, based on the first producer on that branch, since later producers on the same branch start from a snapshot staged by this transaction. Closes #4316 --- pyiceberg/table/__init__.py | 31 +++++++------- tests/table/test_commit_retry.py | 69 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 14 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 40118478ad..e478411892 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1194,17 +1194,10 @@ def commit_transaction(self) -> Table: elif self._snapshot_producers: # An empty staged output (e.g. a delete whose plan matched nothing) skips the # commit loop above, so run concurrency validation explicitly before reporting success. - from pyiceberg.table.update.snapshot import CommitWindow - try: self._table.refresh() - commit_window = CommitWindow.resolve( - self._table.metadata, - self._snapshot_producers[0]._starting_snapshot_id, - self._snapshot_producers[0]._target_branch, - ) + self._resolve_commit_windows() for producer in self._snapshot_producers: - producer._commit_window = commit_window producer._validate_concurrency() except Exception: for producer in self._snapshot_producers: @@ -1224,20 +1217,30 @@ def _cleanup_uncommitted_manifests(self) -> None: for producer in self._snapshot_producers: producer._cleanup_uncommitted() + def _resolve_commit_windows(self) -> None: + """Resolve each producer's commit window against the refreshed metadata of its own branch. + + The window starts where the transaction first wrote to the branch. Later producers on the same + branch start from a snapshot staged by this transaction, which the catalog has never seen. + """ + from pyiceberg.table.update.snapshot import CommitWindow + + windows: dict[str | None, CommitWindow] = {} + for producer in self._snapshot_producers: + branch = producer._target_branch + if branch not in windows: + windows[branch] = CommitWindow.resolve(self._table.metadata, producer._starting_snapshot_id, branch) + producer._commit_window = windows[branch] + def _rebuild_snapshot_updates(self) -> None: """Rebuild snapshot updates for retry by re-executing registered producers.""" from pyiceberg.table.update import AddSnapshotUpdate, AssertRefSnapshotId, SetSnapshotRefUpdate - from pyiceberg.table.update.snapshot import CommitWindow self._updates = tuple(u for u in self._updates if not isinstance(u, (AddSnapshotUpdate, SetSnapshotRefUpdate))) self._requirements = tuple(r for r in self._requirements if not isinstance(r, AssertRefSnapshotId)) - starting_id = self._snapshot_producers[0]._starting_snapshot_id if self._snapshot_producers else None - target_branch = self._snapshot_producers[0]._target_branch if self._snapshot_producers else None - commit_window = CommitWindow.resolve(self._table.metadata, starting_id, target_branch) - + self._resolve_commit_windows() for producer in self._snapshot_producers: - producer._commit_window = commit_window producer._refresh_for_retry() producer._validate_concurrency() updates, requirements = producer._commit() diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index dff1071c9c..7bc30860ba 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -1363,3 +1363,72 @@ def test_transaction_stages_one_ref_assertion_per_ref(catalog: Catalog) -> None: ref_assertions = {r.ref: r.snapshot_id for r in tx._requirements if isinstance(r, AssertRefSnapshotId)} assert ref_assertions == {"b": base_snapshot_id, "main": base_snapshot_id} assert len([r for r in tx._requirements if isinstance(r, AssertRefSnapshotId)]) == 2 + + +def _table_with_branch(catalog: Catalog, identifier: str) -> Any: + import pyarrow as pa + + tbl = catalog.create_table( + identifier, + schema=_test_schema(), + properties={TableProperties.COMMIT_MIN_RETRY_WAIT_MS: "1", TableProperties.COMMIT_MAX_RETRY_WAIT_MS: "2"}, + ) + tbl.append(pa.table({"x": [0, 1]})) + snapshot_id = tbl.metadata.current_snapshot_id + assert snapshot_id is not None + tbl.manage_snapshots().create_branch(snapshot_id=snapshot_id, branch_name="b").commit() + return tbl + + +def test_retry_validates_each_branch_against_its_own_commits(catalog: Catalog) -> None: + """A retried delete on a second branch is validated against that branch, not the first producer's branch.""" + import pyarrow as pa + + catalog.create_namespace("default") + tx = _table_with_branch(catalog, "default.retry_window_per_branch").transaction() + tx.append(pa.table({"x": [9]})) + tx.delete("x == 1", branch="b") + + # A row matching the delete lands on b after the delete was planned; main is untouched. + catalog.load_table("default.retry_window_per_branch").append(pa.table({"x": [1]}), branch="b") + + with pytest.raises(ValidationException): + tx.commit_transaction() + + refreshed = catalog.load_table("default.retry_window_per_branch") + assert sorted(refreshed.scan().use_ref("b").to_arrow()["x"].to_pylist()) == [0, 1, 1] + assert sorted(refreshed.scan().to_arrow()["x"].to_pylist()) == [0, 1] + + +def test_empty_plan_validation_checks_each_branch_against_its_own_commits(catalog: Catalog) -> None: + """Empty-plan deletes on two branches are each validated against their own branch.""" + import pyarrow as pa + + catalog.create_namespace("default") + tx = _table_with_branch(catalog, "default.empty_plan_window_per_branch").transaction() + with pytest.warns(UserWarning): # neither delete matches anything at staging time + tx.delete("x > 45") + tx.delete("x > 45", branch="b") + + catalog.load_table("default.empty_plan_window_per_branch").append(pa.table({"x": [50]}), branch="b") + + with pytest.raises(ValidationException): + tx.commit_transaction() + + +def test_retry_with_two_writes_to_one_branch_succeeds(catalog: Catalog) -> None: + """The second write to a branch starts from a staged snapshot; its retry window must use the committed base.""" + import pyarrow as pa + + catalog.create_namespace("default") + tx = _table_with_branch(catalog, "default.retry_two_writes_one_branch").transaction() + tx.append(pa.table({"x": [9]})) + tx.delete("x == 0") + + # Non-conflicting concurrent append to the same branch forces a retry. + catalog.load_table("default.retry_two_writes_one_branch").append(pa.table({"x": [5]})) + + tx.commit_transaction() + + rows = sorted(catalog.load_table("default.retry_two_writes_one_branch").scan().to_arrow()["x"].to_pylist()) + assert rows == [1, 5, 9]