From 2306ac4d1eb15fac8048374ba96756549a3e7ba2 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 11 Nov 2026 14:50:00 +0530 Subject: [PATCH] Check whether a failed last commit attempt landed before cleanup --- pyiceberg/table/__init__.py | 32 +++++++++++--- tests/table/test_commit_retry.py | 75 +++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 7 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index df7aaf824f..444d7fa2e2 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -34,7 +34,7 @@ from pydantic import Field from pyiceberg.encryption.kms import KeyManagementClient -from pyiceberg.exceptions import CommitFailedException, ValidationException +from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, And, BooleanExpression, EqualTo, IsNull, Or, Reference from pyiceberg.expressions.visitors import ( ResidualEvaluator, @@ -1146,6 +1146,12 @@ def commit_transaction(self) -> Table: except CommitFailedException: elapsed_ms = (time.monotonic() - start_time) * 1000 if attempt == num_retries or not self._snapshot_producers or elapsed_ms >= total_timeout_ms: + # The catalog can apply a commit and still report failure, for example + # when the response is lost. Check before cleaning up, since cleaning + # up a snapshot that landed would leave the table unreadable. + if self._snapshot_producers and self._attempt_landed(): + self._cleanup_uncommitted_manifests() + break raise wait = min(min_wait_ms * (2**attempt), max_wait_ms) @@ -1158,11 +1164,7 @@ def commit_transaction(self) -> Table: ) time.sleep((wait + jitter) / 1000.0) - self._table.refresh() - if all( - self._table.metadata.snapshot_by_id(producer._snapshot_id) is not None - for producer in self._snapshot_producers - ): + if self._attempt_landed(): # A previous attempt actually landed even though it was reported as # failed (for example a lost response that the transport layer retried). # The snapshot id is stable across attempts, so finding it in the @@ -1215,6 +1217,24 @@ def commit_transaction(self) -> Table: return self._table + def _attempt_landed(self) -> bool: + """Refresh the table and return whether a snapshot written by this transaction is present. + + Raises: + CommitStateUnknownException: If the table cannot be refreshed, the outcome is unknown and + the files written for the attempt must not be deleted. + """ + try: + self._table.refresh() + except Exception as e: + raise CommitStateUnknownException( + f"Commit failed and the table could not be refreshed to check whether it landed: {e}" + ) from e + + return all( + self._table.metadata.snapshot_by_id(producer._snapshot_id) is not None for producer in self._snapshot_producers + ) + def _cleanup_uncommitted_manifests(self) -> None: """Clean up manifests from failed retry attempts after a successful commit.""" for producer in self._snapshot_producers: diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index ab95457e23..6a94b9fd1e 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -23,7 +23,7 @@ from pyiceberg.catalog import Catalog from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException from pyiceberg.schema import Schema -from pyiceberg.table import TableProperties, Transaction +from pyiceberg.table import Table, TableProperties, Transaction from pyiceberg.table.snapshots import IsolationLevel, Operation from pyiceberg.types import LongType, NestedField, StringType @@ -1290,3 +1290,76 @@ def test_negative_wait_properties_do_not_mask_commit_failure(catalog: Catalog) - result = catalog.load_table("default.negative_wait_test").scan().to_arrow() assert len(result) == 6 + + +def _commit_outcomes(catalog: Catalog, *outcomes: str) -> Any: + """Simulate a commit failing before ("conflict") or after ("lost") the catalog applied it.""" + real_commit = catalog.commit_table + remaining = iter(outcomes) + + def commit(*args: Any, **kwargs: Any) -> Any: + outcome = next(remaining, "ok") + if outcome == "conflict": + raise CommitFailedException("concurrent update") + result = real_commit(*args, **kwargs) + if outcome == "lost": + raise CommitFailedException("response lost after the commit landed") + return result + + return patch.object(catalog, "commit_table", side_effect=commit) + + +def _assert_single_readable_snapshot(catalog: Catalog, identifier: str, expected: list[dict[str, Any]]) -> None: + table = catalog.load_table(identifier) + assert len(table.snapshots()) == 1 + snapshot = table.current_snapshot() + assert snapshot is not None + assert table.io.new_input(snapshot.manifest_list).exists() + assert table.scan().to_arrow().to_pylist() == expected + + +def test_lost_response_on_the_last_attempt_keeps_the_landed_snapshot(catalog: Catalog) -> None: + """A commit that lands but reports failure on the last attempt is not cleaned up.""" + catalog.create_namespace("default") + table = catalog.create_table( + "default.lost_response_last_attempt", + schema=_test_schema(), + properties={TableProperties.COMMIT_NUM_RETRIES: "0"}, + ) + + import pyarrow as pa + + with _commit_outcomes(catalog, "lost"): + table.append(pa.table({"x": [1, 2, 3]})) + + _assert_single_readable_snapshot(catalog, "default.lost_response_last_attempt", [{"x": 1}, {"x": 2}, {"x": 3}]) + + +def test_lost_response_on_the_final_retry_keeps_the_landed_snapshot(catalog: Catalog) -> None: + """A landed retry is detected even when the retry budget is exhausted.""" + catalog.create_namespace("default") + table = catalog.create_table( + "default.lost_response_final_retry", + schema=_test_schema(), + properties={TableProperties.COMMIT_NUM_RETRIES: "1"}, + ) + + import pyarrow as pa + + with _commit_outcomes(catalog, "conflict", "lost"): + table.append(pa.table({"x": [1, 2, 3]})) + + _assert_single_readable_snapshot(catalog, "default.lost_response_final_retry", [{"x": 1}, {"x": 2}, {"x": 3}]) + + +def test_unreadable_commit_outcome_raises_commit_state_unknown(catalog: Catalog) -> None: + """When the table cannot be refreshed, the outcome is unknown and no file is deleted.""" + catalog.create_namespace("default") + table = catalog.create_table("default.unknown_commit_outcome", schema=_test_schema()) + + import pyarrow as pa + + with _commit_outcomes(catalog, "lost"): + with patch.object(Table, "refresh", side_effect=RuntimeError("catalog unavailable")): + with pytest.raises(CommitStateUnknownException): + table.append(pa.table({"x": [1, 2, 3]}))