From 82d50d5e6149ad701925b401724f4e9c2d57a203 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 01:27:48 +0530 Subject: [PATCH 1/2] Fail a retried upsert instead of inserting a concurrent key A commit retry replays the append that was planned before the concurrent commit, and the fast append producer skips concurrency validation, so a key another writer inserted after the upsert read ends up inserted twice. Record the filter for the rows an upsert inserts on the append producer and validate the files added since the starting snapshot against it when the commit is rebuilt for a retry. The upsert now fails with a CommitFailedException instead of leaving a duplicate key. --- pyiceberg/table/__init__.py | 13 +++++++++++- pyiceberg/table/update/snapshot.py | 34 +++++++++++++++++++++++++++++- tests/table/test_upsert.py | 29 +++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..3ebbbc1c4f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -512,6 +512,8 @@ def append( df: pa.Table | pa.RecordBatchReader, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, + _upsert_insert_filter: BooleanExpression | None = None, + _case_sensitive: bool = True, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. @@ -576,6 +578,9 @@ def append( ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + if _upsert_insert_filter is not None: + append_files.set_upsert_insert_filter(_upsert_insert_filter, _case_sensitive) + # For pa.Table we can short-circuit empty inputs cheaply. For a # RecordBatchReader the stream is consumed lazily by # _dataframe_to_data_files and an empty reader simply yields zero @@ -1014,7 +1019,13 @@ def upsert( if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + self.append( + rows_to_insert, + branch=branch, + snapshot_properties=snapshot_properties, + _upsert_insert_filter=upsert_util.create_match_filter(rows_to_insert, join_cols), + _case_sensitive=case_sensitive, + ) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..69f6bc71fc 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -29,6 +29,7 @@ from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.exceptions import ( + CommitFailedException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, @@ -690,8 +691,39 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): + _upsert_insert_filter: BooleanExpression | None = None + + def set_upsert_insert_filter(self, insert_filter: BooleanExpression, case_sensitive: bool) -> None: + self._upsert_insert_filter = insert_filter + self._case_sensitive = case_sensitive + def _validate_concurrency(self) -> None: - """Skip validation; appends do not conflict with other operations.""" + """Skip validation for plain appends. + + An upsert registers the filter for the rows it inserts, so a retry fails instead of + inserting a key that another writer added after the upsert was planned. + """ + if self._upsert_insert_filter is None or self._commit_window is None or self._commit_window.is_empty(): + return + + from pyiceberg.table.update.validate import _validate_added_data_files + + catalog_head = self._commit_window.head + if catalog_head is None: + return + + try: + _validate_added_data_files( + self._transaction._table, + catalog_head, + self._upsert_insert_filter, + self._commit_window.base, + self._case_sensitive, + ) + except ValidationException as e: + raise CommitFailedException( + "Cannot retry the upsert because a concurrent commit added rows matching the rows to insert" + ) from e def _existing_manifests(self) -> list[ManifestFile]: """To determine if there are any existing manifest files. diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..596dd41d6d 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,7 +23,7 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import CommitFailedException, NoSuchTableError from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral from pyiceberg.io.pyarrow import schema_to_pyarrow @@ -33,7 +33,7 @@ from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -427,6 +427,31 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None: assert upd.rows_inserted == 4 +def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None: + identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema)) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + with pytest.raises(CommitFailedException): + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + def test_create_match_filter_single_condition() -> None: """ Test create_match_filter with a composite key where the source yields exactly one unique key. From e38742b3221d998b1036c836d717dbb1dd31bcf2 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 00:53:33 +0530 Subject: [PATCH 2/2] Validate conflicting data on upsert retries like an update Address the review feedback on the upsert retry check: - Gate the check on write.update.isolation-level, so both halves of an upsert follow the same isolation setting. - Let the ValidationException propagate instead of converting it to a CommitFailedException, matching the update half. - Drop the private Transaction.append arguments and let upsert configure the append producer through validate_no_conflicting_data. The filter's case sensitivity is stored on the producer instead of overwriting its inherited value. --- pyiceberg/table/__init__.py | 48 +++++++++++++++--------------- pyiceberg/table/update/snapshot.py | 41 ++++++++++++------------- tests/table/test_upsert.py | 34 +++++++++++++++++++-- 3 files changed, 76 insertions(+), 47 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ebbbc1c4f..1348380b28 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -512,8 +512,6 @@ def append( df: pa.Table | pa.RecordBatchReader, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, - _upsert_insert_filter: BooleanExpression | None = None, - _case_sensitive: bool = True, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. @@ -564,7 +562,7 @@ def append( except ModuleNotFoundError as e: raise ModuleNotFoundError("For writes PyArrow needs to be installed") from e - from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") @@ -578,20 +576,24 @@ def append( ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: - if _upsert_insert_filter is not None: - append_files.set_upsert_insert_filter(_upsert_insert_filter, _case_sensitive) - - # For pa.Table we can short-circuit empty inputs cheaply. For a - # RecordBatchReader the stream is consumed lazily by - # _dataframe_to_data_files and an empty reader simply yields zero - # data files (the snapshot is still committed for symmetry with the - # pa.Table case where empty inputs also produce a snapshot). - if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0: - data_files = _dataframe_to_data_files( - table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io - ) - for data_file in data_files: - append_files.append_data_file(data_file) + self._write_append_data(df, append_files) + + def _write_append_data(self, df: pa.Table | pa.RecordBatchReader, append_files: _FastAppendFiles) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import _dataframe_to_data_files + + # For pa.Table we can short-circuit empty inputs cheaply. For a + # RecordBatchReader the stream is consumed lazily by + # _dataframe_to_data_files and an empty reader simply yields zero + # data files (the snapshot is still committed for symmetry with the + # pa.Table case where empty inputs also produce a snapshot). + if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0: + data_files = _dataframe_to_data_files( + table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io + ) + for data_file in data_files: + append_files.append_data_file(data_file) def dynamic_partition_overwrite( self, df: pa.Table, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH @@ -1019,13 +1021,11 @@ def upsert( if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) if rows_to_insert: - self.append( - rows_to_insert, - branch=branch, - snapshot_properties=snapshot_properties, - _upsert_insert_filter=upsert_util.create_match_filter(rows_to_insert, join_cols), - _case_sensitive=case_sensitive, - ) + with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + append_files.validate_no_conflicting_data( + upsert_util.create_match_filter(rows_to_insert, join_cols), case_sensitive + ) + self._write_append_data(rows_to_insert, append_files) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 69f6bc71fc..fbd4592401 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -29,7 +29,6 @@ from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.exceptions import ( - CommitFailedException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, @@ -691,39 +690,41 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): - _upsert_insert_filter: BooleanExpression | None = None + _conflict_detection_filter: BooleanExpression | None = None + _conflict_detection_case_sensitive: bool = True - def set_upsert_insert_filter(self, insert_filter: BooleanExpression, case_sensitive: bool) -> None: - self._upsert_insert_filter = insert_filter - self._case_sensitive = case_sensitive + def validate_no_conflicting_data(self, data_filter: BooleanExpression, case_sensitive: bool = True) -> None: + """Fail a retry when a concurrent commit added data matching the filter.""" + self._conflict_detection_filter = data_filter + self._conflict_detection_case_sensitive = case_sensitive def _validate_concurrency(self) -> None: """Skip validation for plain appends. - An upsert registers the filter for the rows it inserts, so a retry fails instead of - inserting a key that another writer added after the upsert was planned. + An append that declares a conflict detection filter fails when a concurrent commit added + data matching it, instead of replaying the append on retry. """ - if self._upsert_insert_filter is None or self._commit_window is None or self._commit_window.is_empty(): + if self._conflict_detection_filter is None or self._commit_window is None or self._commit_window.is_empty(): return + from pyiceberg.table.snapshots import IsolationLevel from pyiceberg.table.update.validate import _validate_added_data_files + table = self._transaction._table + if table.metadata.isolation_level(Operation.OVERWRITE) != IsolationLevel.SERIALIZABLE: + return + catalog_head = self._commit_window.head if catalog_head is None: return - try: - _validate_added_data_files( - self._transaction._table, - catalog_head, - self._upsert_insert_filter, - self._commit_window.base, - self._case_sensitive, - ) - except ValidationException as e: - raise CommitFailedException( - "Cannot retry the upsert because a concurrent commit added rows matching the rows to insert" - ) from e + _validate_added_data_files( + table, + catalog_head, + self._conflict_detection_filter, + self._commit_window.base, + self._conflict_detection_case_sensitive, + ) def _existing_manifests(self) -> list[ManifestFile]: """To determine if there are any existing manifest files. diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 596dd41d6d..3a666d5351 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,13 +23,13 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import CommitFailedException, NoSuchTableError +from pyiceberg.exceptions import NoSuchTableError, ValidationException from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral from pyiceberg.io.pyarrow import schema_to_pyarrow from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, UpsertResult +from pyiceberg.table import Table, TableProperties, UpsertResult from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update from pyiceberg.transforms import DayTransform @@ -443,7 +443,7 @@ def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None writer_b = catalog.load_table(identifier) writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) - with pytest.raises(CommitFailedException): + with pytest.raises(ValidationException): writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ @@ -452,6 +452,34 @@ def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None ] +def test_upsert_retry_under_snapshot_isolation_does_not_validate_inserted_keys(catalog: Catalog) -> None: + """The insert half follows the update isolation level, which tolerates concurrently added rows.""" + identifier = "default.test_upsert_retry_snapshot_isolation" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema, properties={TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "snapshot"}).append( + pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema) + ) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + def test_create_match_filter_single_condition() -> None: """ Test create_match_filter with a composite key where the source yields exactly one unique key.