diff --git a/MAINTENANCE-FORK.md b/MAINTENANCE-FORK.md index 1915335e27..90ec94ba57 100644 --- a/MAINTENANCE-FORK.md +++ b/MAINTENANCE-FORK.md @@ -116,6 +116,7 @@ where the mirror exists only for Zamboni's tests and demo, never in production. | The S3 remote signer is registered on the session, once per signer: concurrent writes no longer go out unsigned (403), and each table signs through its own endpoint. A signed filesystem is shared only while a FileIO holds it, so its client, signer and the catalog's auth session are released with the last table using it, not kept until exit (`_s3`, `_signed_s3_filesystem`, `_SIGNED_FILESYSTEMS`) | [ZMBNI-51](https://github.com/paulcaron16k/Zamboni/issues/51), ELT-1269, [ZMBNI-246](https://github.com/paulcaron16k/Zamboni/issues/246) | `signer.py`, installed into `SCHEME_TO_FS` by `CatalogSession` ([#242](https://github.com/paulcaron16k/Zamboni/pull/242); released when unused since #246) | `_pyiceberg_signer.py`, installed into `SCHEME_TO_FS` before `load_catalog` (ELT-1269); released when unused since ELT-1279 | | S3 requests are signed off the event loop, on a daemon thread per call, at most ten at once and started only once a per-loop slot is free, so one signature no longer stalls every request in the process; and signed filesystems draw on one shared object-store connector and one signing `requests.Session`, while each table keeps its own client and signs through its own endpoint (`_SignOffTheLoop`, `_shared_pool_session`, `_shared_connector`, `_signing_session`). The connector subclass relies on private aiobotocore names, checked first, with a fallback to a pool per table, and reads a client's sessions again under aiobotocore's `_sessions_lock`, so a request queued behind `close()` cannot reopen the client (ZMBNI-260) | [ZMBNI-250](https://github.com/paulcaron16k/Zamboni/issues/250), [ZMBNI-260](https://github.com/paulcaron16k/Zamboni/issues/260), ELT-1281, ELT-1283, ELT-1311 | `signer.py` (#250, #260); stands down only where the behavioural probe `s3_signing_overlaps` also holds | `_pyiceberg_signer.py` (ELT-1283, ELT-1281, ELT-1311); level with this branch since ELT-1311 (omantra/target-iceberg#162, merged 2026-11-03), which brought over the per-loop gate, the conditional `_create_connector` await and the ZMBNI-260 read | | An overwrite can refuse data files concurrently added under a conflict-detection filter, so two upserts of one key cannot both commit (`validate_no_conflicting_data_files`). Like Java's, it reads file bounds, so disjoint keys whose ranges overlap, or a filter of more than 200 keys, are refused once and re-resolved | ELT-1271 | Not needed in production, which has no merge-on-read upsert | `_AppendDeleteFiles._validate_concurrency`; `_OverwriteFiles` on this branch (ELT-1271) | +| An upsert commits one overwrite snapshot instead of an overwrite plus an unvalidated append, validated from the snapshot its matches were read from, with its match filter as the conflict-detection filter for added data and delete files, at the isolation level in the new `write.merge.isolation-level`; so two upserts inserting one new key cannot both commit it (`Transaction.upsert`, `_rewrite_data_files`, `_SnapshotProducer._isolation_level`). As in Java, `snapshot` isolation validates deletes only, and the check reads file bounds | ELT-1277 | Not needed in production: Zamboni does not call `upsert` (only `test_safety.py` does, as a regression test) | Pending — ELT-1277 mirror in `writer.py` / `_pyiceberg_mor.py` | Rows three to five together are what let PyIceberg write a merge-on-read table at all: five rows, a delete file naming two positions, and a scan that returns the other three. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index bd4470d085..73c71ef60d 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -133,6 +133,7 @@ These properties control conflict detection behavior during concurrent writes. | -------------------------------- | -------------------------------- | ------------ | ----------------------------------------------------------------------------------------------- | | `write.delete.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for delete operations. Under `serializable`, concurrent appends to affected partitions cause `ValidationException`. Under `snapshot`, only conflicting deletes are rejected. | | `write.update.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for overwrite operations. Same semantics as `write.delete.isolation-level`. | +| `write.merge.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for upsert operations. Under `serializable`, data or delete files concurrently added that could hold the upserted keys cause `ValidationException`. Under `snapshot`, only conflicting deletes are rejected. | ## FileIO diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7da0e59d26..6ec4fecf06 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -62,6 +62,7 @@ from pyiceberg.table.name_mapping import NameMapping from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef from pyiceberg.table.snapshots import ( + IsolationLevel, Operation, Snapshot, SnapshotLogEntry, @@ -92,7 +93,7 @@ update_table_metadata, ) from pyiceberg.table.update.schema import UpdateSchema -from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles +from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles, _OverwriteFiles from pyiceberg.table.update.sorting import UpdateSortOrder from pyiceberg.table.update.spec import UpdateSpec from pyiceberg.table.update.statistics import UpdateStatistics @@ -243,6 +244,7 @@ class TableProperties: WRITE_DELETE_ISOLATION_LEVEL = "write.delete.isolation-level" WRITE_UPDATE_ISOLATION_LEVEL = "write.update.isolation-level" + WRITE_MERGE_ISOLATION_LEVEL = "write.merge.isolation-level" WRITE_ISOLATION_LEVEL_DEFAULT = "serializable" @@ -767,8 +769,6 @@ def delete( case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive branch: Branch Reference to run the delete operation """ - from pyiceberg.io.pyarrow import ArrowScan, _dataframe_to_data_files, _expression_to_complementary_pyarrow - if ( self.table_metadata.properties.get(TableProperties.DELETE_MODE, TableProperties.DELETE_MODE_DEFAULT) == TableProperties.DELETE_MODE_MERGE_ON_READ @@ -785,52 +785,14 @@ def delete( # Check if there are any files that require an actual rewrite of a data file if delete_snapshot.rewrites_needed is True: - bound_delete_filter = bind(self.table_metadata.schema(), delete_filter, case_sensitive) - preserve_row_filter = _expression_to_complementary_pyarrow(bound_delete_filter, self.table_metadata.schema()) - file_scan = self._scan(row_filter=delete_filter, case_sensitive=case_sensitive) if branch is not None: file_scan = file_scan.use_ref(branch) - files = file_scan.plan_files() commit_uuid = uuid.uuid4() - counter = itertools.count(0) - - replaced_files: list[tuple[DataFile, list[DataFile]]] = [] - # This will load the Parquet file into memory, including: - # - Filter out the rows based on the delete filter - # - Projecting it to the current schema - # - Applying the positional deletes if they are there - # When writing - # - Apply the latest partition-spec - # - And sort order when added - for original_file in files: - df = ArrowScan( - table_metadata=self.table_metadata, - io=self._table.io, - projected_schema=self.table_metadata.schema(), - row_filter=AlwaysTrue(), - ).to_table(tasks=[original_file]) - filtered_df = df.filter(preserve_row_filter) - - # Only rewrite if there are records being deleted - if len(filtered_df) == 0: - replaced_files.append((original_file.file, [])) - elif len(df) != len(filtered_df): - replaced_files.append( - ( - original_file.file, - list( - _dataframe_to_data_files( - io=self._table.io, - df=filtered_df, - table_metadata=self.table_metadata, - write_uuid=commit_uuid, - counter=counter, - ) - ), - ) - ) + replaced_files = self._rewrite_data_files( + file_scan.plan_files(), delete_filter, case_sensitive, commit_uuid, itertools.count(0) + ) if len(replaced_files) > 0: with self.update_snapshot( @@ -849,6 +811,57 @@ def delete( if not delete_snapshot.files_affected and not delete_snapshot.rewrites_needed: warnings.warn("Delete operation did not match any records", stacklevel=2) + def _rewrite_data_files( + self, + tasks: Iterable[FileScanTask], + delete_filter: BooleanExpression, + case_sensitive: bool, + write_uuid: uuid.UUID, + counter: itertools.count[int], + ) -> list[tuple[DataFile, list[DataFile]]]: + """Rewrite the data files of the scan tasks without the rows that match the filter. + + Each file is loaded into memory, with its positional deletes applied and projected to the current + schema, and the remaining rows are written under the latest partition spec. + + Returns: + Each data file holding a matching row, with the data files replacing it (none when no row remains). + """ + from pyiceberg.io.pyarrow import ArrowScan, _dataframe_to_data_files, _expression_to_complementary_pyarrow + + bound_delete_filter = bind(self.table_metadata.schema(), delete_filter, case_sensitive) + preserve_row_filter = _expression_to_complementary_pyarrow(bound_delete_filter, self.table_metadata.schema()) + + replaced_files: list[tuple[DataFile, list[DataFile]]] = [] + for original_file in tasks: + df = ArrowScan( + table_metadata=self.table_metadata, + io=self._table.io, + projected_schema=self.table_metadata.schema(), + row_filter=AlwaysTrue(), + ).to_table(tasks=[original_file]) + filtered_df = df.filter(preserve_row_filter) + + # Only rewrite if there are records being deleted + if len(filtered_df) == 0: + replaced_files.append((original_file.file, [])) + elif len(df) != len(filtered_df): + replaced_files.append( + ( + original_file.file, + list( + _dataframe_to_data_files( + io=self._table.io, + df=filtered_df, + table_metadata=self.table_metadata, + write_uuid=write_uuid, + counter=counter, + ) + ), + ) + ) + return replaced_files + def upsert( self, df: pa.Table, @@ -861,6 +874,11 @@ def upsert( ) -> UpsertResult: """Shorthand API for performing an upsert to an iceberg table. + The rewritten data files, the updated rows and the inserted rows are committed as one snapshot, validated + at the isolation level in `write.merge.isolation-level`. Under `serializable` (the default), a concurrent + commit that added data or delete files which could hold rows with the upserted keys makes it fail with a + ValidationException; under `snapshot`, only delete files are checked. + Args: df: The input dataframe to upsert with the table's data. join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. @@ -900,7 +918,7 @@ def upsert( except ModuleNotFoundError as e: raise ModuleNotFoundError("For writes PyArrow needs to be installed") from e - from pyiceberg.io.pyarrow import expression_to_pyarrow + from pyiceberg.io.pyarrow import _dataframe_to_data_files, expression_to_pyarrow from pyiceberg.table import upsert_util if join_cols is None: @@ -980,20 +998,67 @@ def upsert( update_row_cnt = 0 insert_row_cnt = 0 + commit_uuid = uuid.uuid4() + counter = itertools.count(0) + replaced_files: list[tuple[DataFile, list[DataFile]]] = [] + rows_to_write: list[pa.Table] = [] + if batches_to_overwrite: rows_to_update = pa.concat_tables(batches_to_overwrite) update_row_cnt = len(rows_to_update) - self.overwrite( - rows_to_update, - overwrite_filter=Or(*overwrite_predicates) if len(overwrite_predicates) > 1 else overwrite_predicates[0], - branch=branch, - snapshot_properties=snapshot_properties, + overwrite_filter = Or(*overwrite_predicates) if len(overwrite_predicates) > 1 else overwrite_predicates[0] + # Plan from the scan the matches were read from, so the files rewritten are the files matched + replaced_files = self._rewrite_data_files( + matched_iceberg_record_batches_scan.update(row_filter=overwrite_filter).plan_files(), + overwrite_filter, + case_sensitive, + commit_uuid, + counter, ) + rows_to_write.append(rows_to_update) if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) - if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + if insert_row_cnt > 0: + rows_to_write.append(rows_to_insert) + + if not replaced_files and not rows_to_write: + return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) + + # Commit the rewritten files, the updated rows and the inserted rows as one snapshot, as a copy-on-write + # MERGE does in Java. A separate append for the inserted rows would skip validation: two writers inserting + # the same new key would each commit it, the second replayed onto the first by the commit retry. + with _OverwriteFiles( + operation=Operation.OVERWRITE if replaced_files else Operation.APPEND, + transaction=self, + io=self._table.io, + branch=branch, + commit_uuid=commit_uuid, + snapshot_properties=snapshot_properties, + ) as overwrite_snapshot: + # Validate from the snapshot the matches were read from, with every key matched against as the + # conflict-detection filter, at the isolation level of a merge + scan_snapshot = matched_iceberg_record_batches_scan.snapshot() + overwrite_snapshot._starting_snapshot_id = scan_snapshot.snapshot_id if scan_snapshot is not None else None + overwrite_snapshot._isolation_level = IsolationLevel( + self.table_metadata.properties.get( + TableProperties.WRITE_MERGE_ISOLATION_LEVEL, TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT + ) + ) + overwrite_snapshot.delete_by_predicate(matched_predicate, case_sensitive) + if rows_to_write: + for data_file in _dataframe_to_data_files( + table_metadata=self.table_metadata, + df=pa.concat_tables(rows_to_write), + io=self._table.io, + write_uuid=commit_uuid, + counter=counter, + ): + overwrite_snapshot.append_data_file(data_file) + for original_data_file, replaced_data_files in replaced_files: + overwrite_snapshot.delete_data_file(original_data_file) + for replaced_data_file in replaced_data_files: + overwrite_snapshot.append_data_file(replaced_data_file) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index afceec4643..25faadd885 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -59,6 +59,7 @@ from pyiceberg.schema import Schema from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( + IsolationLevel, Operation, Snapshot, SnapshotSummaryCollector, @@ -159,6 +160,7 @@ class _SnapshotProducer(UpdateTableMetadata[U], Generic[U]): _case_sensitive: bool _commit_window: CommitWindow | None _conflict_detection_filter: BooleanExpression | None + _isolation_level: IsolationLevel | None _written_manifests: list[str] _uncommitted_manifests: list[str] _written_manifest_lists: list[str] @@ -198,6 +200,8 @@ def __init__( self._commit_window = None self._conflict_detection_filter = None self._isolation_operation: Operation = Operation.DELETE + # Set by a command whose isolation level is not keyed by its snapshot operation, such as an upsert + self._isolation_level = None def _validate_target_branch(self, branch: str | None) -> str | None: # if branch is none, write will be written into a staging snapshot @@ -514,7 +518,6 @@ def _validate_concurrency(self) -> None: Subclasses that do not require validation (e.g. fast append) should override with a no-op. """ - from pyiceberg.table.snapshots import IsolationLevel from pyiceberg.table.update.validate import ( _validate_added_data_files, _validate_data_files_exist, @@ -534,7 +537,11 @@ def _validate_concurrency(self) -> None: return table = self._transaction._table - isolation_level = table.metadata.isolation_level(self._isolation_operation) + isolation_level = ( + self._isolation_level + if self._isolation_level is not None + else table.metadata.isolation_level(self._isolation_operation) + ) conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None if isolation_level == IsolationLevel.SERIALIZABLE and conflict_detection_filter is not None: diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index ab95457e23..a100df02be 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -49,6 +49,7 @@ def test_commit_retry_table_properties() -> None: def test_isolation_level_table_properties() -> None: assert TableProperties.WRITE_DELETE_ISOLATION_LEVEL == "write.delete.isolation-level" assert TableProperties.WRITE_UPDATE_ISOLATION_LEVEL == "write.update.isolation-level" + assert TableProperties.WRITE_MERGE_ISOLATION_LEVEL == "write.merge.isolation-level" assert TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT == "serializable" diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 78ddbc7c5c..d54d998bd3 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,17 +23,17 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import NoSuchTableError, ValidationException from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral -from pyiceberg.io.pyarrow import schema_to_pyarrow +from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow, write_positional_deletes from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, UpsertResult +from pyiceberg.table import Table, TableProperties, UpsertResult from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -375,7 +375,8 @@ def test_upsert_with_identifier_fields(catalog: Catalog) -> None: ) upd = tbl.upsert(df) - expected_operations = [Operation.APPEND, Operation.OVERWRITE, Operation.APPEND, Operation.APPEND] + # One snapshot: the rewritten file, the updated row and the inserted row are committed together + expected_operations = [Operation.APPEND, Operation.OVERWRITE] assert upd.rows_updated == 1 assert upd.rows_inserted == 1 @@ -920,10 +921,181 @@ def test_upsert_snapshot_properties(catalog: Catalog) -> None: # Verify properties are on the snapshots created by upsert snapshots = list(tbl.snapshots()) - # Upsert should have created additional snapshots (overwrite + append) - assert len(snapshots) > initial_snapshot_count + # Upsert should have created one additional snapshot + assert len(snapshots) == initial_snapshot_count + 1 # Check that all new snapshots have the snapshot_properties for snapshot in snapshots[initial_snapshot_count:]: assert snapshot.summary is not None assert snapshot.summary.additional_properties.get("test_prop") == "test_value" + + +_KEYED_SCHEMA = Schema( + NestedField(1, "id", LongType(), required=True), + NestedField(2, "name", StringType(), required=True), + identifier_field_ids=[1], +) + + +def _keyed_rows(*rows: tuple[int, str]) -> pa_table: + return pa.Table.from_pylist([{"id": key, "name": name} for key, name in rows], schema=schema_to_pyarrow(_KEYED_SCHEMA)) + + +def _keyed_table(catalog: Catalog, identifier: str, properties: dict[str, str] | None = None) -> Table: + """Three rows in one data file, so the row with key 2 is at position 1.""" + _drop_table(catalog, identifier) + table = catalog.create_table(identifier, schema=_KEYED_SCHEMA, properties={"format-version": "2", **(properties or {})}) + table.append(_keyed_rows((1, "a"), (2, "b"), (3, "c"))) + return table + + +def _rows(catalog: Catalog, identifier: str) -> list[tuple[int, str]]: + return sorted((row["id"], row["name"]) for row in catalog.load_table(identifier).scan().to_arrow().to_pylist()) + + +def _merge_on_read_upsert(table: Table, key: int, name: str, position: int | None) -> None: + """Upsert one row as a merge-on-read writer does: delete the old row by position, if any, and add the new one.""" + tx = table.transaction() + update = tx.update_snapshot().overwrite() + update.validate_no_conflicting_data_files(EqualTo("id", key)) + if position is not None: + data_file = next(iter(table.scan().plan_files())).file + update.append_data_file(write_positional_deletes(table.io, table.metadata, [(data_file.file_path, position)])) + for data_file in _dataframe_to_data_files(table.metadata, _keyed_rows((key, name)), table.io, update.commit_uuid): + update.append_data_file(data_file) + update.commit() + tx.commit_transaction() + + +@pytest.mark.parametrize("merge_on_read", [False, True], ids=["copy_on_write_winner", "merge_on_read_winner"]) +@pytest.mark.parametrize(("key", "position"), [(4, None), (2, 1)], ids=["new_key", "existing_key"]) +def test_concurrent_upserts_of_one_key_commit_it_once( + catalog: Catalog, merge_on_read: bool, key: int, position: int | None +) -> None: + """Two writers resolve the same key against the same base snapshot. + + Replaying the loser onto the winner would commit the key a second time, so the loser's retry is + refused, and its caller re-resolves the upsert against the winner's snapshot. + """ + identifier = "default.test_concurrent_upserts_of_one_key" + table = _keyed_table(catalog, identifier) + loser = catalog.load_table(identifier) + + if merge_on_read: + _merge_on_read_upsert(table, key, "winner", position) + else: + table.upsert(_keyed_rows((key, "winner"))) + snapshot_count = len(catalog.load_table(identifier).snapshots()) + + with pytest.raises(ValidationException): + loser.upsert(_keyed_rows((key, "loser"))) + + expected = {1: "a", 2: "b", 3: "c", key: "winner"} + assert _rows(catalog, identifier) == sorted(expected.items()) + assert len(catalog.load_table(identifier).snapshots()) == snapshot_count + + res = catalog.load_table(identifier).upsert(_keyed_rows((key, "loser"))) + assert_upsert_result(res, expected_updated=1, expected_inserted=0) + assert _rows(catalog, identifier) == sorted({**expected, key: "loser"}.items()) + + +def test_an_upsert_validates_the_keys_it_inserts_as_well_as_those_it_updates(catalog: Catalog) -> None: + """The conflict-detection filter is every key the upsert matched against, not only the keys it updated.""" + identifier = "default.test_upsert_validates_inserted_keys" + table = _keyed_table(catalog, identifier) + loser = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + + with pytest.raises(ValidationException): + loser.upsert(_keyed_rows((2, "loser"), (4, "loser"))) + + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c"), (4, "winner")] + + +def test_concurrent_upserts_of_disjoint_keys_both_commit(catalog: Catalog) -> None: + """The check is scoped to the keys the upsert matched against, so a writer on other keys does not conflict.""" + identifier = "default.test_concurrent_upserts_of_disjoint_keys" + table = _keyed_table(catalog, identifier) + other = catalog.load_table(identifier) + + table.upsert(_keyed_rows((1, "winner"))) + res = other.upsert(_keyed_rows((4, "other"))) + + assert_upsert_result(res, expected_updated=0, expected_inserted=1) + assert _rows(catalog, identifier) == [(1, "winner"), (2, "b"), (3, "c"), (4, "other")] + + +def test_upsert_isolation_level_is_read_from_the_merge_property(catalog: Catalog) -> None: + """An upsert is a merge, so `write.merge.isolation-level` governs it rather than the delete or update levels.""" + identifier = "default.test_upsert_merge_isolation_level" + table = _keyed_table( + catalog, + identifier, + { + TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "serializable", + TableProperties.WRITE_DELETE_ISOLATION_LEVEL: "snapshot", + TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "snapshot", + }, + ) + loser = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + + with pytest.raises(ValidationException, match="Added data files were found matching the filter"): + loser.upsert(_keyed_rows((4, "loser"))) + + +def test_upsert_under_snapshot_isolation_does_not_validate_added_data(catalog: Catalog) -> None: + """As in Spark, snapshot isolation does not check for data added concurrently, so both inserts commit.""" + identifier = "default.test_upsert_snapshot_isolation_added_data" + table = _keyed_table( + catalog, + identifier, + { + TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "snapshot", + TableProperties.WRITE_DELETE_ISOLATION_LEVEL: "serializable", + TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "serializable", + }, + ) + other = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + res = other.upsert(_keyed_rows((4, "other"))) + + assert_upsert_result(res, expected_updated=0, expected_inserted=1) + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c"), (4, "other"), (4, "winner")] + + +def test_upsert_under_snapshot_isolation_still_validates_deletes(catalog: Catalog) -> None: + """As in Spark, snapshot isolation still refuses an upsert whose rows were concurrently deleted.""" + identifier = "default.test_upsert_snapshot_isolation_deletes" + table = _keyed_table(catalog, identifier, {TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "snapshot"}) + loser = catalog.load_table(identifier) + + _merge_on_read_upsert(table, 2, "winner", position=1) + + with pytest.raises(ValidationException, match="Found new conflicting delete files"): + loser.upsert(_keyed_rows((2, "loser"))) + + assert _rows(catalog, identifier) == [(1, "a"), (2, "winner"), (3, "c")] + + +def test_upsert_on_a_branch(catalog: Catalog) -> None: + identifier = "default.test_upsert_on_a_branch" + table = _keyed_table(catalog, identifier) + main_snapshot_id = table.metadata.current_snapshot_id + assert main_snapshot_id is not None + table.manage_snapshots().create_branch(snapshot_id=main_snapshot_id, branch_name="dev").commit() + + res = table.upsert(_keyed_rows((2, "B"), (4, "d")), branch="dev") + + assert_upsert_result(res, expected_updated=1, expected_inserted=1) + assert table.metadata.current_snapshot_id == main_snapshot_id + dev = table.snapshot_by_name("dev") + assert dev is not None + assert dev.parent_snapshot_id == main_snapshot_id + assert dev.summary is not None and dev.summary.operation == Operation.OVERWRITE + rows = table.scan(snapshot_id=dev.snapshot_id).to_arrow().to_pylist() + assert sorted((row["id"], row["name"]) for row in rows) == [(1, "a"), (2, "B"), (3, "c"), (4, "d")] + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c")]