From 9e28f2bc204cde4c48502dfc4549b43aa763d0a8 Mon Sep 17 00:00:00 2001 From: Paul Caron Date: Wed, 11 Nov 2026 14:49:22 -0500 Subject: [PATCH 1/2] Commit an upsert as one validated overwrite snapshot A copy-on-write upsert committed its matched rows through overwrite() and its new rows through a separate append(). The append skips validation, so when two writers insert the same new key, the commit retry replays the loser's append onto the winner's snapshot and the key is committed twice, whether the winner wrote copy-on-write or merge-on-read. Commit the rewritten files, the updated rows and the inserted rows as one overwrite snapshot, as Java's copy-on-write MERGE does: validated from the snapshot the matches were read from, with the match filter as the conflict-detection filter for added data and delete files, at the isolation level in the new write.merge.isolation-level. Under snapshot isolation only deletes are validated, as in Java. Like Java's, the check reads file bounds, so disjoint keys whose ranges overlap are refused once and re-resolved. An upsert that removes no file is labelled an append. The file rewrite moves out of delete() into _rewrite_data_files() so the upsert can share it. Co-Authored-By: Claude Opus 5.5 --- mkdocs/docs/configuration.md | 1 + pyiceberg/table/__init__.py | 169 ++++++++++++++++++-------- pyiceberg/table/update/snapshot.py | 11 +- tests/table/test_commit_retry.py | 1 + tests/table/test_upsert.py | 186 +++++++++++++++++++++++++++-- 5 files changed, 307 insertions(+), 61 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index bd4470d085..73c71ef60d 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -133,6 +133,7 @@ These properties control conflict detection behavior during concurrent writes. | -------------------------------- | -------------------------------- | ------------ | ----------------------------------------------------------------------------------------------- | | `write.delete.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for delete operations. Under `serializable`, concurrent appends to affected partitions cause `ValidationException`. Under `snapshot`, only conflicting deletes are rejected. | | `write.update.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for overwrite operations. Same semantics as `write.delete.isolation-level`. | +| `write.merge.isolation-level` | `{serializable,snapshot}` | serializable | Isolation level for upsert operations. Under `serializable`, data or delete files concurrently added that could hold the upserted keys cause `ValidationException`. Under `snapshot`, only conflicting deletes are rejected. | ## FileIO diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7da0e59d26..6ec4fecf06 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -62,6 +62,7 @@ from pyiceberg.table.name_mapping import NameMapping from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef from pyiceberg.table.snapshots import ( + IsolationLevel, Operation, Snapshot, SnapshotLogEntry, @@ -92,7 +93,7 @@ update_table_metadata, ) from pyiceberg.table.update.schema import UpdateSchema -from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles +from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles, _OverwriteFiles from pyiceberg.table.update.sorting import UpdateSortOrder from pyiceberg.table.update.spec import UpdateSpec from pyiceberg.table.update.statistics import UpdateStatistics @@ -243,6 +244,7 @@ class TableProperties: WRITE_DELETE_ISOLATION_LEVEL = "write.delete.isolation-level" WRITE_UPDATE_ISOLATION_LEVEL = "write.update.isolation-level" + WRITE_MERGE_ISOLATION_LEVEL = "write.merge.isolation-level" WRITE_ISOLATION_LEVEL_DEFAULT = "serializable" @@ -767,8 +769,6 @@ def delete( case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive branch: Branch Reference to run the delete operation """ - from pyiceberg.io.pyarrow import ArrowScan, _dataframe_to_data_files, _expression_to_complementary_pyarrow - if ( self.table_metadata.properties.get(TableProperties.DELETE_MODE, TableProperties.DELETE_MODE_DEFAULT) == TableProperties.DELETE_MODE_MERGE_ON_READ @@ -785,52 +785,14 @@ def delete( # Check if there are any files that require an actual rewrite of a data file if delete_snapshot.rewrites_needed is True: - bound_delete_filter = bind(self.table_metadata.schema(), delete_filter, case_sensitive) - preserve_row_filter = _expression_to_complementary_pyarrow(bound_delete_filter, self.table_metadata.schema()) - file_scan = self._scan(row_filter=delete_filter, case_sensitive=case_sensitive) if branch is not None: file_scan = file_scan.use_ref(branch) - files = file_scan.plan_files() commit_uuid = uuid.uuid4() - counter = itertools.count(0) - - replaced_files: list[tuple[DataFile, list[DataFile]]] = [] - # This will load the Parquet file into memory, including: - # - Filter out the rows based on the delete filter - # - Projecting it to the current schema - # - Applying the positional deletes if they are there - # When writing - # - Apply the latest partition-spec - # - And sort order when added - for original_file in files: - df = ArrowScan( - table_metadata=self.table_metadata, - io=self._table.io, - projected_schema=self.table_metadata.schema(), - row_filter=AlwaysTrue(), - ).to_table(tasks=[original_file]) - filtered_df = df.filter(preserve_row_filter) - - # Only rewrite if there are records being deleted - if len(filtered_df) == 0: - replaced_files.append((original_file.file, [])) - elif len(df) != len(filtered_df): - replaced_files.append( - ( - original_file.file, - list( - _dataframe_to_data_files( - io=self._table.io, - df=filtered_df, - table_metadata=self.table_metadata, - write_uuid=commit_uuid, - counter=counter, - ) - ), - ) - ) + replaced_files = self._rewrite_data_files( + file_scan.plan_files(), delete_filter, case_sensitive, commit_uuid, itertools.count(0) + ) if len(replaced_files) > 0: with self.update_snapshot( @@ -849,6 +811,57 @@ def delete( if not delete_snapshot.files_affected and not delete_snapshot.rewrites_needed: warnings.warn("Delete operation did not match any records", stacklevel=2) + def _rewrite_data_files( + self, + tasks: Iterable[FileScanTask], + delete_filter: BooleanExpression, + case_sensitive: bool, + write_uuid: uuid.UUID, + counter: itertools.count[int], + ) -> list[tuple[DataFile, list[DataFile]]]: + """Rewrite the data files of the scan tasks without the rows that match the filter. + + Each file is loaded into memory, with its positional deletes applied and projected to the current + schema, and the remaining rows are written under the latest partition spec. + + Returns: + Each data file holding a matching row, with the data files replacing it (none when no row remains). + """ + from pyiceberg.io.pyarrow import ArrowScan, _dataframe_to_data_files, _expression_to_complementary_pyarrow + + bound_delete_filter = bind(self.table_metadata.schema(), delete_filter, case_sensitive) + preserve_row_filter = _expression_to_complementary_pyarrow(bound_delete_filter, self.table_metadata.schema()) + + replaced_files: list[tuple[DataFile, list[DataFile]]] = [] + for original_file in tasks: + df = ArrowScan( + table_metadata=self.table_metadata, + io=self._table.io, + projected_schema=self.table_metadata.schema(), + row_filter=AlwaysTrue(), + ).to_table(tasks=[original_file]) + filtered_df = df.filter(preserve_row_filter) + + # Only rewrite if there are records being deleted + if len(filtered_df) == 0: + replaced_files.append((original_file.file, [])) + elif len(df) != len(filtered_df): + replaced_files.append( + ( + original_file.file, + list( + _dataframe_to_data_files( + io=self._table.io, + df=filtered_df, + table_metadata=self.table_metadata, + write_uuid=write_uuid, + counter=counter, + ) + ), + ) + ) + return replaced_files + def upsert( self, df: pa.Table, @@ -861,6 +874,11 @@ def upsert( ) -> UpsertResult: """Shorthand API for performing an upsert to an iceberg table. + The rewritten data files, the updated rows and the inserted rows are committed as one snapshot, validated + at the isolation level in `write.merge.isolation-level`. Under `serializable` (the default), a concurrent + commit that added data or delete files which could hold rows with the upserted keys makes it fail with a + ValidationException; under `snapshot`, only delete files are checked. + Args: df: The input dataframe to upsert with the table's data. join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. @@ -900,7 +918,7 @@ def upsert( except ModuleNotFoundError as e: raise ModuleNotFoundError("For writes PyArrow needs to be installed") from e - from pyiceberg.io.pyarrow import expression_to_pyarrow + from pyiceberg.io.pyarrow import _dataframe_to_data_files, expression_to_pyarrow from pyiceberg.table import upsert_util if join_cols is None: @@ -980,20 +998,67 @@ def upsert( update_row_cnt = 0 insert_row_cnt = 0 + commit_uuid = uuid.uuid4() + counter = itertools.count(0) + replaced_files: list[tuple[DataFile, list[DataFile]]] = [] + rows_to_write: list[pa.Table] = [] + if batches_to_overwrite: rows_to_update = pa.concat_tables(batches_to_overwrite) update_row_cnt = len(rows_to_update) - self.overwrite( - rows_to_update, - overwrite_filter=Or(*overwrite_predicates) if len(overwrite_predicates) > 1 else overwrite_predicates[0], - branch=branch, - snapshot_properties=snapshot_properties, + overwrite_filter = Or(*overwrite_predicates) if len(overwrite_predicates) > 1 else overwrite_predicates[0] + # Plan from the scan the matches were read from, so the files rewritten are the files matched + replaced_files = self._rewrite_data_files( + matched_iceberg_record_batches_scan.update(row_filter=overwrite_filter).plan_files(), + overwrite_filter, + case_sensitive, + commit_uuid, + counter, ) + rows_to_write.append(rows_to_update) if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) - if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + if insert_row_cnt > 0: + rows_to_write.append(rows_to_insert) + + if not replaced_files and not rows_to_write: + return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) + + # Commit the rewritten files, the updated rows and the inserted rows as one snapshot, as a copy-on-write + # MERGE does in Java. A separate append for the inserted rows would skip validation: two writers inserting + # the same new key would each commit it, the second replayed onto the first by the commit retry. + with _OverwriteFiles( + operation=Operation.OVERWRITE if replaced_files else Operation.APPEND, + transaction=self, + io=self._table.io, + branch=branch, + commit_uuid=commit_uuid, + snapshot_properties=snapshot_properties, + ) as overwrite_snapshot: + # Validate from the snapshot the matches were read from, with every key matched against as the + # conflict-detection filter, at the isolation level of a merge + scan_snapshot = matched_iceberg_record_batches_scan.snapshot() + overwrite_snapshot._starting_snapshot_id = scan_snapshot.snapshot_id if scan_snapshot is not None else None + overwrite_snapshot._isolation_level = IsolationLevel( + self.table_metadata.properties.get( + TableProperties.WRITE_MERGE_ISOLATION_LEVEL, TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT + ) + ) + overwrite_snapshot.delete_by_predicate(matched_predicate, case_sensitive) + if rows_to_write: + for data_file in _dataframe_to_data_files( + table_metadata=self.table_metadata, + df=pa.concat_tables(rows_to_write), + io=self._table.io, + write_uuid=commit_uuid, + counter=counter, + ): + overwrite_snapshot.append_data_file(data_file) + for original_data_file, replaced_data_files in replaced_files: + overwrite_snapshot.delete_data_file(original_data_file) + for replaced_data_file in replaced_data_files: + overwrite_snapshot.append_data_file(replaced_data_file) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index afceec4643..25faadd885 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -59,6 +59,7 @@ from pyiceberg.schema import Schema from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( + IsolationLevel, Operation, Snapshot, SnapshotSummaryCollector, @@ -159,6 +160,7 @@ class _SnapshotProducer(UpdateTableMetadata[U], Generic[U]): _case_sensitive: bool _commit_window: CommitWindow | None _conflict_detection_filter: BooleanExpression | None + _isolation_level: IsolationLevel | None _written_manifests: list[str] _uncommitted_manifests: list[str] _written_manifest_lists: list[str] @@ -198,6 +200,8 @@ def __init__( self._commit_window = None self._conflict_detection_filter = None self._isolation_operation: Operation = Operation.DELETE + # Set by a command whose isolation level is not keyed by its snapshot operation, such as an upsert + self._isolation_level = None def _validate_target_branch(self, branch: str | None) -> str | None: # if branch is none, write will be written into a staging snapshot @@ -514,7 +518,6 @@ def _validate_concurrency(self) -> None: Subclasses that do not require validation (e.g. fast append) should override with a no-op. """ - from pyiceberg.table.snapshots import IsolationLevel from pyiceberg.table.update.validate import ( _validate_added_data_files, _validate_data_files_exist, @@ -534,7 +537,11 @@ def _validate_concurrency(self) -> None: return table = self._transaction._table - isolation_level = table.metadata.isolation_level(self._isolation_operation) + isolation_level = ( + self._isolation_level + if self._isolation_level is not None + else table.metadata.isolation_level(self._isolation_operation) + ) conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None if isolation_level == IsolationLevel.SERIALIZABLE and conflict_detection_filter is not None: diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index ab95457e23..a100df02be 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -49,6 +49,7 @@ def test_commit_retry_table_properties() -> None: def test_isolation_level_table_properties() -> None: assert TableProperties.WRITE_DELETE_ISOLATION_LEVEL == "write.delete.isolation-level" assert TableProperties.WRITE_UPDATE_ISOLATION_LEVEL == "write.update.isolation-level" + assert TableProperties.WRITE_MERGE_ISOLATION_LEVEL == "write.merge.isolation-level" assert TableProperties.WRITE_ISOLATION_LEVEL_DEFAULT == "serializable" diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 78ddbc7c5c..d54d998bd3 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,17 +23,17 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import NoSuchTableError, ValidationException from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral -from pyiceberg.io.pyarrow import schema_to_pyarrow +from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow, write_positional_deletes from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, UpsertResult +from pyiceberg.table import Table, TableProperties, UpsertResult from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -375,7 +375,8 @@ def test_upsert_with_identifier_fields(catalog: Catalog) -> None: ) upd = tbl.upsert(df) - expected_operations = [Operation.APPEND, Operation.OVERWRITE, Operation.APPEND, Operation.APPEND] + # One snapshot: the rewritten file, the updated row and the inserted row are committed together + expected_operations = [Operation.APPEND, Operation.OVERWRITE] assert upd.rows_updated == 1 assert upd.rows_inserted == 1 @@ -920,10 +921,181 @@ def test_upsert_snapshot_properties(catalog: Catalog) -> None: # Verify properties are on the snapshots created by upsert snapshots = list(tbl.snapshots()) - # Upsert should have created additional snapshots (overwrite + append) - assert len(snapshots) > initial_snapshot_count + # Upsert should have created one additional snapshot + assert len(snapshots) == initial_snapshot_count + 1 # Check that all new snapshots have the snapshot_properties for snapshot in snapshots[initial_snapshot_count:]: assert snapshot.summary is not None assert snapshot.summary.additional_properties.get("test_prop") == "test_value" + + +_KEYED_SCHEMA = Schema( + NestedField(1, "id", LongType(), required=True), + NestedField(2, "name", StringType(), required=True), + identifier_field_ids=[1], +) + + +def _keyed_rows(*rows: tuple[int, str]) -> pa_table: + return pa.Table.from_pylist([{"id": key, "name": name} for key, name in rows], schema=schema_to_pyarrow(_KEYED_SCHEMA)) + + +def _keyed_table(catalog: Catalog, identifier: str, properties: dict[str, str] | None = None) -> Table: + """Three rows in one data file, so the row with key 2 is at position 1.""" + _drop_table(catalog, identifier) + table = catalog.create_table(identifier, schema=_KEYED_SCHEMA, properties={"format-version": "2", **(properties or {})}) + table.append(_keyed_rows((1, "a"), (2, "b"), (3, "c"))) + return table + + +def _rows(catalog: Catalog, identifier: str) -> list[tuple[int, str]]: + return sorted((row["id"], row["name"]) for row in catalog.load_table(identifier).scan().to_arrow().to_pylist()) + + +def _merge_on_read_upsert(table: Table, key: int, name: str, position: int | None) -> None: + """Upsert one row as a merge-on-read writer does: delete the old row by position, if any, and add the new one.""" + tx = table.transaction() + update = tx.update_snapshot().overwrite() + update.validate_no_conflicting_data_files(EqualTo("id", key)) + if position is not None: + data_file = next(iter(table.scan().plan_files())).file + update.append_data_file(write_positional_deletes(table.io, table.metadata, [(data_file.file_path, position)])) + for data_file in _dataframe_to_data_files(table.metadata, _keyed_rows((key, name)), table.io, update.commit_uuid): + update.append_data_file(data_file) + update.commit() + tx.commit_transaction() + + +@pytest.mark.parametrize("merge_on_read", [False, True], ids=["copy_on_write_winner", "merge_on_read_winner"]) +@pytest.mark.parametrize(("key", "position"), [(4, None), (2, 1)], ids=["new_key", "existing_key"]) +def test_concurrent_upserts_of_one_key_commit_it_once( + catalog: Catalog, merge_on_read: bool, key: int, position: int | None +) -> None: + """Two writers resolve the same key against the same base snapshot. + + Replaying the loser onto the winner would commit the key a second time, so the loser's retry is + refused, and its caller re-resolves the upsert against the winner's snapshot. + """ + identifier = "default.test_concurrent_upserts_of_one_key" + table = _keyed_table(catalog, identifier) + loser = catalog.load_table(identifier) + + if merge_on_read: + _merge_on_read_upsert(table, key, "winner", position) + else: + table.upsert(_keyed_rows((key, "winner"))) + snapshot_count = len(catalog.load_table(identifier).snapshots()) + + with pytest.raises(ValidationException): + loser.upsert(_keyed_rows((key, "loser"))) + + expected = {1: "a", 2: "b", 3: "c", key: "winner"} + assert _rows(catalog, identifier) == sorted(expected.items()) + assert len(catalog.load_table(identifier).snapshots()) == snapshot_count + + res = catalog.load_table(identifier).upsert(_keyed_rows((key, "loser"))) + assert_upsert_result(res, expected_updated=1, expected_inserted=0) + assert _rows(catalog, identifier) == sorted({**expected, key: "loser"}.items()) + + +def test_an_upsert_validates_the_keys_it_inserts_as_well_as_those_it_updates(catalog: Catalog) -> None: + """The conflict-detection filter is every key the upsert matched against, not only the keys it updated.""" + identifier = "default.test_upsert_validates_inserted_keys" + table = _keyed_table(catalog, identifier) + loser = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + + with pytest.raises(ValidationException): + loser.upsert(_keyed_rows((2, "loser"), (4, "loser"))) + + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c"), (4, "winner")] + + +def test_concurrent_upserts_of_disjoint_keys_both_commit(catalog: Catalog) -> None: + """The check is scoped to the keys the upsert matched against, so a writer on other keys does not conflict.""" + identifier = "default.test_concurrent_upserts_of_disjoint_keys" + table = _keyed_table(catalog, identifier) + other = catalog.load_table(identifier) + + table.upsert(_keyed_rows((1, "winner"))) + res = other.upsert(_keyed_rows((4, "other"))) + + assert_upsert_result(res, expected_updated=0, expected_inserted=1) + assert _rows(catalog, identifier) == [(1, "winner"), (2, "b"), (3, "c"), (4, "other")] + + +def test_upsert_isolation_level_is_read_from_the_merge_property(catalog: Catalog) -> None: + """An upsert is a merge, so `write.merge.isolation-level` governs it rather than the delete or update levels.""" + identifier = "default.test_upsert_merge_isolation_level" + table = _keyed_table( + catalog, + identifier, + { + TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "serializable", + TableProperties.WRITE_DELETE_ISOLATION_LEVEL: "snapshot", + TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "snapshot", + }, + ) + loser = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + + with pytest.raises(ValidationException, match="Added data files were found matching the filter"): + loser.upsert(_keyed_rows((4, "loser"))) + + +def test_upsert_under_snapshot_isolation_does_not_validate_added_data(catalog: Catalog) -> None: + """As in Spark, snapshot isolation does not check for data added concurrently, so both inserts commit.""" + identifier = "default.test_upsert_snapshot_isolation_added_data" + table = _keyed_table( + catalog, + identifier, + { + TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "snapshot", + TableProperties.WRITE_DELETE_ISOLATION_LEVEL: "serializable", + TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "serializable", + }, + ) + other = catalog.load_table(identifier) + + table.upsert(_keyed_rows((4, "winner"))) + res = other.upsert(_keyed_rows((4, "other"))) + + assert_upsert_result(res, expected_updated=0, expected_inserted=1) + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c"), (4, "other"), (4, "winner")] + + +def test_upsert_under_snapshot_isolation_still_validates_deletes(catalog: Catalog) -> None: + """As in Spark, snapshot isolation still refuses an upsert whose rows were concurrently deleted.""" + identifier = "default.test_upsert_snapshot_isolation_deletes" + table = _keyed_table(catalog, identifier, {TableProperties.WRITE_MERGE_ISOLATION_LEVEL: "snapshot"}) + loser = catalog.load_table(identifier) + + _merge_on_read_upsert(table, 2, "winner", position=1) + + with pytest.raises(ValidationException, match="Found new conflicting delete files"): + loser.upsert(_keyed_rows((2, "loser"))) + + assert _rows(catalog, identifier) == [(1, "a"), (2, "winner"), (3, "c")] + + +def test_upsert_on_a_branch(catalog: Catalog) -> None: + identifier = "default.test_upsert_on_a_branch" + table = _keyed_table(catalog, identifier) + main_snapshot_id = table.metadata.current_snapshot_id + assert main_snapshot_id is not None + table.manage_snapshots().create_branch(snapshot_id=main_snapshot_id, branch_name="dev").commit() + + res = table.upsert(_keyed_rows((2, "B"), (4, "d")), branch="dev") + + assert_upsert_result(res, expected_updated=1, expected_inserted=1) + assert table.metadata.current_snapshot_id == main_snapshot_id + dev = table.snapshot_by_name("dev") + assert dev is not None + assert dev.parent_snapshot_id == main_snapshot_id + assert dev.summary is not None and dev.summary.operation == Operation.OVERWRITE + rows = table.scan(snapshot_id=dev.snapshot_id).to_arrow().to_pylist() + assert sorted((row["id"], row["name"]) for row in rows) == [(1, "a"), (2, "B"), (3, "c"), (4, "d")] + assert _rows(catalog, identifier) == [(1, "a"), (2, "b"), (3, "c")] From 42b49fdd4e833e11fc63e52897b159c6f617fd66 Mon Sep 17 00:00:00 2001 From: Paul Caron Date: Wed, 11 Nov 2026 14:50:14 -0500 Subject: [PATCH 2/2] Charter: record the one-snapshot upsert (ELT-1277) Co-Authored-By: Claude Opus 5.5 --- MAINTENANCE-FORK.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTENANCE-FORK.md b/MAINTENANCE-FORK.md index 1915335e27..90ec94ba57 100644 --- a/MAINTENANCE-FORK.md +++ b/MAINTENANCE-FORK.md @@ -116,6 +116,7 @@ where the mirror exists only for Zamboni's tests and demo, never in production. | The S3 remote signer is registered on the session, once per signer: concurrent writes no longer go out unsigned (403), and each table signs through its own endpoint. A signed filesystem is shared only while a FileIO holds it, so its client, signer and the catalog's auth session are released with the last table using it, not kept until exit (`_s3`, `_signed_s3_filesystem`, `_SIGNED_FILESYSTEMS`) | [ZMBNI-51](https://github.com/paulcaron16k/Zamboni/issues/51), ELT-1269, [ZMBNI-246](https://github.com/paulcaron16k/Zamboni/issues/246) | `signer.py`, installed into `SCHEME_TO_FS` by `CatalogSession` ([#242](https://github.com/paulcaron16k/Zamboni/pull/242); released when unused since #246) | `_pyiceberg_signer.py`, installed into `SCHEME_TO_FS` before `load_catalog` (ELT-1269); released when unused since ELT-1279 | | S3 requests are signed off the event loop, on a daemon thread per call, at most ten at once and started only once a per-loop slot is free, so one signature no longer stalls every request in the process; and signed filesystems draw on one shared object-store connector and one signing `requests.Session`, while each table keeps its own client and signs through its own endpoint (`_SignOffTheLoop`, `_shared_pool_session`, `_shared_connector`, `_signing_session`). The connector subclass relies on private aiobotocore names, checked first, with a fallback to a pool per table, and reads a client's sessions again under aiobotocore's `_sessions_lock`, so a request queued behind `close()` cannot reopen the client (ZMBNI-260) | [ZMBNI-250](https://github.com/paulcaron16k/Zamboni/issues/250), [ZMBNI-260](https://github.com/paulcaron16k/Zamboni/issues/260), ELT-1281, ELT-1283, ELT-1311 | `signer.py` (#250, #260); stands down only where the behavioural probe `s3_signing_overlaps` also holds | `_pyiceberg_signer.py` (ELT-1283, ELT-1281, ELT-1311); level with this branch since ELT-1311 (omantra/target-iceberg#162, merged 2026-11-03), which brought over the per-loop gate, the conditional `_create_connector` await and the ZMBNI-260 read | | An overwrite can refuse data files concurrently added under a conflict-detection filter, so two upserts of one key cannot both commit (`validate_no_conflicting_data_files`). Like Java's, it reads file bounds, so disjoint keys whose ranges overlap, or a filter of more than 200 keys, are refused once and re-resolved | ELT-1271 | Not needed in production, which has no merge-on-read upsert | `_AppendDeleteFiles._validate_concurrency`; `_OverwriteFiles` on this branch (ELT-1271) | +| An upsert commits one overwrite snapshot instead of an overwrite plus an unvalidated append, validated from the snapshot its matches were read from, with its match filter as the conflict-detection filter for added data and delete files, at the isolation level in the new `write.merge.isolation-level`; so two upserts inserting one new key cannot both commit it (`Transaction.upsert`, `_rewrite_data_files`, `_SnapshotProducer._isolation_level`). As in Java, `snapshot` isolation validates deletes only, and the check reads file bounds | ELT-1277 | Not needed in production: Zamboni does not call `upsert` (only `test_safety.py` does, as a regression test) | Pending — ELT-1277 mirror in `writer.py` / `_pyiceberg_mor.py` | Rows three to five together are what let PyIceberg write a merge-on-read table at all: five rows, a delete file naming two positions, and a scan that returns the other three.