Skip to content

Commit 68697df

Browse files
kotwal-itproclaude
andcommitted
Add Catalog.transaction() context manager for multi-table commits
Stage the changes of each table through tx.table(table) and commit them atomically with commit_transactions when the block exits; nothing is committed if it raises. The per-table Transaction context manager can't be used for this, because its exit commits that table alone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011ZBQVsXRmKT8WXDr2qSfbd
1 parent 80513fc commit 68697df

5 files changed

Lines changed: 154 additions & 2 deletions

File tree

‎mkdocs/docs/api.md‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1411,8 +1411,19 @@ New snapshot summaries automatically include `engine-name` (`pyiceberg`) and `en
14111411
## Multi-table commits
14121412

14131413
With the REST catalog, changes to several tables can be committed atomically: either every table
1414-
is updated, or none of them is. Stage the changes in one transaction per table, then pass the
1415-
transactions to `commit_transactions`:
1414+
is updated, or none of them is. Open a transaction on the catalog and stage the changes of each
1415+
table through `tx.table(...)`. The changes are committed together when the block exits, and nothing
1416+
is committed if it raises:
1417+
1418+
```python
1419+
with catalog.transaction() as tx:
1420+
tx.table(orders).append(new_orders)
1421+
tx.table(items).append(new_items)
1422+
```
1423+
1424+
Don't commit the per-table transactions, or use them as context managers, on their own: that would
1425+
commit that table alone. Without the context manager, stage one transaction per table and pass them
1426+
to `commit_transactions`:
14161427

14171428
```python
14181429
orders_tx = orders.transaction()

‎pyiceberg/catalog/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE,
5050
CommitTableResponse,
5151
CreateTableTransaction,
52+
MultiTableTransaction,
5253
StagedTable,
5354
Table,
5455
TableProperties,
@@ -620,6 +621,15 @@ def commit_table(
620621
CommitStateUnknownException: Failed due to an internal exception on the side of the catalog.
621622
"""
622623

624+
def transaction(self) -> MultiTableTransaction:
625+
"""Create a transaction to stage changes to several tables and commit them atomically.
626+
627+
Returns:
628+
The transaction object. Used as a context manager, it commits with
629+
``commit_transactions`` when the block exits without an exception.
630+
"""
631+
return MultiTableTransaction(self)
632+
623633
def commit_transactions(self, *transactions: Transaction) -> None:
624634
"""Commit the changes of several table transactions atomically.
625635

‎pyiceberg/table/__init__.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1337,6 +1337,65 @@ def commit_transaction(self) -> Table:
13371337
return self._table
13381338

13391339

1340+
class MultiTableTransaction:
1341+
"""Stage changes to several tables of a catalog and commit them atomically.
1342+
1343+
Use it as a context manager: the changes are committed with ``Catalog.commit_transactions``
1344+
when the block exits without an exception, and nothing is committed otherwise.
1345+
"""
1346+
1347+
_catalog: Catalog
1348+
_transactions: dict[Identifier, Transaction]
1349+
1350+
def __init__(self, catalog: Catalog):
1351+
"""Open a transaction over several tables of a catalog.
1352+
1353+
Args:
1354+
catalog: The catalog that holds the tables.
1355+
"""
1356+
self._catalog = catalog
1357+
self._transactions = {}
1358+
1359+
def __enter__(self) -> MultiTableTransaction:
1360+
"""Start a transaction to update several tables."""
1361+
return self
1362+
1363+
def __exit__(self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None) -> None:
1364+
"""Commit the staged changes if no exceptions have been raised."""
1365+
if exctype is None and excinst is None and exctb is None:
1366+
self.commit_transaction()
1367+
1368+
def table(self, table: Table) -> Transaction:
1369+
"""Return the transaction that stages the changes to a table.
1370+
1371+
The same transaction is returned each time for the same table. Don't commit it or use it
1372+
as a context manager on its own: its changes are committed with the other tables.
1373+
1374+
Args:
1375+
table: The table to change.
1376+
1377+
Returns:
1378+
The transaction for the table.
1379+
"""
1380+
if table.catalog is not self._catalog:
1381+
raise ValueError(f"Table {table.name()} belongs to a different catalog")
1382+
if (transaction := self._transactions.get(table.name())) is None:
1383+
transaction = self._transactions[table.name()] = table.transaction()
1384+
return transaction
1385+
1386+
def commit_transaction(self) -> None:
1387+
"""Commit the changes of every table atomically.
1388+
1389+
Raises:
1390+
NotImplementedError: If the catalog doesn't support multi-table commits.
1391+
CommitFailedException: A requirement wasn't met, or a concurrent commit conflicted.
1392+
CommitStateUnknownException: The catalog failed with an internal error; the commit may
1393+
or may not have been applied.
1394+
"""
1395+
if self._transactions:
1396+
self._catalog.commit_transactions(*self._transactions.values())
1397+
1398+
13401399
class Namespace(IcebergRootModel[list[str]]):
13411400
"""Reference to one or more levels of a namespace."""
13421401

‎tests/catalog/test_rest.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3759,6 +3759,64 @@ def test_commit_transactions_sends_one_request(
37593759
assert orders_tx._updates == () and items_tx._updates == ()
37603760

37613761

3762+
def test_catalog_transaction_commits_on_exit(
3763+
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]
3764+
) -> None:
3765+
catalog, (orders, items) = _load_tables_for_multi_table_commit(rest_mock, example_table_metadata_with_snapshot_v1_rest_json)
3766+
commit = rest_mock.post(f"{TEST_URI}v1/transactions/commit", status_code=204, request_headers=TEST_HEADERS)
3767+
3768+
# A block that staged nothing sends no request.
3769+
with catalog.transaction() as tx:
3770+
tx.table(orders)
3771+
assert commit.call_count == 0
3772+
3773+
with catalog.transaction() as tx:
3774+
tx.table(orders).set_properties(owner="sales")
3775+
tx.table(items).set_properties(owner="sales")
3776+
tx.table(orders).set_properties(team="orders")
3777+
assert tx.table(orders) is tx.table(orders)
3778+
assert commit.call_count == 0
3779+
3780+
assert commit.call_count == 1
3781+
assert [(change["identifier"]["name"], change["updates"]) for change in commit.last_request.json()["table-changes"]] == [
3782+
(
3783+
"orders",
3784+
[
3785+
{"action": "set-properties", "updates": {"owner": "sales"}},
3786+
{"action": "set-properties", "updates": {"team": "orders"}},
3787+
],
3788+
),
3789+
("items", [{"action": "set-properties", "updates": {"owner": "sales"}}]),
3790+
]
3791+
3792+
3793+
def test_catalog_transaction_commits_nothing_on_exception(
3794+
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]
3795+
) -> None:
3796+
catalog, (orders, items) = _load_tables_for_multi_table_commit(rest_mock, example_table_metadata_with_snapshot_v1_rest_json)
3797+
commit = rest_mock.post(f"{TEST_URI}v1/transactions/commit", status_code=204, request_headers=TEST_HEADERS)
3798+
3799+
def stage_then_fail() -> None:
3800+
with catalog.transaction() as tx:
3801+
tx.table(orders).set_properties(owner="sales")
3802+
tx.table(items).set_properties(owner="sales")
3803+
raise RuntimeError("stop")
3804+
3805+
with pytest.raises(RuntimeError, match="stop"):
3806+
stage_then_fail()
3807+
assert not commit.called
3808+
3809+
3810+
def test_catalog_transaction_rejects_a_table_of_another_catalog(
3811+
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]
3812+
) -> None:
3813+
_, (orders,) = _load_tables_for_multi_table_commit(rest_mock, example_table_metadata_with_snapshot_v1_rest_json, ("orders",))
3814+
other_catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
3815+
3816+
with pytest.raises(ValueError, match="belongs to a different catalog"):
3817+
other_catalog.transaction().table(orders)
3818+
3819+
37623820
def test_commit_transactions_with_prefix(
37633821
requests_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]
37643822
) -> None:

‎tests/integration/test_rest_catalog.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,20 @@ def test_commit_transactions_appends_to_tables_atomically(catalog: RestCatalog,
146146
assert items.properties["owner"] == "sales"
147147

148148

149+
@pytest.mark.integration
150+
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
151+
def test_catalog_transaction_appends_to_tables_atomically(catalog: RestCatalog, database_name: str) -> None:
152+
rows, (orders, items) = _create_tables_for_multi_table_commit(catalog, database_name)
153+
154+
with catalog.transaction() as tx:
155+
tx.table(orders).append(rows)
156+
tx.table(items).append(rows)
157+
assert orders.current_snapshot() is None
158+
159+
for table in (orders, items):
160+
assert table.scan().to_arrow().num_rows == 3
161+
162+
149163
@pytest.mark.integration
150164
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
151165
def test_commit_transactions_applies_nothing_on_conflict(catalog: RestCatalog, database_name: str) -> None:

0 commit comments

Comments
 (0)