Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1031,9 +1031,9 @@ def upsert(

return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt)

def _find_referenced_data_files(self, file_paths: list[str]) -> list[str]:
def _find_referenced_data_files(self, file_paths: list[str], branch: str | None = MAIN_BRANCH) -> list[str]:
"""Return file_paths already referenced by data files in the current snapshot."""
snapshot = self.table_metadata.current_snapshot()
snapshot = self.table_metadata.snapshot_by_name(branch)
if snapshot is None:
return []

Expand Down Expand Up @@ -1075,7 +1075,7 @@ def add_files(
raise ValueError("File paths must be unique")

if check_duplicate_files:
referenced_files = self._find_referenced_data_files(file_paths)
referenced_files = self._find_referenced_data_files(file_paths, branch)
if referenced_files:
raise ValueError(f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}")

Expand All @@ -1084,6 +1084,16 @@ def add_files(
**{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()}
)
with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
if check_duplicate_files:

def validate_duplicate_files() -> None:
referenced_files = self._find_referenced_data_files(file_paths, branch)
if referenced_files:
raise ValueError(
f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}"
)

append_files._set_retry_validator(validate_duplicate_files)
data_files = _parquet_files_to_data_files(
table_metadata=self.table_metadata, file_paths=file_paths, io=self._table.io
)
Expand Down Expand Up @@ -1200,7 +1210,7 @@ def commit_transaction(self) -> Table:
self._cleanup_uncommitted_manifests()
break
self._rebuild_snapshot_updates()
except (CommitFailedException, ValidationException):
except (CommitFailedException, ValidationException, ValueError):
# These exceptions guarantee the commit did not land, so it is safe to delete the
# files written for it. Any other exception (unknown outcome, or a commit that already
# succeeded) is re-raised without deleting, since those files may be referenced by the
Expand Down Expand Up @@ -1285,6 +1295,7 @@ def _rebuild_snapshot_updates(self) -> None:
self._resolve_commit_windows()
for producer in self._snapshot_producers:
producer._refresh_for_retry()
producer._validate_retry()
producer._validate_concurrency()
Comment on lines +1298 to 1299

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two methods look similar, could we merge the _validate_retry() into _validate_concurrency()

updates, requirements = producer._commit()
self._stage(updates, requirements)
Expand Down
9 changes: 9 additions & 0 deletions pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ class _SnapshotProducer(UpdateTableMetadata[U], Generic[U]):
_uncommitted_manifests: list[str]
_written_manifest_lists: list[str]
_isolation_operation: Operation
_retry_validator: Callable[[], None] | None

def __init__(
self,
Expand Down Expand Up @@ -186,6 +187,14 @@ def __init__(
self._case_sensitive = True
self._commit_window = None
self._isolation_operation: Operation = Operation.DELETE
self._retry_validator = None

def _set_retry_validator(self, validator: Callable[[], None]) -> None:
self._retry_validator = validator

def _validate_retry(self) -> None:
if self._retry_validator is not None:
self._retry_validator()

def _validate_target_branch(self, branch: str | None) -> str | None:
# if branch is none, write will be written into a staging snapshot
Expand Down
24 changes: 24 additions & 0 deletions tests/table/test_commit_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
# specific language governing permissions and limitations
# under the License.
import uuid
from pathlib import Path
from typing import Any
from unittest.mock import patch

import pyarrow.parquet as pq
import pytest

from pyiceberg.catalog import Catalog
Expand Down Expand Up @@ -97,6 +99,28 @@ def counting_rebuild(self_tx: Any) -> None:
assert len(result) == 6


def test_concurrent_add_files_retry_rechecks_duplicate_paths(catalog: Catalog, tmp_path: Path) -> None:
"""A retried add_files call rejects a file committed concurrently."""
catalog.create_namespace("default")
catalog.create_table("default.add_files_retry_test", schema=_test_schema())

import pyarrow as pa

data_file = tmp_path / "data.parquet"
pq.write_table(pa.table({"x": [1, 2]}), data_file)

tbl1 = catalog.load_table("default.add_files_retry_test")
tbl2 = catalog.load_table("default.add_files_retry_test")

tbl1.add_files([str(data_file)])

with pytest.raises(ValueError, match="Cannot add files that are already referenced by table"):
tbl2.add_files([str(data_file)])

refreshed = catalog.load_table("default.add_files_retry_test")
assert refreshed.scan().to_arrow()["x"].to_pylist() == [1, 2]


def test_no_retry_without_snapshot_producers(catalog: Catalog) -> None:
"""Verify that a transaction with no snapshot producers has an empty producer list."""
catalog.create_namespace("default")
Expand Down
Loading