|
23 | 23 | from pyiceberg.catalog import Catalog |
24 | 24 | from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException |
25 | 25 | from pyiceberg.schema import Schema |
26 | | -from pyiceberg.table import TableProperties, Transaction |
| 26 | +from pyiceberg.table import Table, TableProperties, Transaction |
27 | 27 | from pyiceberg.table.snapshots import IsolationLevel, Operation |
28 | | -from pyiceberg.types import LongType, NestedField, StringType |
| 28 | +from pyiceberg.typedef import Record |
| 29 | +from pyiceberg.types import DoubleType, LongType, NestedField, StringType |
29 | 30 |
|
30 | 31 |
|
31 | 32 | def test_isolation_level_enum() -> None: |
@@ -395,6 +396,85 @@ def test_file_overwrite_validates_concurrent_file_delete( |
395 | 396 | assert sorted(result["value"].to_pylist()) == expected_values |
396 | 397 |
|
397 | 398 |
|
| 399 | +def _commit_partition_position_delete(table: Table, partition: Record, file_path: str, pos: int) -> None: |
| 400 | + """PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them.""" |
| 401 | + import pyarrow as pa |
| 402 | + import pyarrow.parquet as pq |
| 403 | + |
| 404 | + from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestContent, ManifestWriter, ManifestWriterV2 |
| 405 | + from pyiceberg.partitioning import PartitionSpec |
| 406 | + from pyiceberg.table.update.snapshot import _OverwriteFiles |
| 407 | + |
| 408 | + class DeleteManifestWriter(ManifestWriterV2): |
| 409 | + def content(self) -> ManifestContent: |
| 410 | + return ManifestContent.DELETES |
| 411 | + |
| 412 | + @property |
| 413 | + def _meta(self) -> dict[str, str]: |
| 414 | + return {**super()._meta, "content": "deletes"} |
| 415 | + |
| 416 | + def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) -> ManifestWriter: |
| 417 | + return DeleteManifestWriter( |
| 418 | + spec, producer.schema(), producer.new_manifest_output(), producer._snapshot_id, producer._compression |
| 419 | + ) |
| 420 | + |
| 421 | + delete_path = table.location_provider().new_data_location(f"pos-delete-{uuid.uuid4()}.parquet") |
| 422 | + with table.io.new_output(delete_path).create() as output: |
| 423 | + pq.write_table(pa.table({"file_path": [file_path], "pos": [pos]}), output) |
| 424 | + delete_file = DataFile.from_args( |
| 425 | + content=DataFileContent.POSITION_DELETES, |
| 426 | + file_path=delete_path, |
| 427 | + file_format=FileFormat.PARQUET, |
| 428 | + partition=partition, |
| 429 | + record_count=1, |
| 430 | + file_size_in_bytes=len(table.io.new_input(delete_path)), |
| 431 | + spec_id=table.spec().spec_id, |
| 432 | + ) |
| 433 | + |
| 434 | + with patch.object(_OverwriteFiles, "new_manifest_writer", new_delete_manifest_writer): |
| 435 | + with table.transaction() as transaction, transaction.update_snapshot().overwrite() as overwrite: |
| 436 | + overwrite.append_data_file(delete_file) |
| 437 | + |
| 438 | + |
| 439 | +def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(catalog: Catalog) -> None: |
| 440 | + """A file replacement in a NaN partition must fail when a delete for that file was committed concurrently.""" |
| 441 | + import pyarrow as pa |
| 442 | + |
| 443 | + from pyiceberg.io.pyarrow import _dataframe_to_data_files |
| 444 | + from pyiceberg.partitioning import PartitionField, PartitionSpec |
| 445 | + from pyiceberg.transforms import IdentityTransform |
| 446 | + |
| 447 | + catalog.create_namespace("default") |
| 448 | + schema = Schema( |
| 449 | + NestedField(1, "id", LongType(), required=False), |
| 450 | + NestedField(2, "value", DoubleType(), required=False), |
| 451 | + ) |
| 452 | + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value")) |
| 453 | + identifier = "default.concurrent_nan_position_delete" |
| 454 | + table = catalog.create_table(identifier, schema=schema, partition_spec=spec, properties={"format-version": "2"}) |
| 455 | + rows = pa.table({"id": pa.array([1, 2], pa.int64()), "value": [float("nan"), float("nan")]}) |
| 456 | + table.append(rows) |
| 457 | + file_to_replace = next(task.file for task in table.scan().plan_files()) |
| 458 | + |
| 459 | + replacing_table = catalog.load_table(identifier) |
| 460 | + deleting_table = catalog.load_table(identifier) |
| 461 | + |
| 462 | + replacement_file = list( |
| 463 | + _dataframe_to_data_files(table_metadata=replacing_table.metadata, df=rows, io=replacing_table.io, write_uuid=uuid.uuid4()) |
| 464 | + )[0] |
| 465 | + replacing_transaction = replacing_table.transaction() |
| 466 | + with replacing_transaction.update_snapshot().overwrite() as overwrite: |
| 467 | + overwrite.delete_data_file(file_to_replace) |
| 468 | + overwrite.append_data_file(replacement_file) |
| 469 | + |
| 470 | + _commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0) |
| 471 | + |
| 472 | + with pytest.raises(ValidationException, match="found new delete for replace data file"): |
| 473 | + replacing_transaction.commit_transaction() |
| 474 | + |
| 475 | + assert catalog.load_table(identifier).scan().to_arrow()["id"].to_pylist() == [2] |
| 476 | + |
| 477 | + |
398 | 478 | def test_concurrent_overwrite_append_retries_successfully(catalog: Catalog) -> None: |
399 | 479 | """Append after a concurrent overwrite should succeed via retry.""" |
400 | 480 | catalog.create_namespace("default") |
|
0 commit comments