|
31 | 31 | from pyiceberg.encryption.key_metadata import StandardKeyMetadata |
32 | 32 | from pyiceberg.encryption.manager import EncryptionManager |
33 | 33 | from pyiceberg.encryption.stream import _encrypt_ags1 |
| 34 | +from pyiceberg.exceptions import ValidationError |
34 | 35 | from pyiceberg.io import load_file_io |
35 | 36 | from pyiceberg.io.pyarrow import PyArrowFileIO |
36 | 37 | from pyiceberg.manifest import ( |
@@ -1653,3 +1654,104 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon |
1653 | 1654 | finally: |
1654 | 1655 | monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False) |
1655 | 1656 | importlib.reload(manifest_module) |
| 1657 | + |
| 1658 | + |
| 1659 | +@pytest.mark.parametrize("content", [ManifestContent.DATA, ManifestContent.DELETES]) |
| 1660 | +def test_write_manifest_content( |
| 1661 | + generated_manifest_file_file_v2: str, |
| 1662 | + test_schema: Schema, |
| 1663 | + test_partition_spec: PartitionSpec, |
| 1664 | + content: ManifestContent, |
| 1665 | +) -> None: |
| 1666 | + """A v2 manifest must record the content it was asked to write. |
| 1667 | +
|
| 1668 | + Without this the writer always claims `data`, so a delete manifest written by |
| 1669 | + PyIceberg would be read back as a data manifest and its entries applied as |
| 1670 | + live data files. |
| 1671 | + """ |
| 1672 | + io = load_file_io() |
| 1673 | + snapshot = Snapshot( |
| 1674 | + snapshot_id=25, |
| 1675 | + parent_snapshot_id=19, |
| 1676 | + timestamp_ms=1602638573590, |
| 1677 | + manifest_list=generated_manifest_file_file_v2, |
| 1678 | + summary=Summary(Operation.APPEND), |
| 1679 | + schema_id=3, |
| 1680 | + ) |
| 1681 | + manifest_entries = snapshot.manifests(io)[0].fetch_manifest_entry(io) |
| 1682 | + |
| 1683 | + with TemporaryDirectory() as tmpdir: |
| 1684 | + tmp_avro_file = tmpdir + "/test_write_manifest_content.avro" |
| 1685 | + with write_manifest( |
| 1686 | + format_version=2, |
| 1687 | + spec=test_partition_spec, |
| 1688 | + schema=test_schema, |
| 1689 | + output_file=io.new_output(tmp_avro_file), |
| 1690 | + snapshot_id=8744736658442914487, |
| 1691 | + avro_compression="deflate", |
| 1692 | + content=content, |
| 1693 | + ) as writer: |
| 1694 | + for entry in manifest_entries: |
| 1695 | + writer.add_entry(entry) |
| 1696 | + new_manifest = writer.to_manifest_file() |
| 1697 | + |
| 1698 | + assert new_manifest.content == content |
| 1699 | + _verify_metadata_with_fastavro( |
| 1700 | + tmp_avro_file, |
| 1701 | + {"content": "data" if content == ManifestContent.DATA else "deletes"}, |
| 1702 | + ) |
| 1703 | + |
| 1704 | + |
| 1705 | +def test_write_manifest_defaults_to_data_content( |
| 1706 | + generated_manifest_file_file_v2: str, |
| 1707 | + test_schema: Schema, |
| 1708 | + test_partition_spec: PartitionSpec, |
| 1709 | +) -> None: |
| 1710 | + """Callers that do not ask for a content type still get a data manifest.""" |
| 1711 | + io = load_file_io() |
| 1712 | + snapshot = Snapshot( |
| 1713 | + snapshot_id=25, |
| 1714 | + parent_snapshot_id=19, |
| 1715 | + timestamp_ms=1602638573590, |
| 1716 | + manifest_list=generated_manifest_file_file_v2, |
| 1717 | + summary=Summary(Operation.APPEND), |
| 1718 | + schema_id=3, |
| 1719 | + ) |
| 1720 | + manifest_entries = snapshot.manifests(io)[0].fetch_manifest_entry(io) |
| 1721 | + |
| 1722 | + with TemporaryDirectory() as tmpdir: |
| 1723 | + tmp_avro_file = tmpdir + "/test_write_manifest_default_content.avro" |
| 1724 | + with write_manifest( |
| 1725 | + format_version=2, |
| 1726 | + spec=test_partition_spec, |
| 1727 | + schema=test_schema, |
| 1728 | + output_file=io.new_output(tmp_avro_file), |
| 1729 | + snapshot_id=8744736658442914487, |
| 1730 | + avro_compression="deflate", |
| 1731 | + ) as writer: |
| 1732 | + for entry in manifest_entries: |
| 1733 | + writer.add_entry(entry) |
| 1734 | + assert writer.to_manifest_file().content == ManifestContent.DATA |
| 1735 | + |
| 1736 | + |
| 1737 | +def test_write_manifest_v1_rejects_delete_content( |
| 1738 | + test_schema: Schema, |
| 1739 | + test_partition_spec: PartitionSpec, |
| 1740 | +) -> None: |
| 1741 | + """v1 has no delete files, so asking for a delete manifest must fail loudly. |
| 1742 | +
|
| 1743 | + `ManifestListWriterV1` already refuses to store one; rejecting it at the |
| 1744 | + writer keeps a v1 table from producing a manifest it could never reference. |
| 1745 | + """ |
| 1746 | + io = load_file_io() |
| 1747 | + with TemporaryDirectory() as tmpdir: |
| 1748 | + with pytest.raises(ValidationError, match="Cannot write delete manifests in a v1 table"): |
| 1749 | + write_manifest( |
| 1750 | + format_version=1, |
| 1751 | + spec=test_partition_spec, |
| 1752 | + schema=test_schema, |
| 1753 | + output_file=io.new_output(tmpdir + "/test_write_manifest_v1_deletes.avro"), |
| 1754 | + snapshot_id=8744736658442914487, |
| 1755 | + avro_compression="deflate", |
| 1756 | + content=ManifestContent.DELETES, |
| 1757 | + ) |
0 commit comments