Skip to content

Commit ffd5e38

Browse files
committed
feat(manifest): support writing delete manifests in v2
ManifestWriterV2 hardcoded content() to ManifestContent.DATA and wrote "content": "data" into the Avro metadata, so PyIceberg could not produce a delete manifest even though it reads them and ManifestListWriterV1 already guards against storing one in a v1 table. Thread an optional content through write_manifest and ManifestWriterV2, defaulting to DATA so existing callers are unaffected, and reject a non-data content for format version 1.
1 parent 0008d65 commit ffd5e38

2 files changed

Lines changed: 112 additions & 3 deletions

File tree

‎pyiceberg/manifest.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1323,18 +1323,22 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13231323

13241324

13251325
class ManifestWriterV2(ManifestWriter):
1326+
_content: ManifestContent
1327+
13261328
def __init__(
13271329
self,
13281330
spec: PartitionSpec,
13291331
schema: Schema,
13301332
output_file: OutputFile,
13311333
snapshot_id: int,
13321334
avro_compression: AvroCompressionCodec,
1335+
content: ManifestContent = ManifestContent.DATA,
13331336
):
13341337
super().__init__(spec, schema, output_file, snapshot_id, avro_compression)
1338+
self._content = content
13351339

13361340
def content(self) -> ManifestContent:
1337-
return ManifestContent.DATA
1341+
return self._content
13381342

13391343
@property
13401344
def version(self) -> TableVersion:
@@ -1344,7 +1348,7 @@ def version(self) -> TableVersion:
13441348
def _meta(self) -> dict[str, str]:
13451349
return {
13461350
**super()._meta,
1347-
"content": "data",
1351+
"content": "data" if self._content == ManifestContent.DATA else "deletes",
13481352
}
13491353

13501354
def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
@@ -1363,11 +1367,14 @@ def write_manifest(
13631367
output_file: OutputFile,
13641368
snapshot_id: int,
13651369
avro_compression: AvroCompressionCodec,
1370+
content: ManifestContent = ManifestContent.DATA,
13661371
) -> ManifestWriter:
13671372
if format_version == 1:
1373+
if content != ManifestContent.DATA:
1374+
raise ValidationError("Cannot write delete manifests in a v1 table")
13681375
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
13691376
elif format_version == 2:
1370-
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
1377+
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression, content)
13711378
else:
13721379
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13731380

‎tests/utils/test_manifest.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
3232
from pyiceberg.encryption.manager import EncryptionManager
3333
from pyiceberg.encryption.stream import _encrypt_ags1
34+
from pyiceberg.exceptions import ValidationError
3435
from pyiceberg.io import load_file_io
3536
from pyiceberg.io.pyarrow import PyArrowFileIO
3637
from pyiceberg.manifest import (
@@ -1653,3 +1654,104 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
16531654
finally:
16541655
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
16551656
importlib.reload(manifest_module)
1657+
1658+
1659+
@pytest.mark.parametrize("content", [ManifestContent.DATA, ManifestContent.DELETES])
1660+
def test_write_manifest_content(
1661+
generated_manifest_file_file_v2: str,
1662+
test_schema: Schema,
1663+
test_partition_spec: PartitionSpec,
1664+
content: ManifestContent,
1665+
) -> None:
1666+
"""A v2 manifest must record the content it was asked to write.
1667+
1668+
Without this the writer always claims `data`, so a delete manifest written by
1669+
PyIceberg would be read back as a data manifest and its entries applied as
1670+
live data files.
1671+
"""
1672+
io = load_file_io()
1673+
snapshot = Snapshot(
1674+
snapshot_id=25,
1675+
parent_snapshot_id=19,
1676+
timestamp_ms=1602638573590,
1677+
manifest_list=generated_manifest_file_file_v2,
1678+
summary=Summary(Operation.APPEND),
1679+
schema_id=3,
1680+
)
1681+
manifest_entries = snapshot.manifests(io)[0].fetch_manifest_entry(io)
1682+
1683+
with TemporaryDirectory() as tmpdir:
1684+
tmp_avro_file = tmpdir + "/test_write_manifest_content.avro"
1685+
with write_manifest(
1686+
format_version=2,
1687+
spec=test_partition_spec,
1688+
schema=test_schema,
1689+
output_file=io.new_output(tmp_avro_file),
1690+
snapshot_id=8744736658442914487,
1691+
avro_compression="deflate",
1692+
content=content,
1693+
) as writer:
1694+
for entry in manifest_entries:
1695+
writer.add_entry(entry)
1696+
new_manifest = writer.to_manifest_file()
1697+
1698+
assert new_manifest.content == content
1699+
_verify_metadata_with_fastavro(
1700+
tmp_avro_file,
1701+
{"content": "data" if content == ManifestContent.DATA else "deletes"},
1702+
)
1703+
1704+
1705+
def test_write_manifest_defaults_to_data_content(
1706+
generated_manifest_file_file_v2: str,
1707+
test_schema: Schema,
1708+
test_partition_spec: PartitionSpec,
1709+
) -> None:
1710+
"""Callers that do not ask for a content type still get a data manifest."""
1711+
io = load_file_io()
1712+
snapshot = Snapshot(
1713+
snapshot_id=25,
1714+
parent_snapshot_id=19,
1715+
timestamp_ms=1602638573590,
1716+
manifest_list=generated_manifest_file_file_v2,
1717+
summary=Summary(Operation.APPEND),
1718+
schema_id=3,
1719+
)
1720+
manifest_entries = snapshot.manifests(io)[0].fetch_manifest_entry(io)
1721+
1722+
with TemporaryDirectory() as tmpdir:
1723+
tmp_avro_file = tmpdir + "/test_write_manifest_default_content.avro"
1724+
with write_manifest(
1725+
format_version=2,
1726+
spec=test_partition_spec,
1727+
schema=test_schema,
1728+
output_file=io.new_output(tmp_avro_file),
1729+
snapshot_id=8744736658442914487,
1730+
avro_compression="deflate",
1731+
) as writer:
1732+
for entry in manifest_entries:
1733+
writer.add_entry(entry)
1734+
assert writer.to_manifest_file().content == ManifestContent.DATA
1735+
1736+
1737+
def test_write_manifest_v1_rejects_delete_content(
1738+
test_schema: Schema,
1739+
test_partition_spec: PartitionSpec,
1740+
) -> None:
1741+
"""v1 has no delete files, so asking for a delete manifest must fail loudly.
1742+
1743+
`ManifestListWriterV1` already refuses to store one; rejecting it at the
1744+
writer keeps a v1 table from producing a manifest it could never reference.
1745+
"""
1746+
io = load_file_io()
1747+
with TemporaryDirectory() as tmpdir:
1748+
with pytest.raises(ValidationError, match="Cannot write delete manifests in a v1 table"):
1749+
write_manifest(
1750+
format_version=1,
1751+
spec=test_partition_spec,
1752+
schema=test_schema,
1753+
output_file=io.new_output(tmpdir + "/test_write_manifest_v1_deletes.avro"),
1754+
snapshot_id=8744736658442914487,
1755+
avro_compression="deflate",
1756+
content=ManifestContent.DELETES,
1757+
)

0 commit comments

Comments
 (0)