Skip to content

Commit c8d6723

Browse files
authored
feat(encryption): [10/N] Read encrypted manifests (#4308)
Read encrypted manifests A manifest list entry's key_metadata is the plaintext key metadata of its manifest, so fetch_manifest_entry decrypts with it when it is set, as iceberg-rust's ManifestReader does.
1 parent 2ed79b6 commit c8d6723

4 files changed

Lines changed: 62 additions & 2 deletions

File tree

‎pyiceberg/manifest.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -900,7 +900,7 @@ def fetch_manifest_entry(
900900
entry_filter: Callable[[ManifestEntry], bool] | None = None,
901901
) -> list[ManifestEntry]:
902902
"""
903-
Read the manifest entries from the manifest file.
903+
Read the manifest entries from the manifest file, decrypting it when it has key metadata.
904904
905905
Args:
906906
io: The FileIO to fetch the file.
@@ -911,6 +911,8 @@ def fetch_manifest_entry(
911911
An Iterator of manifest entries.
912912
"""
913913
input_file = io.new_input(self.manifest_path)
914+
if self.key_metadata is not None:
915+
input_file = EncryptedInputFile(input_file, StandardKeyMetadata.from_bytes(self.key_metadata))
914916
with AvroFile[ManifestEntry](
915917
input_file,
916918
MANIFEST_ENTRY_SCHEMAS[DEFAULT_READ_VERSION],

‎tests/table/encryption/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@
1717
~ under the License.
1818
-->
1919

20-
# Encrypted manifest list test data
20+
# Encrypted metadata test data
2121

2222
`manifest-list-v3-encrypted.avro` is an empty manifest list encrypted by iceberg-rust, copied from its
2323
`crates/iceberg/testdata/manifests_lists/`. It was written by iceberg-rust's `EncryptionManager`, with a memory KMS
2424
holding master key `master-1` = bytes `0x00` to `0x0f`. Its keys and snapshot are in the
2525
`example_table_metadata_v3_encrypted` fixture in `tests/conftest.py`.
26+
27+
`manifest-v3-encrypted.avro` is a v3 data manifest with two added entries, written and encrypted by iceberg-rust's
28+
`ManifestWriter` with key `0123456789abcdef` and AAD prefix `fedcba9876543210`. The key metadata that its manifest list
29+
entry would record is in `test_read_manifest_encrypted_by_iceberg_rust` in `tests/utils/test_manifest.py`.
3.36 KB
Binary file not shown.

‎tests/utils/test_manifest.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import pyiceberg.manifest as manifest_module
2828
from pyiceberg.avro.codecs import AvroCompressionCodec
2929
from pyiceberg.avro.file import DEFAULT_SYNC_INTERVAL, AvroFile, AvroOutputFile
30+
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
3031
from pyiceberg.encryption.manager import EncryptionManager
3132
from pyiceberg.encryption.stream import _encrypt_ags1
3233
from pyiceberg.io import load_file_io
@@ -548,6 +549,59 @@ def test_read_encrypted_manifest_list_without_encryption_manager(generated_manif
548549
snapshot.manifests(load_file_io())
549550

550551

552+
def test_read_encrypted_manifest(generated_manifest_entry_file: str, tmp_path: Path) -> None:
553+
io = load_file_io()
554+
key_metadata = StandardKeyMetadata.generate()
555+
encrypted = _encrypt_ags1(Path(generated_manifest_entry_file).read_bytes(), key_metadata)
556+
encrypted_path = tmp_path / "manifest.avro"
557+
encrypted_path.write_bytes(encrypted)
558+
key_metadata = replace(key_metadata, file_length=len(encrypted))
559+
560+
def manifest(path: str, key_metadata: bytes | None) -> ManifestFile:
561+
return ManifestFile.from_args(
562+
manifest_path=path,
563+
manifest_length=0,
564+
partition_spec_id=0,
565+
added_snapshot_id=0,
566+
sequence_number=0,
567+
partitions=[],
568+
key_metadata=key_metadata,
569+
)
570+
571+
expected = manifest(generated_manifest_entry_file, None).fetch_manifest_entry(io, discard_deleted=False)
572+
assert manifest(str(encrypted_path), key_metadata.to_bytes()).fetch_manifest_entry(io, discard_deleted=False) == expected
573+
574+
wrong_key = replace(key_metadata, encryption_key=bytes(16))
575+
with pytest.raises(ValueError, match="GCM tag check failed"):
576+
manifest(str(encrypted_path), wrong_key.to_bytes()).fetch_manifest_entry(io)
577+
578+
with pytest.raises(ValueError, match="No schema found in Avro file headers"):
579+
manifest(str(encrypted_path), None).fetch_manifest_entry(io)
580+
581+
582+
def test_read_manifest_encrypted_by_iceberg_rust() -> None:
583+
"""The manifest was encrypted by iceberg-rust, see tests/table/encryption/README.md."""
584+
manifest = ManifestFile.from_args(
585+
manifest_path=str(Path(__file__).parent.parent / "table" / "encryption" / "manifest-v3-encrypted.avro"),
586+
manifest_length=3444,
587+
partition_spec_id=0,
588+
content=ManifestContent.DATA,
589+
sequence_number=1,
590+
min_sequence_number=1,
591+
added_snapshot_id=1,
592+
partitions=[],
593+
key_metadata=bytes.fromhex("01203031323334353637383961626364656602206665646362613938373635343332313002e835"),
594+
)
595+
596+
entries = manifest.fetch_manifest_entry(load_file_io())
597+
598+
assert [(entry.data_file.file_path, entry.data_file.record_count) for entry in entries] == [
599+
("s3://bucket/table/data/00000.parquet", 3),
600+
("s3://bucket/table/data/00001.parquet", 5),
601+
]
602+
assert all(entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == 1 for entry in entries)
603+
604+
551605
def test_read_manifest_cache(generated_manifest_file_file_v2: str) -> None:
552606
"""Test that ManifestFile objects are cached and reused across multiple reads.
553607

0 commit comments

Comments
 (0)