|
27 | 27 | import pyiceberg.manifest as manifest_module |
28 | 28 | from pyiceberg.avro.codecs import AvroCompressionCodec |
29 | 29 | from pyiceberg.avro.file import DEFAULT_SYNC_INTERVAL, AvroFile, AvroOutputFile |
| 30 | +from pyiceberg.encryption.key_metadata import StandardKeyMetadata |
30 | 31 | from pyiceberg.encryption.manager import EncryptionManager |
31 | 32 | from pyiceberg.encryption.stream import _encrypt_ags1 |
32 | 33 | from pyiceberg.io import load_file_io |
@@ -548,6 +549,59 @@ def test_read_encrypted_manifest_list_without_encryption_manager(generated_manif |
548 | 549 | snapshot.manifests(load_file_io()) |
549 | 550 |
|
550 | 551 |
|
| 552 | +def test_read_encrypted_manifest(generated_manifest_entry_file: str, tmp_path: Path) -> None: |
| 553 | + io = load_file_io() |
| 554 | + key_metadata = StandardKeyMetadata.generate() |
| 555 | + encrypted = _encrypt_ags1(Path(generated_manifest_entry_file).read_bytes(), key_metadata) |
| 556 | + encrypted_path = tmp_path / "manifest.avro" |
| 557 | + encrypted_path.write_bytes(encrypted) |
| 558 | + key_metadata = replace(key_metadata, file_length=len(encrypted)) |
| 559 | + |
| 560 | + def manifest(path: str, key_metadata: bytes | None) -> ManifestFile: |
| 561 | + return ManifestFile.from_args( |
| 562 | + manifest_path=path, |
| 563 | + manifest_length=0, |
| 564 | + partition_spec_id=0, |
| 565 | + added_snapshot_id=0, |
| 566 | + sequence_number=0, |
| 567 | + partitions=[], |
| 568 | + key_metadata=key_metadata, |
| 569 | + ) |
| 570 | + |
| 571 | + expected = manifest(generated_manifest_entry_file, None).fetch_manifest_entry(io, discard_deleted=False) |
| 572 | + assert manifest(str(encrypted_path), key_metadata.to_bytes()).fetch_manifest_entry(io, discard_deleted=False) == expected |
| 573 | + |
| 574 | + wrong_key = replace(key_metadata, encryption_key=bytes(16)) |
| 575 | + with pytest.raises(ValueError, match="GCM tag check failed"): |
| 576 | + manifest(str(encrypted_path), wrong_key.to_bytes()).fetch_manifest_entry(io) |
| 577 | + |
| 578 | + with pytest.raises(ValueError, match="No schema found in Avro file headers"): |
| 579 | + manifest(str(encrypted_path), None).fetch_manifest_entry(io) |
| 580 | + |
| 581 | + |
| 582 | +def test_read_manifest_encrypted_by_iceberg_rust() -> None: |
| 583 | + """The manifest was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" |
| 584 | + manifest = ManifestFile.from_args( |
| 585 | + manifest_path=str(Path(__file__).parent.parent / "table" / "encryption" / "manifest-v3-encrypted.avro"), |
| 586 | + manifest_length=3444, |
| 587 | + partition_spec_id=0, |
| 588 | + content=ManifestContent.DATA, |
| 589 | + sequence_number=1, |
| 590 | + min_sequence_number=1, |
| 591 | + added_snapshot_id=1, |
| 592 | + partitions=[], |
| 593 | + key_metadata=bytes.fromhex("01203031323334353637383961626364656602206665646362613938373635343332313002e835"), |
| 594 | + ) |
| 595 | + |
| 596 | + entries = manifest.fetch_manifest_entry(load_file_io()) |
| 597 | + |
| 598 | + assert [(entry.data_file.file_path, entry.data_file.record_count) for entry in entries] == [ |
| 599 | + ("s3://bucket/table/data/00000.parquet", 3), |
| 600 | + ("s3://bucket/table/data/00001.parquet", 5), |
| 601 | + ] |
| 602 | + assert all(entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == 1 for entry in entries) |
| 603 | + |
| 604 | + |
551 | 605 | def test_read_manifest_cache(generated_manifest_file_file_v2: str) -> None: |
552 | 606 | """Test that ManifestFile objects are cached and reused across multiple reads. |
553 | 607 |
|
|
0 commit comments