From 221519c4239264364cc051a21179e28fe71ee944 Mon Sep 17 00:00:00 2001 From: Xander Date: Sun, 8 Nov 2026 12:20:24 -0500 Subject: [PATCH] Read encrypted manifests A manifest list entry's key_metadata is the plaintext key metadata of its manifest, so fetch_manifest_entry decrypts with it when it is set, as iceberg-rust's ManifestReader does. --- pyiceberg/manifest.py | 4 +- tests/table/encryption/README.md | 6 +- .../encryption/manifest-v3-encrypted.avro | Bin 0 -> 3444 bytes tests/utils/test_manifest.py | 54 ++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tests/table/encryption/manifest-v3-encrypted.avro diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 3975c9c2a9..aacd61dd79 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -900,7 +900,7 @@ def fetch_manifest_entry( entry_filter: Callable[[ManifestEntry], bool] | None = None, ) -> list[ManifestEntry]: """ - Read the manifest entries from the manifest file. + Read the manifest entries from the manifest file, decrypting it when it has key metadata. Args: io: The FileIO to fetch the file. @@ -911,6 +911,8 @@ def fetch_manifest_entry( An Iterator of manifest entries. """ input_file = io.new_input(self.manifest_path) + if self.key_metadata is not None: + input_file = EncryptedInputFile(input_file, StandardKeyMetadata.from_bytes(self.key_metadata)) with AvroFile[ManifestEntry]( input_file, MANIFEST_ENTRY_SCHEMAS[DEFAULT_READ_VERSION], diff --git a/tests/table/encryption/README.md b/tests/table/encryption/README.md index ae20c650c8..53f1d04210 100644 --- a/tests/table/encryption/README.md +++ b/tests/table/encryption/README.md @@ -17,9 +17,13 @@ ~ under the License. --> -# Encrypted manifest list test data +# Encrypted metadata test data `manifest-list-v3-encrypted.avro` is an empty manifest list encrypted by iceberg-rust, copied from its `crates/iceberg/testdata/manifests_lists/`. It was written by iceberg-rust's `EncryptionManager`, with a memory KMS holding master key `master-1` = bytes `0x00` to `0x0f`. Its keys and snapshot are in the `example_table_metadata_v3_encrypted` fixture in `tests/conftest.py`. + +`manifest-v3-encrypted.avro` is a v3 data manifest with two added entries, written and encrypted by iceberg-rust's +`ManifestWriter` with key `0123456789abcdef` and AAD prefix `fedcba9876543210`. The key metadata that its manifest list +entry would record is in `test_read_manifest_encrypted_by_iceberg_rust` in `tests/utils/test_manifest.py`. diff --git a/tests/table/encryption/manifest-v3-encrypted.avro b/tests/table/encryption/manifest-v3-encrypted.avro new file mode 100644 index 0000000000000000000000000000000000000000..fba73668b73b0e7467eb02c56da2e0b50eb53da9 GIT binary patch literal 3444 zcmV-)4U6(YM^iBX01yD2xeBdOLz&V&$s0dNX<+>@F>t^mJW$^+NkvRd>bLthooGE1 z9}Gdny`aQIA2pEW5+#_wh^S_4ec%LD8$D3ii)W4Keb*<|V;(CpO+98Dl6mYr2~OBX z_qwS9gl&G*Q`)GTQ^fkJ{EiPe+iHu*5+e5KltZ){kfxjAyQox0d}B=H?rPcVcL!;A z@T4R3-=EVmqADU%#z2s*n`hV4Wy#+YoT*5vP3#kSdj^l3=_Bdk4o=r)LBDZXTM?k( zd*s3NgL!d#v0d8Q~xqRms`yt?JNnB3{UUF%+OFi$kkWEJ@Q#S^agdAF0X7|?%A31zBA zGs@%;q*!*TyFLKYI{-DtxGJ*i)#blvzt}9j@_v3@$z{6&z?&8JHtX> zNNR1&KL({vd~oa|Q*!pSKazH4kFX**=7oHIz!$gq7d-bnUZUu3n4>ak4T?FRtW@sg z3XElj5ufj{4UCDHXo3@_zMj>UoWi2k{}-n^01tg?`p+$wji|Ars6iDYonA_fk|^y- z2Ccehy9g=K7rv7m9+wQN{tp!k82E(!u%J zcuK;y+=!jUS=`Gs)x^3H*n2kf=kY2Hj#9eAp&vpSsLUwBTHZ;O&(DRAdoPHcsjeS? zJ-<;R@|@`D^=~0q^fXg;%0=~;2k)svxXtD*wrCzrLlxF=9BSAp{I)~|^1$aJy(<$E z(Bd>$(M$MK%80PTx^6#&4$v9KohXan8USj4n_LkQCE%=}69M|nFr9iRRX~z@ zt!Gk&itwW~Dvv%TmLiv^YF2d`rmgzYwcMar4)bO#MDN70-NY)VP2hzZrdFh6C?EYR zYI7z$1gNEZ8LgJPrslM8zTKf+;6N{w9+B=o_7hY9~>oWAO|IB zBd8zeW2o8nu;47=+VM_JQ@@xhWlXY$=!j*+9l4?8tbZ|>X|g^nQ1 zdl}DxBx-n60^FjJCyT(sHX|%h_YX)z>%xPH?{d%9AE)L+$M)6n*G0JKxO1;ziEJC2 zb+>gIaB0xq&f1?y;z7!I%`oYnA(@NlMMq2FV&#B6R>{jWWdmlEt?RP8BS*XYwT^Dk zM>_IeK6x0`w)oB>@W+3Y`1nx5Jll&VZZ@>R`+7wO;#&)ggIw((a@E20+3NtOq#2?1 zNq_cZ`AbJG;`>^->u;e9Qv7CC08fEa#<8+gBt}Y8oTZ|Iq;J=f2*c_D)rUkIR2AZH zGEqJvkr@ltj!V&=5IYrKsq`k@zT-_)isYmHU&Km>F)J_UO+N=Xy>_jo5a^!xkHm4y zLS$LGYjCSd)iVJpl``ZZF_2^-Rd}ccGTV*Q0~MPkj9bDqS@>l#N_ZzAW#Jh6yBVZW zIPbeCXf=Jj8D#rjTqi$ujw=78e8aJnA;*}pcHmdz@<7-VRV_Bu7ey&{!fm4jIo1-@ zo+Jm?R+Ntn*MCqK@%DRpeOgV8W;FE&-Z$!@u3hdn8CufHEytLHIWSNwrZ_1tYf6a{ z_%I~0AqcuRZhTu4auD5#)E^1uYYt}QUV(qzMk!Ev4WRGQ>J*R#fy#cIlaH*`yE8co zE1Ny?O0hS3VZpfjhTag{ovNxPaOqmA#fh9UR&VW`*bgy6l(?B^+FPqSCZFqkKP51nRW06j_KV(T2hzo*-GL0=6ftYrO95oU~%{_r^l zL~9Y4Ye53kevV@)qZ0ZF@ob@~`==q`ggIo0b8AWT4-C~lD;)R3JuhK2*mTOG^=valF5WL-N>A!}*$&Hy6R&^~( zL8COUx#{dCs)?>?vE5D70_u;0oR|@;f1GR_q{Rj7hrBwKsZ(yzP!zvI{L{11xT_(f z&~AjPmbOvi*+BbF0=wt8QdQNgGQ!PFRmuT;MJruXyI!YG$rXhoFwJtAQeDn1-pUKQ zVq*d8eB_ym+cnIYV<6C~)u*dk^wE$}1n?0{vP))0Rnihd-sU>m`lV&x-z7Tl18kHM z^qfpsyyUF|y6C0EzR`)*3l)9?tX|=3wrBB zo7|daX-2bF=G@qWMz&N1wsFTcNjlS<@^2AS2WdrFC^EuM8zyT&hA$s9Kjk09ul!5U z*0DEyPI&GuS*fhJ_mcUngg69T%-GTX6k=bu&XXzo?E-Y6`K1`aZH&`Y zZ8N^21+fN*Bpq-3*Wm|NLj3ZwR@-1@TiiQ8KwS#LYTJ3EEVitJAU_;1GkyE0)3%0* zM)SP)CEda&tSHVnR-G)noQV1eq_GCjBjdR9gb#0{|KQnsudtBlU`Y;=u24Ez;N#9C zW8c4_dvEpn7sXyD%pS7B$~0KrS2IRF?mSfi&V?0?Gil|1@-nP+oqYPRz(9(RK(a1P zZ7AF`1Tew4*efy3bQ|!@9LAj8g2tF;!|=o_UZ-p5!o2u`Bq}Uw*EN_sS;T5q^Mq01 z3frOw{A`wyMd6Lqw~3hAvjK{2^8^2nl13Wy@vS~;kYPca#7X5!Y?jpB>YRQb^%*~t z4GQYS3CFhUje7I_$>5H)qz)rOS1hWq27Mu$GS`FN2zDOIM>`CLMh!pf`^`;lsh7x= z!TNl8@3YowmFhWn9Eq=h8t6RgT?d@3A)FGTnX1XLL<=UX!|%*!I?aICb_LDN zx%Pz!jEee2ey#MVX})i6qn;6#ET|{V;-=}56`>vrLN~!4 z7AOJ}+3-_##(YTLNtJ2}lPUyUzCz^#Qo3ECKF_$^kf;bTsGw<+I!cn-~u)o1L( zLs*@VJd|ePFH^9&mgcx+F*?xr73n{w8Tkvv-VlV4Gi447$0{1MkOBQ)d*=x?5Qfu* znHGd3m}y^ZwgR<+Mufwc;f%Vg22Gnf|L6Yl^48NFa#nAZl#;Ei^Ow~Z6o=2B^w`*> zJ3R_9+Wu#IU zpIb({LP_!|Ueo}nik=f-$r2o{y zhsu>?I7Ast7_A4fvdf=nrKVNbcS$@z=B_3t1h3%_pHETR5^##IlXrzTHl+)c3sL?2 z?Ukh6kF^(%w9=pmP$RLq-y*o?%Gp#mquK+qcnhIREUNJQaDjZLZgyn7&=F859I21(B%7(I@FJ}u%n_!5W>)< z>69{!9?oJ6xK6=zarHX|pGj{uFje$fblNOwU~7$mX22>Y#f2<~%*~gMvTrUGgLV(X zxA37MUe8D;^CqI52)?ZZ*91pHs|TF`9a}sp8BuSgO6IEcDp^2I2cqX(s;^>k-ox|L zCIR~>+Mkjo{y;zyk0~);^kWp$qZL3^;lLV9vQ$ROtB5hVm^d-^1UQcv!9B}EI#5iI z4(YHtAh(!$UY_7#@jnkOC!@7aZ_1?NHa9^FzsJrlaSJfprqsRh);@@)%+zK8gMeNC z#xWuKOafG}5CiCkhe-xybk!<|F&2n-Cf0J4TF6KWPU?5<2u!9B3Ut?=^1;FY literal 0 HcmV?d00001 diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index accfb07f57..a73d0feeea 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -27,6 +27,7 @@ import pyiceberg.manifest as manifest_module from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.avro.file import DEFAULT_SYNC_INTERVAL, AvroFile, AvroOutputFile +from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.encryption.manager import EncryptionManager from pyiceberg.encryption.stream import _encrypt_ags1 from pyiceberg.io import load_file_io @@ -548,6 +549,59 @@ def test_read_encrypted_manifest_list_without_encryption_manager(generated_manif snapshot.manifests(load_file_io()) +def test_read_encrypted_manifest(generated_manifest_entry_file: str, tmp_path: Path) -> None: + io = load_file_io() + key_metadata = StandardKeyMetadata.generate() + encrypted = _encrypt_ags1(Path(generated_manifest_entry_file).read_bytes(), key_metadata) + encrypted_path = tmp_path / "manifest.avro" + encrypted_path.write_bytes(encrypted) + key_metadata = replace(key_metadata, file_length=len(encrypted)) + + def manifest(path: str, key_metadata: bytes | None) -> ManifestFile: + return ManifestFile.from_args( + manifest_path=path, + manifest_length=0, + partition_spec_id=0, + added_snapshot_id=0, + sequence_number=0, + partitions=[], + key_metadata=key_metadata, + ) + + expected = manifest(generated_manifest_entry_file, None).fetch_manifest_entry(io, discard_deleted=False) + assert manifest(str(encrypted_path), key_metadata.to_bytes()).fetch_manifest_entry(io, discard_deleted=False) == expected + + wrong_key = replace(key_metadata, encryption_key=bytes(16)) + with pytest.raises(ValueError, match="GCM tag check failed"): + manifest(str(encrypted_path), wrong_key.to_bytes()).fetch_manifest_entry(io) + + with pytest.raises(ValueError, match="No schema found in Avro file headers"): + manifest(str(encrypted_path), None).fetch_manifest_entry(io) + + +def test_read_manifest_encrypted_by_iceberg_rust() -> None: + """The manifest was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" + manifest = ManifestFile.from_args( + manifest_path=str(Path(__file__).parent.parent / "table" / "encryption" / "manifest-v3-encrypted.avro"), + manifest_length=3444, + partition_spec_id=0, + content=ManifestContent.DATA, + sequence_number=1, + min_sequence_number=1, + added_snapshot_id=1, + partitions=[], + key_metadata=bytes.fromhex("01203031323334353637383961626364656602206665646362613938373635343332313002e835"), + ) + + entries = manifest.fetch_manifest_entry(load_file_io()) + + assert [(entry.data_file.file_path, entry.data_file.record_count) for entry in entries] == [ + ("s3://bucket/table/data/00000.parquet", 3), + ("s3://bucket/table/data/00001.parquet", 5), + ] + assert all(entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == 1 for entry in entries) + + def test_read_manifest_cache(generated_manifest_file_file_v2: str) -> None: """Test that ManifestFile objects are cached and reused across multiple reads.