From a715467fe83b54859586e59fba53c6648c19b84c Mon Sep 17 00:00:00 2001 From: Xander Date: Sun, 8 Nov 2026 13:15:43 -0500 Subject: [PATCH 1/2] Read encrypted Parquet data and position delete files A data file's key_metadata holds its Parquet encryption key and AAD prefix, so scans decrypt with pyarrow's direct key API when it is set, as iceberg-rust does. That API needs pyarrow 25. --- pyiceberg/io/pyarrow.py | 32 +++++- tests/io/test_pyarrow.py | 108 ++++++++++++++++++ tests/table/encryption/README.md | 6 +- .../encryption/data-v3-encrypted.parquet | Bin 0 -> 1407 bytes 4 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 tests/table/encryption/data-v3-encrypted.parquet diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..59055f0eb8 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -68,6 +68,7 @@ from typing_extensions import override from pyiceberg.conversions import to_bytes +from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.exceptions import ResolveError from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or from pyiceberg.expressions.literals import Literal @@ -1158,11 +1159,34 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi raise ValueError(f"Unsupported file format: {file_format}") +def _get_data_file_format(data_file: DataFile, **kwargs: Any) -> ds.FileFormat: + if data_file.key_metadata is None: + return _get_file_format(data_file.file_format, **kwargs) + if data_file.file_format != FileFormat.PARQUET: + raise ValueError(f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet is supported") + return ds.ParquetFileFormat(decryption_properties=_parquet_decryption_properties(data_file.key_metadata), **kwargs) + + +def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties: + from packaging import version + from pyarrow.parquet import encryption + + MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION = "25.0.0" + if version.parse(pyarrow.__version__) < version.parse(MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION): + raise ImportError( + f"pyarrow version >= {MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION} required to read encrypted Parquet files, " + f"but found version {pyarrow.__version__}." + ) + + metadata = StandardKeyMetadata.from_bytes(key_metadata) + return encryption.create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix) + + def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: if data_file.file_format == FileFormat.PARQUET: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_file_format( - data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE + delete_fragment = _get_data_file_format( + data_file, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE ).make_fragment(fi) table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() table = table.unify_dictionaries() @@ -1172,7 +1196,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray] } elif data_file.file_format == FileFormat.ORC: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi) + delete_fragment = _get_data_file_format(data_file).make_fragment(fi) table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() # For ORC, file_path columns are not dictionary-encoded, so we use unique() directly return { @@ -1715,7 +1739,7 @@ def _task_to_record_batches( format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8} if dictionary_columns and task.file.file_format == FileFormat.PARQUET: format_kwargs["dictionary_columns"] = dictionary_columns - arrow_format = _get_file_format(task.file.file_format, **format_kwargs) + arrow_format = _get_data_file_format(task.file, **format_kwargs) with io.new_input(task.file.file_path).open() as fin: fragment = arrow_format.make_fragment(fin) physical_schema = fragment.physical_schema diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index abb7d384d5..06f62f4f99 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -25,6 +25,7 @@ import warnings import zlib from collections.abc import Iterator +from dataclasses import replace from datetime import UTC, date, datetime from pathlib import Path from typing import Any, Literal @@ -42,6 +43,7 @@ from pyroaring import BitMap from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile +from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.exceptions import ResolveError from pyiceberg.expressions import ( AlwaysFalse, @@ -81,6 +83,7 @@ _check_pyarrow_schema_compatible, _ConvertToArrowSchema, _determine_partitions, + _get_data_file_format, _primitive_to_physical, _read_all_delete_files, _read_deletes, @@ -4132,6 +4135,111 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp assert result_batches == [] +def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: + """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=str(Path(__file__).parent.parent / "table" / "encryption" / "data-v3-encrypted.parquet"), + file_format=FileFormat.PARQUET, + partition={}, + record_count=3, + file_size_in_bytes=1407, + key_metadata=bytes.fromhex("01207d30accaf199ff6f49ae7f3d65cbd8c6022033a904147e53c08d6f71c21c6638852a00"), + ) + table_schema = Schema(NestedField(1, "id", LongType(), required=True), NestedField(2, "data", StringType(), required=False)) + + batches = _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file), + bound_row_filter=AlwaysTrue(), + projected_schema=table_schema, + table_schema=table_schema, + projected_field_ids={1, 2}, + positional_deletes=None, + case_sensitive=True, + ) + + assert pa.Table.from_batches(batches).to_pylist() == [{"id": 1, "data": "a"}, {"id": 2, "data": None}, {"id": 3, "data": "c"}] + + +def test_read_encrypted_parquet_with_position_deletes(tmp_path: Path) -> None: + from pyarrow.parquet import encryption + + def write_encrypted(path: str, table: pa.Table, content: DataFileContent) -> DataFile: + key_metadata = StandardKeyMetadata.generate() + # Java does not store the AAD prefix in the file, so it must come from key metadata + encryption_properties = encryption.create_encryption_properties( + key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False + ) + pq.write_table(table, path, encryption_properties=encryption_properties) + return DataFile.from_args( + content=content, + file_path=path, + file_format=FileFormat.PARQUET, + partition={}, + record_count=len(table), + file_size_in_bytes=os.path.getsize(path), + key_metadata=key_metadata.to_bytes(), + ) + + arrow_schema = pa.schema((pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),)) + data_path = str(tmp_path / "data.parquet") + data_file = write_encrypted( + data_path, pa.table([pa.array([1, 2, 3, 4], type=pa.int32())], schema=arrow_schema), DataFileContent.DATA + ) + delete_file = write_encrypted( + str(tmp_path / "deletes.parquet"), + pa.table({"file_path": [data_path], "pos": pa.array([2], type=pa.int64())}), + DataFileContent.POSITION_DELETES, + ) + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + + deletes = _read_deletes(PyArrowFileIO(), delete_file) + batches = _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file), + bound_row_filter=AlwaysTrue(), + projected_schema=table_schema, + table_schema=table_schema, + projected_field_ids={1}, + positional_deletes=[deletes[data_path]], + case_sensitive=True, + ) + + assert pa.Table.from_batches(batches).column("id").to_pylist() == [1, 2, 4] + + +def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None: + from pyarrow.parquet import encryption + + key_metadata = StandardKeyMetadata.generate() + encrypted_path = str(tmp_path / "encrypted.parquet") + pq.write_table( + pa.table({"id": [1]}), + encrypted_path, + encryption_properties=encryption.create_encryption_properties( + key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False + ), + ) + plaintext_path = str(tmp_path / "plaintext.parquet") + pq.write_table(pa.table({"id": [1]}), plaintext_path) + + for path, key, error in [ + (encrypted_path, replace(key_metadata, encryption_key=bytes(16)).to_bytes(), "Failed decryption finalization"), + (encrypted_path, replace(key_metadata, aad_prefix=bytes(16)).to_bytes(), "Failed decryption finalization"), + (encrypted_path, None, "no decryption found in reader's properties"), + (plaintext_path, key_metadata.to_bytes(), "Applying decryption properties on plaintext file"), + ]: + data_file = DataFile.from_args(file_path=path, file_format=FileFormat.PARQUET, key_metadata=key) + with pytest.raises(OSError, match=error): + with open(path, "rb") as f: + _get_data_file_format(data_file).make_fragment(f).to_table() + + orc_file = DataFile.from_args(file_path="file.orc", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes()) + with pytest.raises(ValueError, match="only Parquet is supported"): + _get_data_file_format(orc_file) + + def test_parse_location_defaults() -> None: """Test that parse_location uses defaults.""" diff --git a/tests/table/encryption/README.md b/tests/table/encryption/README.md index 53f1d04210..d88061c682 100644 --- a/tests/table/encryption/README.md +++ b/tests/table/encryption/README.md @@ -17,7 +17,7 @@ ~ under the License. --> -# Encrypted metadata test data +# Encrypted test data `manifest-list-v3-encrypted.avro` is an empty manifest list encrypted by iceberg-rust, copied from its `crates/iceberg/testdata/manifests_lists/`. It was written by iceberg-rust's `EncryptionManager`, with a memory KMS @@ -27,3 +27,7 @@ holding master key `master-1` = bytes `0x00` to `0x0f`. Its keys and snapshot ar `manifest-v3-encrypted.avro` is a v3 data manifest with two added entries, written and encrypted by iceberg-rust's `ManifestWriter` with key `0123456789abcdef` and AAD prefix `fedcba9876543210`. The key metadata that its manifest list entry would record is in `test_read_manifest_encrypted_by_iceberg_rust` in `tests/utils/test_manifest.py`. + +`data-v3-encrypted.parquet` is a Parquet data file with three rows, written and encrypted by iceberg-rust's +`ParquetWriter` with a key from its `EncryptionManager`. The key metadata that its manifest entry would record is in +`test_task_to_record_batches_encrypted_by_iceberg_rust` in `tests/io/test_pyarrow.py`. diff --git a/tests/table/encryption/data-v3-encrypted.parquet b/tests/table/encryption/data-v3-encrypted.parquet new file mode 100644 index 0000000000000000000000000000000000000000..973728192b4e0b10b2d1734585aa4605482def33 GIT binary patch literal 1407 zcmYk+c{~#g7{KuvCT*!z?-n^fYRya}8gtIE!je(3lpMo5P3|$ddgUmJMDB@Ep%bhzMs$Y{P8??ID1PJ008i6#?OQ|Rmy@PtzsG0qBGUX#wFDVMb5LeqoCkTf9s{d9n(fn&YZg}h zOBg4l0ZxiQ%PtTS9g6e3dZ#V^?z_wa>N&}S!QKH<=yqNVd#S3w>M!0j&_EqG2Rj{i4Hbl?UzhO;5Vi;!<)A%S@3HXTG)4YD;~25hGQZ~*UZ0yK98>ZGaU zW}Xu9){rEaJmA4cG?y76w)OlMlkIi^gTL@ks33!i{dBo`E-$J2Z8b2{_!9m*rC>!@ z0VzV`^OelYh((e!8u$r?_47}7H|n_8GDn?8Dp_%1K05Lla_?F=ShHDO)Ed*b4W9W9 z9vJN~skk*TeN)Uuw}%4F`nqRlY;+{a<~Gr5)&Ij92}20!ke5g9SvE#)elJ&`LPb^wiLOsDq2Sm zZdigSl}P`CJ<@WZYt7W`1=;{_cV$hc&%Lp3d-y#k+}4Ui<7ug_2{ycL0h$(*y zjeT4k`KTs6n29;H#sfy_K3Z~83VM3DKJt0Y!R_JeJM(^QlYvl;FG`uJPh3x)CWaP6 zm|W{Zfmt%g*gmLbrc|%xn5B37vcg++O4a=wFOC7U*(uj*VE;wwqQ_J;Ppjr)+g zR`)e7zqTjWk7`6GNTkxBq|RBnz6M-aRPi+A=_GizFiz+d6(LQ&0~3+uCGf&L}Oar|A^9`ii&hr>KW(uJ8f&jis3c^48+d z;^1(N+Ohn`f`@2R@7IFrhwG?iY2DE5N~>fZWkYNAW*$7dd2ln*UR&vn*D%Syb?Nma zwi6m9o64~!A9CsIg7l3f0C>k37iUZ#36B&ih4F?LKvQ9}T)A3Hx&PSaiDg&_#Qc1J zVwM7I$J7viV9vS5X+#5C`J`r}E4C~j$a?Cx1Efp!WhyISRPx;W={YL$A7dMQ8~s5P xxq0%>H=ghwq!9tZ6F;XKSj@_4oA(VU@^R0>!K5Z-ZMesuEWt%*U?+OL{{xD`i@yK> literal 0 HcmV?d00001 From 0b2f6e56879fc736c8a68d290c69d0ae4d364448 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 9 Nov 2026 12:28:54 -0800 Subject: [PATCH 2/2] Simplify encrypted read format and fail closed on unsupported formats Merge format selection and Parquet decryption into _create_arrow_read_format, dropping the lru_cache so per-file keys are not retained. Detect PyArrow decryption support by import instead of parsing the version. Reject encrypted non-Parquet files, including deletion vectors, with the file path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyiceberg/io/pyarrow.py | 41 ++++++++++++++++++++-------------------- tests/io/test_pyarrow.py | 12 ++++++------ 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 59055f0eb8..9f1e4c9b58 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1147,9 +1147,10 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema return expression_to_pyarrow(preserve_expr, schema) -@lru_cache -def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.FileFormat: +def _create_arrow_read_format(file_format: FileFormat, key_metadata: bytes | None, **kwargs: Any) -> ds.FileFormat: if file_format == FileFormat.PARQUET: + if key_metadata is not None: + kwargs["decryption_properties"] = _parquet_decryption_properties(key_metadata) return ds.ParquetFileFormat(**kwargs) elif file_format == FileFormat.ORC: # ORC doesn't support pre_buffer and buffer_size parameters @@ -1159,34 +1160,31 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi raise ValueError(f"Unsupported file format: {file_format}") -def _get_data_file_format(data_file: DataFile, **kwargs: Any) -> ds.FileFormat: - if data_file.key_metadata is None: - return _get_file_format(data_file.file_format, **kwargs) - if data_file.file_format != FileFormat.PARQUET: +def _check_encryption_supported(data_file: DataFile) -> None: + if data_file.key_metadata is not None and data_file.file_format != FileFormat.PARQUET: raise ValueError(f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet is supported") - return ds.ParquetFileFormat(decryption_properties=_parquet_decryption_properties(data_file.key_metadata), **kwargs) def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties: - from packaging import version - from pyarrow.parquet import encryption - - MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION = "25.0.0" - if version.parse(pyarrow.__version__) < version.parse(MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION): - raise ImportError( - f"pyarrow version >= {MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION} required to read encrypted Parquet files, " - f"but found version {pyarrow.__version__}." - ) + try: + from pyarrow.parquet.encryption import create_decryption_properties + except ImportError as e: + raise ImportError(f"pyarrow >= 25.0.0 required to read encrypted Parquet files, found {pyarrow.__version__}") from e metadata = StandardKeyMetadata.from_bytes(key_metadata) - return encryption.create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix) + return create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix) def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]: + _check_encryption_supported(data_file) if data_file.file_format == FileFormat.PARQUET: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_data_file_format( - data_file, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE + delete_fragment = _create_arrow_read_format( + data_file.file_format, + data_file.key_metadata, + dictionary_columns=("file_path",), + pre_buffer=True, + buffer_size=ONE_MEGABYTE, ).make_fragment(fi) table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() table = table.unify_dictionaries() @@ -1196,7 +1194,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray] } elif data_file.file_format == FileFormat.ORC: with io.new_input(data_file.file_path).open() as fi: - delete_fragment = _get_data_file_format(data_file).make_fragment(fi) + delete_fragment = _create_arrow_read_format(data_file.file_format, data_file.key_metadata).make_fragment(fi) table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table() # For ORC, file_path columns are not dictionary-encoded, so we use unique() directly return { @@ -1736,10 +1734,11 @@ def _task_to_record_batches( downcast_ns_timestamp_to_us: bool | None = None, dictionary_columns: tuple[str, ...] = (), ) -> Iterator[pa.RecordBatch]: + _check_encryption_supported(task.file) format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8} if dictionary_columns and task.file.file_format == FileFormat.PARQUET: format_kwargs["dictionary_columns"] = dictionary_columns - arrow_format = _get_data_file_format(task.file, **format_kwargs) + arrow_format = _create_arrow_read_format(task.file.file_format, task.file.key_metadata, **format_kwargs) with io.new_input(task.file.file_path).open() as fin: fragment = arrow_format.make_fragment(fin) physical_schema = fragment.physical_schema diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 06f62f4f99..7fd809e1f1 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -82,8 +82,8 @@ StatsAggregator, _check_pyarrow_schema_compatible, _ConvertToArrowSchema, + _create_arrow_read_format, _determine_partitions, - _get_data_file_format, _primitive_to_physical, _read_all_delete_files, _read_deletes, @@ -4230,14 +4230,14 @@ def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None: (encrypted_path, None, "no decryption found in reader's properties"), (plaintext_path, key_metadata.to_bytes(), "Applying decryption properties on plaintext file"), ]: - data_file = DataFile.from_args(file_path=path, file_format=FileFormat.PARQUET, key_metadata=key) with pytest.raises(OSError, match=error): with open(path, "rb") as f: - _get_data_file_format(data_file).make_fragment(f).to_table() + _create_arrow_read_format(FileFormat.PARQUET, key).make_fragment(f).to_table() - orc_file = DataFile.from_args(file_path="file.orc", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes()) - with pytest.raises(ValueError, match="only Parquet is supported"): - _get_data_file_format(orc_file) + for file_format in [FileFormat.ORC, FileFormat.PUFFIN]: + delete_file = DataFile.from_args(file_path="deletes", file_format=file_format, key_metadata=key_metadata.to_bytes()) + with pytest.raises(ValueError, match="file deletes: only Parquet is supported"): + _read_deletes(PyArrowFileIO(), delete_file) def test_parse_location_defaults() -> None: