Skip to content

Commit c0e9d04

Browse files
committed
Read encrypted Parquet data and position delete files
A data file's key_metadata holds its Parquet encryption key and AAD prefix, so scans decrypt with pyarrow's direct key API when it is set, as iceberg-rust does. That API needs pyarrow 25.
1 parent 1c800eb commit c0e9d04

4 files changed

Lines changed: 141 additions & 5 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
from typing_extensions import override
6969

7070
from pyiceberg.conversions import to_bytes
71+
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
7172
from pyiceberg.exceptions import ResolveError
7273
from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or
7374
from pyiceberg.expressions.literals import Literal
@@ -1158,11 +1159,34 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi
11581159
raise ValueError(f"Unsupported file format: {file_format}")
11591160

11601161

1162+
def _get_data_file_format(data_file: DataFile, **kwargs: Any) -> ds.FileFormat:
1163+
if data_file.key_metadata is None:
1164+
return _get_file_format(data_file.file_format, **kwargs)
1165+
if data_file.file_format != FileFormat.PARQUET:
1166+
raise ValueError(f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet is supported")
1167+
return ds.ParquetFileFormat(decryption_properties=_parquet_decryption_properties(data_file.key_metadata), **kwargs)
1168+
1169+
1170+
def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties:
1171+
from packaging import version
1172+
from pyarrow.parquet import encryption
1173+
1174+
MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION = "25.0.0"
1175+
if version.parse(pyarrow.__version__) < version.parse(MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION):
1176+
raise ImportError(
1177+
f"pyarrow version >= {MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION} required to read encrypted Parquet files, "
1178+
f"but found version {pyarrow.__version__}."
1179+
)
1180+
1181+
metadata = StandardKeyMetadata.from_bytes(key_metadata)
1182+
return encryption.create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix)
1183+
1184+
11611185
def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]:
11621186
if data_file.file_format == FileFormat.PARQUET:
11631187
with io.new_input(data_file.file_path).open() as fi:
1164-
delete_fragment = _get_file_format(
1165-
data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE
1188+
delete_fragment = _get_data_file_format(
1189+
data_file, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE
11661190
).make_fragment(fi)
11671191
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
11681192
table = table.unify_dictionaries()
@@ -1172,7 +1196,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11721196
}
11731197
elif data_file.file_format == FileFormat.ORC:
11741198
with io.new_input(data_file.file_path).open() as fi:
1175-
delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi)
1199+
delete_fragment = _get_data_file_format(data_file).make_fragment(fi)
11761200
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
11771201
# For ORC, file_path columns are not dictionary-encoded, so we use unique() directly
11781202
return {
@@ -1715,7 +1739,7 @@ def _task_to_record_batches(
17151739
format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8}
17161740
if dictionary_columns and task.file.file_format == FileFormat.PARQUET:
17171741
format_kwargs["dictionary_columns"] = dictionary_columns
1718-
arrow_format = _get_file_format(task.file.file_format, **format_kwargs)
1742+
arrow_format = _get_data_file_format(task.file, **format_kwargs)
17191743
with io.new_input(task.file.file_path).open() as fin:
17201744
fragment = arrow_format.make_fragment(fin)
17211745
physical_schema = fragment.physical_schema

‎tests/io/test_pyarrow.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import warnings
2626
import zlib
2727
from collections.abc import Iterator
28+
from dataclasses import replace
2829
from datetime import UTC, date, datetime
2930
from pathlib import Path
3031
from typing import Any, Literal
@@ -42,6 +43,7 @@
4243
from pyroaring import BitMap
4344

4445
from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile
46+
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
4547
from pyiceberg.exceptions import ResolveError
4648
from pyiceberg.expressions import (
4749
AlwaysFalse,
@@ -81,6 +83,7 @@
8183
_check_pyarrow_schema_compatible,
8284
_ConvertToArrowSchema,
8385
_determine_partitions,
86+
_get_data_file_format,
8487
_primitive_to_physical,
8588
_read_all_delete_files,
8689
_read_deletes,
@@ -4132,6 +4135,111 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp
41324135
assert result_batches == []
41334136

41344137

4138+
def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None:
4139+
"""The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md."""
4140+
data_file = DataFile.from_args(
4141+
content=DataFileContent.DATA,
4142+
file_path=str(Path(__file__).parent.parent / "table" / "encryption" / "data-v3-encrypted.parquet"),
4143+
file_format=FileFormat.PARQUET,
4144+
partition={},
4145+
record_count=3,
4146+
file_size_in_bytes=1407,
4147+
key_metadata=bytes.fromhex("01207d30accaf199ff6f49ae7f3d65cbd8c6022033a904147e53c08d6f71c21c6638852a00"),
4148+
)
4149+
table_schema = Schema(NestedField(1, "id", LongType(), required=True), NestedField(2, "data", StringType(), required=False))
4150+
4151+
batches = _task_to_record_batches(
4152+
PyArrowFileIO(),
4153+
FileScanTask(data_file),
4154+
bound_row_filter=AlwaysTrue(),
4155+
projected_schema=table_schema,
4156+
table_schema=table_schema,
4157+
projected_field_ids={1, 2},
4158+
positional_deletes=None,
4159+
case_sensitive=True,
4160+
)
4161+
4162+
assert pa.Table.from_batches(batches).to_pylist() == [{"id": 1, "data": "a"}, {"id": 2, "data": None}, {"id": 3, "data": "c"}]
4163+
4164+
4165+
def test_read_encrypted_parquet_with_position_deletes(tmp_path: Path) -> None:
4166+
from pyarrow.parquet import encryption
4167+
4168+
def write_encrypted(path: str, table: pa.Table, content: DataFileContent) -> DataFile:
4169+
key_metadata = StandardKeyMetadata.generate()
4170+
# Java does not store the AAD prefix in the file, so it must come from key metadata
4171+
encryption_properties = encryption.create_encryption_properties(
4172+
key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False
4173+
)
4174+
pq.write_table(table, path, encryption_properties=encryption_properties)
4175+
return DataFile.from_args(
4176+
content=content,
4177+
file_path=path,
4178+
file_format=FileFormat.PARQUET,
4179+
partition={},
4180+
record_count=len(table),
4181+
file_size_in_bytes=os.path.getsize(path),
4182+
key_metadata=key_metadata.to_bytes(),
4183+
)
4184+
4185+
arrow_schema = pa.schema((pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),))
4186+
data_path = str(tmp_path / "data.parquet")
4187+
data_file = write_encrypted(
4188+
data_path, pa.table([pa.array([1, 2, 3, 4], type=pa.int32())], schema=arrow_schema), DataFileContent.DATA
4189+
)
4190+
delete_file = write_encrypted(
4191+
str(tmp_path / "deletes.parquet"),
4192+
pa.table({"file_path": [data_path], "pos": pa.array([2], type=pa.int64())}),
4193+
DataFileContent.POSITION_DELETES,
4194+
)
4195+
table_schema = Schema(NestedField(1, "id", IntegerType(), required=False))
4196+
4197+
deletes = _read_deletes(PyArrowFileIO(), delete_file)
4198+
batches = _task_to_record_batches(
4199+
PyArrowFileIO(),
4200+
FileScanTask(data_file),
4201+
bound_row_filter=AlwaysTrue(),
4202+
projected_schema=table_schema,
4203+
table_schema=table_schema,
4204+
projected_field_ids={1},
4205+
positional_deletes=[deletes[data_path]],
4206+
case_sensitive=True,
4207+
)
4208+
4209+
assert pa.Table.from_batches(batches).column("id").to_pylist() == [1, 2, 4]
4210+
4211+
4212+
def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None:
4213+
from pyarrow.parquet import encryption
4214+
4215+
key_metadata = StandardKeyMetadata.generate()
4216+
encrypted_path = str(tmp_path / "encrypted.parquet")
4217+
pq.write_table(
4218+
pa.table({"id": [1]}),
4219+
encrypted_path,
4220+
encryption_properties=encryption.create_encryption_properties(
4221+
key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False
4222+
),
4223+
)
4224+
plaintext_path = str(tmp_path / "plaintext.parquet")
4225+
pq.write_table(pa.table({"id": [1]}), plaintext_path)
4226+
4227+
for path, key, error in [
4228+
(encrypted_path, replace(key_metadata, encryption_key=bytes(16)).to_bytes(), "Failed decryption finalization"),
4229+
(encrypted_path, replace(key_metadata, aad_prefix=bytes(16)).to_bytes(), "Failed decryption finalization"),
4230+
(encrypted_path, None, "no decryption found in reader's properties"),
4231+
(plaintext_path, key_metadata.to_bytes(), "Applying decryption properties on plaintext file"),
4232+
]:
4233+
data_file = DataFile.from_args(file_path=path, file_format=FileFormat.PARQUET, key_metadata=key)
4234+
with pytest.raises(OSError, match=error):
4235+
with open(path, "rb") as f:
4236+
_get_data_file_format(data_file).make_fragment(f).to_table()
4237+
4238+
orc_file = DataFile.from_args(file_path="file.orc", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes())
4239+
with pytest.raises(ValueError, match="only Parquet is supported"):
4240+
_get_data_file_format(orc_file)
4241+
4242+
41354243
def test_parse_location_defaults() -> None:
41364244
"""Test that parse_location uses defaults."""
41374245

‎tests/table/encryption/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@
1717
~ under the License.
1818
-->
1919

20-
# Encrypted manifest list test data
20+
# Encrypted test data
2121

2222
`manifest-list-v3-encrypted.avro` is an empty manifest list encrypted by iceberg-rust, copied from its
2323
`crates/iceberg/testdata/manifests_lists/`. It was written by iceberg-rust's `EncryptionManager`, with a memory KMS
2424
holding master key `master-1` = bytes `0x00` to `0x0f`. Its keys and snapshot are in the
2525
`example_table_metadata_v3_encrypted` fixture in `tests/conftest.py`.
26+
27+
`data-v3-encrypted.parquet` is a Parquet data file with three rows, written and encrypted by iceberg-rust's
28+
`ParquetWriter` with a key from its `EncryptionManager`. The key metadata that its manifest entry would record is in
29+
`test_task_to_record_batches_encrypted_by_iceberg_rust` in `tests/io/test_pyarrow.py`.
1.37 KB
Binary file not shown.

0 commit comments

Comments
 (0)