Skip to content

Commit 0b2f6e5

Browse files
kevinjqliuCopilot
authored andcommitted
Simplify encrypted read format and fail closed on unsupported formats
Merge format selection and Parquet decryption into _create_arrow_read_format, dropping the lru_cache so per-file keys are not retained. Detect PyArrow decryption support by import instead of parsing the version. Reject encrypted non-Parquet files, including deletion vectors, with the file path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent a715467 commit 0b2f6e5

2 files changed

Lines changed: 26 additions & 27 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,9 +1147,10 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
11471147
return expression_to_pyarrow(preserve_expr, schema)
11481148

11491149

1150-
@lru_cache
1151-
def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.FileFormat:
1150+
def _create_arrow_read_format(file_format: FileFormat, key_metadata: bytes | None, **kwargs: Any) -> ds.FileFormat:
11521151
if file_format == FileFormat.PARQUET:
1152+
if key_metadata is not None:
1153+
kwargs["decryption_properties"] = _parquet_decryption_properties(key_metadata)
11531154
return ds.ParquetFileFormat(**kwargs)
11541155
elif file_format == FileFormat.ORC:
11551156
# ORC doesn't support pre_buffer and buffer_size parameters
@@ -1159,34 +1160,31 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi
11591160
raise ValueError(f"Unsupported file format: {file_format}")
11601161

11611162

1162-
def _get_data_file_format(data_file: DataFile, **kwargs: Any) -> ds.FileFormat:
1163-
if data_file.key_metadata is None:
1164-
return _get_file_format(data_file.file_format, **kwargs)
1165-
if data_file.file_format != FileFormat.PARQUET:
1163+
def _check_encryption_supported(data_file: DataFile) -> None:
1164+
if data_file.key_metadata is not None and data_file.file_format != FileFormat.PARQUET:
11661165
raise ValueError(f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet is supported")
1167-
return ds.ParquetFileFormat(decryption_properties=_parquet_decryption_properties(data_file.key_metadata), **kwargs)
11681166

11691167

11701168
def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties:
1171-
from packaging import version
1172-
from pyarrow.parquet import encryption
1173-
1174-
MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION = "25.0.0"
1175-
if version.parse(pyarrow.__version__) < version.parse(MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION):
1176-
raise ImportError(
1177-
f"pyarrow version >= {MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION} required to read encrypted Parquet files, "
1178-
f"but found version {pyarrow.__version__}."
1179-
)
1169+
try:
1170+
from pyarrow.parquet.encryption import create_decryption_properties
1171+
except ImportError as e:
1172+
raise ImportError(f"pyarrow >= 25.0.0 required to read encrypted Parquet files, found {pyarrow.__version__}") from e
11801173

11811174
metadata = StandardKeyMetadata.from_bytes(key_metadata)
1182-
return encryption.create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix)
1175+
return create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix)
11831176

11841177

11851178
def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]:
1179+
_check_encryption_supported(data_file)
11861180
if data_file.file_format == FileFormat.PARQUET:
11871181
with io.new_input(data_file.file_path).open() as fi:
1188-
delete_fragment = _get_data_file_format(
1189-
data_file, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE
1182+
delete_fragment = _create_arrow_read_format(
1183+
data_file.file_format,
1184+
data_file.key_metadata,
1185+
dictionary_columns=("file_path",),
1186+
pre_buffer=True,
1187+
buffer_size=ONE_MEGABYTE,
11901188
).make_fragment(fi)
11911189
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
11921190
table = table.unify_dictionaries()
@@ -1196,7 +1194,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11961194
}
11971195
elif data_file.file_format == FileFormat.ORC:
11981196
with io.new_input(data_file.file_path).open() as fi:
1199-
delete_fragment = _get_data_file_format(data_file).make_fragment(fi)
1197+
delete_fragment = _create_arrow_read_format(data_file.file_format, data_file.key_metadata).make_fragment(fi)
12001198
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
12011199
# For ORC, file_path columns are not dictionary-encoded, so we use unique() directly
12021200
return {
@@ -1736,10 +1734,11 @@ def _task_to_record_batches(
17361734
downcast_ns_timestamp_to_us: bool | None = None,
17371735
dictionary_columns: tuple[str, ...] = (),
17381736
) -> Iterator[pa.RecordBatch]:
1737+
_check_encryption_supported(task.file)
17391738
format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8}
17401739
if dictionary_columns and task.file.file_format == FileFormat.PARQUET:
17411740
format_kwargs["dictionary_columns"] = dictionary_columns
1742-
arrow_format = _get_data_file_format(task.file, **format_kwargs)
1741+
arrow_format = _create_arrow_read_format(task.file.file_format, task.file.key_metadata, **format_kwargs)
17431742
with io.new_input(task.file.file_path).open() as fin:
17441743
fragment = arrow_format.make_fragment(fin)
17451744
physical_schema = fragment.physical_schema

‎tests/io/test_pyarrow.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,8 @@
8282
StatsAggregator,
8383
_check_pyarrow_schema_compatible,
8484
_ConvertToArrowSchema,
85+
_create_arrow_read_format,
8586
_determine_partitions,
86-
_get_data_file_format,
8787
_primitive_to_physical,
8888
_read_all_delete_files,
8989
_read_deletes,
@@ -4230,14 +4230,14 @@ def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None:
42304230
(encrypted_path, None, "no decryption found in reader's properties"),
42314231
(plaintext_path, key_metadata.to_bytes(), "Applying decryption properties on plaintext file"),
42324232
]:
4233-
data_file = DataFile.from_args(file_path=path, file_format=FileFormat.PARQUET, key_metadata=key)
42344233
with pytest.raises(OSError, match=error):
42354234
with open(path, "rb") as f:
4236-
_get_data_file_format(data_file).make_fragment(f).to_table()
4235+
_create_arrow_read_format(FileFormat.PARQUET, key).make_fragment(f).to_table()
42374236

4238-
orc_file = DataFile.from_args(file_path="file.orc", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes())
4239-
with pytest.raises(ValueError, match="only Parquet is supported"):
4240-
_get_data_file_format(orc_file)
4237+
for file_format in [FileFormat.ORC, FileFormat.PUFFIN]:
4238+
delete_file = DataFile.from_args(file_path="deletes", file_format=file_format, key_metadata=key_metadata.to_bytes())
4239+
with pytest.raises(ValueError, match="file deletes: only Parquet is supported"):
4240+
_read_deletes(PyArrowFileIO(), delete_file)
42414241

42424242

42434243
def test_parse_location_defaults() -> None:

0 commit comments

Comments
 (0)