@@ -1147,9 +1147,10 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
11471147 return expression_to_pyarrow (preserve_expr , schema )
11481148
11491149
1150- @lru_cache
1151- def _get_file_format (file_format : FileFormat , ** kwargs : dict [str , Any ]) -> ds .FileFormat :
1150+ def _create_arrow_read_format (file_format : FileFormat , key_metadata : bytes | None , ** kwargs : Any ) -> ds .FileFormat :
11521151 if file_format == FileFormat .PARQUET :
1152+ if key_metadata is not None :
1153+ kwargs ["decryption_properties" ] = _parquet_decryption_properties (key_metadata )
11531154 return ds .ParquetFileFormat (** kwargs )
11541155 elif file_format == FileFormat .ORC :
11551156 # ORC doesn't support pre_buffer and buffer_size parameters
@@ -1159,34 +1160,31 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi
11591160 raise ValueError (f"Unsupported file format: { file_format } " )
11601161
11611162
1162- def _get_data_file_format (data_file : DataFile , ** kwargs : Any ) -> ds .FileFormat :
1163- if data_file .key_metadata is None :
1164- return _get_file_format (data_file .file_format , ** kwargs )
1165- if data_file .file_format != FileFormat .PARQUET :
1163+ def _check_encryption_supported (data_file : DataFile ) -> None :
1164+ if data_file .key_metadata is not None and data_file .file_format != FileFormat .PARQUET :
11661165 raise ValueError (f"Cannot read encrypted { data_file .file_format } file { data_file .file_path } : only Parquet is supported" )
1167- return ds .ParquetFileFormat (decryption_properties = _parquet_decryption_properties (data_file .key_metadata ), ** kwargs )
11681166
11691167
11701168def _parquet_decryption_properties (key_metadata : bytes ) -> pq .FileDecryptionProperties :
1171- from packaging import version
1172- from pyarrow .parquet import encryption
1173-
1174- MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION = "25.0.0"
1175- if version .parse (pyarrow .__version__ ) < version .parse (MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION ):
1176- raise ImportError (
1177- f"pyarrow version >= { MIN_PYARROW_VERSION_SUPPORTING_PARQUET_DECRYPTION } required to read encrypted Parquet files, "
1178- f"but found version { pyarrow .__version__ } ."
1179- )
1169+ try :
1170+ from pyarrow .parquet .encryption import create_decryption_properties
1171+ except ImportError as e :
1172+ raise ImportError (f"pyarrow >= 25.0.0 required to read encrypted Parquet files, found { pyarrow .__version__ } " ) from e
11801173
11811174 metadata = StandardKeyMetadata .from_bytes (key_metadata )
1182- return encryption . create_decryption_properties (metadata .encryption_key , aad_prefix = metadata .aad_prefix )
1175+ return create_decryption_properties (metadata .encryption_key , aad_prefix = metadata .aad_prefix )
11831176
11841177
11851178def _read_deletes (io : FileIO , data_file : DataFile ) -> dict [str , pa .ChunkedArray ]:
1179+ _check_encryption_supported (data_file )
11861180 if data_file .file_format == FileFormat .PARQUET :
11871181 with io .new_input (data_file .file_path ).open () as fi :
1188- delete_fragment = _get_data_file_format (
1189- data_file , dictionary_columns = ("file_path" ,), pre_buffer = True , buffer_size = ONE_MEGABYTE
1182+ delete_fragment = _create_arrow_read_format (
1183+ data_file .file_format ,
1184+ data_file .key_metadata ,
1185+ dictionary_columns = ("file_path" ,),
1186+ pre_buffer = True ,
1187+ buffer_size = ONE_MEGABYTE ,
11901188 ).make_fragment (fi )
11911189 table = ds .Scanner .from_fragment (fragment = delete_fragment ).to_table ()
11921190 table = table .unify_dictionaries ()
@@ -1196,7 +1194,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11961194 }
11971195 elif data_file .file_format == FileFormat .ORC :
11981196 with io .new_input (data_file .file_path ).open () as fi :
1199- delete_fragment = _get_data_file_format (data_file ).make_fragment (fi )
1197+ delete_fragment = _create_arrow_read_format (data_file . file_format , data_file . key_metadata ).make_fragment (fi )
12001198 table = ds .Scanner .from_fragment (fragment = delete_fragment ).to_table ()
12011199 # For ORC, file_path columns are not dictionary-encoded, so we use unique() directly
12021200 return {
@@ -1736,10 +1734,11 @@ def _task_to_record_batches(
17361734 downcast_ns_timestamp_to_us : bool | None = None ,
17371735 dictionary_columns : tuple [str , ...] = (),
17381736) -> Iterator [pa .RecordBatch ]:
1737+ _check_encryption_supported (task .file )
17391738 format_kwargs : dict [str , Any ] = {"pre_buffer" : True , "buffer_size" : ONE_MEGABYTE * 8 }
17401739 if dictionary_columns and task .file .file_format == FileFormat .PARQUET :
17411740 format_kwargs ["dictionary_columns" ] = dictionary_columns
1742- arrow_format = _get_data_file_format (task .file , ** format_kwargs )
1741+ arrow_format = _create_arrow_read_format (task .file . file_format , task . file . key_metadata , ** format_kwargs )
17431742 with io .new_input (task .file .file_path ).open () as fin :
17441743 fragment = arrow_format .make_fragment (fin )
17451744 physical_schema = fragment .physical_schema
0 commit comments