|
25 | 25 | import warnings |
26 | 26 | import zlib |
27 | 27 | from collections.abc import Iterator |
| 28 | +from dataclasses import replace |
28 | 29 | from datetime import UTC, date, datetime |
29 | 30 | from pathlib import Path |
30 | 31 | from typing import Any, Literal |
|
42 | 43 | from pyroaring import BitMap |
43 | 44 |
|
44 | 45 | from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile |
| 46 | +from pyiceberg.encryption.key_metadata import StandardKeyMetadata |
45 | 47 | from pyiceberg.exceptions import ResolveError |
46 | 48 | from pyiceberg.expressions import ( |
47 | 49 | AlwaysFalse, |
|
81 | 83 | _check_pyarrow_schema_compatible, |
82 | 84 | _ConvertToArrowSchema, |
83 | 85 | _determine_partitions, |
| 86 | + _get_data_file_format, |
84 | 87 | _primitive_to_physical, |
85 | 88 | _read_all_delete_files, |
86 | 89 | _read_deletes, |
@@ -4132,6 +4135,111 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp |
4132 | 4135 | assert result_batches == [] |
4133 | 4136 |
|
4134 | 4137 |
|
| 4138 | +def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: |
| 4139 | + """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" |
| 4140 | + data_file = DataFile.from_args( |
| 4141 | + content=DataFileContent.DATA, |
| 4142 | + file_path=str(Path(__file__).parent.parent / "table" / "encryption" / "data-v3-encrypted.parquet"), |
| 4143 | + file_format=FileFormat.PARQUET, |
| 4144 | + partition={}, |
| 4145 | + record_count=3, |
| 4146 | + file_size_in_bytes=1407, |
| 4147 | + key_metadata=bytes.fromhex("01207d30accaf199ff6f49ae7f3d65cbd8c6022033a904147e53c08d6f71c21c6638852a00"), |
| 4148 | + ) |
| 4149 | + table_schema = Schema(NestedField(1, "id", LongType(), required=True), NestedField(2, "data", StringType(), required=False)) |
| 4150 | + |
| 4151 | + batches = _task_to_record_batches( |
| 4152 | + PyArrowFileIO(), |
| 4153 | + FileScanTask(data_file), |
| 4154 | + bound_row_filter=AlwaysTrue(), |
| 4155 | + projected_schema=table_schema, |
| 4156 | + table_schema=table_schema, |
| 4157 | + projected_field_ids={1, 2}, |
| 4158 | + positional_deletes=None, |
| 4159 | + case_sensitive=True, |
| 4160 | + ) |
| 4161 | + |
| 4162 | + assert pa.Table.from_batches(batches).to_pylist() == [{"id": 1, "data": "a"}, {"id": 2, "data": None}, {"id": 3, "data": "c"}] |
| 4163 | + |
| 4164 | + |
| 4165 | +def test_read_encrypted_parquet_with_position_deletes(tmp_path: Path) -> None: |
| 4166 | + from pyarrow.parquet import encryption |
| 4167 | + |
| 4168 | + def write_encrypted(path: str, table: pa.Table, content: DataFileContent) -> DataFile: |
| 4169 | + key_metadata = StandardKeyMetadata.generate() |
| 4170 | + # Java does not store the AAD prefix in the file, so it must come from key metadata |
| 4171 | + encryption_properties = encryption.create_encryption_properties( |
| 4172 | + key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False |
| 4173 | + ) |
| 4174 | + pq.write_table(table, path, encryption_properties=encryption_properties) |
| 4175 | + return DataFile.from_args( |
| 4176 | + content=content, |
| 4177 | + file_path=path, |
| 4178 | + file_format=FileFormat.PARQUET, |
| 4179 | + partition={}, |
| 4180 | + record_count=len(table), |
| 4181 | + file_size_in_bytes=os.path.getsize(path), |
| 4182 | + key_metadata=key_metadata.to_bytes(), |
| 4183 | + ) |
| 4184 | + |
| 4185 | + arrow_schema = pa.schema((pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),)) |
| 4186 | + data_path = str(tmp_path / "data.parquet") |
| 4187 | + data_file = write_encrypted( |
| 4188 | + data_path, pa.table([pa.array([1, 2, 3, 4], type=pa.int32())], schema=arrow_schema), DataFileContent.DATA |
| 4189 | + ) |
| 4190 | + delete_file = write_encrypted( |
| 4191 | + str(tmp_path / "deletes.parquet"), |
| 4192 | + pa.table({"file_path": [data_path], "pos": pa.array([2], type=pa.int64())}), |
| 4193 | + DataFileContent.POSITION_DELETES, |
| 4194 | + ) |
| 4195 | + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) |
| 4196 | + |
| 4197 | + deletes = _read_deletes(PyArrowFileIO(), delete_file) |
| 4198 | + batches = _task_to_record_batches( |
| 4199 | + PyArrowFileIO(), |
| 4200 | + FileScanTask(data_file), |
| 4201 | + bound_row_filter=AlwaysTrue(), |
| 4202 | + projected_schema=table_schema, |
| 4203 | + table_schema=table_schema, |
| 4204 | + projected_field_ids={1}, |
| 4205 | + positional_deletes=[deletes[data_path]], |
| 4206 | + case_sensitive=True, |
| 4207 | + ) |
| 4208 | + |
| 4209 | + assert pa.Table.from_batches(batches).column("id").to_pylist() == [1, 2, 4] |
| 4210 | + |
| 4211 | + |
| 4212 | +def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None: |
| 4213 | + from pyarrow.parquet import encryption |
| 4214 | + |
| 4215 | + key_metadata = StandardKeyMetadata.generate() |
| 4216 | + encrypted_path = str(tmp_path / "encrypted.parquet") |
| 4217 | + pq.write_table( |
| 4218 | + pa.table({"id": [1]}), |
| 4219 | + encrypted_path, |
| 4220 | + encryption_properties=encryption.create_encryption_properties( |
| 4221 | + key_metadata.encryption_key, aad_prefix=key_metadata.aad_prefix, store_aad_prefix=False |
| 4222 | + ), |
| 4223 | + ) |
| 4224 | + plaintext_path = str(tmp_path / "plaintext.parquet") |
| 4225 | + pq.write_table(pa.table({"id": [1]}), plaintext_path) |
| 4226 | + |
| 4227 | + for path, key, error in [ |
| 4228 | + (encrypted_path, replace(key_metadata, encryption_key=bytes(16)).to_bytes(), "Failed decryption finalization"), |
| 4229 | + (encrypted_path, replace(key_metadata, aad_prefix=bytes(16)).to_bytes(), "Failed decryption finalization"), |
| 4230 | + (encrypted_path, None, "no decryption found in reader's properties"), |
| 4231 | + (plaintext_path, key_metadata.to_bytes(), "Applying decryption properties on plaintext file"), |
| 4232 | + ]: |
| 4233 | + data_file = DataFile.from_args(file_path=path, file_format=FileFormat.PARQUET, key_metadata=key) |
| 4234 | + with pytest.raises(OSError, match=error): |
| 4235 | + with open(path, "rb") as f: |
| 4236 | + _get_data_file_format(data_file).make_fragment(f).to_table() |
| 4237 | + |
| 4238 | + orc_file = DataFile.from_args(file_path="file.orc", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes()) |
| 4239 | + with pytest.raises(ValueError, match="only Parquet is supported"): |
| 4240 | + _get_data_file_format(orc_file) |
| 4241 | + |
| 4242 | + |
4135 | 4243 | def test_parse_location_defaults() -> None: |
4136 | 4244 | """Test that parse_location uses defaults.""" |
4137 | 4245 |
|
|
0 commit comments