|
44 | 44 |
|
45 | 45 | from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile |
46 | 46 | from pyiceberg.encryption.key_metadata import StandardKeyMetadata |
| 47 | +from pyiceberg.encryption.stream import _encrypt_ags1 |
47 | 48 | from pyiceberg.exceptions import ResolveError |
48 | 49 | from pyiceberg.expressions import ( |
49 | 50 | AlwaysFalse, |
@@ -2083,6 +2084,68 @@ def test_read_deletion_vector_blob_from_content_range(tmp_path: Path) -> None: |
2083 | 2084 | assert deletes == {referenced_data_file: pa.chunked_array([[1, 3, 5]])} |
2084 | 2085 |
|
2085 | 2086 |
|
| 2087 | +def _puffin_delete_file(file_path: str, **kwargs: Any) -> DataFile: |
| 2088 | + return DataFile.from_args( |
| 2089 | + _table_format_version=3, |
| 2090 | + content=DataFileContent.POSITION_DELETES, |
| 2091 | + file_path=file_path, |
| 2092 | + file_format=FileFormat.PUFFIN, |
| 2093 | + record_count=3, |
| 2094 | + **kwargs, |
| 2095 | + ) |
| 2096 | + |
| 2097 | + |
| 2098 | +def test_read_deletes_from_puffin_file_encrypted_by_iceberg_rust() -> None: |
| 2099 | + """The deletion vector was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" |
| 2100 | + file_path = str(Path(__file__).parent.parent / "table" / "encryption" / "dv-v3-encrypted.puffin") |
| 2101 | + referenced_data_file = "s3://bucket/table/data/00000.parquet" |
| 2102 | + key_metadata = bytes.fromhex("01203031323334353637383961626364656602206665646362613938373635343332313002ea04") |
| 2103 | + |
| 2104 | + for dv in [ |
| 2105 | + _puffin_delete_file(file_path, key_metadata=key_metadata), |
| 2106 | + _puffin_delete_file( |
| 2107 | + file_path, |
| 2108 | + key_metadata=key_metadata, |
| 2109 | + referenced_data_file=referenced_data_file, |
| 2110 | + content_offset=4, |
| 2111 | + content_size_in_bytes=46, |
| 2112 | + ), |
| 2113 | + ]: |
| 2114 | + assert _read_deletes(PyArrowFileIO(), dv) == {referenced_data_file: pa.chunked_array([[1, 3, 5]])} |
| 2115 | + |
| 2116 | + |
| 2117 | +def test_read_deletes_from_encrypted_puffin_file(tmp_path: Path) -> None: |
| 2118 | + referenced_data_file = f"{tmp_path}/data.parquet" |
| 2119 | + delete_file_path = f"{tmp_path}/deletes.puffin" |
| 2120 | + puffin_payload = _deletion_vector_puffin_payload(referenced_data_file) |
| 2121 | + blob = PuffinFile(puffin_payload).footer.blobs[0] |
| 2122 | + key_metadata = StandardKeyMetadata.generate() |
| 2123 | + encrypted = _encrypt_ags1(puffin_payload, key_metadata) |
| 2124 | + key_metadata = replace(key_metadata, file_length=len(encrypted)) |
| 2125 | + wrong_key = replace(key_metadata, encryption_key=bytes(16)) |
| 2126 | + |
| 2127 | + with open(delete_file_path, "wb") as f: |
| 2128 | + f.write(encrypted) |
| 2129 | + |
| 2130 | + for content_reference, plaintext_error in [ |
| 2131 | + ({}, "Incorrect magic bytes"), |
| 2132 | + ( |
| 2133 | + {"referenced_data_file": referenced_data_file, "content_offset": blob.offset, "content_size_in_bytes": blob.length}, |
| 2134 | + "Invalid bitmap data length", |
| 2135 | + ), |
| 2136 | + ]: |
| 2137 | + dv = _puffin_delete_file(delete_file_path, key_metadata=key_metadata.to_bytes(), **content_reference) |
| 2138 | + assert _read_deletes(PyArrowFileIO(), dv) == {referenced_data_file: pa.chunked_array([[1, 3, 5]])} |
| 2139 | + |
| 2140 | + with pytest.raises(ValueError, match="GCM tag check failed"): |
| 2141 | + _read_deletes( |
| 2142 | + PyArrowFileIO(), _puffin_delete_file(delete_file_path, key_metadata=wrong_key.to_bytes(), **content_reference) |
| 2143 | + ) |
| 2144 | + |
| 2145 | + with pytest.raises(ValueError, match=plaintext_error): |
| 2146 | + _read_deletes(PyArrowFileIO(), _puffin_delete_file(delete_file_path, **content_reference)) |
| 2147 | + |
| 2148 | + |
2086 | 2149 | def test_read_all_delete_files_keeps_multiple_dv_content_ranges_for_same_path(tmp_path: Path) -> None: |
2087 | 2150 | referenced_data_files = [f"{tmp_path}/data-{index}.parquet" for index in range(2)] |
2088 | 2151 | first_dv_blob = _deletion_vector_blob(_deletion_vector_bitmap_payload([1, 3])) |
@@ -4234,10 +4297,9 @@ def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None: |
4234 | 4297 | with open(path, "rb") as f: |
4235 | 4298 | _create_arrow_read_format(FileFormat.PARQUET, key).make_fragment(f).to_table() |
4236 | 4299 |
|
4237 | | - for file_format in [FileFormat.ORC, FileFormat.PUFFIN]: |
4238 | | - delete_file = DataFile.from_args(file_path="deletes", file_format=file_format, key_metadata=key_metadata.to_bytes()) |
4239 | | - with pytest.raises(ValueError, match="file deletes: only Parquet is supported"): |
4240 | | - _read_deletes(PyArrowFileIO(), delete_file) |
| 4300 | + delete_file = DataFile.from_args(file_path="deletes", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes()) |
| 4301 | + with pytest.raises(ValueError, match="file deletes: only Parquet and Puffin are supported"): |
| 4302 | + _read_deletes(PyArrowFileIO(), delete_file) |
4241 | 4303 |
|
4242 | 4304 |
|
4243 | 4305 | def test_parse_location_defaults() -> None: |
|
0 commit comments