Skip to content

Commit c8e7781

Browse files
committed
Read encrypted deletion vectors
A Puffin delete file's key_metadata holds the plaintext key metadata of its AGS1 stream, so deletion vector reads decrypt with it when it is set, as iceberg-rust does. Content offsets refer to the decrypted file.
1 parent 1310a19 commit c8e7781

5 files changed

Lines changed: 88 additions & 9 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1161,8 +1161,10 @@ def _create_arrow_read_format(file_format: FileFormat, key_metadata: bytes | Non
11611161

11621162

11631163
def _check_encryption_supported(data_file: DataFile) -> None:
1164-
if data_file.key_metadata is not None and data_file.file_format != FileFormat.PARQUET:
1165-
raise ValueError(f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet is supported")
1164+
if data_file.key_metadata is not None and data_file.file_format not in (FileFormat.PARQUET, FileFormat.PUFFIN):
1165+
raise ValueError(
1166+
f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet and Puffin are supported"
1167+
)
11661168

11671169

11681170
def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties:

‎pyiceberg/table/deletion_vector.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@
2121

2222
from pyroaring import BitMap, FrozenBitMap
2323

24+
from pyiceberg.encryption.io import EncryptedInputFile
25+
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
2426
from pyiceberg.table.puffin import PuffinFile
2527

2628
if TYPE_CHECKING:
2729
import pyarrow as pa
2830

29-
from pyiceberg.io import FileIO
31+
from pyiceberg.io import FileIO, InputFile
3032
from pyiceberg.manifest import DataFile
3133

3234
EMPTY_BITMAP = FrozenBitMap()
@@ -152,10 +154,18 @@ def _has_deletion_vector_content_reference(dv: "DataFile") -> bool:
152154
return dv.content_offset is not None or dv.content_size_in_bytes is not None or dv.referenced_data_file is not None
153155

154156

157+
def _new_input(io: "FileIO", dv: "DataFile") -> "InputFile":
158+
"""Return the Puffin file to read, decrypting it when it has key metadata so content offsets refer to the plaintext."""
159+
input_file = io.new_input(dv.file_path)
160+
if dv.key_metadata is not None:
161+
return EncryptedInputFile(input_file, StandardKeyMetadata.from_bytes(dv.key_metadata))
162+
return input_file
163+
164+
155165
def _read_deletion_vector(io: "FileIO", dv: "DataFile") -> DeletionVector:
156166
content_offset, content_size_in_bytes, referenced_data_file = _validate_deletion_vector_content(dv)
157167

158-
with io.new_input(dv.file_path).open() as fi:
168+
with _new_input(io, dv).open() as fi:
159169
fi.seek(content_offset)
160170
payload = fi.read(content_size_in_bytes)
161171

@@ -173,7 +183,7 @@ def read_deletion_vectors(io: "FileIO", dv: "DataFile") -> list[DeletionVector]:
173183
if _has_deletion_vector_content_reference(dv):
174184
return [_read_deletion_vector(io, dv)]
175185

176-
with io.new_input(dv.file_path).open() as fi:
186+
with _new_input(io, dv).open() as fi:
177187
return deletion_vectors_from_puffin_file(PuffinFile(fi.read()))
178188

179189

‎tests/io/test_pyarrow.py‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444

4545
from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile
4646
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
47+
from pyiceberg.encryption.stream import _encrypt_ags1
4748
from pyiceberg.exceptions import ResolveError
4849
from pyiceberg.expressions import (
4950
AlwaysFalse,
@@ -2083,6 +2084,68 @@ def test_read_deletion_vector_blob_from_content_range(tmp_path: Path) -> None:
20832084
assert deletes == {referenced_data_file: pa.chunked_array([[1, 3, 5]])}
20842085

20852086

2087+
def _puffin_delete_file(file_path: str, **kwargs: Any) -> DataFile:
2088+
return DataFile.from_args(
2089+
_table_format_version=3,
2090+
content=DataFileContent.POSITION_DELETES,
2091+
file_path=file_path,
2092+
file_format=FileFormat.PUFFIN,
2093+
record_count=3,
2094+
**kwargs,
2095+
)
2096+
2097+
2098+
def test_read_deletes_from_puffin_file_encrypted_by_iceberg_rust() -> None:
2099+
"""The deletion vector was encrypted by iceberg-rust, see tests/table/encryption/README.md."""
2100+
file_path = str(Path(__file__).parent.parent / "table" / "encryption" / "dv-v3-encrypted.puffin")
2101+
referenced_data_file = "s3://bucket/table/data/00000.parquet"
2102+
key_metadata = bytes.fromhex("01203031323334353637383961626364656602206665646362613938373635343332313002ea04")
2103+
2104+
for dv in [
2105+
_puffin_delete_file(file_path, key_metadata=key_metadata),
2106+
_puffin_delete_file(
2107+
file_path,
2108+
key_metadata=key_metadata,
2109+
referenced_data_file=referenced_data_file,
2110+
content_offset=4,
2111+
content_size_in_bytes=46,
2112+
),
2113+
]:
2114+
assert _read_deletes(PyArrowFileIO(), dv) == {referenced_data_file: pa.chunked_array([[1, 3, 5]])}
2115+
2116+
2117+
def test_read_deletes_from_encrypted_puffin_file(tmp_path: Path) -> None:
2118+
referenced_data_file = f"{tmp_path}/data.parquet"
2119+
delete_file_path = f"{tmp_path}/deletes.puffin"
2120+
puffin_payload = _deletion_vector_puffin_payload(referenced_data_file)
2121+
blob = PuffinFile(puffin_payload).footer.blobs[0]
2122+
key_metadata = StandardKeyMetadata.generate()
2123+
encrypted = _encrypt_ags1(puffin_payload, key_metadata)
2124+
key_metadata = replace(key_metadata, file_length=len(encrypted))
2125+
wrong_key = replace(key_metadata, encryption_key=bytes(16))
2126+
2127+
with open(delete_file_path, "wb") as f:
2128+
f.write(encrypted)
2129+
2130+
for content_reference, plaintext_error in [
2131+
({}, "Incorrect magic bytes"),
2132+
(
2133+
{"referenced_data_file": referenced_data_file, "content_offset": blob.offset, "content_size_in_bytes": blob.length},
2134+
"Invalid bitmap data length",
2135+
),
2136+
]:
2137+
dv = _puffin_delete_file(delete_file_path, key_metadata=key_metadata.to_bytes(), **content_reference)
2138+
assert _read_deletes(PyArrowFileIO(), dv) == {referenced_data_file: pa.chunked_array([[1, 3, 5]])}
2139+
2140+
with pytest.raises(ValueError, match="GCM tag check failed"):
2141+
_read_deletes(
2142+
PyArrowFileIO(), _puffin_delete_file(delete_file_path, key_metadata=wrong_key.to_bytes(), **content_reference)
2143+
)
2144+
2145+
with pytest.raises(ValueError, match=plaintext_error):
2146+
_read_deletes(PyArrowFileIO(), _puffin_delete_file(delete_file_path, **content_reference))
2147+
2148+
20862149
def test_read_all_delete_files_keeps_multiple_dv_content_ranges_for_same_path(tmp_path: Path) -> None:
20872150
referenced_data_files = [f"{tmp_path}/data-{index}.parquet" for index in range(2)]
20882151
first_dv_blob = _deletion_vector_blob(_deletion_vector_bitmap_payload([1, 3]))
@@ -4234,10 +4297,9 @@ def test_read_encrypted_parquet_fails_closed(tmp_path: Path) -> None:
42344297
with open(path, "rb") as f:
42354298
_create_arrow_read_format(FileFormat.PARQUET, key).make_fragment(f).to_table()
42364299

4237-
for file_format in [FileFormat.ORC, FileFormat.PUFFIN]:
4238-
delete_file = DataFile.from_args(file_path="deletes", file_format=file_format, key_metadata=key_metadata.to_bytes())
4239-
with pytest.raises(ValueError, match="file deletes: only Parquet is supported"):
4240-
_read_deletes(PyArrowFileIO(), delete_file)
4300+
delete_file = DataFile.from_args(file_path="deletes", file_format=FileFormat.ORC, key_metadata=key_metadata.to_bytes())
4301+
with pytest.raises(ValueError, match="file deletes: only Parquet and Puffin are supported"):
4302+
_read_deletes(PyArrowFileIO(), delete_file)
42414303

42424304

42434305
def test_parse_location_defaults() -> None:

‎tests/table/encryption/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,8 @@ entry would record is in `test_read_manifest_encrypted_by_iceberg_rust` in `test
3131
`data-v3-encrypted.parquet` is a Parquet data file with three rows, written and encrypted by iceberg-rust's
3232
`ParquetWriter` with a key from its `EncryptionManager`. The key metadata that its manifest entry would record is in
3333
`test_task_to_record_batches_encrypted_by_iceberg_rust` in `tests/io/test_pyarrow.py`.
34+
35+
`dv-v3-encrypted.puffin` is a Puffin file with one deletion vector, deleting positions 1, 3 and 5 of
36+
`s3://bucket/table/data/00000.parquet`. It was written and encrypted by iceberg-rust's `PuffinWriter` with key
37+
`0123456789abcdef` and AAD prefix `fedcba9876543210`. The key metadata that its manifest entry would record is in
38+
`test_read_deletes_from_puffin_file_encrypted_by_iceberg_rust` in `tests/io/test_pyarrow.py`.
309 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)