Skip to content

Commit 046e584

Browse files
xanderbaileykevinjqliuCopilot
authored
feat(encryption): [14/N] Add EncryptedOutputFile (#4380)
* Add EncryptedOutputFile Writes a file, such as a manifest, as an AGS1 stream and records its key metadata with the written file length. * Close EncryptedOutputFile stream even if the write fails Otherwise garbage collection calls close again and retries the write. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Add @OverRide to EncryptedOutputFile methods Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Kevin Liu <kevin.jq.liu@gmail.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 87d4143 commit 046e584

2 files changed

Lines changed: 102 additions & 4 deletions

File tree

‎pyiceberg/encryption/io.py‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,19 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
"""Input files for metadata encrypted with AGS1 streams."""
17+
"""Input and output files for metadata encrypted with AGS1 streams."""
1818

1919
from __future__ import annotations
2020

2121
import io
22+
from collections.abc import Callable
23+
from dataclasses import replace
24+
25+
from typing_extensions import override
2226

2327
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
24-
from pyiceberg.encryption.stream import _decrypt_ags1
25-
from pyiceberg.io import InputFile, InputStream
28+
from pyiceberg.encryption.stream import _decrypt_ags1, _encrypt_ags1
29+
from pyiceberg.io import InputFile, InputStream, OutputFile, OutputStream
2630

2731

2832
class EncryptedInputFile(InputFile):
@@ -48,3 +52,63 @@ def open(self, seekable: bool = True) -> InputStream:
4852
def _decrypt(self) -> bytes:
4953
with self._input_file.open(seekable=False) as stream:
5054
return _decrypt_ags1(stream.read(), self._key_metadata)
55+
56+
57+
class EncryptedOutputFile(OutputFile):
58+
"""Encrypts a file, such as a manifest, into an AGS1 stream when its output stream is closed."""
59+
60+
def __init__(self, output_file: OutputFile, key_metadata: StandardKeyMetadata) -> None:
61+
super().__init__(output_file.location)
62+
self._output_file = output_file
63+
self._key_metadata = key_metadata
64+
self._file_length: int | None = None
65+
66+
@property
67+
def key_metadata(self) -> StandardKeyMetadata:
68+
"""The key metadata to record for the file, including the length of the written file."""
69+
if self._file_length is None:
70+
raise ValueError(f"Cannot get key metadata before the file is written: {self.location}")
71+
return replace(self._key_metadata, file_length=self._file_length)
72+
73+
@override
74+
def __len__(self) -> int:
75+
"""Return the length of the encrypted file, in bytes."""
76+
return self._file_length if self._file_length is not None else len(self._output_file)
77+
78+
@override
79+
def exists(self) -> bool:
80+
"""Check whether the file exists."""
81+
return self._output_file.exists()
82+
83+
@override
84+
def to_input_file(self) -> InputFile:
85+
"""Return an input file that decrypts the written file."""
86+
return EncryptedInputFile(self._output_file.to_input_file(), self.key_metadata)
87+
88+
@override
89+
def create(self, overwrite: bool = False) -> OutputStream:
90+
"""Return a stream that buffers the file, then encrypts and writes it when closed."""
91+
return _EncryptingOutputStream(self._output_file.create(overwrite=overwrite), self._key_metadata, self._set_file_length)
92+
93+
def _set_file_length(self, file_length: int) -> None:
94+
self._file_length = file_length
95+
96+
97+
class _EncryptingOutputStream(io.BytesIO):
98+
def __init__(self, output_stream: OutputStream, key_metadata: StandardKeyMetadata, on_close: Callable[[int], None]) -> None:
99+
super().__init__()
100+
self._output_stream = output_stream
101+
self._key_metadata = key_metadata
102+
self._on_close = on_close
103+
104+
def close(self) -> None:
105+
if self.closed:
106+
return
107+
try:
108+
encrypted = _encrypt_ags1(self.getvalue(), self._key_metadata)
109+
with self._output_stream:
110+
self._output_stream.write(encrypted)
111+
self._on_close(len(encrypted))
112+
finally:
113+
# Close even if the write fails, so garbage collection does not retry it.
114+
super().close()

‎tests/encryption/test_stream.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18+
import io
1819
from dataclasses import replace
1920
from pathlib import Path
2021

2122
import pytest
2223

23-
from pyiceberg.encryption.io import EncryptedInputFile
24+
from pyiceberg.encryption.io import EncryptedInputFile, EncryptedOutputFile, _EncryptingOutputStream
2425
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
2526
from pyiceberg.encryption.stream import (
2627
_BLOCK_OVERHEAD,
@@ -234,3 +235,36 @@ def test_encrypted_input_file_rejects_a_truncated_file(tmp_path: Path) -> None:
234235

235236
with pytest.raises(ValueError, match="key metadata expects"):
236237
input_file.open()
238+
239+
240+
def test_encrypted_output_file(tmp_path: Path) -> None:
241+
path = tmp_path / "manifest.avro"
242+
output_file = EncryptedOutputFile(PyArrowFileIO().new_output(str(path)), KEY_METADATA)
243+
244+
with pytest.raises(ValueError, match="Cannot get key metadata before the file is written"):
245+
_ = output_file.key_metadata
246+
247+
with output_file.create() as stream:
248+
stream.write(b"manifest contents")
249+
250+
encrypted = path.read_bytes()
251+
assert encrypted[:4] == b"AGS1"
252+
assert output_file.key_metadata == replace(KEY_METADATA, file_length=len(encrypted))
253+
assert len(output_file) == len(encrypted)
254+
with output_file.to_input_file().open() as stream:
255+
assert stream.read() == b"manifest contents"
256+
257+
with pytest.raises(FileExistsError):
258+
output_file.create()
259+
260+
261+
def test_encrypting_output_stream_closes_when_the_write_fails() -> None:
262+
class FailingStream(io.BytesIO):
263+
def write(self, b: object) -> int:
264+
raise OSError("write failed")
265+
266+
stream = _EncryptingOutputStream(FailingStream(), KEY_METADATA, lambda _: None)
267+
with pytest.raises(OSError, match="write failed"):
268+
stream.close()
269+
270+
assert stream.closed

0 commit comments

Comments
 (0)