Skip to content

Commit 07c243f

Browse files
authored
feat(encryption): [1/N] spec model (#3930)
* feat(encryption): [1/N] spec model * test * validate * assert missing when empty
1 parent 7dc3a60 commit 07c243f

5 files changed

Lines changed: 213 additions & 2 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,10 @@ class TableProperties:
198198
FORMAT_VERSION = "format-version"
199199
DEFAULT_FORMAT_VERSION: TableVersion = 2
200200

201+
ENCRYPTION_KEY_ID = "encryption.key-id"
202+
ENCRYPTION_DATA_KEY_LENGTH = "encryption.data-key-length"
203+
ENCRYPTION_DATA_KEY_LENGTH_DEFAULT = 16
204+
201205
MANIFEST_TARGET_SIZE_BYTES = "commit.manifest.target-size-bytes"
202206
MANIFEST_TARGET_SIZE_BYTES_DEFAULT = 8 * 1024 * 1024 # 8 MB
203207

‎pyiceberg/table/metadata.py‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19+
import base64
1920
import datetime
2021
import uuid
2122
from collections.abc import Iterable
2223
from copy import copy
23-
from typing import Annotated, Any, Literal
24+
from typing import TYPE_CHECKING, Annotated, Any, Literal
2425

25-
from pydantic import Field, field_serializer, field_validator, model_validator
26+
from pydantic import Field, field_serializer, field_validator, model_serializer, model_validator
2627
from pydantic import ValidationError as PydanticValidationError
2728

2829
from pyiceberg.exceptions import ValidationError
@@ -48,6 +49,9 @@
4849
from pyiceberg.utils.config import Config
4950
from pyiceberg.utils.datetime import datetime_to_millis
5051

52+
if TYPE_CHECKING:
53+
from pydantic.functional_serializers import ModelWrapSerializerWithoutInfo
54+
5155
CURRENT_SNAPSHOT_ID = "current-snapshot-id"
5256
CURRENT_SCHEMA_ID = "current-schema-id"
5357
SCHEMAS = "schemas"
@@ -125,6 +129,36 @@ def construct_refs(table_metadata: TableMetadata) -> TableMetadata:
125129
return table_metadata
126130

127131

132+
class EncryptedKey(IcebergBaseModel):
133+
"""A key used for table encryption, tracked in v3 metadata under `encryption-keys`.
134+
135+
https://iceberg.apache.org/spec/#encryption-keys
136+
"""
137+
138+
key_id: str = Field(alias="key-id")
139+
"""ID of the encryption key."""
140+
141+
encrypted_key_metadata: bytes = Field(alias="encrypted-key-metadata")
142+
"""The encrypted key and metadata, base64 encoded in JSON."""
143+
144+
encrypted_by_id: str | None = Field(alias="encrypted-by-id", default=None)
145+
"""ID of the key used to encrypt or wrap `encrypted-key-metadata`."""
146+
147+
properties: dict[str, str] = Field(default_factory=dict)
148+
"""Additional metadata used by the table's encryption scheme."""
149+
150+
@field_validator("encrypted_key_metadata", mode="before")
151+
def decode_encrypted_key_metadata(cls, encrypted_key_metadata: Any) -> Any:
152+
# validate=True so that malformed base64 raises instead of silently discarding characters
153+
if isinstance(encrypted_key_metadata, str):
154+
return base64.b64decode(encrypted_key_metadata, validate=True)
155+
return encrypted_key_metadata
156+
157+
@field_serializer("encrypted_key_metadata")
158+
def serialize_encrypted_key_metadata(self, encrypted_key_metadata: bytes) -> str:
159+
return base64.b64encode(encrypted_key_metadata).decode("utf-8")
160+
161+
128162
class TableMetadataCommonFields(IcebergBaseModel):
129163
"""Metadata for an Iceberg table as specified in the Apache Iceberg spec.
130164
@@ -584,6 +618,17 @@ def construct_refs(self) -> TableMetadata:
584618
next_row_id: int | None = Field(alias="next-row-id", default=None)
585619
"""A long higher than all assigned row IDs; the next snapshot's `first-row-id`."""
586620

621+
encryption_keys: list[EncryptedKey] = Field(alias="encryption-keys", default_factory=list)
622+
"""An optional list of encryption keys used for table encryption."""
623+
624+
@model_serializer(mode="wrap")
625+
def serialize_model(self, handler: ModelWrapSerializerWithoutInfo) -> dict[str, Any]:
626+
"""Set custom serializer to leave out `encryption-keys` when it is empty."""
627+
serialized: dict[str, Any] = handler(self)
628+
if not self.encryption_keys:
629+
serialized.pop("encryption-keys", None)
630+
return serialized
631+
587632
def model_dump_json(self, exclude_none: bool = True, exclude: Any | None = None, by_alias: bool = True, **kwargs: Any) -> str:
588633
raise NotImplementedError("Writing V3 is not yet supported, see: https://github.com/apache/iceberg-python/issues/1551")
589634

‎pyiceberg/table/snapshots.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,9 @@ class Snapshot(IcebergBaseModel):
259259
added_rows: int | None = Field(
260260
alias="added-rows", default=None, description="The upper bound of the number of rows with assigned row IDs"
261261
)
262+
key_id: str | None = Field(
263+
alias="key-id", default=None, description="ID of the encryption key that encrypts the manifest list key metadata"
264+
)
262265

263266
def __str__(self) -> str:
264267
"""Return the string representation of the Snapshot class."""
@@ -280,6 +283,7 @@ def __repr__(self) -> str:
280283
f"schema_id={self.schema_id}" if self.schema_id is not None else None,
281284
f"first_row_id={self.first_row_id}" if self.first_row_id is not None else None,
282285
f"added_rows={self.added_rows}" if self.added_rows is not None else None,
286+
f"key_id='{self.key_id}'" if self.key_id is not None else None,
283287
]
284288
filtered_fields = [field for field in fields if field is not None]
285289
return f"Snapshot({', '.join(filtered_fields)})"

‎tests/table/test_metadata.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,14 @@
2424
from uuid import UUID
2525

2626
import pytest
27+
from pydantic import ValidationError as PydanticValidationError
2728

2829
from pyiceberg.exceptions import ValidationError
2930
from pyiceberg.partitioning import PartitionField, PartitionSpec
3031
from pyiceberg.schema import Schema
3132
from pyiceberg.serializers import FromByteStream
3233
from pyiceberg.table.metadata import (
34+
EncryptedKey,
3335
TableMetadataUtil,
3436
TableMetadataV1,
3537
TableMetadataV2,
@@ -876,3 +878,117 @@ def test_new_table_metadata_format_v2_with_v3_schema_fails(field_type: Primitive
876878
location="s3://some_v1_location/",
877879
properties={"format-version": "2"},
878880
)
881+
882+
883+
def test_encrypted_key_minimal() -> None:
884+
# Mirrors Java's TestEncryptedKeyParser, where the key metadata is base64 of b"key"
885+
key = EncryptedKey.model_validate_json('{"key-id": "a", "encrypted-key-metadata": "a2V5"}')
886+
887+
assert key.key_id == "a"
888+
assert key.encrypted_key_metadata == b"key"
889+
assert key.encrypted_by_id is None
890+
assert key.properties == {}
891+
892+
893+
def test_encrypted_key_full() -> None:
894+
key = EncryptedKey.model_validate_json(
895+
'{"key-id": "a", "encrypted-key-metadata": "a2V5", "encrypted-by-id": "b", "properties": {"test": "value"}}'
896+
)
897+
898+
assert key.key_id == "a"
899+
assert key.encrypted_key_metadata == b"key"
900+
assert key.encrypted_by_id == "b"
901+
assert key.properties == {"test": "value"}
902+
903+
904+
def test_encrypted_key_serialize() -> None:
905+
key = EncryptedKey(key_id="a", encrypted_key_metadata=b"key", encrypted_by_id="b", properties={"test": "value"})
906+
907+
expected = '{"key-id":"a","encrypted-key-metadata":"a2V5","encrypted-by-id":"b","properties":{"test":"value"}}'
908+
assert key.model_dump_json() == expected
909+
910+
911+
def test_encrypted_key_serialize_minimal() -> None:
912+
key = EncryptedKey(key_id="a", encrypted_key_metadata=b"key")
913+
914+
assert key.model_dump_json() == '{"key-id":"a","encrypted-key-metadata":"a2V5","properties":{}}'
915+
916+
917+
@pytest.mark.parametrize(
918+
"payload, missing",
919+
[
920+
('{"encrypted-key-metadata": "a2V5"}', "key-id"),
921+
('{"key-id": "a"}', "encrypted-key-metadata"),
922+
],
923+
)
924+
def test_encrypted_key_missing_required_field(payload: str, missing: str) -> None:
925+
with pytest.raises(PydanticValidationError) as exc_info:
926+
EncryptedKey.model_validate_json(payload)
927+
928+
assert missing in str(exc_info.value)
929+
930+
931+
@pytest.mark.parametrize(
932+
"encrypted_key_metadata",
933+
[
934+
"a2V*5", # character outside the base64 alphabet
935+
"a2V5=extra",
936+
"a2V", # incorrect padding
937+
"not base64",
938+
],
939+
)
940+
def test_encrypted_key_malformed_base64(encrypted_key_metadata: str) -> None:
941+
with pytest.raises(PydanticValidationError, match="encrypted-key-metadata"):
942+
EncryptedKey.model_validate({"key-id": "a", "encrypted-key-metadata": encrypted_key_metadata})
943+
944+
945+
def test_v3_metadata_parsing_encryption_keys(example_table_metadata_v3: dict[str, Any]) -> None:
946+
metadata = {
947+
**example_table_metadata_v3,
948+
"encryption-keys": [
949+
{
950+
"key-id": "table-key-1",
951+
"encrypted-key-metadata": "a2V5",
952+
"encrypted-by-id": "external-key-1",
953+
"properties": {"a": "b"},
954+
},
955+
{"key-id": "table-key-2", "encrypted-key-metadata": "a2V5", "encrypted-by-id": "table-key-1"},
956+
],
957+
"snapshots": [
958+
{**snapshot, "key-id": "table-key-2"} if snapshot["snapshot-id"] == 3055729675574597004 else snapshot
959+
for snapshot in example_table_metadata_v3["snapshots"]
960+
],
961+
}
962+
963+
table_metadata = TableMetadataUtil.parse_obj(metadata)
964+
965+
assert isinstance(table_metadata, TableMetadataV3)
966+
assert [key.key_id for key in table_metadata.encryption_keys] == ["table-key-1", "table-key-2"]
967+
assert table_metadata.encryption_keys[0].encrypted_key_metadata == b"key"
968+
assert table_metadata.encryption_keys[0].properties == {"a": "b"}
969+
assert table_metadata.encryption_keys[1].encrypted_by_id == "table-key-1"
970+
971+
current_snapshot = table_metadata.snapshot_by_id(3055729675574597004)
972+
assert current_snapshot is not None
973+
assert current_snapshot.key_id == "table-key-2"
974+
975+
976+
def test_v3_metadata_without_encryption_keys(example_table_metadata_v3: dict[str, Any]) -> None:
977+
table_metadata = TableMetadataUtil.parse_obj(example_table_metadata_v3)
978+
979+
assert isinstance(table_metadata, TableMetadataV3)
980+
assert table_metadata.encryption_keys == []
981+
for snapshot in table_metadata.snapshots:
982+
assert snapshot.key_id is None
983+
984+
assert "encryption-keys" not in table_metadata.model_dump(mode="json")
985+
986+
987+
def test_v3_metadata_with_encryption_keys_serializes_them(example_table_metadata_v3: dict[str, Any]) -> None:
988+
table_metadata = TableMetadataUtil.parse_obj(
989+
{**example_table_metadata_v3, "encryption-keys": [{"key-id": "a", "encrypted-key-metadata": "a2V5"}]}
990+
)
991+
992+
assert table_metadata.model_dump(mode="json")["encryption-keys"] == [
993+
{"key-id": "a", "encrypted-key-metadata": "a2V5", "properties": {}}
994+
]

‎tests/table/test_snapshots.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,48 @@ def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot) -> No
176176
assert snapshot_with_properties == eval(repr(snapshot_with_properties))
177177

178178

179+
@pytest.fixture
180+
def snapshot_with_key_id() -> Snapshot:
181+
return Snapshot(
182+
snapshot_id=25,
183+
parent_snapshot_id=19,
184+
sequence_number=200,
185+
timestamp_ms=1602638573590,
186+
manifest_list="s3:/a/b/c.avro",
187+
summary=Summary(Operation.APPEND),
188+
schema_id=3,
189+
key_id="table-key-1",
190+
)
191+
192+
193+
def test_serialize_snapshot_with_key_id(snapshot_with_key_id: Snapshot) -> None:
194+
assert snapshot_with_key_id.model_dump_json() == (
195+
'{"snapshot-id":25,"parent-snapshot-id":19,"sequence-number":200,"timestamp-ms":1602638573590,'
196+
'"manifest-list":"s3:/a/b/c.avro","summary":{"operation":"append"},"schema-id":3,"key-id":"table-key-1"}'
197+
)
198+
199+
200+
def test_deserialize_snapshot_with_key_id(snapshot_with_key_id: Snapshot) -> None:
201+
payload = (
202+
'{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, '
203+
'"manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3, "key-id": "table-key-1"}'
204+
)
205+
assert Snapshot.model_validate_json(payload) == snapshot_with_key_id
206+
207+
208+
def test_snapshot_without_key_id_omits_it(snapshot: Snapshot) -> None:
209+
assert snapshot.key_id is None
210+
assert "key-id" not in snapshot.model_dump_json()
211+
212+
213+
def test_snapshot_with_key_id_repr(snapshot_with_key_id: Snapshot) -> None:
214+
assert repr(snapshot_with_key_id) == (
215+
"Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, "
216+
"manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND), schema_id=3, key_id='table-key-1')"
217+
)
218+
assert snapshot_with_key_id == eval(repr(snapshot_with_key_id))
219+
220+
179221
@pytest.fixture
180222
def manifest_file() -> ManifestFile:
181223
return ManifestFile.from_args(

0 commit comments

Comments
 (0)