Skip to content

Commit 35bb9bb

Browse files
authored
feat(encryption): [7/N] Add encryption key updates (#4148)
feat(encryption): Add encryption key updates
1 parent 2ecd87b commit 35bb9bb

2 files changed

Lines changed: 119 additions & 4 deletions

File tree

‎pyiceberg/table/update/__init__.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,13 @@
2929
from pyiceberg.exceptions import CommitFailedException
3030
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec
3131
from pyiceberg.schema import Schema
32-
from pyiceberg.table.metadata import SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil
32+
from pyiceberg.table.metadata import (
33+
SUPPORTED_TABLE_FORMAT_VERSION,
34+
EncryptedKey,
35+
TableMetadata,
36+
TableMetadataUtil,
37+
TableMetadataV3,
38+
)
3339
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
3440
from pyiceberg.table.snapshots import (
3541
MetadataLogEntry,
@@ -222,6 +228,16 @@ class RemovePartitionStatisticsUpdate(IcebergBaseModel):
222228
snapshot_id: int = Field(alias="snapshot-id")
223229

224230

231+
class AddEncryptionKeyUpdate(IcebergBaseModel):
232+
action: Literal["add-encryption-key"] = Field(default="add-encryption-key")
233+
encryption_key: EncryptedKey = Field(alias="encryption-key")
234+
235+
236+
class RemoveEncryptionKeyUpdate(IcebergBaseModel):
237+
action: Literal["remove-encryption-key"] = Field(default="remove-encryption-key")
238+
key_id: str = Field(alias="key-id")
239+
240+
225241
TableUpdate = Annotated[
226242
AssignUUIDUpdate
227243
| UpgradeFormatVersionUpdate
@@ -243,7 +259,9 @@ class RemovePartitionStatisticsUpdate(IcebergBaseModel):
243259
| RemovePartitionSpecsUpdate
244260
| RemoveSchemasUpdate
245261
| SetPartitionStatisticsUpdate
246-
| RemovePartitionStatisticsUpdate,
262+
| RemovePartitionStatisticsUpdate
263+
| AddEncryptionKeyUpdate
264+
| RemoveEncryptionKeyUpdate,
247265
Field(discriminator="action"),
248266
]
249267

@@ -680,6 +698,31 @@ def _(
680698
return base_metadata.model_copy(update={"partition_statistics": statistics})
681699

682700

701+
@_apply_table_update.register(AddEncryptionKeyUpdate)
702+
def _(update: AddEncryptionKeyUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata:
703+
if not isinstance(base_metadata, TableMetadataV3):
704+
raise ValueError(f"Encryption keys are not supported in format version {base_metadata.format_version}")
705+
706+
if any(key.key_id == update.encryption_key.key_id for key in base_metadata.encryption_keys):
707+
return base_metadata
708+
709+
context.add_update(update)
710+
return base_metadata.model_copy(update={"encryption_keys": base_metadata.encryption_keys + [update.encryption_key]})
711+
712+
713+
@_apply_table_update.register(RemoveEncryptionKeyUpdate)
714+
def _(update: RemoveEncryptionKeyUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext) -> TableMetadata:
715+
if not isinstance(base_metadata, TableMetadataV3):
716+
raise ValueError(f"Encryption keys are not supported in format version {base_metadata.format_version}")
717+
718+
encryption_keys = [key for key in base_metadata.encryption_keys if key.key_id != update.key_id]
719+
if len(encryption_keys) == len(base_metadata.encryption_keys):
720+
return base_metadata
721+
722+
context.add_update(update)
723+
return base_metadata.model_copy(update={"encryption_keys": encryption_keys})
724+
725+
683726
def update_table_metadata(
684727
base_metadata: TableMetadata,
685728
updates: tuple[TableUpdate, ...],

‎tests/table/test_init.py‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from typing import Any
2323

2424
import pytest
25-
from pydantic import BaseModel, ValidationError
25+
from pydantic import BaseModel, TypeAdapter, ValidationError
2626
from pytest_lazy_fixtures import lf
2727

2828
from pyiceberg.catalog import Catalog
@@ -46,7 +46,7 @@
4646
Table,
4747
TableIdentifier,
4848
)
49-
from pyiceberg.table.metadata import TableMetadataUtil, TableMetadataV1, TableMetadataV2, _generate_snapshot_id
49+
from pyiceberg.table.metadata import EncryptedKey, TableMetadataUtil, TableMetadataV1, TableMetadataV2, _generate_snapshot_id
5050
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
5151
from pyiceberg.table.snapshots import (
5252
MetadataLogEntry,
@@ -63,6 +63,7 @@
6363
)
6464
from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile
6565
from pyiceberg.table.update import (
66+
AddEncryptionKeyUpdate,
6667
AddPartitionSpecUpdate,
6768
AddSnapshotUpdate,
6869
AddSortOrderUpdate,
@@ -74,6 +75,7 @@
7475
AssertLastAssignedPartitionId,
7576
AssertRefSnapshotId,
7677
AssertTableUUID,
78+
RemoveEncryptionKeyUpdate,
7779
RemovePartitionSpecsUpdate,
7880
RemovePartitionStatisticsUpdate,
7981
RemovePropertiesUpdate,
@@ -86,6 +88,7 @@
8688
SetPropertiesUpdate,
8789
SetSnapshotRefUpdate,
8890
SetStatisticsUpdate,
91+
TableUpdate,
8992
_apply_table_update,
9093
_TableMetadataUpdateContext,
9194
update_table_metadata,
@@ -1951,6 +1954,75 @@ def test_remove_partition_statistics_update_with_invalid_snapshot_id(table_v2_wi
19511954
)
19521955

19531956

1957+
def test_add_encryption_key_update(table_v3: Table) -> None:
1958+
kek = EncryptedKey(key_id="kek", encrypted_key_metadata=b"wrapped-kek", encrypted_by_id="master", properties={"a": "b"})
1959+
manifest_list_key = EncryptedKey(key_id="manifest-list-key", encrypted_key_metadata=b"wrapped-dek", encrypted_by_id="kek")
1960+
1961+
new_metadata = update_table_metadata(
1962+
table_v3.metadata,
1963+
(AddEncryptionKeyUpdate(encryption_key=kek), AddEncryptionKeyUpdate(encryption_key=manifest_list_key)),
1964+
)
1965+
1966+
assert new_metadata.encryption_keys == [kek, manifest_list_key]
1967+
1968+
1969+
def test_add_encryption_key_update_with_existing_key_id(table_v3: Table) -> None:
1970+
key = EncryptedKey(key_id="kek", encrypted_key_metadata=b"wrapped-kek")
1971+
base_metadata = update_table_metadata(table_v3.metadata, (AddEncryptionKeyUpdate(encryption_key=key),))
1972+
1973+
context = _TableMetadataUpdateContext()
1974+
duplicate = EncryptedKey(key_id="kek", encrypted_key_metadata=b"other")
1975+
new_metadata = _apply_table_update(AddEncryptionKeyUpdate(encryption_key=duplicate), base_metadata, context)
1976+
1977+
assert new_metadata.encryption_keys == [key]
1978+
assert not context.has_changes()
1979+
1980+
1981+
def test_remove_encryption_key_update(table_v3: Table) -> None:
1982+
first = EncryptedKey(key_id="first", encrypted_key_metadata=b"first")
1983+
second = EncryptedKey(key_id="second", encrypted_key_metadata=b"second")
1984+
base_metadata = update_table_metadata(
1985+
table_v3.metadata, (AddEncryptionKeyUpdate(encryption_key=first), AddEncryptionKeyUpdate(encryption_key=second))
1986+
)
1987+
1988+
new_metadata = update_table_metadata(base_metadata, (RemoveEncryptionKeyUpdate(key_id="first"),))
1989+
1990+
assert new_metadata.encryption_keys == [second]
1991+
1992+
1993+
def test_remove_encryption_key_update_with_missing_key_id(table_v3: Table) -> None:
1994+
context = _TableMetadataUpdateContext()
1995+
new_metadata = _apply_table_update(RemoveEncryptionKeyUpdate(key_id="missing"), table_v3.metadata, context)
1996+
1997+
assert new_metadata == table_v3.metadata
1998+
assert not context.has_changes()
1999+
2000+
2001+
@pytest.mark.parametrize(
2002+
"update",
2003+
[
2004+
AddEncryptionKeyUpdate(encryption_key=EncryptedKey(key_id="a", encrypted_key_metadata=b"key")),
2005+
RemoveEncryptionKeyUpdate(key_id="a"),
2006+
],
2007+
)
2008+
def test_encryption_key_updates_require_v3(table_v2: Table, update: TableUpdate) -> None:
2009+
with pytest.raises(ValueError, match="Encryption keys are not supported in format version 2"):
2010+
update_table_metadata(table_v2.metadata, (update,))
2011+
2012+
2013+
def test_serialize_encryption_key_updates() -> None:
2014+
add = AddEncryptionKeyUpdate(encryption_key=EncryptedKey(key_id="a", encrypted_key_metadata=b"key", encrypted_by_id="b"))
2015+
remove = RemoveEncryptionKeyUpdate(key_id="a")
2016+
2017+
assert json.loads(add.model_dump_json()) == {
2018+
"action": "add-encryption-key",
2019+
"encryption-key": {"key-id": "a", "encrypted-key-metadata": "a2V5", "encrypted-by-id": "b", "properties": {}},
2020+
}
2021+
assert json.loads(remove.model_dump_json()) == {"action": "remove-encryption-key", "key-id": "a"}
2022+
assert TypeAdapter(TableUpdate).validate_json(add.model_dump_json()) == add
2023+
assert TypeAdapter(TableUpdate).validate_json(remove.model_dump_json()) == remove
2024+
2025+
19542026
def test_add_snapshot_update_fails_without_first_row_id(table_v3: Table) -> None:
19552027
new_snapshot = Snapshot(
19562028
snapshot_id=25,

0 commit comments

Comments
 (0)