Skip to content

Commit 1c800eb

Browse files
authored
Reject encryption properties on pre-v3 tables (#4173)
Spec compliance indicates that encryption (and its properties) aren't allowable on pre-v3 tables.
1 parent 9febaf3 commit 1c800eb

4 files changed

Lines changed: 74 additions & 0 deletions

File tree

‎pyiceberg/table/metadata.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,28 @@ def model_dump_json(self, exclude_none: bool = True, exclude: Any | None = None,
636636
TableMetadata = Annotated[TableMetadataV1 | TableMetadataV2 | TableMetadataV3, Field(discriminator="format_version")]
637637

638638

639+
def _check_properties_format_version_compatibility(properties: Properties, format_version: int) -> None:
640+
"""Check that the table properties are compatible for the given table format version.
641+
642+
Args:
643+
properties: The table properties to check.
644+
format_version: The Iceberg table format version.
645+
646+
Raises:
647+
ValueError: If the properties are not compatible for the format version.
648+
"""
649+
from pyiceberg.table import TableProperties
650+
651+
if format_version >= 3:
652+
return
653+
654+
encryption_properties = sorted(
655+
{TableProperties.ENCRYPTION_KEY_ID, TableProperties.ENCRYPTION_DATA_KEY_LENGTH} & properties.keys()
656+
)
657+
if encryption_properties:
658+
raise ValueError(f"Invalid properties for v{format_version}: {encryption_properties}")
659+
660+
639661
def new_table_metadata(
640662
schema: Schema,
641663
partition_spec: PartitionSpec,
@@ -650,6 +672,7 @@ def new_table_metadata(
650672
format_version = int(properties.pop(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION))
651673

652674
schema.check_format_version_compatibility(format_version)
675+
_check_properties_format_version_compatibility(properties, format_version)
653676

654677
fresh_schema = assign_fresh_schema_ids(schema)
655678
fresh_partition_spec = assign_fresh_partition_spec_ids(partition_spec, schema, fresh_schema)

‎pyiceberg/table/update/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
TableMetadata,
3636
TableMetadataUtil,
3737
TableMetadataV3,
38+
_check_properties_format_version_compatibility,
3839
)
3940
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
4041
from pyiceberg.table.snapshots import (
@@ -350,6 +351,8 @@ def _(update: SetPropertiesUpdate, base_metadata: TableMetadata, context: _Table
350351
if len(update.updates) == 0:
351352
return base_metadata
352353

354+
_check_properties_format_version_compatibility(update.updates, base_metadata.format_version)
355+
353356
properties = dict(base_metadata.properties)
354357
properties.update(update.updates)
355358

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,22 @@ def test_create_v1_table(catalog: Catalog, table_schema_nested: Schema, test_tab
210210
catalog.drop_table(test_table_identifier)
211211

212212

213+
@pytest.mark.parametrize("format_version", [1, 2])
214+
def test_create_table_rejects_encryption_properties_before_v3(
215+
catalog: Catalog, table_schema_nested: Schema, test_table_identifier: Identifier, format_version: int
216+
) -> None:
217+
catalog.create_namespace(Catalog.namespace_from(test_table_identifier))
218+
219+
with pytest.raises(ValueError, match=rf"Invalid properties for v{format_version}: \['encryption.key-id'\]"):
220+
catalog.create_table(
221+
test_table_identifier,
222+
table_schema_nested,
223+
properties={"format-version": str(format_version), "encryption.key-id": "key"},
224+
)
225+
226+
assert not catalog.table_exists(test_table_identifier)
227+
228+
213229
def test_create_table_custom_sort_order(catalog: Catalog, table_schema_nested: Schema, test_table_identifier: Identifier) -> None:
214230
namespace = Catalog.namespace_from(test_table_identifier)
215231
catalog.create_namespace(namespace)

‎tests/table/test_init.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1036,6 +1036,38 @@ def test_apply_set_properties_update(table_v2: Table) -> None:
10361036
assert new_metadata_add_only.last_updated_ms > base_metadata.last_updated_ms
10371037

10381038

1039+
@pytest.mark.parametrize("table_fixture", ["table_v1", "table_v2"])
1040+
@pytest.mark.parametrize("property_name", ["encryption.key-id", "encryption.data-key-length"])
1041+
def test_apply_set_properties_update_rejects_encryption_before_v3(
1042+
table_fixture: str, property_name: str, request: pytest.FixtureRequest
1043+
) -> None:
1044+
table: Table = request.getfixturevalue(table_fixture)
1045+
1046+
with pytest.raises(ValueError, match=rf"Invalid properties for v{table.format_version}: \['{property_name}'\]"):
1047+
update_table_metadata(table.metadata, (SetPropertiesUpdate(updates={property_name: "value"}),))
1048+
1049+
1050+
def test_apply_set_properties_update_allows_encryption_on_v3(table_v3: Table) -> None:
1051+
new_metadata = update_table_metadata(
1052+
table_v3.metadata,
1053+
(SetPropertiesUpdate(updates={"encryption.key-id": "key", "encryption.data-key-length": "32"}),),
1054+
)
1055+
1056+
assert new_metadata.properties["encryption.key-id"] == "key"
1057+
assert new_metadata.properties["encryption.data-key-length"] == "32"
1058+
1059+
1060+
def test_apply_remove_encryption_property_before_v3(table_v2: Table) -> None:
1061+
# A pre-v3 table that already has encryption.key-id must be repairable
1062+
base_metadata = table_v2.metadata.model_copy(
1063+
update={"properties": {**table_v2.metadata.properties, "encryption.key-id": "key"}}
1064+
)
1065+
1066+
new_metadata = update_table_metadata(base_metadata, (RemovePropertiesUpdate(removals=["encryption.key-id"]),))
1067+
1068+
assert "encryption.key-id" not in new_metadata.properties
1069+
1070+
10391071
def test_apply_remove_properties_update(table_v2: Table) -> None:
10401072
base_metadata = update_table_metadata(
10411073
table_v2.metadata,

0 commit comments

Comments
 (0)