Skip to content

Commit 07216ed

Browse files
kevinjqliuCopilot
andauthored
Add AWS KMS client and Spark-encrypted table read tests (#4431)
* Add integration tests for reading Spark-encrypted tables Spark writes encrypted v3 tables through its hive catalog with Java's UnitestKMS, and PyIceberg reads them with a matching in-memory KMS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Check that Spark encrypts Parquet data files Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Use AWS KMS backed by moto for encryption integration tests Adds AwsKeyManagementClient and a moto KMS service, replacing UnitestKMS and the iceberg-core tests jar in the Spark image. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent b84df8d commit 07216ed

5 files changed

Lines changed: 253 additions & 0 deletions

File tree

‎dev/docker-compose-integration.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,15 @@ services:
101101
command:
102102
- -c
103103
- "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
104+
kms:
105+
image: motoserver/moto:5.0.3
106+
container_name: pyiceberg-kms
107+
networks:
108+
iceberg_net:
109+
# Use port 5001 instead of default 5000 to avoid conflict with macOS AirPlay Receiver.
110+
command: ["-p", "5001"]
111+
ports:
112+
- 5001:5001
104113
hive:
105114
image: pyiceberg-hive:latest
106115
build:

‎dev/spark/spark-defaults.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ spark.sql.catalog.hive.uri thrift://hive:9083
3333
spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO
3434
spark.sql.catalog.hive.warehouse s3://warehouse/hive/
3535
spark.sql.catalog.hive.s3.endpoint http://object-store:9000
36+
# Table encryption uses AWS KMS, mocked by moto in the kms service
37+
spark.sql.catalog.hive.encryption.kms-type aws
38+
spark.sql.catalog.hive.kms.endpoint http://kms:5001
3639

3740
# Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
3841
spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog

‎pyiceberg/encryption/aws_kms.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""AWS KMS key management client for table encryption."""
18+
19+
from __future__ import annotations
20+
21+
import boto3
22+
23+
from pyiceberg.encryption.kms import KeyManagementClient
24+
from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
25+
from pyiceberg.typedef import EMPTY_DICT, Properties
26+
from pyiceberg.utils.transient import _TransientStateMixin
27+
28+
KMS_ENDPOINT = "kms.endpoint"
29+
30+
31+
class AwsKeyManagementClient(_TransientStateMixin, KeyManagementClient):
32+
"""Wraps and unwraps keys with master keys held by AWS KMS."""
33+
34+
_transient_attrs = ("_client",)
35+
36+
def __init__(self, properties: Properties = EMPTY_DICT) -> None:
37+
super().__init__(properties)
38+
self._init_transient()
39+
40+
def _init_transient(self) -> None:
41+
session = boto3.Session(
42+
profile_name=self.properties.get(AWS_PROFILE_NAME),
43+
region_name=self.properties.get(AWS_REGION),
44+
aws_access_key_id=self.properties.get(AWS_ACCESS_KEY_ID),
45+
aws_secret_access_key=self.properties.get(AWS_SECRET_ACCESS_KEY),
46+
aws_session_token=self.properties.get(AWS_SESSION_TOKEN),
47+
)
48+
self._client = session.client("kms", endpoint_url=self.properties.get(KMS_ENDPOINT))
49+
50+
def wrap_key(self, key: bytes, wrapping_key_id: str) -> bytes:
51+
"""Encrypt `key` with the KMS key `wrapping_key_id`."""
52+
return self._client.encrypt(KeyId=wrapping_key_id, Plaintext=key)["CiphertextBlob"]
53+
54+
def unwrap_key(self, wrapped_key: bytes, wrapping_key_id: str) -> bytes:
55+
"""Decrypt `wrapped_key` with the KMS key `wrapping_key_id`."""
56+
return self._client.decrypt(KeyId=wrapping_key_id, CiphertextBlob=wrapped_key)["Plaintext"]

‎tests/encryption/test_kms.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,14 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18+
import pickle
19+
20+
import boto3
1821
import pytest
1922
from memory_kms import MemoryKeyManagementClient
23+
from moto import mock_aws
2024

25+
from pyiceberg.encryption.aws_kms import AwsKeyManagementClient
2126
from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey
2227
from pyiceberg.encryption.kms import GeneratedKey, KeyManagementClient
2328

@@ -145,3 +150,14 @@ def test_repr_redacts_master_keys(kms: MemoryKeyManagementClient) -> None:
145150

146151
assert repr(kms) == "MemoryKeyManagementClient(master_key_size=<AesKeySize.BITS_128: 128>, key_count=2)"
147152
assert repr(MASTER_KEY.key) not in repr(kms)
153+
154+
155+
@mock_aws
156+
def test_aws_kms_wrap_unwrap_round_trip(_aws_credentials: None) -> None:
157+
master_key_id = boto3.client("kms").create_key()["KeyMetadata"]["KeyId"]
158+
kms = AwsKeyManagementClient()
159+
160+
wrapped = kms.wrap_key(DEK, master_key_id)
161+
162+
assert kms.unwrap_key(wrapped, master_key_id) == DEK
163+
assert pickle.loads(pickle.dumps(kms)).unwrap_key(wrapped, master_key_id) == DEK
Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
import boto3
18+
import pyarrow.parquet as pq
19+
import pytest
20+
from pyspark.sql import SparkSession
21+
22+
from pyiceberg.catalog import Catalog
23+
from pyiceberg.encryption.aws_kms import KMS_ENDPOINT, AwsKeyManagementClient
24+
from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY
25+
from pyiceberg.table import Table
26+
27+
# AES-128 and AES-256 data keys
28+
DATA_KEY_LENGTHS = [16, 32]
29+
30+
ROWS = [
31+
{"id": 1, "category": "a", "payload": "alpha"},
32+
{"id": 2, "category": "b", "payload": None},
33+
{"id": 3, "category": "a", "payload": "gamma"},
34+
{"id": 4, "category": "b", "payload": "delta"},
35+
]
36+
EXTRA_ROWS = [
37+
{"id": 5, "category": "a", "payload": "epsilon"},
38+
{"id": 6, "category": "b", "payload": "zeta"},
39+
]
40+
41+
42+
@pytest.mark.integration
43+
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
44+
def test_scan_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
45+
identifier = "default.test_scan_encrypted_table"
46+
_create_encrypted_table(spark, identifier, data_key_length)
47+
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
48+
# Delete a row from each partition, so Spark writes a deletion vector per partition
49+
spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)")
50+
table = _load_table(session_catalog_hive, identifier)
51+
52+
expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}]
53+
scan = table.scan()
54+
assert scan.to_arrow().sort_by("id").to_pylist() == expected
55+
assert scan.to_arrow_batch_reader().read_all().sort_by("id").to_pylist() == expected
56+
assert scan.count() == len(expected)
57+
58+
filtered = table.scan(row_filter="category == 'a'", selected_fields=("id", "payload")).to_arrow()
59+
assert filtered.to_pylist() == [{"id": 3, "payload": "gamma"}]
60+
61+
62+
@pytest.mark.integration
63+
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
64+
def test_scan_encrypted_table_snapshots(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
65+
identifier = "default.test_scan_encrypted_table_snapshots"
66+
_create_encrypted_table(spark, identifier, data_key_length)
67+
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
68+
spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append()
69+
table = _load_table(session_catalog_hive, identifier)
70+
71+
first, second = table.snapshots()
72+
assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS
73+
assert table.scan(snapshot_id=second.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + EXTRA_ROWS
74+
75+
76+
@pytest.mark.integration
77+
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
78+
def test_scan_encrypted_table_requires_master_key(
79+
spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int
80+
) -> None:
81+
identifier = "default.test_scan_encrypted_table_requires_master_key"
82+
_create_encrypted_table(spark, identifier, data_key_length)
83+
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
84+
85+
with pytest.raises(ValueError, match="no key management client was provided"):
86+
session_catalog_hive.load_table(identifier).scan().to_arrow()
87+
88+
# Spark also encrypts the Parquet data files, so they can't be read without their keys
89+
table = _load_table(session_catalog_hive, identifier)
90+
data_files = [task.file.file_path for task in table.scan().plan_files()]
91+
assert data_files
92+
for data_file in data_files:
93+
with table.io.new_input(data_file).open() as stream:
94+
with pytest.raises(OSError, match="no decryption found"):
95+
pq.read_table(stream)
96+
97+
98+
@pytest.mark.integration
99+
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
100+
def test_inspect_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
101+
identifier = "default.test_inspect_encrypted_table"
102+
_create_encrypted_table(spark, identifier, data_key_length)
103+
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
104+
# Delete a row from each partition, so Spark writes a deletion vector per partition
105+
spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)")
106+
inspect = _load_table(session_catalog_hive, identifier).inspect
107+
108+
files = inspect.files()
109+
spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()]
110+
assert sorted(files["file_path"].to_pylist()) == sorted(spark_file_paths)
111+
assert all(key_metadata is not None for key_metadata in files["key_metadata"].to_pylist())
112+
assert sorted(entry["data_file"]["file_path"] for entry in inspect.entries().to_pylist()) == sorted(spark_file_paths)
113+
114+
spark_manifest_paths = [row.path for row in spark.table(f"hive.{identifier}.manifests").collect()]
115+
assert sorted(inspect.manifests()["path"].to_pylist()) == sorted(spark_manifest_paths)
116+
117+
# Each partition has two rows, and a deletion vector for one of them
118+
partitions = [
119+
(row["partition"]["category"], row["record_count"], row["position_delete_record_count"])
120+
for row in inspect.partitions().to_pylist()
121+
]
122+
assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)]
123+
124+
125+
def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None:
126+
# As of Iceberg Java 1.12, only the hive catalog supports table encryption.
127+
# Spark's hive catalog uses the mock KMS configured in dev/spark/spark-defaults.conf.
128+
kms = boto3.client(
129+
"kms",
130+
endpoint_url="http://localhost:5001",
131+
region_name="us-east-1",
132+
aws_access_key_id="admin",
133+
aws_secret_access_key="password",
134+
)
135+
master_key_id = kms.create_key()["KeyMetadata"]["KeyId"]
136+
137+
spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}")
138+
spark.sql(
139+
f"""
140+
CREATE TABLE hive.{identifier} (id bigint, category string, payload string)
141+
USING iceberg
142+
PARTITIONED BY (category)
143+
TBLPROPERTIES (
144+
'format-version'='3',
145+
'encryption.key-id'='{master_key_id}',
146+
'encryption.data-key-length'='{data_key_length}',
147+
'write.delete.mode'='merge-on-read'
148+
)
149+
"""
150+
)
151+
152+
153+
def _load_table(catalog: Catalog, identifier: str) -> Table:
154+
table = catalog.load_table(identifier)
155+
return Table(
156+
identifier=table.name(),
157+
metadata=table.metadata,
158+
metadata_location=table.metadata_location,
159+
io=table.io,
160+
catalog=catalog,
161+
kms_client=AwsKeyManagementClient(
162+
{
163+
KMS_ENDPOINT: "http://localhost:5001",
164+
AWS_REGION: "us-east-1",
165+
AWS_ACCESS_KEY_ID: "admin",
166+
AWS_SECRET_ACCESS_KEY: "password",
167+
}
168+
),
169+
)

0 commit comments

Comments
 (0)