|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | +import boto3 |
| 18 | +import pyarrow.parquet as pq |
| 19 | +import pytest |
| 20 | +from pyspark.sql import SparkSession |
| 21 | + |
| 22 | +from pyiceberg.catalog import Catalog |
| 23 | +from pyiceberg.encryption.aws_kms import KMS_ENDPOINT, AwsKeyManagementClient |
| 24 | +from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY |
| 25 | +from pyiceberg.table import Table |
| 26 | + |
| 27 | +# AES-128 and AES-256 data keys |
| 28 | +DATA_KEY_LENGTHS = [16, 32] |
| 29 | + |
| 30 | +ROWS = [ |
| 31 | + {"id": 1, "category": "a", "payload": "alpha"}, |
| 32 | + {"id": 2, "category": "b", "payload": None}, |
| 33 | + {"id": 3, "category": "a", "payload": "gamma"}, |
| 34 | + {"id": 4, "category": "b", "payload": "delta"}, |
| 35 | +] |
| 36 | +EXTRA_ROWS = [ |
| 37 | + {"id": 5, "category": "a", "payload": "epsilon"}, |
| 38 | + {"id": 6, "category": "b", "payload": "zeta"}, |
| 39 | +] |
| 40 | + |
| 41 | + |
| 42 | +@pytest.mark.integration |
| 43 | +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) |
| 44 | +def test_scan_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: |
| 45 | + identifier = "default.test_scan_encrypted_table" |
| 46 | + _create_encrypted_table(spark, identifier, data_key_length) |
| 47 | + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() |
| 48 | + # Delete a row from each partition, so Spark writes a deletion vector per partition |
| 49 | + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") |
| 50 | + table = _load_table(session_catalog_hive, identifier) |
| 51 | + |
| 52 | + expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}] |
| 53 | + scan = table.scan() |
| 54 | + assert scan.to_arrow().sort_by("id").to_pylist() == expected |
| 55 | + assert scan.to_arrow_batch_reader().read_all().sort_by("id").to_pylist() == expected |
| 56 | + assert scan.count() == len(expected) |
| 57 | + |
| 58 | + filtered = table.scan(row_filter="category == 'a'", selected_fields=("id", "payload")).to_arrow() |
| 59 | + assert filtered.to_pylist() == [{"id": 3, "payload": "gamma"}] |
| 60 | + |
| 61 | + |
| 62 | +@pytest.mark.integration |
| 63 | +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) |
| 64 | +def test_scan_encrypted_table_snapshots(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: |
| 65 | + identifier = "default.test_scan_encrypted_table_snapshots" |
| 66 | + _create_encrypted_table(spark, identifier, data_key_length) |
| 67 | + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() |
| 68 | + spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append() |
| 69 | + table = _load_table(session_catalog_hive, identifier) |
| 70 | + |
| 71 | + first, second = table.snapshots() |
| 72 | + assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS |
| 73 | + assert table.scan(snapshot_id=second.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + EXTRA_ROWS |
| 74 | + |
| 75 | + |
| 76 | +@pytest.mark.integration |
| 77 | +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) |
| 78 | +def test_scan_encrypted_table_requires_master_key( |
| 79 | + spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int |
| 80 | +) -> None: |
| 81 | + identifier = "default.test_scan_encrypted_table_requires_master_key" |
| 82 | + _create_encrypted_table(spark, identifier, data_key_length) |
| 83 | + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() |
| 84 | + |
| 85 | + with pytest.raises(ValueError, match="no key management client was provided"): |
| 86 | + session_catalog_hive.load_table(identifier).scan().to_arrow() |
| 87 | + |
| 88 | + # Spark also encrypts the Parquet data files, so they can't be read without their keys |
| 89 | + table = _load_table(session_catalog_hive, identifier) |
| 90 | + data_files = [task.file.file_path for task in table.scan().plan_files()] |
| 91 | + assert data_files |
| 92 | + for data_file in data_files: |
| 93 | + with table.io.new_input(data_file).open() as stream: |
| 94 | + with pytest.raises(OSError, match="no decryption found"): |
| 95 | + pq.read_table(stream) |
| 96 | + |
| 97 | + |
| 98 | +@pytest.mark.integration |
| 99 | +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) |
| 100 | +def test_inspect_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: |
| 101 | + identifier = "default.test_inspect_encrypted_table" |
| 102 | + _create_encrypted_table(spark, identifier, data_key_length) |
| 103 | + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() |
| 104 | + # Delete a row from each partition, so Spark writes a deletion vector per partition |
| 105 | + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") |
| 106 | + inspect = _load_table(session_catalog_hive, identifier).inspect |
| 107 | + |
| 108 | + files = inspect.files() |
| 109 | + spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()] |
| 110 | + assert sorted(files["file_path"].to_pylist()) == sorted(spark_file_paths) |
| 111 | + assert all(key_metadata is not None for key_metadata in files["key_metadata"].to_pylist()) |
| 112 | + assert sorted(entry["data_file"]["file_path"] for entry in inspect.entries().to_pylist()) == sorted(spark_file_paths) |
| 113 | + |
| 114 | + spark_manifest_paths = [row.path for row in spark.table(f"hive.{identifier}.manifests").collect()] |
| 115 | + assert sorted(inspect.manifests()["path"].to_pylist()) == sorted(spark_manifest_paths) |
| 116 | + |
| 117 | + # Each partition has two rows, and a deletion vector for one of them |
| 118 | + partitions = [ |
| 119 | + (row["partition"]["category"], row["record_count"], row["position_delete_record_count"]) |
| 120 | + for row in inspect.partitions().to_pylist() |
| 121 | + ] |
| 122 | + assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)] |
| 123 | + |
| 124 | + |
| 125 | +def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None: |
| 126 | + # As of Iceberg Java 1.12, only the hive catalog supports table encryption. |
| 127 | + # Spark's hive catalog uses the mock KMS configured in dev/spark/spark-defaults.conf. |
| 128 | + kms = boto3.client( |
| 129 | + "kms", |
| 130 | + endpoint_url="http://localhost:5001", |
| 131 | + region_name="us-east-1", |
| 132 | + aws_access_key_id="admin", |
| 133 | + aws_secret_access_key="password", |
| 134 | + ) |
| 135 | + master_key_id = kms.create_key()["KeyMetadata"]["KeyId"] |
| 136 | + |
| 137 | + spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}") |
| 138 | + spark.sql( |
| 139 | + f""" |
| 140 | + CREATE TABLE hive.{identifier} (id bigint, category string, payload string) |
| 141 | + USING iceberg |
| 142 | + PARTITIONED BY (category) |
| 143 | + TBLPROPERTIES ( |
| 144 | + 'format-version'='3', |
| 145 | + 'encryption.key-id'='{master_key_id}', |
| 146 | + 'encryption.data-key-length'='{data_key_length}', |
| 147 | + 'write.delete.mode'='merge-on-read' |
| 148 | + ) |
| 149 | + """ |
| 150 | + ) |
| 151 | + |
| 152 | + |
| 153 | +def _load_table(catalog: Catalog, identifier: str) -> Table: |
| 154 | + table = catalog.load_table(identifier) |
| 155 | + return Table( |
| 156 | + identifier=table.name(), |
| 157 | + metadata=table.metadata, |
| 158 | + metadata_location=table.metadata_location, |
| 159 | + io=table.io, |
| 160 | + catalog=catalog, |
| 161 | + kms_client=AwsKeyManagementClient( |
| 162 | + { |
| 163 | + KMS_ENDPOINT: "http://localhost:5001", |
| 164 | + AWS_REGION: "us-east-1", |
| 165 | + AWS_ACCESS_KEY_ID: "admin", |
| 166 | + AWS_SECRET_ACCESS_KEY: "password", |
| 167 | + } |
| 168 | + ), |
| 169 | + ) |
0 commit comments