From a3a0732b3d06e0136e76cd37458a6c1230d3e36b Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Fri, 6 Nov 2026 15:23:38 +0800 Subject: [PATCH 1/2] feat(parquet/encryption): KMS support for reading PKMT1 key material New optional parquet/encryption package with a KmsClient interface and a KmsKeyRetriever that plugs into parquet.WithKeyRetriever, so files written by the parquet-cpp/pyarrow CryptoFactory can be read: PKMT1 key metadata is parsed, and DEKs are resolved via the KMS client (single wrapping) or by unwrapping the KEK through the KMS and decrypting the locally wrapped DEK with AES-GCM (double wrapping, AAD = decoded KEK ID bytes, matching parquet-cpp EncryptKeyLocally: nonce || ciphertext || tag). Interop tests against pyarrow CryptoFactory output cover both wrapping modes; the test KMS wraps with plain base64 so no extra python dependencies are needed and the existing cgo python CI job runs them unchanged. Also addresses the review nits from #1439 (renamed the requirePyarrow flag, dropped the unused footer key metadata, deduped a doc comment). Part of #566 --- parquet/encryption/doc.go | 30 +++++ parquet/encryption/example_test.go | 72 ++++++++++ parquet/encryption/key_material.go | 80 +++++++++++ parquet/encryption/key_material_test.go | 150 +++++++++++++++++++++ parquet/encryption/kms.go | 123 +++++++++++++++++ parquet/encryption_interop_pyarrow.py | 48 +++++++ parquet/encryption_interop_pyarrow_test.go | 74 ++++++++-- 7 files changed, 569 insertions(+), 8 deletions(-) create mode 100644 parquet/encryption/doc.go create mode 100644 parquet/encryption/example_test.go create mode 100644 parquet/encryption/key_material.go create mode 100644 parquet/encryption/key_material_test.go create mode 100644 parquet/encryption/kms.go diff --git a/parquet/encryption/doc.go b/parquet/encryption/doc.go new file mode 100644 index 000000000..e00945a9e --- /dev/null +++ b/parquet/encryption/doc.go @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package encryption provides optional KMS integration for reading parquet +// files encrypted with PKMT1 key material, the format written by the +// CryptoFactory of parquet-cpp and pyarrow (both single and double key +// wrapping). +// +// It plugs into the existing low level API as a +// parquet.DecryptionKeyRetriever: pass a KmsClient implementation to +// NewKmsKeyRetriever and set the retriever with +// parquet.WithKeyRetriever. Files using explicit keys or custom key +// metadata continue to work without this package. +// +// This package is optional: it carries no effect on code that does not +// import it. +package encryption diff --git a/parquet/encryption/example_test.go b/parquet/encryption/example_test.go new file mode 100644 index 000000000..47f56af2e --- /dev/null +++ b/parquet/encryption/example_test.go @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption_test + +import ( + "encoding/base64" + "fmt" + + "github.com/apache/arrow-go/v18/parquet/encryption" +) + +// toyKMS is a stand-in for a real key management service client: the wrapped +// key representation is opaque to the library, so anything reversible works +// for demonstration. +type toyKMS struct{} + +func (toyKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (toyKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// ExampleKmsKeyRetriever shows how to read files written by the +// parquet-cpp/pyarrow CryptoFactory: implement KmsClient, wrap it in a +// KmsKeyRetriever and set it on the decryption properties with +// parquet.WithKeyRetriever. Without this package, files whose key metadata +// carries PKMT1 key material cannot be read. +func ExampleKmsKeyRetriever() { + retriever := encryption.NewKmsKeyRetriever(toyKMS{}) + + // key metadata as found in the footer of a CryptoFactory-written file + // (single wrapping mode) + metadata := []byte(`{ + "keyMaterialType": "PKMT1", + "internalStorage": true, + "isFooterKey": true, + "kmsInstanceID": "kms-instance", + "kmsInstanceURL": "https://kms.example", + "masterKeyID": "footer_master", + "wrappedDEK": "c2VjcmV0LWRlaw==", + "doubleWrapping": false + }`) + + fmt.Printf("%q\n", retriever.GetKey(metadata)) + + // To open an encrypted file, pass the retriever to the reader: + // + // props := parquet.NewReaderProperties(memory.DefaultAllocator) + // props.FileDecryptProps = parquet.NewFileDecryptionProperties( + // parquet.WithKeyRetriever(retriever)) + // fr, err := file.OpenParquetFile(path, false, file.WithReadProps(props)) + // + // Code that does not import this package is unaffected. + + // Output: "secret-dek" +} diff --git a/parquet/encryption/key_material.go b/parquet/encryption/key_material.go new file mode 100644 index 000000000..74d485dfb --- /dev/null +++ b/parquet/encryption/key_material.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "encoding/json" + "fmt" +) + +// keyMaterialType1 is the only key material type defined by the parquet key +// management tools specification ("parquet key management tools, version 1"). +const keyMaterialType1 = "PKMT1" + +// KeyMaterial is the PKMT1 key material carried in the key metadata of a +// parquet file written with envelope encryption. With internal key storage, +// the material is stored as a flat JSON object in the footer and column key +// metadata of the file. +type KeyMaterial struct { + // KeyMaterialType must be "PKMT1". + KeyMaterialType string `json:"keyMaterialType"` + // InternalStorage reports whether the key material is stored inside the + // parquet file. External key material stores are not supported. + InternalStorage bool `json:"internalStorage"` + // IsFooterKey reports whether this material belongs to the footer key. + // Only footer key material carries KmsInstanceID and KmsInstanceURL. + IsFooterKey bool `json:"isFooterKey"` + KmsInstanceID string `json:"kmsInstanceID"` + KmsInstanceURL string `json:"kmsInstanceURL"` + // MasterKeyID identifies the master key used to wrap the key. + MasterKeyID string `json:"masterKeyID"` + // WrappedDEK is the base64 encoded wrapped data encryption key. + WrappedDEK string `json:"wrappedDEK"` + // DoubleWrapping reports whether the DEK is wrapped by a key encryption + // key that is itself wrapped by the master key. + DoubleWrapping bool `json:"doubleWrapping"` + // KeyEncryptionKeyID is the base64 encoded key encryption key ID, set + // only in double wrapping mode. + KeyEncryptionKeyID string `json:"keyEncryptionKeyID"` + // WrappedKEK is the base64 encoded wrapped key encryption key, set only + // in double wrapping mode. + WrappedKEK string `json:"wrappedKEK"` +} + +// ParseKeyMetadata parses the key metadata JSON of a footer or column key. +// Only internal key storage (the parquet-cpp/pyarrow CryptoFactory default) +// is supported; external key material stores are rejected. +func ParseKeyMetadata(metadata []byte) (KeyMaterial, error) { + var material KeyMaterial + if err := json.Unmarshal(metadata, &material); err != nil { + return KeyMaterial{}, fmt.Errorf("parquet: parse key metadata: %w", err) + } + if material.KeyMaterialType != keyMaterialType1 { + return KeyMaterial{}, fmt.Errorf("parquet: unsupported key material type %q (want %q)", + material.KeyMaterialType, keyMaterialType1) + } + if !material.InternalStorage { + return KeyMaterial{}, fmt.Errorf("parquet: external key material storage is not supported") + } + if material.MasterKeyID == "" || material.WrappedDEK == "" { + return KeyMaterial{}, fmt.Errorf("parquet: key metadata missing masterKeyID or wrappedDEK") + } + if material.DoubleWrapping && (material.KeyEncryptionKeyID == "" || material.WrappedKEK == "") { + return KeyMaterial{}, fmt.Errorf("parquet: double wrapped key material missing keyEncryptionKeyID or wrappedKEK") + } + return material, nil +} diff --git a/parquet/encryption/key_material_test.go b/parquet/encryption/key_material_test.go new file mode 100644 index 000000000..5645121d7 --- /dev/null +++ b/parquet/encryption/key_material_test.go @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mapKMS resolves master key IDs to raw keys, standing in for a real KMS. +type mapKMS struct { + keys map[string][]byte +} + +func (m mapKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return "", nil // unused by these tests +} + +func (m mapKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + key, ok := m.keys[masterKeyID] + if !ok { + return nil, assert.AnError + } + return key, nil +} + +// encryptKeyLocally mirrors parquet-cpp's EncryptKeyLocally for test +// fixtures: AES-GCM, nonce || ciphertext || tag, base64 encoded. +func encryptKeyLocally(t *testing.T, key, master, aad []byte) string { + t.Helper() + block, err := aes.NewCipher(master) + require.NoError(t, err) + gcm, err := cipher.NewGCM(block) + require.NoError(t, err) + nonce := make([]byte, gcm.NonceSize()) + _, err = rand.Read(nonce) + require.NoError(t, err) + sealed := gcm.Seal(nonce, nonce, key, aad) + return base64.StdEncoding.EncodeToString(sealed) +} + +func footerMetadata(t *testing.T, double bool) []byte { + t.Helper() + material := KeyMaterial{ + KeyMaterialType: "PKMT1", + InternalStorage: true, + IsFooterKey: true, + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + MasterKeyID: "mk1", + DoubleWrapping: double, + } + if double { + kek := bytes.Repeat([]byte{0x2a}, 16) + material.KeyEncryptionKeyID = base64.StdEncoding.EncodeToString([]byte("kek-id")) + material.WrappedKEK = "wrapped-kek" + // a real wrapped DEK so the unwrapper has something valid to decrypt + material.WrappedDEK = encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), kek, []byte("kek-id")) + } else { + material.WrappedDEK = "wrapped-dek" + } + encoded, err := json.Marshal(material) + require.NoError(t, err) + return encoded +} + +func TestParseKeyMetadata(t *testing.T) { + material, err := ParseKeyMetadata(footerMetadata(t, false)) + require.NoError(t, err) + assert.Equal(t, "PKMT1", material.KeyMaterialType) + assert.True(t, material.InternalStorage) + assert.True(t, material.IsFooterKey) + assert.Equal(t, "mk1", material.MasterKeyID) + assert.False(t, material.DoubleWrapping) +} + +func TestParseKeyMetadataErrors(t *testing.T) { + _, err := ParseKeyMetadata([]byte(`{"keyMaterialType":"KM2","internalStorage":true}`)) + assert.ErrorContains(t, err, "unsupported key material type") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":false,"keyReference":"ref"}`)) + assert.ErrorContains(t, err, "external key material storage") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":true}`)) + assert.ErrorContains(t, err, "missing masterKeyID or wrappedDEK") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":true,"masterKeyID":"m","wrappedDEK":"d","doubleWrapping":true}`)) + assert.ErrorContains(t, err, "missing keyEncryptionKeyID or wrappedKEK") + + _, err = ParseKeyMetadata([]byte(`{`)) + assert.ErrorContains(t, err, "parse key metadata") +} + +func TestKmsKeyRetrieverSingleWrapping(t *testing.T) { + dek := bytes.Repeat([]byte{0x42}, 16) + kms := mapKMS{keys: map[string][]byte{"mk1": dek}} + retriever := NewKmsKeyRetriever(kms) + + key, err := retriever.UnwrapKeyMetadata(footerMetadata(t, false)) + require.NoError(t, err) + assert.Equal(t, dek, key) + assert.Equal(t, string(dek), retriever.GetKey(footerMetadata(t, false))) +} + +func TestKmsKeyRetrieverDoubleWrapping(t *testing.T) { + kek := bytes.Repeat([]byte{0x2a}, 16) + kms := mapKMS{keys: map[string][]byte{"mk1": kek}} + retriever := NewKmsKeyRetriever(kms) + + key, err := retriever.UnwrapKeyMetadata(footerMetadata(t, true)) + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) +} + +func TestKmsKeyRetrieverGetKeyHidesErrors(t *testing.T) { + retriever := NewKmsKeyRetriever(mapKMS{keys: map[string][]byte{}}) + assert.Empty(t, retriever.GetKey(footerMetadata(t, false))) +} + +func TestDecryptKeyLocallyRejectsTampering(t *testing.T) { + kek := bytes.Repeat([]byte{0x2a}, 16) + wrapped, err := base64.StdEncoding.DecodeString( + encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), kek, []byte("kek-id"))) + require.NoError(t, err) + wrapped[0] ^= 0xff + _, err = decryptKeyLocally(wrapped, kek, []byte("kek-id")) + assert.Error(t, err) +} diff --git a/parquet/encryption/kms.go b/parquet/encryption/kms.go new file mode 100644 index 000000000..512b709f6 --- /dev/null +++ b/parquet/encryption/kms.go @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "fmt" +) + +// KmsClient wraps and unwraps data encryption keys with master keys managed +// by a key management service. It mirrors the parquet-cpp/pyarrow KmsClient +// contract: implementations are provided by users, and the wrapped key +// representation is opaque to the library. Implementations must be safe for +// concurrent use. +type KmsClient interface { + // WrapKey encrypts key with the master key identified by masterKeyID and + // returns the KMS-specific wrapped representation. + WrapKey(key []byte, masterKeyID string) (string, error) + + // UnwrapKey reverses WrapKey, returning the raw key bytes. + UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) +} + +// KmsKeyRetriever implements parquet.DecryptionKeyRetriever for files whose +// footer and column key metadata carry PKMT1 key material (internal key +// storage), as written by the parquet-cpp/pyarrow CryptoFactory. +type KmsKeyRetriever struct { + client KmsClient +} + +// NewKmsKeyRetriever returns a retriever that resolves keys through client. +func NewKmsKeyRetriever(client KmsClient) *KmsKeyRetriever { + return &KmsKeyRetriever{client: client} +} + +// GetKey implements parquet.DecryptionKeyRetriever. Parse and unwrap errors +// surface as an empty key, which the reader reports as a missing key; use +// UnwrapKeyMetadata when the underlying error is needed. +func (r *KmsKeyRetriever) GetKey(keyMetadata []byte) string { + key, err := r.UnwrapKeyMetadata(keyMetadata) + if err != nil { + return "" + } + return string(key) +} + +// UnwrapKeyMetadata parses PKMT1 key metadata and returns the data +// encryption key it carries: +// +// - single wrapping: the wrapped DEK is unwrapped by the KmsClient with +// the master key; +// - double wrapping: the KmsClient first unwraps the key encryption key +// with the master key, and the DEK is then decrypted locally with +// AES-GCM, using the KEK as the key and the (base64 decoded) KEK ID +// bytes as the AAD, matching parquet-cpp's EncryptKeyLocally format: +// nonce || ciphertext || tag. +func (r *KmsKeyRetriever) UnwrapKeyMetadata(keyMetadata []byte) ([]byte, error) { + material, err := ParseKeyMetadata(keyMetadata) + if err != nil { + return nil, err + } + if !material.DoubleWrapping { + dek, err := r.client.UnwrapKey(material.WrappedDEK, material.MasterKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: unwrap data encryption key: %w", err) + } + return dek, nil + } + + kek, err := r.client.UnwrapKey(material.WrappedKEK, material.MasterKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: unwrap key encryption key: %w", err) + } + + // parquet-cpp base64 encodes the KEK ID in the key material but uses the + // decoded ID bytes as the AES-GCM AAD for the locally wrapped DEK. + aad, err := base64.StdEncoding.DecodeString(material.KeyEncryptionKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: decode key encryption key id: %w", err) + } + wrappedDEK, err := base64.StdEncoding.DecodeString(material.WrappedDEK) + if err != nil { + return nil, fmt.Errorf("parquet: decode wrapped data encryption key: %w", err) + } + dek, err := decryptKeyLocally(wrappedDEK, kek, aad) + if err != nil { + return nil, fmt.Errorf("parquet: decrypt data encryption key: %w", err) + } + return dek, nil +} + +// decryptKeyLocally reverses parquet-cpp's EncryptKeyLocally: AES-GCM with a +// 12-byte nonce prefix and no length framing (nonce || ciphertext || tag). +func decryptKeyLocally(wrapped, key, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + if len(wrapped) < gcm.NonceSize()+gcm.Overhead() { + return nil, fmt.Errorf("wrapped key too short (%d bytes)", len(wrapped)) + } + return gcm.Open(nil, wrapped[:gcm.NonceSize()], wrapped[gcm.NonceSize():], aad) +} diff --git a/parquet/encryption_interop_pyarrow.py b/parquet/encryption_interop_pyarrow.py index 6fc1ceadb..54b723766 100644 --- a/parquet/encryption_interop_pyarrow.py +++ b/parquet/encryption_interop_pyarrow.py @@ -20,6 +20,7 @@ Usage: python3 encryption_interop_pyarrow.py probe python3 encryption_interop_pyarrow.py gen + python3 encryption_interop_pyarrow.py gen-kms python3 encryption_interop_pyarrow.py read """ import sys @@ -56,6 +57,50 @@ def write(name, fep): create_encryption_properties(footer_key, plaintext_footer=True)) +def gen_kms(outdir): + """Write PKMT1 key material files with a CryptoFactory and a test-only + in-memory KMS client whose wrapping is plain base64 (no cryptography + dependency). The Go side mirrors the client to unwrap the keys.""" + import base64 as b64 + import os + + import pyarrow.parquet as pq + import pyarrow.parquet.encryption as pe + + class Base64KmsClient(pe.KmsClient): + def __init__(self, config): + super().__init__() + + def wrap_key(self, key_bytes, master_key_identifier): + return b64.b64encode(key_bytes).decode("ascii") + + def unwrap_key(self, wrapped_key, master_key_identifier): + return b64.b64decode(wrapped_key) + + config = pe.KmsConnectionConfig( + kms_instance_id="in-memory", kms_instance_url="local", + key_access_token="token") + + def client_factory(kms_connection_config): + return Base64KmsClient(kms_connection_config) + + try: + factory = pe.CryptoFactory(client_factory) # pyarrow >= 26 + except TypeError: + factory = pe.CryptoFactory() # older pyarrow + factory.register_kms_client_factory(client_factory) + + table = build_table() + for name, double_wrapping in (("kms_double", True), ("kms_single", False)): + encryption = pe.EncryptionConfiguration( + footer_key="footer_master", column_keys={"column_master": ["a"]}, + double_wrapping=double_wrapping) + properties = factory.file_encryption_properties(config, encryption) + path = os.path.join(outdir, name + ".parquet") + pq.write_table(table, path, encryption_properties=properties) + print(name) + + def read(path, footer_key): import pyarrow.parquet as pq from pyarrow.parquet.encryption import create_decryption_properties @@ -82,6 +127,9 @@ def main(): if mode == "gen": gen(sys.argv[2], sys.argv[3].encode()) return 0 + if mode == "gen-kms": + gen_kms(sys.argv[2]) + return 0 if mode == "read": read(sys.argv[2], sys.argv[3].encode()) return 0 diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index 59a0c9855..b72028376 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -21,6 +21,7 @@ import ( "bytes" "context" _ "embed" + "encoding/base64" "fmt" "os" "os/exec" @@ -31,6 +32,7 @@ import ( "github.com/apache/arrow-go/v18/arrow/memory" "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/encryption" "github.com/apache/arrow-go/v18/parquet/file" "github.com/apache/arrow-go/v18/parquet/schema" "github.com/stretchr/testify/assert" @@ -70,8 +72,8 @@ func runInteropPython(t *testing.T, args ...string) (string, error) { return stdout.String(), nil } -// interopPython runs the embedded script and fails the test on a non-zero -// exit. A non-zero exit is a test failure. +// interopPython runs the embedded script and fails the test when it exits +// non-zero. func interopPython(t *testing.T, args ...string) string { t.Helper() @@ -90,15 +92,15 @@ func interopPython(t *testing.T, args ...string) string { func requirePyarrow(t *testing.T) { t.Helper() - require := os.Getenv("PARQUET_TEST_REQUIRE_PYARROW") != "" + mustHavePyarrow := os.Getenv("PARQUET_TEST_REQUIRE_PYARROW") != "" if _, err := exec.LookPath("python3"); err != nil { - if require { + if mustHavePyarrow { t.Fatalf("PARQUET_TEST_REQUIRE_PYARROW is set but python3 not found in PATH: %v", err) } t.Skip("python3 not found in PATH, skipping pyarrow interop tests") } if _, err := runInteropPython(t, "probe"); err != nil { - if require { + if mustHavePyarrow { t.Fatalf("PARQUET_TEST_REQUIRE_PYARROW is set but pyarrow is not available: %v", err) } t.Skipf("pyarrow not available for python3, skipping interop tests: %v", err) @@ -204,8 +206,65 @@ func TestGoWrittenEncryptionReadByPyarrow(t *testing.T) { assert.Equal(t, "0 10 20", got["int32_field"]) } +// interopBase64KMS mirrors the Base64KmsClient in encryption_interop_pyarrow.py: +// wrapping is plain base64, so the interop tests need no real KMS. +type interopBase64KMS struct{} + +func (interopBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (interopBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// TestPyarrowKmsWrittenConfigs reads files written by pyarrow's CryptoFactory +// with PKMT1 key material, in both double (the pyarrow default) and single +// wrapping mode. The footer key and the column key of field "a" are both +// resolved through the KMS retriever. +func TestPyarrowKmsWrittenConfigs(t *testing.T) { + requirePyarrow(t) + + dir := t.TempDir() + interopPython(t, "gen-kms", dir) + + for _, name := range []string{"kms_double", "kms_single"} { + t.Run(name, func(t *testing.T) { + props := parquet.NewReaderProperties(memory.DefaultAllocator) + props.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(interopBase64KMS{}))) + + fr, err := file.OpenParquetFile(filepath.Join(dir, name+".parquet"), false, + file.WithReadProps(props)) + require.NoError(t, err) + defer fr.Close() + + assert.Equal(t, int64(3), fr.MetaData().NumRows) + + rg := fr.RowGroup(0) + a, err := rg.Column(0) + require.NoError(t, err) + f64 := a.(*file.Float64ColumnChunkReader) + vals := make([]float64, 8) + n, _, err := f64.ReadBatch(8, vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float64{1, 2, 3}, vals[:3]) + + b, err := rg.Column(1) + require.NoError(t, err) + f32 := b.(*file.Float32ColumnChunkReader) + f32vals := make([]float32, 8) + n, _, err = f32.ReadBatch(8, f32vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float32{4, 5, 6}, f32vals[:3]) + }) + } +} + // writeInteropEncryptedFile writes a small encrypted parquet file with a footer -// key and key metadata, mirroring what pyarrow needs to decrypt it. +// key for pyarrow to read back. func writeInteropEncryptedFile(t *testing.T, path string) { t.Helper() @@ -220,8 +279,7 @@ func writeInteropEncryptedFile(t *testing.T, path string) { out, err := os.Create(path) require.NoError(t, err) - encProps := parquet.NewFileEncryptionProperties(interopFooterKey, - parquet.WithFooterKeyMetadata("kf")) + encProps := parquet.NewFileEncryptionProperties(interopFooterKey) props := parquet.NewWriterProperties(parquet.WithEncryptionProperties(encProps)) w := file.NewParquetWriter(out, root, file.WithWriterProps(props)) From 72d4edb8ba7c6f674fcd4cc22a48e8f21042e7b1 Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Tue, 10 Nov 2026 16:21:50 +0800 Subject: [PATCH 2/2] feat(parquet/encryption): cache unwrapped key encryption keys The retriever now caches unwrapped KEKs, so a file with double wrapping costs one KMS call per KEK instead of one per encrypted column plus the footer, matching parquet-cpp. Entries live for 10 minutes (a rotated KEK written under the same ID becomes visible after that) and the cache is bounded at 64 keys with LRU eviction; concurrent misses for the same KEK share a single KMS call. Also addresses the review feedback on #1478: - the interop script calls the CryptoFactory constructor directly; pyarrow has no register_kms_client_factory, so the fallback branch could never run - the example no longer claims PKMT1 files cannot be read without this package; it says the package handles the parsing and unwrapping that would otherwise need a custom retriever - the doc.go sentence about code that does not import the package is dropped, the preceding paragraph already says it - the cache clones the KEK before storing it, so a KmsClient that reuses or zeroes the buffer it returned cannot corrupt the cache - the retriever docs state it wraps a single KmsClient identity and should be shared only among callers with the same KMS access (parquet-cpp scopes its cache per access token) The interop test now counts KMS unwrap calls: the generated files have three key slots under two master keys (pyarrow generates one KEK per master key), so double wrapping mode resolves them in two calls instead of three, single wrapping mode in three. On the previous commit the double wrapping case makes three calls and the test fails. --- parquet/encryption/doc.go | 3 - parquet/encryption/example_test.go | 4 +- parquet/encryption/key_material_test.go | 121 +++++++++++++++ parquet/encryption/kms.go | 164 ++++++++++++++++++++- parquet/encryption_interop_pyarrow.py | 8 +- parquet/encryption_interop_pyarrow_test.go | 41 ++++-- 6 files changed, 318 insertions(+), 23 deletions(-) diff --git a/parquet/encryption/doc.go b/parquet/encryption/doc.go index e00945a9e..2959010ff 100644 --- a/parquet/encryption/doc.go +++ b/parquet/encryption/doc.go @@ -24,7 +24,4 @@ // NewKmsKeyRetriever and set the retriever with // parquet.WithKeyRetriever. Files using explicit keys or custom key // metadata continue to work without this package. -// -// This package is optional: it carries no effect on code that does not -// import it. package encryption diff --git a/parquet/encryption/example_test.go b/parquet/encryption/example_test.go index 47f56af2e..fe1417287 100644 --- a/parquet/encryption/example_test.go +++ b/parquet/encryption/example_test.go @@ -39,8 +39,8 @@ func (toyKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { // ExampleKmsKeyRetriever shows how to read files written by the // parquet-cpp/pyarrow CryptoFactory: implement KmsClient, wrap it in a // KmsKeyRetriever and set it on the decryption properties with -// parquet.WithKeyRetriever. Without this package, files whose key metadata -// carries PKMT1 key material cannot be read. +// parquet.WithKeyRetriever. This package handles the PKMT1 parsing and +// unwrapping that would otherwise need a custom retriever. func ExampleKmsKeyRetriever() { retriever := encryption.NewKmsKeyRetriever(toyKMS{}) diff --git a/parquet/encryption/key_material_test.go b/parquet/encryption/key_material_test.go index 5645121d7..0082b1bb9 100644 --- a/parquet/encryption/key_material_test.go +++ b/parquet/encryption/key_material_test.go @@ -23,7 +23,10 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -148,3 +151,121 @@ func TestDecryptKeyLocallyRejectsTampering(t *testing.T) { _, err = decryptKeyLocally(wrapped, kek, []byte("kek-id")) assert.Error(t, err) } + +// countingKMS counts UnwrapKey calls to observe the KEK cache. +type countingKMS struct { + KmsClient + unwraps atomic.Int32 +} + +func (c *countingKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + c.unwraps.Add(1) + return c.KmsClient.UnwrapKey(wrappedKey, masterKeyID) +} + +// doubleWrapMetadata returns double wrapping key metadata carrying kekID, as +// two files sharing one KEK would. +func doubleWrapMetadata(t *testing.T, kekID string) []byte { + t.Helper() + material := KeyMaterial{ + KeyMaterialType: "PKMT1", + InternalStorage: true, + IsFooterKey: true, + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + MasterKeyID: "mk1", + DoubleWrapping: true, + KeyEncryptionKeyID: base64.StdEncoding.EncodeToString([]byte(kekID)), + WrappedKEK: "wrapped-kek", + WrappedDEK: encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), + bytes.Repeat([]byte{0x2a}, 16), []byte(kekID)), + } + encoded, err := json.Marshal(material) + require.NoError(t, err) + return encoded +} + +func newCountingRetriever() (*KmsKeyRetriever, *countingKMS) { + kms := &countingKMS{KmsClient: mapKMS{keys: map[string][]byte{ + "mk1": bytes.Repeat([]byte{0x2a}, 16), + }}} + return NewKmsKeyRetriever(kms), kms +} + +func TestKmsKeyRetrieverCachesKEKs(t *testing.T) { + retriever, kms := newCountingRetriever() + + // two files sharing a KEK: same KEK ID, each with its own wrapped DEK + for i := 0; i < 2; i++ { + key, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) + } + assert.Equal(t, int32(1), kms.unwraps.Load(), "second file must hit the KEK cache") +} + +func TestKmsKeyRetrieverKekCacheExpires(t *testing.T) { + retriever, kms := newCountingRetriever() + + now := time.Now() + retriever.keks.now = func() time.Time { return now } + + _, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + + // a rotated KEK written under the same ID becomes visible after the TTL + now = now.Add(kekCacheTTL + time.Second) + _, err = retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.unwraps.Load(), "expired entry must miss the cache") +} + +func TestKmsKeyRetrieverKekCacheSeparatesKeys(t *testing.T) { + retriever, kms := newCountingRetriever() + + for _, kekID := range []string{"kek-id-1", "kek-id-2"} { + _, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, kekID)) + require.NoError(t, err) + } + assert.Equal(t, int32(2), kms.unwraps.Load(), "distinct KEK IDs must not share a cache entry") +} + +func TestKmsKeyRetrieverKekCacheCollapsesConcurrentMisses(t *testing.T) { + retriever, kms := newCountingRetriever() + + // the footer and column reads of one file resolve the same KEK + // concurrently: a single KMS call must serve all of them + metadata := doubleWrapMetadata(t, "kek-id") + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + key, err := retriever.UnwrapKeyMetadata(metadata) + assert.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) + }() + } + wg.Wait() + assert.Equal(t, int32(1), kms.unwraps.Load()) +} + +func TestKekCacheEviction(t *testing.T) { + cache := newKekCache(time.Minute, 2) + now := time.Now() + cache.now = func() time.Time { return now } + + cache.put(kekCacheKey{"mk", "k1"}, []byte{1}) + now = now.Add(2 * time.Minute) // k1 is now expired + + cache.put(kekCacheKey{"mk", "k2"}, []byte{2}) + assert.Len(t, cache.entries, 1, "inserting must drop expired entries first") + + cache.put(kekCacheKey{"mk", "k3"}, []byte{3}) + cache.put(kekCacheKey{"mk", "k4"}, []byte{4}) // full of live entries: evicts LRU k2 + + assert.Nil(t, cache.get(kekCacheKey{"mk", "k2"}), "LRU entry must be evicted") + assert.Equal(t, []byte{3}, cache.get(kekCacheKey{"mk", "k3"})) + assert.Equal(t, []byte{4}, cache.get(kekCacheKey{"mk", "k4"})) + assert.Len(t, cache.entries, 2) +} diff --git a/parquet/encryption/kms.go b/parquet/encryption/kms.go index 512b709f6..88b82eb99 100644 --- a/parquet/encryption/kms.go +++ b/parquet/encryption/kms.go @@ -17,10 +17,16 @@ package encryption import ( + "bytes" + "container/list" "crypto/aes" "crypto/cipher" "encoding/base64" "fmt" + "sync" + "time" + + "golang.org/x/sync/singleflight" ) // KmsClient wraps and unwraps data encryption keys with master keys managed @@ -40,13 +46,29 @@ type KmsClient interface { // KmsKeyRetriever implements parquet.DecryptionKeyRetriever for files whose // footer and column key metadata carry PKMT1 key material (internal key // storage), as written by the parquet-cpp/pyarrow CryptoFactory. +// +// In double wrapping mode the retriever caches unwrapped key encryption +// keys, so reading a file costs one KMS call per KEK instead of one per +// encrypted column plus the footer, matching parquet-cpp's KEK cache. +// Entries expire after kekCacheTTL and the cache holds at most +// kekCacheCapacity keys. +// +// A retriever wraps a single KmsClient identity, so share it only among +// callers with the same KMS access; parquet-cpp scopes its cache per +// access token instead. type KmsKeyRetriever struct { client KmsClient + keks *kekCache } // NewKmsKeyRetriever returns a retriever that resolves keys through client. +// The retriever is safe for concurrent use and meant to be shared across +// files read with the same KMS access. func NewKmsKeyRetriever(client KmsClient) *KmsKeyRetriever { - return &KmsKeyRetriever{client: client} + return &KmsKeyRetriever{ + client: client, + keks: newKekCache(kekCacheTTL, kekCacheCapacity), + } } // GetKey implements parquet.DecryptionKeyRetriever. Parse and unwrap errors @@ -83,7 +105,9 @@ func (r *KmsKeyRetriever) UnwrapKeyMetadata(keyMetadata []byte) ([]byte, error) return dek, nil } - kek, err := r.client.UnwrapKey(material.WrappedKEK, material.MasterKeyID) + kek, err := r.keks.unwrap(kekCacheKey{material.MasterKeyID, material.KeyEncryptionKeyID}, func() ([]byte, error) { + return r.client.UnwrapKey(material.WrappedKEK, material.MasterKeyID) + }) if err != nil { return nil, fmt.Errorf("parquet: unwrap key encryption key: %w", err) } @@ -121,3 +145,139 @@ func decryptKeyLocally(wrapped, key, aad []byte) ([]byte, error) { } return gcm.Open(nil, wrapped[:gcm.NonceSize()], wrapped[gcm.NonceSize():], aad) } + +// kekCacheTTL bounds how long an unwrapped KEK survives a key rotation +// unnoticed; kekCacheCapacity bounds memory for long-lived retrievers that +// read many files, each carrying its own KEK. +const ( + kekCacheTTL = 10 * time.Minute + kekCacheCapacity = 64 +) + +// kekCacheKey identifies a KEK by the master key that wrapped it and the KEK +// ID carried in the key material, so KEKs from different master keys can +// never collide. +type kekCacheKey struct { + masterKeyID string + keyEncryptionKeyID string +} + +type kekCacheEntry struct { + key kekCacheKey + kek []byte + expiresAt time.Time + element *list.Element // recency, list front = most recently used +} + +// kekCache caches unwrapped KEKs (expiry plus LRU eviction) and collapses +// concurrent misses for the same key into one call. +type kekCache struct { + mu sync.Mutex + ttl time.Duration + capacity int + now func() time.Time + entries map[kekCacheKey]*kekCacheEntry + recency list.List + inflight singleflight.Group +} + +func newKekCache(ttl time.Duration, capacity int) *kekCache { + return &kekCache{ + ttl: ttl, + capacity: capacity, + now: time.Now, + entries: make(map[kekCacheKey]*kekCacheEntry), + } +} + +func (c *kekCache) flightID(k kekCacheKey) string { + // %q escaping keeps the mapping injective: a plain separator can be + // ambiguous when an ID itself contains it + return fmt.Sprintf("%q\x00%q", k.masterKeyID, k.keyEncryptionKeyID) +} + +// get returns the cached KEK for key, or nil when absent or expired. +func (c *kekCache) get(k kekCacheKey) []byte { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.entries[k] + if !ok { + return nil + } + if c.now().After(entry.expiresAt) { + c.removeLocked(entry) + return nil + } + c.recency.MoveToFront(entry.element) + return entry.kek +} + +// put caches kek for key, replacing any previous entry. The KEK is cloned: +// a KmsClient may reuse or zero the buffer it returned. +func (c *kekCache) put(k kekCacheKey, kek []byte) { + kek = bytes.Clone(kek) + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.now() + if entry, ok := c.entries[k]; ok { + entry.kek = kek + entry.expiresAt = now.Add(c.ttl) + c.recency.MoveToFront(entry.element) + return + } + // expired entries go first; only evict live ones when they alone fill + // the cache + for _, entry := range c.entries { + if now.After(entry.expiresAt) { + c.removeLocked(entry) + } + } + for c.recency.Len() >= c.capacity { + c.removeLocked(c.recency.Back().Value.(*kekCacheEntry)) + } + entry := &kekCacheEntry{ + key: k, + kek: kek, + expiresAt: now.Add(c.ttl), + } + entry.element = c.recency.PushFront(entry) + c.entries[k] = entry +} + +// removeLocked drops entry from the map and the recency list; c.mu must be +// held. +func (c *kekCache) removeLocked(entry *kekCacheEntry) { + delete(c.entries, entry.key) + c.recency.Remove(entry.element) +} + +// unwrap returns the cached KEK for key, calling unwrapKEK only on a miss. +// Concurrent misses for the same key share one unwrapKEK call. +func (c *kekCache) unwrap(k kekCacheKey, unwrapKEK func() ([]byte, error)) ([]byte, error) { + if kek := c.get(k); kek != nil { + return kek, nil + } + value, err, _ := c.inflight.Do(c.flightID(k), func() (any, error) { + // the flight may have been queued behind another that already + // filled the cache + if kek := c.get(k); kek != nil { + return kek, nil + } + kek, err := unwrapKEK() + if err != nil { + return nil, err + } + if len(kek) == 0 { + return nil, fmt.Errorf("parquet: KMS returned an empty key encryption key for %q", k.masterKeyID) + } + c.put(k, kek) + return kek, nil + }) + if err != nil { + return nil, err + } + return value.([]byte), nil +} diff --git a/parquet/encryption_interop_pyarrow.py b/parquet/encryption_interop_pyarrow.py index 54b723766..0339cc2ff 100644 --- a/parquet/encryption_interop_pyarrow.py +++ b/parquet/encryption_interop_pyarrow.py @@ -84,16 +84,12 @@ def unwrap_key(self, wrapped_key, master_key_identifier): def client_factory(kms_connection_config): return Base64KmsClient(kms_connection_config) - try: - factory = pe.CryptoFactory(client_factory) # pyarrow >= 26 - except TypeError: - factory = pe.CryptoFactory() # older pyarrow - factory.register_kms_client_factory(client_factory) + factory = pe.CryptoFactory(client_factory) table = build_table() for name, double_wrapping in (("kms_double", True), ("kms_single", False)): encryption = pe.EncryptionConfiguration( - footer_key="footer_master", column_keys={"column_master": ["a"]}, + footer_key="footer_master", column_keys={"column_master": ["a", "b"]}, double_wrapping=double_wrapping) properties = factory.file_encryption_properties(config, encryption) path = os.path.join(outdir, name + ".parquet") diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index b72028376..ff222b635 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -27,6 +27,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -207,34 +208,51 @@ func TestGoWrittenEncryptionReadByPyarrow(t *testing.T) { } // interopBase64KMS mirrors the Base64KmsClient in encryption_interop_pyarrow.py: -// wrapping is plain base64, so the interop tests need no real KMS. -type interopBase64KMS struct{} +// wrapping is plain base64, so the interop tests need no real KMS. It counts +// KMS unwrap calls to check the KEK cache against real CryptoFactory files. +type interopBase64KMS struct { + unwraps atomic.Int32 +} -func (interopBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { +func (k *interopBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { return base64.StdEncoding.EncodeToString(key), nil } -func (interopBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { +func (k *interopBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + k.unwraps.Add(1) return base64.StdEncoding.DecodeString(wrappedKey) } // TestPyarrowKmsWrittenConfigs reads files written by pyarrow's CryptoFactory // with PKMT1 key material, in both double (the pyarrow default) and single -// wrapping mode. The footer key and the column key of field "a" are both -// resolved through the KMS retriever. +// wrapping mode. The footer key and the two column keys are all resolved +// through the KMS retriever. The files have three key slots under two master +// keys, and pyarrow generates one KEK per master key in double wrapping +// mode: the KEK cache must collapse the three slots into two KMS calls in +// double wrapping mode, while single wrapping mode unwraps every DEK +// separately (three calls). func TestPyarrowKmsWrittenConfigs(t *testing.T) { requirePyarrow(t) dir := t.TempDir() interopPython(t, "gen-kms", dir) - for _, name := range []string{"kms_double", "kms_single"} { - t.Run(name, func(t *testing.T) { + configs := []struct { + name string + expectedUnwraps int32 + }{ + {"kms_double", 2}, + {"kms_single", 3}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + kms := &interopBase64KMS{} props := parquet.NewReaderProperties(memory.DefaultAllocator) props.FileDecryptProps = parquet.NewFileDecryptionProperties( - parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(interopBase64KMS{}))) + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) - fr, err := file.OpenParquetFile(filepath.Join(dir, name+".parquet"), false, + fr, err := file.OpenParquetFile(filepath.Join(dir, config.name+".parquet"), false, file.WithReadProps(props)) require.NoError(t, err) defer fr.Close() @@ -259,6 +277,9 @@ func TestPyarrowKmsWrittenConfigs(t *testing.T) { require.NoError(t, err) assert.Equal(t, int64(3), n) assert.Equal(t, []float32{4, 5, 6}, f32vals[:3]) + + assert.Equal(t, config.expectedUnwraps, kms.unwraps.Load(), + "KMS unwrap calls while reading "+config.name) }) } }