diff --git a/parquet/encryption/doc.go b/parquet/encryption/doc.go new file mode 100644 index 000000000..2959010ff --- /dev/null +++ b/parquet/encryption/doc.go @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package encryption provides optional KMS integration for reading parquet +// files encrypted with PKMT1 key material, the format written by the +// CryptoFactory of parquet-cpp and pyarrow (both single and double key +// wrapping). +// +// It plugs into the existing low level API as a +// parquet.DecryptionKeyRetriever: pass a KmsClient implementation to +// NewKmsKeyRetriever and set the retriever with +// parquet.WithKeyRetriever. Files using explicit keys or custom key +// metadata continue to work without this package. +package encryption diff --git a/parquet/encryption/example_test.go b/parquet/encryption/example_test.go new file mode 100644 index 000000000..fe1417287 --- /dev/null +++ b/parquet/encryption/example_test.go @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption_test + +import ( + "encoding/base64" + "fmt" + + "github.com/apache/arrow-go/v18/parquet/encryption" +) + +// toyKMS is a stand-in for a real key management service client: the wrapped +// key representation is opaque to the library, so anything reversible works +// for demonstration. +type toyKMS struct{} + +func (toyKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (toyKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// ExampleKmsKeyRetriever shows how to read files written by the +// parquet-cpp/pyarrow CryptoFactory: implement KmsClient, wrap it in a +// KmsKeyRetriever and set it on the decryption properties with +// parquet.WithKeyRetriever. This package handles the PKMT1 parsing and +// unwrapping that would otherwise need a custom retriever. +func ExampleKmsKeyRetriever() { + retriever := encryption.NewKmsKeyRetriever(toyKMS{}) + + // key metadata as found in the footer of a CryptoFactory-written file + // (single wrapping mode) + metadata := []byte(`{ + "keyMaterialType": "PKMT1", + "internalStorage": true, + "isFooterKey": true, + "kmsInstanceID": "kms-instance", + "kmsInstanceURL": "https://kms.example", + "masterKeyID": "footer_master", + "wrappedDEK": "c2VjcmV0LWRlaw==", + "doubleWrapping": false + }`) + + fmt.Printf("%q\n", retriever.GetKey(metadata)) + + // To open an encrypted file, pass the retriever to the reader: + // + // props := parquet.NewReaderProperties(memory.DefaultAllocator) + // props.FileDecryptProps = parquet.NewFileDecryptionProperties( + // parquet.WithKeyRetriever(retriever)) + // fr, err := file.OpenParquetFile(path, false, file.WithReadProps(props)) + // + // Code that does not import this package is unaffected. + + // Output: "secret-dek" +} diff --git a/parquet/encryption/key_material.go b/parquet/encryption/key_material.go new file mode 100644 index 000000000..74d485dfb --- /dev/null +++ b/parquet/encryption/key_material.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "encoding/json" + "fmt" +) + +// keyMaterialType1 is the only key material type defined by the parquet key +// management tools specification ("parquet key management tools, version 1"). +const keyMaterialType1 = "PKMT1" + +// KeyMaterial is the PKMT1 key material carried in the key metadata of a +// parquet file written with envelope encryption. With internal key storage, +// the material is stored as a flat JSON object in the footer and column key +// metadata of the file. +type KeyMaterial struct { + // KeyMaterialType must be "PKMT1". + KeyMaterialType string `json:"keyMaterialType"` + // InternalStorage reports whether the key material is stored inside the + // parquet file. External key material stores are not supported. + InternalStorage bool `json:"internalStorage"` + // IsFooterKey reports whether this material belongs to the footer key. + // Only footer key material carries KmsInstanceID and KmsInstanceURL. + IsFooterKey bool `json:"isFooterKey"` + KmsInstanceID string `json:"kmsInstanceID"` + KmsInstanceURL string `json:"kmsInstanceURL"` + // MasterKeyID identifies the master key used to wrap the key. + MasterKeyID string `json:"masterKeyID"` + // WrappedDEK is the base64 encoded wrapped data encryption key. + WrappedDEK string `json:"wrappedDEK"` + // DoubleWrapping reports whether the DEK is wrapped by a key encryption + // key that is itself wrapped by the master key. + DoubleWrapping bool `json:"doubleWrapping"` + // KeyEncryptionKeyID is the base64 encoded key encryption key ID, set + // only in double wrapping mode. + KeyEncryptionKeyID string `json:"keyEncryptionKeyID"` + // WrappedKEK is the base64 encoded wrapped key encryption key, set only + // in double wrapping mode. + WrappedKEK string `json:"wrappedKEK"` +} + +// ParseKeyMetadata parses the key metadata JSON of a footer or column key. +// Only internal key storage (the parquet-cpp/pyarrow CryptoFactory default) +// is supported; external key material stores are rejected. +func ParseKeyMetadata(metadata []byte) (KeyMaterial, error) { + var material KeyMaterial + if err := json.Unmarshal(metadata, &material); err != nil { + return KeyMaterial{}, fmt.Errorf("parquet: parse key metadata: %w", err) + } + if material.KeyMaterialType != keyMaterialType1 { + return KeyMaterial{}, fmt.Errorf("parquet: unsupported key material type %q (want %q)", + material.KeyMaterialType, keyMaterialType1) + } + if !material.InternalStorage { + return KeyMaterial{}, fmt.Errorf("parquet: external key material storage is not supported") + } + if material.MasterKeyID == "" || material.WrappedDEK == "" { + return KeyMaterial{}, fmt.Errorf("parquet: key metadata missing masterKeyID or wrappedDEK") + } + if material.DoubleWrapping && (material.KeyEncryptionKeyID == "" || material.WrappedKEK == "") { + return KeyMaterial{}, fmt.Errorf("parquet: double wrapped key material missing keyEncryptionKeyID or wrappedKEK") + } + return material, nil +} diff --git a/parquet/encryption/key_material_test.go b/parquet/encryption/key_material_test.go new file mode 100644 index 000000000..0082b1bb9 --- /dev/null +++ b/parquet/encryption/key_material_test.go @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/json" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mapKMS resolves master key IDs to raw keys, standing in for a real KMS. +type mapKMS struct { + keys map[string][]byte +} + +func (m mapKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return "", nil // unused by these tests +} + +func (m mapKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + key, ok := m.keys[masterKeyID] + if !ok { + return nil, assert.AnError + } + return key, nil +} + +// encryptKeyLocally mirrors parquet-cpp's EncryptKeyLocally for test +// fixtures: AES-GCM, nonce || ciphertext || tag, base64 encoded. +func encryptKeyLocally(t *testing.T, key, master, aad []byte) string { + t.Helper() + block, err := aes.NewCipher(master) + require.NoError(t, err) + gcm, err := cipher.NewGCM(block) + require.NoError(t, err) + nonce := make([]byte, gcm.NonceSize()) + _, err = rand.Read(nonce) + require.NoError(t, err) + sealed := gcm.Seal(nonce, nonce, key, aad) + return base64.StdEncoding.EncodeToString(sealed) +} + +func footerMetadata(t *testing.T, double bool) []byte { + t.Helper() + material := KeyMaterial{ + KeyMaterialType: "PKMT1", + InternalStorage: true, + IsFooterKey: true, + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + MasterKeyID: "mk1", + DoubleWrapping: double, + } + if double { + kek := bytes.Repeat([]byte{0x2a}, 16) + material.KeyEncryptionKeyID = base64.StdEncoding.EncodeToString([]byte("kek-id")) + material.WrappedKEK = "wrapped-kek" + // a real wrapped DEK so the unwrapper has something valid to decrypt + material.WrappedDEK = encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), kek, []byte("kek-id")) + } else { + material.WrappedDEK = "wrapped-dek" + } + encoded, err := json.Marshal(material) + require.NoError(t, err) + return encoded +} + +func TestParseKeyMetadata(t *testing.T) { + material, err := ParseKeyMetadata(footerMetadata(t, false)) + require.NoError(t, err) + assert.Equal(t, "PKMT1", material.KeyMaterialType) + assert.True(t, material.InternalStorage) + assert.True(t, material.IsFooterKey) + assert.Equal(t, "mk1", material.MasterKeyID) + assert.False(t, material.DoubleWrapping) +} + +func TestParseKeyMetadataErrors(t *testing.T) { + _, err := ParseKeyMetadata([]byte(`{"keyMaterialType":"KM2","internalStorage":true}`)) + assert.ErrorContains(t, err, "unsupported key material type") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":false,"keyReference":"ref"}`)) + assert.ErrorContains(t, err, "external key material storage") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":true}`)) + assert.ErrorContains(t, err, "missing masterKeyID or wrappedDEK") + + _, err = ParseKeyMetadata([]byte(`{"keyMaterialType":"PKMT1","internalStorage":true,"masterKeyID":"m","wrappedDEK":"d","doubleWrapping":true}`)) + assert.ErrorContains(t, err, "missing keyEncryptionKeyID or wrappedKEK") + + _, err = ParseKeyMetadata([]byte(`{`)) + assert.ErrorContains(t, err, "parse key metadata") +} + +func TestKmsKeyRetrieverSingleWrapping(t *testing.T) { + dek := bytes.Repeat([]byte{0x42}, 16) + kms := mapKMS{keys: map[string][]byte{"mk1": dek}} + retriever := NewKmsKeyRetriever(kms) + + key, err := retriever.UnwrapKeyMetadata(footerMetadata(t, false)) + require.NoError(t, err) + assert.Equal(t, dek, key) + assert.Equal(t, string(dek), retriever.GetKey(footerMetadata(t, false))) +} + +func TestKmsKeyRetrieverDoubleWrapping(t *testing.T) { + kek := bytes.Repeat([]byte{0x2a}, 16) + kms := mapKMS{keys: map[string][]byte{"mk1": kek}} + retriever := NewKmsKeyRetriever(kms) + + key, err := retriever.UnwrapKeyMetadata(footerMetadata(t, true)) + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) +} + +func TestKmsKeyRetrieverGetKeyHidesErrors(t *testing.T) { + retriever := NewKmsKeyRetriever(mapKMS{keys: map[string][]byte{}}) + assert.Empty(t, retriever.GetKey(footerMetadata(t, false))) +} + +func TestDecryptKeyLocallyRejectsTampering(t *testing.T) { + kek := bytes.Repeat([]byte{0x2a}, 16) + wrapped, err := base64.StdEncoding.DecodeString( + encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), kek, []byte("kek-id"))) + require.NoError(t, err) + wrapped[0] ^= 0xff + _, err = decryptKeyLocally(wrapped, kek, []byte("kek-id")) + assert.Error(t, err) +} + +// countingKMS counts UnwrapKey calls to observe the KEK cache. +type countingKMS struct { + KmsClient + unwraps atomic.Int32 +} + +func (c *countingKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + c.unwraps.Add(1) + return c.KmsClient.UnwrapKey(wrappedKey, masterKeyID) +} + +// doubleWrapMetadata returns double wrapping key metadata carrying kekID, as +// two files sharing one KEK would. +func doubleWrapMetadata(t *testing.T, kekID string) []byte { + t.Helper() + material := KeyMaterial{ + KeyMaterialType: "PKMT1", + InternalStorage: true, + IsFooterKey: true, + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + MasterKeyID: "mk1", + DoubleWrapping: true, + KeyEncryptionKeyID: base64.StdEncoding.EncodeToString([]byte(kekID)), + WrappedKEK: "wrapped-kek", + WrappedDEK: encryptKeyLocally(t, bytes.Repeat([]byte{0x11}, 16), + bytes.Repeat([]byte{0x2a}, 16), []byte(kekID)), + } + encoded, err := json.Marshal(material) + require.NoError(t, err) + return encoded +} + +func newCountingRetriever() (*KmsKeyRetriever, *countingKMS) { + kms := &countingKMS{KmsClient: mapKMS{keys: map[string][]byte{ + "mk1": bytes.Repeat([]byte{0x2a}, 16), + }}} + return NewKmsKeyRetriever(kms), kms +} + +func TestKmsKeyRetrieverCachesKEKs(t *testing.T) { + retriever, kms := newCountingRetriever() + + // two files sharing a KEK: same KEK ID, each with its own wrapped DEK + for i := 0; i < 2; i++ { + key, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) + } + assert.Equal(t, int32(1), kms.unwraps.Load(), "second file must hit the KEK cache") +} + +func TestKmsKeyRetrieverKekCacheExpires(t *testing.T) { + retriever, kms := newCountingRetriever() + + now := time.Now() + retriever.keks.now = func() time.Time { return now } + + _, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + + // a rotated KEK written under the same ID becomes visible after the TTL + now = now.Add(kekCacheTTL + time.Second) + _, err = retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, "kek-id")) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.unwraps.Load(), "expired entry must miss the cache") +} + +func TestKmsKeyRetrieverKekCacheSeparatesKeys(t *testing.T) { + retriever, kms := newCountingRetriever() + + for _, kekID := range []string{"kek-id-1", "kek-id-2"} { + _, err := retriever.UnwrapKeyMetadata(doubleWrapMetadata(t, kekID)) + require.NoError(t, err) + } + assert.Equal(t, int32(2), kms.unwraps.Load(), "distinct KEK IDs must not share a cache entry") +} + +func TestKmsKeyRetrieverKekCacheCollapsesConcurrentMisses(t *testing.T) { + retriever, kms := newCountingRetriever() + + // the footer and column reads of one file resolve the same KEK + // concurrently: a single KMS call must serve all of them + metadata := doubleWrapMetadata(t, "kek-id") + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + key, err := retriever.UnwrapKeyMetadata(metadata) + assert.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte{0x11}, 16), key) + }() + } + wg.Wait() + assert.Equal(t, int32(1), kms.unwraps.Load()) +} + +func TestKekCacheEviction(t *testing.T) { + cache := newKekCache(time.Minute, 2) + now := time.Now() + cache.now = func() time.Time { return now } + + cache.put(kekCacheKey{"mk", "k1"}, []byte{1}) + now = now.Add(2 * time.Minute) // k1 is now expired + + cache.put(kekCacheKey{"mk", "k2"}, []byte{2}) + assert.Len(t, cache.entries, 1, "inserting must drop expired entries first") + + cache.put(kekCacheKey{"mk", "k3"}, []byte{3}) + cache.put(kekCacheKey{"mk", "k4"}, []byte{4}) // full of live entries: evicts LRU k2 + + assert.Nil(t, cache.get(kekCacheKey{"mk", "k2"}), "LRU entry must be evicted") + assert.Equal(t, []byte{3}, cache.get(kekCacheKey{"mk", "k3"})) + assert.Equal(t, []byte{4}, cache.get(kekCacheKey{"mk", "k4"})) + assert.Len(t, cache.entries, 2) +} diff --git a/parquet/encryption/kms.go b/parquet/encryption/kms.go new file mode 100644 index 000000000..88b82eb99 --- /dev/null +++ b/parquet/encryption/kms.go @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "bytes" + "container/list" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "fmt" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// KmsClient wraps and unwraps data encryption keys with master keys managed +// by a key management service. It mirrors the parquet-cpp/pyarrow KmsClient +// contract: implementations are provided by users, and the wrapped key +// representation is opaque to the library. Implementations must be safe for +// concurrent use. +type KmsClient interface { + // WrapKey encrypts key with the master key identified by masterKeyID and + // returns the KMS-specific wrapped representation. + WrapKey(key []byte, masterKeyID string) (string, error) + + // UnwrapKey reverses WrapKey, returning the raw key bytes. + UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) +} + +// KmsKeyRetriever implements parquet.DecryptionKeyRetriever for files whose +// footer and column key metadata carry PKMT1 key material (internal key +// storage), as written by the parquet-cpp/pyarrow CryptoFactory. +// +// In double wrapping mode the retriever caches unwrapped key encryption +// keys, so reading a file costs one KMS call per KEK instead of one per +// encrypted column plus the footer, matching parquet-cpp's KEK cache. +// Entries expire after kekCacheTTL and the cache holds at most +// kekCacheCapacity keys. +// +// A retriever wraps a single KmsClient identity, so share it only among +// callers with the same KMS access; parquet-cpp scopes its cache per +// access token instead. +type KmsKeyRetriever struct { + client KmsClient + keks *kekCache +} + +// NewKmsKeyRetriever returns a retriever that resolves keys through client. +// The retriever is safe for concurrent use and meant to be shared across +// files read with the same KMS access. +func NewKmsKeyRetriever(client KmsClient) *KmsKeyRetriever { + return &KmsKeyRetriever{ + client: client, + keks: newKekCache(kekCacheTTL, kekCacheCapacity), + } +} + +// GetKey implements parquet.DecryptionKeyRetriever. Parse and unwrap errors +// surface as an empty key, which the reader reports as a missing key; use +// UnwrapKeyMetadata when the underlying error is needed. +func (r *KmsKeyRetriever) GetKey(keyMetadata []byte) string { + key, err := r.UnwrapKeyMetadata(keyMetadata) + if err != nil { + return "" + } + return string(key) +} + +// UnwrapKeyMetadata parses PKMT1 key metadata and returns the data +// encryption key it carries: +// +// - single wrapping: the wrapped DEK is unwrapped by the KmsClient with +// the master key; +// - double wrapping: the KmsClient first unwraps the key encryption key +// with the master key, and the DEK is then decrypted locally with +// AES-GCM, using the KEK as the key and the (base64 decoded) KEK ID +// bytes as the AAD, matching parquet-cpp's EncryptKeyLocally format: +// nonce || ciphertext || tag. +func (r *KmsKeyRetriever) UnwrapKeyMetadata(keyMetadata []byte) ([]byte, error) { + material, err := ParseKeyMetadata(keyMetadata) + if err != nil { + return nil, err + } + if !material.DoubleWrapping { + dek, err := r.client.UnwrapKey(material.WrappedDEK, material.MasterKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: unwrap data encryption key: %w", err) + } + return dek, nil + } + + kek, err := r.keks.unwrap(kekCacheKey{material.MasterKeyID, material.KeyEncryptionKeyID}, func() ([]byte, error) { + return r.client.UnwrapKey(material.WrappedKEK, material.MasterKeyID) + }) + if err != nil { + return nil, fmt.Errorf("parquet: unwrap key encryption key: %w", err) + } + + // parquet-cpp base64 encodes the KEK ID in the key material but uses the + // decoded ID bytes as the AES-GCM AAD for the locally wrapped DEK. + aad, err := base64.StdEncoding.DecodeString(material.KeyEncryptionKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: decode key encryption key id: %w", err) + } + wrappedDEK, err := base64.StdEncoding.DecodeString(material.WrappedDEK) + if err != nil { + return nil, fmt.Errorf("parquet: decode wrapped data encryption key: %w", err) + } + dek, err := decryptKeyLocally(wrappedDEK, kek, aad) + if err != nil { + return nil, fmt.Errorf("parquet: decrypt data encryption key: %w", err) + } + return dek, nil +} + +// decryptKeyLocally reverses parquet-cpp's EncryptKeyLocally: AES-GCM with a +// 12-byte nonce prefix and no length framing (nonce || ciphertext || tag). +func decryptKeyLocally(wrapped, key, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + if len(wrapped) < gcm.NonceSize()+gcm.Overhead() { + return nil, fmt.Errorf("wrapped key too short (%d bytes)", len(wrapped)) + } + return gcm.Open(nil, wrapped[:gcm.NonceSize()], wrapped[gcm.NonceSize():], aad) +} + +// kekCacheTTL bounds how long an unwrapped KEK survives a key rotation +// unnoticed; kekCacheCapacity bounds memory for long-lived retrievers that +// read many files, each carrying its own KEK. +const ( + kekCacheTTL = 10 * time.Minute + kekCacheCapacity = 64 +) + +// kekCacheKey identifies a KEK by the master key that wrapped it and the KEK +// ID carried in the key material, so KEKs from different master keys can +// never collide. +type kekCacheKey struct { + masterKeyID string + keyEncryptionKeyID string +} + +type kekCacheEntry struct { + key kekCacheKey + kek []byte + expiresAt time.Time + element *list.Element // recency, list front = most recently used +} + +// kekCache caches unwrapped KEKs (expiry plus LRU eviction) and collapses +// concurrent misses for the same key into one call. +type kekCache struct { + mu sync.Mutex + ttl time.Duration + capacity int + now func() time.Time + entries map[kekCacheKey]*kekCacheEntry + recency list.List + inflight singleflight.Group +} + +func newKekCache(ttl time.Duration, capacity int) *kekCache { + return &kekCache{ + ttl: ttl, + capacity: capacity, + now: time.Now, + entries: make(map[kekCacheKey]*kekCacheEntry), + } +} + +func (c *kekCache) flightID(k kekCacheKey) string { + // %q escaping keeps the mapping injective: a plain separator can be + // ambiguous when an ID itself contains it + return fmt.Sprintf("%q\x00%q", k.masterKeyID, k.keyEncryptionKeyID) +} + +// get returns the cached KEK for key, or nil when absent or expired. +func (c *kekCache) get(k kekCacheKey) []byte { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.entries[k] + if !ok { + return nil + } + if c.now().After(entry.expiresAt) { + c.removeLocked(entry) + return nil + } + c.recency.MoveToFront(entry.element) + return entry.kek +} + +// put caches kek for key, replacing any previous entry. The KEK is cloned: +// a KmsClient may reuse or zero the buffer it returned. +func (c *kekCache) put(k kekCacheKey, kek []byte) { + kek = bytes.Clone(kek) + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.now() + if entry, ok := c.entries[k]; ok { + entry.kek = kek + entry.expiresAt = now.Add(c.ttl) + c.recency.MoveToFront(entry.element) + return + } + // expired entries go first; only evict live ones when they alone fill + // the cache + for _, entry := range c.entries { + if now.After(entry.expiresAt) { + c.removeLocked(entry) + } + } + for c.recency.Len() >= c.capacity { + c.removeLocked(c.recency.Back().Value.(*kekCacheEntry)) + } + entry := &kekCacheEntry{ + key: k, + kek: kek, + expiresAt: now.Add(c.ttl), + } + entry.element = c.recency.PushFront(entry) + c.entries[k] = entry +} + +// removeLocked drops entry from the map and the recency list; c.mu must be +// held. +func (c *kekCache) removeLocked(entry *kekCacheEntry) { + delete(c.entries, entry.key) + c.recency.Remove(entry.element) +} + +// unwrap returns the cached KEK for key, calling unwrapKEK only on a miss. +// Concurrent misses for the same key share one unwrapKEK call. +func (c *kekCache) unwrap(k kekCacheKey, unwrapKEK func() ([]byte, error)) ([]byte, error) { + if kek := c.get(k); kek != nil { + return kek, nil + } + value, err, _ := c.inflight.Do(c.flightID(k), func() (any, error) { + // the flight may have been queued behind another that already + // filled the cache + if kek := c.get(k); kek != nil { + return kek, nil + } + kek, err := unwrapKEK() + if err != nil { + return nil, err + } + if len(kek) == 0 { + return nil, fmt.Errorf("parquet: KMS returned an empty key encryption key for %q", k.masterKeyID) + } + c.put(k, kek) + return kek, nil + }) + if err != nil { + return nil, err + } + return value.([]byte), nil +} diff --git a/parquet/encryption_interop_pyarrow.py b/parquet/encryption_interop_pyarrow.py index 6fc1ceadb..0339cc2ff 100644 --- a/parquet/encryption_interop_pyarrow.py +++ b/parquet/encryption_interop_pyarrow.py @@ -20,6 +20,7 @@ Usage: python3 encryption_interop_pyarrow.py probe python3 encryption_interop_pyarrow.py gen + python3 encryption_interop_pyarrow.py gen-kms python3 encryption_interop_pyarrow.py read """ import sys @@ -56,6 +57,46 @@ def write(name, fep): create_encryption_properties(footer_key, plaintext_footer=True)) +def gen_kms(outdir): + """Write PKMT1 key material files with a CryptoFactory and a test-only + in-memory KMS client whose wrapping is plain base64 (no cryptography + dependency). The Go side mirrors the client to unwrap the keys.""" + import base64 as b64 + import os + + import pyarrow.parquet as pq + import pyarrow.parquet.encryption as pe + + class Base64KmsClient(pe.KmsClient): + def __init__(self, config): + super().__init__() + + def wrap_key(self, key_bytes, master_key_identifier): + return b64.b64encode(key_bytes).decode("ascii") + + def unwrap_key(self, wrapped_key, master_key_identifier): + return b64.b64decode(wrapped_key) + + config = pe.KmsConnectionConfig( + kms_instance_id="in-memory", kms_instance_url="local", + key_access_token="token") + + def client_factory(kms_connection_config): + return Base64KmsClient(kms_connection_config) + + factory = pe.CryptoFactory(client_factory) + + table = build_table() + for name, double_wrapping in (("kms_double", True), ("kms_single", False)): + encryption = pe.EncryptionConfiguration( + footer_key="footer_master", column_keys={"column_master": ["a", "b"]}, + double_wrapping=double_wrapping) + properties = factory.file_encryption_properties(config, encryption) + path = os.path.join(outdir, name + ".parquet") + pq.write_table(table, path, encryption_properties=properties) + print(name) + + def read(path, footer_key): import pyarrow.parquet as pq from pyarrow.parquet.encryption import create_decryption_properties @@ -82,6 +123,9 @@ def main(): if mode == "gen": gen(sys.argv[2], sys.argv[3].encode()) return 0 + if mode == "gen-kms": + gen_kms(sys.argv[2]) + return 0 if mode == "read": read(sys.argv[2], sys.argv[3].encode()) return 0 diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index 59a0c9855..ff222b635 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -21,16 +21,19 @@ import ( "bytes" "context" _ "embed" + "encoding/base64" "fmt" "os" "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" "time" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/encryption" "github.com/apache/arrow-go/v18/parquet/file" "github.com/apache/arrow-go/v18/parquet/schema" "github.com/stretchr/testify/assert" @@ -70,8 +73,8 @@ func runInteropPython(t *testing.T, args ...string) (string, error) { return stdout.String(), nil } -// interopPython runs the embedded script and fails the test on a non-zero -// exit. A non-zero exit is a test failure. +// interopPython runs the embedded script and fails the test when it exits +// non-zero. func interopPython(t *testing.T, args ...string) string { t.Helper() @@ -90,15 +93,15 @@ func interopPython(t *testing.T, args ...string) string { func requirePyarrow(t *testing.T) { t.Helper() - require := os.Getenv("PARQUET_TEST_REQUIRE_PYARROW") != "" + mustHavePyarrow := os.Getenv("PARQUET_TEST_REQUIRE_PYARROW") != "" if _, err := exec.LookPath("python3"); err != nil { - if require { + if mustHavePyarrow { t.Fatalf("PARQUET_TEST_REQUIRE_PYARROW is set but python3 not found in PATH: %v", err) } t.Skip("python3 not found in PATH, skipping pyarrow interop tests") } if _, err := runInteropPython(t, "probe"); err != nil { - if require { + if mustHavePyarrow { t.Fatalf("PARQUET_TEST_REQUIRE_PYARROW is set but pyarrow is not available: %v", err) } t.Skipf("pyarrow not available for python3, skipping interop tests: %v", err) @@ -204,8 +207,85 @@ func TestGoWrittenEncryptionReadByPyarrow(t *testing.T) { assert.Equal(t, "0 10 20", got["int32_field"]) } +// interopBase64KMS mirrors the Base64KmsClient in encryption_interop_pyarrow.py: +// wrapping is plain base64, so the interop tests need no real KMS. It counts +// KMS unwrap calls to check the KEK cache against real CryptoFactory files. +type interopBase64KMS struct { + unwraps atomic.Int32 +} + +func (k *interopBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (k *interopBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + k.unwraps.Add(1) + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// TestPyarrowKmsWrittenConfigs reads files written by pyarrow's CryptoFactory +// with PKMT1 key material, in both double (the pyarrow default) and single +// wrapping mode. The footer key and the two column keys are all resolved +// through the KMS retriever. The files have three key slots under two master +// keys, and pyarrow generates one KEK per master key in double wrapping +// mode: the KEK cache must collapse the three slots into two KMS calls in +// double wrapping mode, while single wrapping mode unwraps every DEK +// separately (three calls). +func TestPyarrowKmsWrittenConfigs(t *testing.T) { + requirePyarrow(t) + + dir := t.TempDir() + interopPython(t, "gen-kms", dir) + + configs := []struct { + name string + expectedUnwraps int32 + }{ + {"kms_double", 2}, + {"kms_single", 3}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + kms := &interopBase64KMS{} + props := parquet.NewReaderProperties(memory.DefaultAllocator) + props.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) + + fr, err := file.OpenParquetFile(filepath.Join(dir, config.name+".parquet"), false, + file.WithReadProps(props)) + require.NoError(t, err) + defer fr.Close() + + assert.Equal(t, int64(3), fr.MetaData().NumRows) + + rg := fr.RowGroup(0) + a, err := rg.Column(0) + require.NoError(t, err) + f64 := a.(*file.Float64ColumnChunkReader) + vals := make([]float64, 8) + n, _, err := f64.ReadBatch(8, vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float64{1, 2, 3}, vals[:3]) + + b, err := rg.Column(1) + require.NoError(t, err) + f32 := b.(*file.Float32ColumnChunkReader) + f32vals := make([]float32, 8) + n, _, err = f32.ReadBatch(8, f32vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float32{4, 5, 6}, f32vals[:3]) + + assert.Equal(t, config.expectedUnwraps, kms.unwraps.Load(), + "KMS unwrap calls while reading "+config.name) + }) + } +} + // writeInteropEncryptedFile writes a small encrypted parquet file with a footer -// key and key metadata, mirroring what pyarrow needs to decrypt it. +// key for pyarrow to read back. func writeInteropEncryptedFile(t *testing.T, path string) { t.Helper() @@ -220,8 +300,7 @@ func writeInteropEncryptedFile(t *testing.T, path string) { out, err := os.Create(path) require.NoError(t, err) - encProps := parquet.NewFileEncryptionProperties(interopFooterKey, - parquet.WithFooterKeyMetadata("kf")) + encProps := parquet.NewFileEncryptionProperties(interopFooterKey) props := parquet.NewWriterProperties(parquet.WithEncryptionProperties(encProps)) w := file.NewParquetWriter(out, root, file.WithWriterProps(props))