From cdf455848342dd6575aae2670b9b58654178d211 Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Sat, 14 Nov 2026 16:52:12 +0800 Subject: [PATCH 1/4] feat(parquet/encryption): KMS support for writing PKMT1 key material CryptoFactory builds parquet.FileEncryptionProperties whose keys are wrapped by a KmsClient, closing the loop with the read side of #1478: fresh 16-byte DEKs are generated for the footer and every configured column, wrapped single (straight through the KMS) or double (a cached per-master-key KEK wraps DEKs locally with AES-GCM, nonce || ciphertext || tag, AAD = KEK ID bytes, and only the KEK goes through the KMS), and serialized as PKMT1 key material in the footer and column key metadata. The material matches parquet-cpp byte for byte: field order, omitted kmsInstance fields on column keys (KeyMaterial now marshals them, and the single-wrapping KEK fields, with omitempty), a fresh DEK per column, one KEK per master key reused across files, and KMS call counts symmetric with the read side (2 unwraps for two master keys in double wrapping mode, 1 per key slot in single wrapping mode). Interop: the harness gains a read-kms mode, and Go-written files read back through pyarrow's CryptoFactory in double, single, and plaintext footer configurations; a Go write/read round trip resolves every key through KmsKeyRetriever alone. --- parquet/encryption/crypto_factory.go | 303 +++++++++++++++++ parquet/encryption/crypto_factory_test.go | 370 +++++++++++++++++++++ parquet/encryption/doc.go | 13 +- parquet/encryption/example_test.go | 38 +++ parquet/encryption/key_material.go | 13 +- parquet/encryption_interop_pyarrow.py | 39 ++- parquet/encryption_interop_pyarrow_test.go | 92 +++++ 7 files changed, 851 insertions(+), 17 deletions(-) create mode 100644 parquet/encryption/crypto_factory.go create mode 100644 parquet/encryption/crypto_factory_test.go diff --git a/parquet/encryption/crypto_factory.go b/parquet/encryption/crypto_factory.go new file mode 100644 index 000000000..ac2366caf --- /dev/null +++ b/parquet/encryption/crypto_factory.go @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "sync" + + "github.com/apache/arrow-go/v18/parquet" +) + +// Key sizes for generated keys, matching the parquet-cpp/pyarrow +// CryptoFactory (AES-128 data and key encryption keys). +const ( + dataKeySize = 16 + kekSize = 16 + kekIDSize = 16 +) + +// KmsClientFactory creates KmsClients for a connection configuration, +// mirroring parquet-cpp's KmsClientFactory. +type KmsClientFactory func(KmsConnectionConfig) (KmsClient, error) + +// KmsConnectionConfig identifies the KMS instance and access credentials +// used for a file's key material. It is passed to the client factory and +// serialized into the footer key material, mirroring parquet-cpp/pyarrow. +type KmsConnectionConfig struct { + KmsInstanceID string + KmsInstanceURL string + KeyAccessToken string +} + +// EncryptionConfiguration describes which master keys encrypt what in a +// file, mirroring parquet-cpp/pyarrow's EncryptionConfiguration. +type EncryptionConfiguration struct { + // FooterKeyID is the master key id that wraps the footer key (and, in + // uniform encryption, the single key encrypting every column). Required. + FooterKeyID string + + // ColumnKeys maps a master key id to the column paths it encrypts; every + // listed column gets its own data encryption key wrapped by that master + // key. Columns not listed stay in plaintext. When empty (or nil) the + // configuration is uniform: all columns are encrypted with the footer + // key. + ColumnKeys map[string][]string + + // SingleWrapping wraps each data encryption key directly with its master + // key. The default (false) is double wrapping (the parquet-cpp/pyarrow + // default), where a per-master-key key encryption key wraps the data + // keys locally and only it is wrapped by the KMS. + SingleWrapping bool + + // AadPrefix is the AAD prefix identifier stored in the file (unless + // DisableAadPrefixStorage is set) so readers can verify file identity. + AadPrefix string + + // DisableAadPrefixStorage keeps the AadPrefix out of the file; readers + // must then supply it themselves. + DisableAadPrefixStorage bool + + // PlaintextFooter leaves the footer unencrypted (still authenticated + // with a signature). + PlaintextFooter bool +} + +// CryptoFactory builds parquet.FileEncryptionProperties whose keys are +// wrapped by a key management service, producing the PKMT1 key material +// that KmsKeyRetriever (and parquet-cpp/pyarrow readers) resolve. +// +// In double wrapping mode the factory keeps one key encryption key per +// (KMS instance, master key) and reuses it across files, so writing many +// files costs one KEK wrap per master key instead of one per key slot, +// matching the parquet-cpp/pyarrow CryptoFactory. The cache key does not +// include the access token: configs sharing a KMS instance id also share +// KEKs, which is safe when the KMS binds wraps to the master key. +type CryptoFactory struct { + clientFactory KmsClientFactory + + mu sync.Mutex + keks map[kekReuseKey]cachedKek +} + +// kekReuseKey identifies the KEK cached for a master key under one KMS +// instance. +type kekReuseKey struct { + kmsInstanceID string + masterKeyID string +} + +// cachedKek is a generated key encryption key together with its KMS-wrapped +// form, ready to be stamped into key material. +type cachedKek struct { + id []byte // raw KEK id; also the AAD for local data key wrapping + kek []byte + wrapped string // KmsClient-wrapped KEK +} + +// NewCryptoFactory returns a factory that creates KmsClients with +// clientFactory. A single factory can encrypt many files; call +// FileEncryptionProperties once per file, since the returned properties are +// single use (the parquet writer wipes their keys). +func NewCryptoFactory(clientFactory KmsClientFactory) *CryptoFactory { + return &CryptoFactory{clientFactory: clientFactory} +} + +// FileEncryptionProperties generates fresh data encryption keys for the +// footer and the configured columns, wraps them through the KMS client, and +// returns file encryption properties carrying the PKMT1 key material in the +// footer and column key metadata. +func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, + cfg EncryptionConfiguration) (*parquet.FileEncryptionProperties, error) { + if cfg.FooterKeyID == "" { + return nil, fmt.Errorf("parquet: encryption configuration needs a footer key id") + } + if f.clientFactory == nil { + return nil, fmt.Errorf("parquet: CryptoFactory needs a client factory, construct it with NewCryptoFactory") + } + if cfg.DisableAadPrefixStorage && cfg.AadPrefix == "" { + return nil, fmt.Errorf("parquet: DisableAadPrefixStorage is set without an AadPrefix") + } + client, err := f.clientFactory(kmsCfg) + if err != nil { + return nil, fmt.Errorf("parquet: create kms client: %w", err) + } + + footerDEK, err := randomKey(dataKeySize) + if err != nil { + return nil, err + } + material, err := f.wrapDataKey(client, kmsCfg, cfg, footerDEK, cfg.FooterKeyID) + if err != nil { + return nil, err + } + material.IsFooterKey = true + material.KmsInstanceID = kmsCfg.KmsInstanceID + material.KmsInstanceURL = kmsCfg.KmsInstanceURL + footerJSON, err := json.Marshal(material) + if err != nil { + return nil, err + } + + opts := []parquet.EncryptOption{ + parquet.WithFooterKeyMetadata(string(footerJSON)), + } + + cols := parquet.ColumnPathToEncryptionPropsMap{} + for masterKeyID, paths := range cfg.ColumnKeys { + if masterKeyID == "" { + return nil, fmt.Errorf("parquet: column keys map needs a master key id") + } + for _, path := range paths { + if path == "" { + return nil, fmt.Errorf("parquet: column keys for master key %q contain an empty column path", masterKeyID) + } + dek, err := randomKey(dataKeySize) + if err != nil { + return nil, err + } + material, err := f.wrapDataKey(client, kmsCfg, cfg, dek, masterKeyID) + if err != nil { + return nil, err + } + colJSON, err := json.Marshal(material) + if err != nil { + return nil, err + } + if _, exists := cols[path]; exists { + return nil, fmt.Errorf("parquet: column %q is listed under more than one master key", path) + } + cols[path] = parquet.NewColumnEncryptionProperties(path, + parquet.WithKey(string(dek)), parquet.WithKeyMetadata(string(colJSON))) + } + } + if len(cols) > 0 { + opts = append(opts, parquet.WithEncryptedColumns(cols)) + } + + if cfg.AadPrefix != "" { + opts = append(opts, parquet.WithAadPrefix(cfg.AadPrefix)) + if cfg.DisableAadPrefixStorage { + opts = append(opts, parquet.DisableAadPrefixStorage()) + } + } + if cfg.PlaintextFooter { + opts = append(opts, parquet.WithPlaintextFooter()) + } + return parquet.NewFileEncryptionProperties(string(footerDEK), opts...), nil +} + +// randomKey returns n cryptographically random bytes. +func randomKey(n int) ([]byte, error) { + key := make([]byte, n) + if _, err := rand.Read(key); err != nil { + return nil, fmt.Errorf("parquet: generate key: %w", err) + } + return key, nil +} + +// wrapDataKey wraps dek with the master key masterKeyID and returns the +// PKMT1 key material for it. +func (f *CryptoFactory) wrapDataKey(client KmsClient, kmsCfg KmsConnectionConfig, + cfg EncryptionConfiguration, dek []byte, masterKeyID string) (KeyMaterial, error) { + + material := KeyMaterial{ + KeyMaterialType: keyMaterialType1, + InternalStorage: true, + MasterKeyID: masterKeyID, + } + if cfg.SingleWrapping { + wrapped, err := client.WrapKey(dek, masterKeyID) + if err != nil { + return KeyMaterial{}, fmt.Errorf("parquet: wrap data encryption key with master key %q: %w", masterKeyID, err) + } + material.WrappedDEK = wrapped + return material, nil + } + + // double wrapping: a key encryption key wraps data keys locally and only + // it is wrapped by the KMS + kek, err := f.kekFor(client, kmsCfg, masterKeyID) + if err != nil { + return KeyMaterial{}, err + } + wrappedDEK, err := wrapKeyLocally(dek, kek.kek, kek.id) + if err != nil { + return KeyMaterial{}, err + } + material.DoubleWrapping = true + material.WrappedDEK = base64.StdEncoding.EncodeToString(wrappedDEK) + material.KeyEncryptionKeyID = base64.StdEncoding.EncodeToString(kek.id) + material.WrappedKEK = kek.wrapped + return material, nil +} + +// kekFor returns the cached key encryption key for masterKeyID, generating +// and wrapping a fresh one on first use. The factory lock is held across the +// KMS call so concurrent files sharing a master key get one wrap. +func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, + masterKeyID string) (cachedKek, error) { + key := kekReuseKey{kmsInstanceID: kmsCfg.KmsInstanceID, masterKeyID: masterKeyID} + f.mu.Lock() + defer f.mu.Unlock() + if kek, ok := f.keks[key]; ok { + return kek, nil + } + id, err := randomKey(kekIDSize) + if err != nil { + return cachedKek{}, err + } + kekKey, err := randomKey(kekSize) + if err != nil { + return cachedKek{}, err + } + wrapped, err := client.WrapKey(kekKey, masterKeyID) + if err != nil { + return cachedKek{}, fmt.Errorf("parquet: wrap key encryption key with master key %q: %w", masterKeyID, err) + } + kek := cachedKek{id: id, kek: kekKey, wrapped: wrapped} + if f.keks == nil { + f.keks = make(map[kekReuseKey]cachedKek) + } + f.keks[key] = kek + return kek, nil +} + +// wrapKeyLocally is parquet-cpp's EncryptKeyLocally: AES-GCM with a +// 12-byte nonce prefix and no length framing (nonce || ciphertext || tag), +// producing the format decryptKeyLocally unwinds. +func wrapKeyLocally(key, kek, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(kek) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("parquet: generate nonce: %w", err) + } + return gcm.Seal(nonce, nonce, key, aad), nil +} diff --git a/parquet/encryption/crypto_factory_test.go b/parquet/encryption/crypto_factory_test.go new file mode 100644 index 000000000..373f6af7a --- /dev/null +++ b/parquet/encryption/crypto_factory_test.go @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption_test + +import ( + "encoding/base64" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/encryption" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingBase64KMS mirrors the test KMS client of the interop harness: +// wrapping is plain base64. It counts calls so tests can assert how many +// KMS round trips key material generation costs. +type countingBase64KMS struct { + wraps atomic.Int32 + unwraps atomic.Int32 +} + +func (k *countingBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { + k.wraps.Add(1) + return base64.StdEncoding.EncodeToString(key), nil +} + +func (k *countingBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + k.unwraps.Add(1) + return base64.StdEncoding.DecodeString(wrappedKey) +} + +func countingFactory(t *testing.T, kms *countingBase64KMS) *encryption.CryptoFactory { + t.Helper() + return encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) +} + +var testKmsConfig = encryption.KmsConnectionConfig{ + KmsInstanceID: "in-memory", + KmsInstanceURL: "local", +} + +// In uniform encryption every column is encrypted with the footer key, so +// only the footer key material is generated. Single wrapping hands each data +// encryption key straight to the KMS client. +func TestFileEncryptionPropertiesUniformSingleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + SingleWrapping: true, + }) + require.NoError(t, err) + + // the footer key is a fresh 16-byte data key + assert.Len(t, props.FooterKey(), 16) + + material, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, "footer_master", material.MasterKeyID) + assert.True(t, material.IsFooterKey) + assert.False(t, material.DoubleWrapping) + assert.Equal(t, "in-memory", material.KmsInstanceID) + assert.Equal(t, "local", material.KmsInstanceURL) + + // the retriever must recover the footer key from the metadata alone + key, err := encryption.NewKmsKeyRetriever(kms).UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, props.FooterKey(), string(key)) + + // uniform encryption: no column properties, columns fall back to the + // footer key + assert.Empty(t, props.EncryptedColumns()) + assert.True(t, props.ColumnEncryptionProperties("a").IsEncryptedWithFooterKey()) + + // one DEK wrapped directly by the KMS + assert.Equal(t, int32(1), kms.wraps.Load()) +} + +// Double wrapping wraps only a per-master-key key encryption key with the +// KMS; data keys are wrapped locally. The KEK is reused across files, so a +// second properties object costs no extra KMS call (the pyarrow behavior). +func TestFileEncryptionPropertiesDoubleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master"} + + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Len(t, props.FooterKey(), 16) + + material, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.True(t, material.DoubleWrapping) + assert.True(t, material.IsFooterKey) + assert.Equal(t, "footer_master", material.MasterKeyID) + + // the wrapped DEK carries the AES-GCM framing: 12 byte nonce + 16 byte + // key + 16 byte tag + wrappedDEK, err := base64.StdEncoding.DecodeString(material.WrappedDEK) + require.NoError(t, err) + assert.Len(t, wrappedDEK, 44) + kekID, err := base64.StdEncoding.DecodeString(material.KeyEncryptionKeyID) + require.NoError(t, err) + assert.Len(t, kekID, 16) + + // the retriever must recover the footer key from the metadata alone, + // unwrapping the KEK with the KMS and the DEK locally + key, err := encryption.NewKmsKeyRetriever(kms).UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, props.FooterKey(), string(key)) + assert.Equal(t, int32(1), kms.unwraps.Load(), "one KMS call to unwrap the KEK") + + // only the KEK is wrapped by the KMS, and it is reused for the next file + _, err = factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load(), "KEK must be reused across files") + + // a fresh factory wraps a fresh KEK + other := &countingBase64KMS{} + _, err = countingFactory(t, other).FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), other.wraps.Load()) +} + +// With column keys every listed column gets its own data encryption key +// wrapped by its master key; columns under the same master key share the +// key encryption key, and unlisted columns stay in plaintext. +func TestFileEncryptionPropertiesColumnKeys(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + retriever := encryption.NewKmsKeyRetriever(kms) + + cfg := encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "float_field"}}, + } + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + + for _, path := range []string{"double_field", "float_field"} { + col := props.ColumnEncryptionProperties(path) + require.NotNil(t, col, path) + assert.True(t, col.IsEncrypted(), path) + assert.False(t, col.IsEncryptedWithFooterKey(), path) + assert.Len(t, col.Key(), 16, path) + + material, err := encryption.ParseKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err, path) + assert.Equal(t, "column_master", material.MasterKeyID, path) + assert.False(t, material.IsFooterKey, path) + + // column key material carries no KMS instance fields, matching + // parquet-cpp + assert.NotContains(t, col.KeyMetadata(), "kmsInstanceID", path) + + key, err := retriever.UnwrapKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err, path) + assert.Equal(t, col.Key(), string(key), path) + } + + // every column gets a fresh data key, but both share the column master + // key's KEK, distinct from the footer KEK + a := props.ColumnEncryptionProperties("double_field") + b := props.ColumnEncryptionProperties("float_field") + assert.NotEqual(t, a.Key(), b.Key()) + am, err := encryption.ParseKeyMetadata([]byte(a.KeyMetadata())) + require.NoError(t, err) + bm, err := encryption.ParseKeyMetadata([]byte(b.KeyMetadata())) + require.NoError(t, err) + assert.Equal(t, am.KeyEncryptionKeyID, bm.KeyEncryptionKeyID) + fm, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.NotEqual(t, am.KeyEncryptionKeyID, fm.KeyEncryptionKeyID) + + // unlisted columns stay in plaintext + assert.Nil(t, props.ColumnEncryptionProperties("boolean_field")) + + // two master keys wrapped once each: the footer KEK and the column KEK + assert.Equal(t, int32(2), kms.wraps.Load()) +} + +// Single wrapping with column keys wraps every data key with the KMS. +func TestFileEncryptionPropertiesColumnKeysSingleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + retriever := encryption.NewKmsKeyRetriever(kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "float_field"}}, + SingleWrapping: true, + }) + require.NoError(t, err) + + // footer + one data key per column: three KMS wraps + assert.Equal(t, int32(3), kms.wraps.Load()) + + col := props.ColumnEncryptionProperties("double_field") + key, err := retriever.UnwrapKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err) + assert.Equal(t, col.Key(), string(key)) +} + +func TestFileEncryptionPropertiesErrors(t *testing.T) { + factory := countingFactory(t, &countingBase64KMS{}) + + _, err := factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{FooterKeyID: ""}) + assert.ErrorContains(t, err, "footer key id") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"": {"a"}}, + }) + assert.ErrorContains(t, err, "master key id") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", ""}}, + }) + assert.ErrorContains(t, err, "column path") + + // a column listed under two master keys would otherwise win + // nondeterministically (map iteration order) + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{ + "column_master": {"a"}, + "other_master": {"a"}, + }, + }) + assert.ErrorContains(t, err, "more than one master key") + + // parquet-cpp rejects the combination too + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + DisableAadPrefixStorage: true, + }) + assert.ErrorContains(t, err, "DisableAadPrefixStorage") +} + +// writeKmsEncryptedFile writes a file with an encrypted float64 column and a +// plaintext boolean column, using keys from the CryptoFactory. +func writeKmsEncryptedFile(t *testing.T, path string, + props *parquet.FileEncryptionProperties) { + t.Helper() + + fields := schema.FieldList{ + schema.Must(schema.NewPrimitiveNode("double_field", parquet.Repetitions.Required, + parquet.Types.Double, 0, -1)), + schema.NewBooleanNode("boolean_field", parquet.Repetitions.Required, -1), + } + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Repeated, fields, -1) + require.NoError(t, err) + + out, err := os.Create(path) + require.NoError(t, err) + + writerProps := parquet.NewWriterProperties(parquet.WithEncryptionProperties(props)) + w := file.NewParquetWriter(out, root, file.WithWriterProps(writerProps)) + + rgr, err := w.AppendBufferedRowGroupChecked() + require.NoError(t, err) + + dw, err := rgr.Column(0) + require.NoError(t, err) + doubleWriter := dw.(*file.Float64ColumnChunkWriter) + for i := 0; i < 3; i++ { + _, err := doubleWriter.WriteBatch([]float64{float64(i) * 10}, nil, nil) + require.NoError(t, err) + } + + bw, err := rgr.Column(1) + require.NoError(t, err) + boolWriter := bw.(*file.BooleanColumnChunkWriter) + for i := 0; i < 3; i++ { + _, err := boolWriter.WriteBatch([]bool{i%2 == 0}, nil, nil) + require.NoError(t, err) + } + + require.NoError(t, w.Close()) +} + +// A file written with CryptoFactory keys must read back through the KMS key +// retriever alone, with no explicit keys anywhere. +func TestCryptoFactoryWriteReadRoundTrip(t *testing.T) { + configs := []struct { + name string + singleWrapping bool + expectedUnwraps int32 + }{ + {"double_wrapping", false, 2}, + {"single_wrapping", true, 3}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "boolean_field"}}, + SingleWrapping: config.singleWrapping, + }) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "kms_written.parquet") + writeKmsEncryptedFile(t, path, props) + + readProps := parquet.NewReaderProperties(memory.DefaultAllocator) + readProps.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) + + fr, err := file.OpenParquetFile(path, false, file.WithReadProps(readProps)) + require.NoError(t, err) + defer fr.Close() + + assert.Equal(t, int64(3), fr.MetaData().NumRows) + + rg := fr.RowGroup(0) + d, err := rg.Column(0) + require.NoError(t, err) + doubleReader := d.(*file.Float64ColumnChunkReader) + vals := make([]float64, 8) + n, _, err := doubleReader.ReadBatch(8, vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float64{0, 10, 20}, vals[:3]) + + b, err := rg.Column(1) + require.NoError(t, err) + boolReader := b.(*file.BooleanColumnChunkReader) + bools := make([]bool, 8) + n, _, err = boolReader.ReadBatch(8, bools, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []bool{true, false, true}, bools[:3]) + + assert.Equal(t, config.expectedUnwraps, kms.unwraps.Load(), + "KMS unwrap calls while reading") + }) + } +} diff --git a/parquet/encryption/doc.go b/parquet/encryption/doc.go index 2959010ff..8417addf4 100644 --- a/parquet/encryption/doc.go +++ b/parquet/encryption/doc.go @@ -14,14 +14,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package encryption provides optional KMS integration for reading parquet -// files encrypted with PKMT1 key material, the format written by the +// Package encryption provides optional KMS integration for parquet +// encryption with PKMT1 key material, the format written by the // CryptoFactory of parquet-cpp and pyarrow (both single and double key // wrapping). // -// It plugs into the existing low level API as a +// Reading plugs into the existing low level API as a // parquet.DecryptionKeyRetriever: pass a KmsClient implementation to // NewKmsKeyRetriever and set the retriever with // parquet.WithKeyRetriever. Files using explicit keys or custom key // metadata continue to work without this package. +// +// Writing uses NewCryptoFactory: FileEncryptionProperties generates the +// data encryption keys and their PKMT1 key material and returns +// parquet.FileEncryptionProperties for parquet.WithEncryptionProperties, +// producing files readable by the same factory on the parquet-cpp/pyarrow +// side. Writing with explicit keys via parquet.NewFileEncryptionProperties +// continues to work without this package. package encryption diff --git a/parquet/encryption/example_test.go b/parquet/encryption/example_test.go index fe1417287..e1dcb78c3 100644 --- a/parquet/encryption/example_test.go +++ b/parquet/encryption/example_test.go @@ -70,3 +70,41 @@ func ExampleKmsKeyRetriever() { // Output: "secret-dek" } + +// ExampleCryptoFactory shows how to write files whose keys are wrapped by a +// key management service: implement KmsClient, wrap a client factory in a +// CryptoFactory, and pass the properties it builds to the writer with +// parquet.WithEncryptionProperties. The generated PKMT1 key material is read +// back by KmsKeyRetriever and by parquet-cpp/pyarrow readers. +func ExampleCryptoFactory() { + factory := encryption.NewCryptoFactory( + func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return toyKMS{}, nil + }) + + props, err := factory.FileEncryptionProperties( + encryption.KmsConnectionConfig{ + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + }, + encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + // every listed column gets its own data key wrapped by this + // master key; columns left out stay in plaintext + ColumnKeys: map[string][]string{"column_master": {"sensitive_column"}}, + // SingleWrapping: true wraps each data key directly with its + // master key instead of a per-master-key key encryption key + }) + if err != nil { + panic(err) + } + fmt.Printf("footer key is %d bytes\n", len(props.FooterKey())) + + // The properties are single use: build fresh ones for every file. + // + // writerProps := parquet.NewWriterProperties( + // parquet.WithEncryptionProperties(props)) + // w := file.NewParquetWriter(out, schema, file.WithWriterProps(writerProps)) + + // Output: footer key is 16 bytes +} diff --git a/parquet/encryption/key_material.go b/parquet/encryption/key_material.go index 74d485dfb..78257bafa 100644 --- a/parquet/encryption/key_material.go +++ b/parquet/encryption/key_material.go @@ -37,9 +37,12 @@ type KeyMaterial struct { InternalStorage bool `json:"internalStorage"` // IsFooterKey reports whether this material belongs to the footer key. // Only footer key material carries KmsInstanceID and KmsInstanceURL. - IsFooterKey bool `json:"isFooterKey"` - KmsInstanceID string `json:"kmsInstanceID"` - KmsInstanceURL string `json:"kmsInstanceURL"` + IsFooterKey bool `json:"isFooterKey"` + // KmsInstanceID and KmsInstanceURL identify the KMS instance that wrote + // the footer key material; they are omitted from column key material, + // matching parquet-cpp. + KmsInstanceID string `json:"kmsInstanceID,omitempty"` + KmsInstanceURL string `json:"kmsInstanceURL,omitempty"` // MasterKeyID identifies the master key used to wrap the key. MasterKeyID string `json:"masterKeyID"` // WrappedDEK is the base64 encoded wrapped data encryption key. @@ -49,10 +52,10 @@ type KeyMaterial struct { DoubleWrapping bool `json:"doubleWrapping"` // KeyEncryptionKeyID is the base64 encoded key encryption key ID, set // only in double wrapping mode. - KeyEncryptionKeyID string `json:"keyEncryptionKeyID"` + KeyEncryptionKeyID string `json:"keyEncryptionKeyID,omitempty"` // WrappedKEK is the base64 encoded wrapped key encryption key, set only // in double wrapping mode. - WrappedKEK string `json:"wrappedKEK"` + WrappedKEK string `json:"wrappedKEK,omitempty"` } // ParseKeyMetadata parses the key metadata JSON of a footer or column key. diff --git a/parquet/encryption_interop_pyarrow.py b/parquet/encryption_interop_pyarrow.py index 0339cc2ff..1f41a69d0 100644 --- a/parquet/encryption_interop_pyarrow.py +++ b/parquet/encryption_interop_pyarrow.py @@ -22,6 +22,7 @@ python3 encryption_interop_pyarrow.py gen python3 encryption_interop_pyarrow.py gen-kms python3 encryption_interop_pyarrow.py read + python3 encryption_interop_pyarrow.py read-kms """ import sys @@ -57,14 +58,12 @@ def write(name, fep): create_encryption_properties(footer_key, plaintext_footer=True)) -def gen_kms(outdir): - """Write PKMT1 key material files with a CryptoFactory and a test-only - in-memory KMS client whose wrapping is plain base64 (no cryptography - dependency). The Go side mirrors the client to unwrap the keys.""" +def _kms_crypto_factory(): + """A CryptoFactory over a test-only in-memory KMS client whose wrapping + is plain base64 (no cryptography dependency), shared by gen-kms and + read-kms.""" import base64 as b64 - import os - import pyarrow.parquet as pq import pyarrow.parquet.encryption as pe class Base64KmsClient(pe.KmsClient): @@ -80,11 +79,18 @@ def unwrap_key(self, wrapped_key, master_key_identifier): config = pe.KmsConnectionConfig( kms_instance_id="in-memory", kms_instance_url="local", key_access_token="token") + return pe.CryptoFactory(lambda cfg: Base64KmsClient(cfg)), config + - def client_factory(kms_connection_config): - return Base64KmsClient(kms_connection_config) +def gen_kms(outdir): + """Write PKMT1 key material files with a CryptoFactory. The Go side + mirrors the client to unwrap the keys.""" + import os - factory = pe.CryptoFactory(client_factory) + import pyarrow.parquet as pq + import pyarrow.parquet.encryption as pe + + factory, config = _kms_crypto_factory() table = build_table() for name, double_wrapping in (("kms_double", True), ("kms_single", False)): @@ -97,6 +103,18 @@ def client_factory(kms_connection_config): print(name) +def read_kms(path): + """Read a file whose keys were wrapped by the test KMS client, resolving + them through the CryptoFactory.""" + import pyarrow.parquet as pq + + factory, config = _kms_crypto_factory() + table = pq.read_table(path, + decryption_properties=factory.file_decryption_properties(config)) + for name, col in zip(table.column_names, table.columns): + print(name + "\t" + " ".join(str(v) for v in col.to_pylist())) + + def read(path, footer_key): import pyarrow.parquet as pq from pyarrow.parquet.encryption import create_decryption_properties @@ -126,6 +144,9 @@ def main(): if mode == "gen-kms": gen_kms(sys.argv[2]) return 0 + if mode == "read-kms": + read_kms(sys.argv[2]) + return 0 if mode == "read": read(sys.argv[2], sys.argv[3].encode()) return 0 diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index ff222b635..93dc4c962 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -325,3 +325,95 @@ func writeInteropEncryptedFile(t *testing.T, path string) { require.NoError(t, w.Close()) } + +// writeGoKmsFile writes an encrypted file whose keys come from the Go +// CryptoFactory, mirroring the pyarrow gen-kms layout: a float64 and a +// float32 column under a column master key, plus the footer key. +func writeGoKmsFile(t *testing.T, path string, cfg encryption.EncryptionConfiguration) { + t.Helper() + + kms := &interopBase64KMS{} + factory := encryption.NewCryptoFactory( + func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) + props, err := factory.FileEncryptionProperties( + encryption.KmsConnectionConfig{KmsInstanceID: "in-memory", KmsInstanceURL: "local"}, cfg) + require.NoError(t, err) + + fields := schema.FieldList{ + schema.Must(schema.NewPrimitiveNode("a", parquet.Repetitions.Required, + parquet.Types.Double, 0, -1)), + schema.Must(schema.NewPrimitiveNode("b", parquet.Repetitions.Required, + parquet.Types.Float, 0, -1)), + } + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Repeated, fields, -1) + require.NoError(t, err) + + out, err := os.Create(path) + require.NoError(t, err) + + writerProps := parquet.NewWriterProperties(parquet.WithEncryptionProperties(props)) + w := file.NewParquetWriter(out, root, file.WithWriterProps(writerProps)) + + rgr, err := w.AppendBufferedRowGroupChecked() + require.NoError(t, err) + + aw, err := rgr.Column(0) + require.NoError(t, err) + doubleWriter := aw.(*file.Float64ColumnChunkWriter) + _, err = doubleWriter.WriteBatch([]float64{1, 2, 3}, nil, nil) + require.NoError(t, err) + + bw, err := rgr.Column(1) + require.NoError(t, err) + floatWriter := bw.(*file.Float32ColumnChunkWriter) + _, err = floatWriter.WriteBatch([]float32{4, 5, 6}, nil, nil) + require.NoError(t, err) + + require.NoError(t, w.Close()) +} + +// TestGoKmsWrittenReadByPyarrow writes files with the Go CryptoFactory and +// reads them back with pyarrow's, resolving every key through the test KMS. +func TestGoKmsWrittenReadByPyarrow(t *testing.T) { + requirePyarrow(t) + + configs := []struct { + name string + cfg encryption.EncryptionConfiguration + }{ + {"double", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + }}, + {"single", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + SingleWrapping: true, + }}, + {"plaintext_footer", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + PlaintextFooter: true, + }}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "go_kms.parquet") + writeGoKmsFile(t, path, config.cfg) + + out := interopPython(t, "read-kms", path) + got := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + parts := strings.SplitN(line, " ", 2) + if len(parts) == 2 { + got[parts[0]] = parts[1] + } + } + assert.Equal(t, "1.0 2.0 3.0", got["a"]) + assert.Equal(t, "4.0 5.0 6.0", got["b"]) + }) + } +} From 8012ac10204f0f66bb2066c7514d9430700070b7 Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Tue, 17 Nov 2026 14:58:55 +0800 Subject: [PATCH 2/4] fix(parquet/encryption): address review on KEK cache and config - key the KEK write cache by (access token, master key) like parquet-cpp's per-token two-level cache, and expire entries after a configurable lifetime (default 10m, cache_lifetime_seconds) - lock per cache entry instead of across the whole factory, so KMS wraps under different master keys run concurrently - add DataKeyLengthBits (128/192/256, default 128) and Algorithm (AesGcm/AesCtr) to EncryptionConfiguration - require an explicit UniformEncryption flag; an empty ColumnKeys map is now a configuration error, as in parquet-cpp --- parquet/encryption/crypto_factory.go | 146 ++++++++++++----- parquet/encryption/crypto_factory_test.go | 191 +++++++++++++++++++++- 2 files changed, 296 insertions(+), 41 deletions(-) diff --git a/parquet/encryption/crypto_factory.go b/parquet/encryption/crypto_factory.go index ac2366caf..e6484bd25 100644 --- a/parquet/encryption/crypto_factory.go +++ b/parquet/encryption/crypto_factory.go @@ -25,18 +25,26 @@ import ( "encoding/json" "fmt" "sync" + "time" "github.com/apache/arrow-go/v18/parquet" ) // Key sizes for generated keys, matching the parquet-cpp/pyarrow -// CryptoFactory (AES-128 data and key encryption keys). +// CryptoFactory. Data key length is configurable via +// EncryptionConfiguration.DataKeyLengthBits; the KEK is always AES-128, +// as in parquet-cpp (kKekLength = 16). const ( - dataKeySize = 16 kekSize = 16 kekIDSize = 16 + defaultBits = 128 ) +// DefaultKekCacheLifetime is how long a cached key encryption key is +// reused before it is wrapped again, matching parquet-cpp's +// cache_lifetime_seconds default of 600. +const DefaultKekCacheLifetime = 10 * time.Minute + // KmsClientFactory creates KmsClients for a connection configuration, // mirroring parquet-cpp's KmsClientFactory. type KmsClientFactory func(KmsConnectionConfig) (KmsClient, error) @@ -57,13 +65,25 @@ type EncryptionConfiguration struct { // uniform encryption, the single key encrypting every column). Required. FooterKeyID string + // UniformEncryption encrypts every column with the footer key. Either + // it or ColumnKeys must be set, never both — an empty ColumnKeys map + // is a configuration error, not an implicit uniform layout (the + // parquet-cpp rule). + UniformEncryption bool + // ColumnKeys maps a master key id to the column paths it encrypts; every // listed column gets its own data encryption key wrapped by that master - // key. Columns not listed stay in plaintext. When empty (or nil) the - // configuration is uniform: all columns are encrypted with the footer - // key. + // key. Columns not listed stay in plaintext. ColumnKeys map[string][]string + // DataKeyLengthBits is the data encryption key length in bits: 128, + // 192 or 256. Zero means 128, the parquet-cpp default. + DataKeyLengthBits int + + // Algorithm picks the Parquet cipher: parquet.AesGcm (the zero value + // and the parquet-cpp default) or parquet.AesGcmCtr. + Algorithm parquet.Cipher + // SingleWrapping wraps each data encryption key directly with its master // key. The default (false) is double wrapping (the parquet-cpp/pyarrow // default), where a per-master-key key encryption key wraps the data @@ -88,39 +108,59 @@ type EncryptionConfiguration struct { // that KmsKeyRetriever (and parquet-cpp/pyarrow readers) resolve. // // In double wrapping mode the factory keeps one key encryption key per -// (KMS instance, master key) and reuses it across files, so writing many -// files costs one KEK wrap per master key instead of one per key slot, -// matching the parquet-cpp/pyarrow CryptoFactory. The cache key does not -// include the access token: configs sharing a KMS instance id also share -// KEKs, which is safe when the KMS binds wraps to the master key. +// (key access token, master key) and reuses it across files, matching +// parquet-cpp's per-token KEK write cache: writing many files costs one +// KEK wrap per master key per token. Entries expire after the cache +// lifetime (DefaultKekCacheLifetime, tunable with WithKekCacheLifetime) +// and are then wrapped again. Generation is serialized per cache entry, +// so files under different master keys wrap concurrently; only writers +// sharing a KEK wait for one another. type CryptoFactory struct { clientFactory KmsClientFactory + kekLifetime time.Duration - mu sync.Mutex - keks map[kekReuseKey]cachedKek + mu sync.Mutex // guards the keks map only + keks map[kekReuseKey]*cachedKek } -// kekReuseKey identifies the KEK cached for a master key under one KMS -// instance. +// kekReuseKey identifies the KEK cached for a master key under one key +// access token, as in parquet-cpp's two-level write cache. type kekReuseKey struct { - kmsInstanceID string - masterKeyID string + accessToken string + masterKeyID string } // cachedKek is a generated key encryption key together with its KMS-wrapped -// form, ready to be stamped into key material. +// form, ready to be stamped into key material. Its mutex serializes +// (re)generation so the KMS call is made once per entry. type cachedKek struct { + mu sync.Mutex + createdAt time.Time + id []byte // raw KEK id; also the AAD for local data key wrapping kek []byte wrapped string // KmsClient-wrapped KEK } +// A CryptoFactoryOption configures a CryptoFactory. +type CryptoFactoryOption func(*CryptoFactory) + +// WithKekCacheLifetime sets how long cached key encryption keys are +// reused before being wrapped again. +func WithKekCacheLifetime(d time.Duration) CryptoFactoryOption { + return func(f *CryptoFactory) { f.kekLifetime = d } +} + // NewCryptoFactory returns a factory that creates KmsClients with // clientFactory. A single factory can encrypt many files; call // FileEncryptionProperties once per file, since the returned properties are // single use (the parquet writer wipes their keys). -func NewCryptoFactory(clientFactory KmsClientFactory) *CryptoFactory { - return &CryptoFactory{clientFactory: clientFactory} +func NewCryptoFactory(clientFactory KmsClientFactory, opts ...CryptoFactoryOption) *CryptoFactory { + f := &CryptoFactory{clientFactory: clientFactory} + for _, opt := range opts { + opt(f) + } + return f } // FileEncryptionProperties generates fresh data encryption keys for the @@ -132,6 +172,20 @@ func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, if cfg.FooterKeyID == "" { return nil, fmt.Errorf("parquet: encryption configuration needs a footer key id") } + if cfg.UniformEncryption && len(cfg.ColumnKeys) > 0 { + return nil, fmt.Errorf("parquet: cannot set both uniform encryption and column keys") + } + if !cfg.UniformEncryption && len(cfg.ColumnKeys) == 0 { + return nil, fmt.Errorf("parquet: either column keys or uniform encryption must be set") + } + bits := cfg.DataKeyLengthBits + if bits == 0 { + bits = defaultBits + } + if bits != 128 && bits != 192 && bits != 256 { + return nil, fmt.Errorf("parquet: data key length must be 128, 192 or 256 bits, got %d", bits) + } + keyBytes := bits / 8 if f.clientFactory == nil { return nil, fmt.Errorf("parquet: CryptoFactory needs a client factory, construct it with NewCryptoFactory") } @@ -143,7 +197,7 @@ func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, return nil, fmt.Errorf("parquet: create kms client: %w", err) } - footerDEK, err := randomKey(dataKeySize) + footerDEK, err := randomKey(keyBytes) if err != nil { return nil, err } @@ -161,6 +215,7 @@ func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, opts := []parquet.EncryptOption{ parquet.WithFooterKeyMetadata(string(footerJSON)), + parquet.WithAlg(cfg.Algorithm), } cols := parquet.ColumnPathToEncryptionPropsMap{} @@ -172,7 +227,7 @@ func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, if path == "" { return nil, fmt.Errorf("parquet: column keys for master key %q contain an empty column path", masterKeyID) } - dek, err := randomKey(dataKeySize) + dek, err := randomKey(keyBytes) if err != nil { return nil, err } @@ -252,35 +307,50 @@ func (f *CryptoFactory) wrapDataKey(client KmsClient, kmsCfg KmsConnectionConfig return material, nil } -// kekFor returns the cached key encryption key for masterKeyID, generating -// and wrapping a fresh one on first use. The factory lock is held across the -// KMS call so concurrent files sharing a master key get one wrap. +// kekFor returns the cached key encryption key for masterKeyID under the +// config's key access token, generating and wrapping a fresh one on first +// use or once the cache lifetime has passed. Each cache entry has its own +// lock, so only writers sharing a KEK wait for its KMS wrap; the map lock +// is never held across a KMS call. func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, - masterKeyID string) (cachedKek, error) { - key := kekReuseKey{kmsInstanceID: kmsCfg.KmsInstanceID, masterKeyID: masterKeyID} + masterKeyID string) (*cachedKek, error) { + key := kekReuseKey{accessToken: kmsCfg.KeyAccessToken, masterKeyID: masterKeyID} + f.mu.Lock() - defer f.mu.Unlock() - if kek, ok := f.keks[key]; ok { - return kek, nil + entry, ok := f.keks[key] + if !ok { + entry = &cachedKek{} + if f.keks == nil { + f.keks = make(map[kekReuseKey]*cachedKek) + } + f.keks[key] = entry + } + f.mu.Unlock() + + entry.mu.Lock() + defer entry.mu.Unlock() + lifetime := f.kekLifetime + if lifetime <= 0 { + lifetime = DefaultKekCacheLifetime + } + if !entry.createdAt.IsZero() && time.Since(entry.createdAt) <= lifetime { + return entry, nil } id, err := randomKey(kekIDSize) if err != nil { - return cachedKek{}, err + return nil, err } kekKey, err := randomKey(kekSize) if err != nil { - return cachedKek{}, err + return nil, err } wrapped, err := client.WrapKey(kekKey, masterKeyID) if err != nil { - return cachedKek{}, fmt.Errorf("parquet: wrap key encryption key with master key %q: %w", masterKeyID, err) - } - kek := cachedKek{id: id, kek: kekKey, wrapped: wrapped} - if f.keks == nil { - f.keks = make(map[kekReuseKey]cachedKek) + return nil, fmt.Errorf("parquet: wrap key encryption key with master key %q: %w", masterKeyID, err) } - f.keks[key] = kek - return kek, nil + entry.id, entry.kek, entry.wrapped = id, kekKey, wrapped + entry.createdAt = time.Now() + return entry, nil } // wrapKeyLocally is parquet-cpp's EncryptKeyLocally: AES-GCM with a diff --git a/parquet/encryption/crypto_factory_test.go b/parquet/encryption/crypto_factory_test.go index 373f6af7a..478926d41 100644 --- a/parquet/encryption/crypto_factory_test.go +++ b/parquet/encryption/crypto_factory_test.go @@ -19,10 +19,13 @@ package encryption_test import ( "encoding/base64" + "fmt" "os" "path/filepath" + "sync" "sync/atomic" "testing" + "time" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/apache/arrow-go/v18/parquet" @@ -72,8 +75,9 @@ func TestFileEncryptionPropertiesUniformSingleWrapping(t *testing.T) { props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ - FooterKeyID: "footer_master", - SingleWrapping: true, + FooterKeyID: "footer_master", + UniformEncryption: true, + SingleWrapping: true, }) require.NoError(t, err) @@ -108,7 +112,7 @@ func TestFileEncryptionPropertiesUniformSingleWrapping(t *testing.T) { func TestFileEncryptionPropertiesDoubleWrapping(t *testing.T) { kms := &countingBase64KMS{} factory := countingFactory(t, kms) - cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master"} + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) require.NoError(t, err) @@ -234,6 +238,26 @@ func TestFileEncryptionPropertiesErrors(t *testing.T) { encryption.EncryptionConfiguration{FooterKeyID: ""}) assert.ErrorContains(t, err, "footer key id") + // an empty ColumnKeys map is a configuration error, not an implicit + // uniform layout (the parquet-cpp rule) + _, err = factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{FooterKeyID: "footer_master"}) + assert.ErrorContains(t, err, "either column keys or uniform encryption") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + ColumnKeys: map[string][]string{"column_master": {"a"}}, + }) + assert.ErrorContains(t, err, "cannot set both") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + DataKeyLengthBits: 100, + }) + assert.ErrorContains(t, err, "128, 192 or 256 bits") + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ FooterKeyID: "footer_master", ColumnKeys: map[string][]string{"": {"a"}}, @@ -260,6 +284,7 @@ func TestFileEncryptionPropertiesErrors(t *testing.T) { // parquet-cpp rejects the combination too _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ FooterKeyID: "footer_master", + UniformEncryption: true, DisableAadPrefixStorage: true, }) assert.ErrorContains(t, err, "DisableAadPrefixStorage") @@ -368,3 +393,163 @@ func TestCryptoFactoryWriteReadRoundTrip(t *testing.T) { }) } } + +// slowKMS counts wraps like countingBase64KMS but takes delay per call, to +// observe which KMS wraps the factory actually serializes. +type slowKMS struct { + wraps atomic.Int32 + delay time.Duration +} + +func (k *slowKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + k.wraps.Add(1) + time.Sleep(k.delay) + return base64.StdEncoding.EncodeToString(key), nil +} + +func (k *slowKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// The KEK write cache is per (key access token, master key), as in +// parquet-cpp's two-level cache: two tokens over one master key get distinct +// KEKs, and revisiting a token costs nothing new. +func TestKekCacheIsPerAccessToken(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + tokenA := testKmsConfig + tokenA.KeyAccessToken = "token-a" + tokenB := testKmsConfig + tokenB.KeyAccessToken = "token-b" + + _, err := factory.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load()) + + _, err = factory.FileEncryptionProperties(tokenB, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "second access token must get its own KEK") + + _, err = factory.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "revisiting a token reuses its cached KEK") +} + +// Cached KEKs expire after the cache lifetime and are wrapped again. +func TestKekCacheEntryLifetime(t *testing.T) { + kms := &countingBase64KMS{} + factory := encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }, encryption.WithKekCacheLifetime(50*time.Millisecond)) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + first, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load()) + + time.Sleep(80 * time.Millisecond) + + second, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "expired KEK must be wrapped again") + assert.NotEqual(t, first.FooterKeyMetadata(), second.FooterKeyMetadata(), + "the second file carries a fresh KEK") +} + +// KEK generation locks per cache entry: files under different master keys +// wrap concurrently, while files sharing a master key wait for one wrap. +func TestKekWrappingLocksPerMasterKey(t *testing.T) { + const delay = 100 * time.Millisecond + newFactory := func(kms *slowKMS) *encryption.CryptoFactory { + return encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) + } + + t.Run("same master key serializes to one wrap", func(t *testing.T) { + kms := &slowKMS{delay: delay} + factory := newFactory(kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + var wg sync.WaitGroup + metas := make([]string, 4) + for i := range metas { + wg.Add(1) + go func(i int) { + defer wg.Done() + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + if err == nil { + metas[i] = props.FooterKeyMetadata() + } + }(i) + } + wg.Wait() + + assert.Equal(t, int32(1), kms.wraps.Load(), "concurrent files sharing a master key get one KEK wrap") + firstMaterial, err := encryption.ParseKeyMetadata([]byte(metas[0])) + require.NoError(t, err) + for _, meta := range metas[1:] { + material, err := encryption.ParseKeyMetadata([]byte(meta)) + require.NoError(t, err) + assert.Equal(t, firstMaterial.KeyEncryptionKeyID, material.KeyEncryptionKeyID, + "all files carry the same KEK") + } + }) + + t.Run("different master keys wrap concurrently", func(t *testing.T) { + kms := &slowKMS{delay: delay} + factory := newFactory(kms) + + var wg sync.WaitGroup + start := time.Now() + for i := 0; i < 4; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: fmt.Sprintf("footer_master_%d", i), + UniformEncryption: true, + }) + assert.NoError(t, err) + }(i) + } + wg.Wait() + + assert.Equal(t, int32(4), kms.wraps.Load()) + elapsed := time.Since(start) + assert.Less(t, elapsed, 2*delay, + "wraps under different master keys must not serialize (took %v)", elapsed) + }) +} + +// DataKeyLengthBits and Algorithm reach the generated properties: keys get +// the configured length, the cipher is passed through, and the file still +// reads back through the key retriever. +func TestDataKeyLengthAndAlgorithm(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field"}}, + DataKeyLengthBits: 256, + Algorithm: parquet.AesCtr, + }) + require.NoError(t, err) + assert.Len(t, props.FooterKey(), 32) + + assert.Equal(t, parquet.AesCtr, props.Algorithm().Algo) + + path := filepath.Join(t.TempDir(), "kms_ctr256.parquet") + writeKmsEncryptedFile(t, path, props) + + readProps := parquet.NewReaderProperties(memory.DefaultAllocator) + readProps.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) + fr, err := file.OpenParquetFile(path, false, file.WithReadProps(readProps)) + require.NoError(t, err) + defer fr.Close() + assert.Equal(t, int64(3), fr.MetaData().NumRows) +} From 45fef3aea255973dc246b725db72600f5072b125 Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Tue, 17 Nov 2026 15:11:50 +0800 Subject: [PATCH 3/4] fix(parquet/encryption): keep KMS instance in the KEK cache key The instance stays in the cache key alongside the access token: the same master key id can resolve to different keys on different KMS deployments, so wraps must not be shared across them. Also validate the cipher value, relax the per-entry lock timing bound for CI noise, and read a CTR + 256-bit file back through pyarrow in the interop test. --- parquet/encryption/crypto_factory.go | 36 ++++++++++++++-------- parquet/encryption/crypto_factory_test.go | 10 +++++- parquet/encryption_interop_pyarrow_test.go | 6 ++++ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/parquet/encryption/crypto_factory.go b/parquet/encryption/crypto_factory.go index e6484bd25..a4627b626 100644 --- a/parquet/encryption/crypto_factory.go +++ b/parquet/encryption/crypto_factory.go @@ -108,13 +108,15 @@ type EncryptionConfiguration struct { // that KmsKeyRetriever (and parquet-cpp/pyarrow readers) resolve. // // In double wrapping mode the factory keeps one key encryption key per -// (key access token, master key) and reuses it across files, matching -// parquet-cpp's per-token KEK write cache: writing many files costs one -// KEK wrap per master key per token. Entries expire after the cache -// lifetime (DefaultKekCacheLifetime, tunable with WithKekCacheLifetime) -// and are then wrapped again. Generation is serialized per cache entry, -// so files under different master keys wrap concurrently; only writers -// sharing a KEK wait for one another. +// (KMS instance, key access token, master key) and reuses it across +// files, so writing many files costs one KEK wrap per master key per +// token — parquet-cpp's per-token KEK write cache, additionally scoped +// to the KMS instance so wraps are never shared across deployments. +// Entries expire after the cache lifetime (DefaultKekCacheLifetime, +// tunable with WithKekCacheLifetime) and are then wrapped again. +// Generation is serialized per cache entry, so files under different +// master keys wrap concurrently; only writers sharing a KEK wait for +// one another. type CryptoFactory struct { clientFactory KmsClientFactory kekLifetime time.Duration @@ -123,11 +125,14 @@ type CryptoFactory struct { keks map[kekReuseKey]*cachedKek } -// kekReuseKey identifies the KEK cached for a master key under one key -// access token, as in parquet-cpp's two-level write cache. +// kekReuseKey identifies the KEK cached for a master key under one KMS +// instance and key access token: the token separates tenants the KMS +// authenticates, the instance keeps wraps from being shared across KMS +// deployments that may bind the same master key id to different keys. type kekReuseKey struct { - accessToken string - masterKeyID string + kmsInstanceID string + accessToken string + masterKeyID string } // cachedKek is a generated key encryption key together with its KMS-wrapped @@ -185,6 +190,9 @@ func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, if bits != 128 && bits != 192 && bits != 256 { return nil, fmt.Errorf("parquet: data key length must be 128, 192 or 256 bits, got %d", bits) } + if cfg.Algorithm != parquet.AesGcm && cfg.Algorithm != parquet.AesCtr { + return nil, fmt.Errorf("parquet: encryption algorithm must be parquet.AesGcm or parquet.AesCtr, got %d", cfg.Algorithm) + } keyBytes := bits / 8 if f.clientFactory == nil { return nil, fmt.Errorf("parquet: CryptoFactory needs a client factory, construct it with NewCryptoFactory") @@ -314,7 +322,11 @@ func (f *CryptoFactory) wrapDataKey(client KmsClient, kmsCfg KmsConnectionConfig // is never held across a KMS call. func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, masterKeyID string) (*cachedKek, error) { - key := kekReuseKey{accessToken: kmsCfg.KeyAccessToken, masterKeyID: masterKeyID} + key := kekReuseKey{ + kmsInstanceID: kmsCfg.KmsInstanceID, + accessToken: kmsCfg.KeyAccessToken, + masterKeyID: masterKeyID, + } f.mu.Lock() entry, ok := f.keks[key] diff --git a/parquet/encryption/crypto_factory_test.go b/parquet/encryption/crypto_factory_test.go index 478926d41..0e2086ece 100644 --- a/parquet/encryption/crypto_factory_test.go +++ b/parquet/encryption/crypto_factory_test.go @@ -435,6 +435,14 @@ func TestKekCacheIsPerAccessToken(t *testing.T) { _, err = factory.FileEncryptionProperties(tokenA, cfg) require.NoError(t, err) assert.Equal(t, int32(2), kms.wraps.Load(), "revisiting a token reuses its cached KEK") + + // a different KMS instance never shares wraps, even with the same + // token: the same master key id may resolve to a different key there + otherInstance := tokenA + otherInstance.KmsInstanceID = "other-instance" + _, err = factory.FileEncryptionProperties(otherInstance, cfg) + require.NoError(t, err) + assert.Equal(t, int32(3), kms.wraps.Load(), "another KMS instance gets its own KEK") } // Cached KEKs expire after the cache lifetime and are wrapped again. @@ -519,7 +527,7 @@ func TestKekWrappingLocksPerMasterKey(t *testing.T) { assert.Equal(t, int32(4), kms.wraps.Load()) elapsed := time.Since(start) - assert.Less(t, elapsed, 2*delay, + assert.Less(t, elapsed, 3*delay, "wraps under different master keys must not serialize (took %v)", elapsed) }) } diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index 93dc4c962..30327e128 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -397,6 +397,12 @@ func TestGoKmsWrittenReadByPyarrow(t *testing.T) { ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, PlaintextFooter: true, }}, + {"ctr_256", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + DataKeyLengthBits: 256, + Algorithm: parquet.AesCtr, + }}, } for _, config := range configs { From fa5c175508a65757582bf100981cf5a1a29b788c Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Wed, 18 Nov 2026 15:39:24 +0800 Subject: [PATCH 4/4] fix(parquet/encryption): hand out immutable KEKs and evict expired cache entries kekFor returned the shared *cachedKek after releasing its lock, and wrapDataKey then read kek/id/wrapped unlocked while a concurrent refresh rewrote those fields in place. Under -race that is reported between the refresh write and the reader; worse, a torn entry can wrap a data key with one KEK while the key material records another KEK id or wrapped value, producing files that cannot be decrypted. Refreshes now publish a brand-new immutable cachedKek through a per-key slot, so entries handed to writers never change underneath them, and a concurrent-refresh test covers the window with a 1ns lifetime. Expired entries are also dropped from the map when a new entry joins it (parquet-cpp CheckCacheForExpiredTokens), so rotating access tokens no longer pin every stale plaintext KEK in memory. Also document that a zero or negative WithKekCacheLifetime falls back to the default rather than disabling caching, and fix the Algorithm doc to name parquet.AesCtr. --- parquet/encryption/crypto_factory.go | 97 ++++++++++++++----- .../crypto_factory_internal_test.go | 65 +++++++++++++ parquet/encryption/crypto_factory_test.go | 46 +++++++++ 3 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 parquet/encryption/crypto_factory_internal_test.go diff --git a/parquet/encryption/crypto_factory.go b/parquet/encryption/crypto_factory.go index a4627b626..5c80d4fa8 100644 --- a/parquet/encryption/crypto_factory.go +++ b/parquet/encryption/crypto_factory.go @@ -81,7 +81,7 @@ type EncryptionConfiguration struct { DataKeyLengthBits int // Algorithm picks the Parquet cipher: parquet.AesGcm (the zero value - // and the parquet-cpp default) or parquet.AesGcmCtr. + // and the parquet-cpp default) or parquet.AesCtr. Algorithm parquet.Cipher // SingleWrapping wraps each data encryption key directly with its master @@ -113,7 +113,10 @@ type EncryptionConfiguration struct { // token — parquet-cpp's per-token KEK write cache, additionally scoped // to the KMS instance so wraps are never shared across deployments. // Entries expire after the cache lifetime (DefaultKekCacheLifetime, -// tunable with WithKekCacheLifetime) and are then wrapped again. +// tunable with WithKekCacheLifetime) and are then wrapped again; a new +// entry joining the map sweeps out expired ones (parquet-cpp's +// CheckCacheForExpiredTokens), so rotating access tokens don't keep +// stale plaintext KEKs in memory. // Generation is serialized per cache entry, so files under different // master keys wrap concurrently; only writers sharing a KEK wait for // one another. @@ -122,7 +125,7 @@ type CryptoFactory struct { kekLifetime time.Duration mu sync.Mutex // guards the keks map only - keks map[kekReuseKey]*cachedKek + keks map[kekReuseKey]*kekSlot } // kekReuseKey identifies the KEK cached for a master key under one KMS @@ -136,10 +139,11 @@ type kekReuseKey struct { } // cachedKek is a generated key encryption key together with its KMS-wrapped -// form, ready to be stamped into key material. Its mutex serializes -// (re)generation so the KMS call is made once per entry. +// form, ready to be stamped into key material. Entries are immutable once +// published: a refresh installs a brand-new value instead of mutating the +// old one, so a KEK handed to a writer can never change — let alone tear — +// underneath the unlocked reads in wrapDataKey. type cachedKek struct { - mu sync.Mutex createdAt time.Time id []byte // raw KEK id; also the AAD for local data key wrapping @@ -147,11 +151,21 @@ type cachedKek struct { wrapped string // KmsClient-wrapped KEK } +// kekSlot owns the KEK cached for one kekReuseKey. Its mutex serializes +// (re)generation so the KMS call is made once per refresh and guards the +// cur pointer; the cachedKek it holds is immutable once published. +type kekSlot struct { + mu sync.Mutex + cur *cachedKek +} + // A CryptoFactoryOption configures a CryptoFactory. type CryptoFactoryOption func(*CryptoFactory) // WithKekCacheLifetime sets how long cached key encryption keys are -// reused before being wrapped again. +// reused before being wrapped again. A zero or negative duration falls +// back to DefaultKekCacheLifetime — caching cannot be disabled by +// passing 0. func WithKekCacheLifetime(d time.Duration) CryptoFactoryOption { return func(f *CryptoFactory) { f.kekLifetime = d } } @@ -317,9 +331,11 @@ func (f *CryptoFactory) wrapDataKey(client KmsClient, kmsCfg KmsConnectionConfig // kekFor returns the cached key encryption key for masterKeyID under the // config's key access token, generating and wrapping a fresh one on first -// use or once the cache lifetime has passed. Each cache entry has its own -// lock, so only writers sharing a KEK wait for its KMS wrap; the map lock -// is never held across a KMS call. +// use or once the cache lifetime has passed. The returned entry is +// immutable: refreshes replace it wholesale, so the unlocked reads in +// wrapDataKey can never race a refresh. Each cache key has its own lock, +// so only writers sharing a KEK wait for its KMS wrap; the map lock is +// never held across a KMS call. func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, masterKeyID string) (*cachedKek, error) { key := kekReuseKey{ @@ -329,24 +345,21 @@ func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, } f.mu.Lock() - entry, ok := f.keks[key] + slot, ok := f.keks[key] if !ok { - entry = &cachedKek{} + slot = &kekSlot{} if f.keks == nil { - f.keks = make(map[kekReuseKey]*cachedKek) + f.keks = make(map[kekReuseKey]*kekSlot) } - f.keks[key] = entry + f.keks[key] = slot + f.removeExpiredLocked(key) } f.mu.Unlock() - entry.mu.Lock() - defer entry.mu.Unlock() - lifetime := f.kekLifetime - if lifetime <= 0 { - lifetime = DefaultKekCacheLifetime - } - if !entry.createdAt.IsZero() && time.Since(entry.createdAt) <= lifetime { - return entry, nil + slot.mu.Lock() + defer slot.mu.Unlock() + if slot.cur != nil && time.Since(slot.cur.createdAt) <= f.kekLifetimeOrDefault() { + return slot.cur, nil } id, err := randomKey(kekIDSize) if err != nil { @@ -360,9 +373,43 @@ func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, if err != nil { return nil, fmt.Errorf("parquet: wrap key encryption key with master key %q: %w", masterKeyID, err) } - entry.id, entry.kek, entry.wrapped = id, kekKey, wrapped - entry.createdAt = time.Now() - return entry, nil + fresh := &cachedKek{ + createdAt: time.Now(), + id: id, + kek: kekKey, + wrapped: wrapped, + } + slot.cur = fresh + return fresh, nil +} + +// removeExpiredLocked deletes cache entries whose lifetime has passed, so +// rotating access tokens don't keep every stale plaintext KEK in memory — +// parquet-cpp's CheckCacheForExpiredTokens, triggered when a new entry +// joins the cache. The caller must hold f.mu; keep is the entry the caller +// is inserting. Slots busy with a regeneration are left alone (their next +// value is fresh by construction). +func (f *CryptoFactory) removeExpiredLocked(keep kekReuseKey) { + lifetime := f.kekLifetimeOrDefault() + now := time.Now() + for k, slot := range f.keks { + if k == keep || !slot.mu.TryLock() { + continue + } + if slot.cur == nil || now.Sub(slot.cur.createdAt) > lifetime { + delete(f.keks, k) + } + slot.mu.Unlock() + } +} + +// kekLifetimeOrDefault is the configured cache lifetime, falling back to +// DefaultKekCacheLifetime for zero or negative values. +func (f *CryptoFactory) kekLifetimeOrDefault() time.Duration { + if f.kekLifetime <= 0 { + return DefaultKekCacheLifetime + } + return f.kekLifetime } // wrapKeyLocally is parquet-cpp's EncryptKeyLocally: AES-GCM with a diff --git a/parquet/encryption/crypto_factory_internal_test.go b/parquet/encryption/crypto_factory_internal_test.go new file mode 100644 index 000000000..b07b5b1ff --- /dev/null +++ b/parquet/encryption/crypto_factory_internal_test.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// evictKMS is the smallest working KmsClient: symmetric base64 wrapping. +type evictKMS struct{} + +func (evictKMS) WrapKey(key []byte, _ string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (evictKMS) UnwrapKey(wrappedKey string, _ string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// Expired cache entries must leave the map when a new entry joins it +// (parquet-cpp's CheckCacheForExpiredTokens): with rotating access tokens +// the cache must not keep every stale plaintext KEK alive forever. +func TestKekCacheEvictsExpiredEntries(t *testing.T) { + f := NewCryptoFactory(func(KmsConnectionConfig) (KmsClient, error) { + return evictKMS{}, nil + }, WithKekCacheLifetime(time.Nanosecond)) + cfg := EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + tokenA := KmsConnectionConfig{KmsInstanceID: "in-memory", KeyAccessToken: "token-a"} + _, err := f.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + + // the entry expires; a different token adds a new cache entry, which + // is the sweep trigger + time.Sleep(2 * time.Millisecond) + tokenB := KmsConnectionConfig{KmsInstanceID: "in-memory", KeyAccessToken: "token-b"} + _, err = f.FileEncryptionProperties(tokenB, cfg) + require.NoError(t, err) + + f.mu.Lock() + defer f.mu.Unlock() + assert.Len(t, f.keks, 1, "the expired token-a entry must be swept") + _, ok := f.keks[kekReuseKey{kmsInstanceID: "in-memory", accessToken: "token-b", masterKeyID: "footer_master"}] + assert.True(t, ok, "the fresh token-b entry must survive the sweep") +} diff --git a/parquet/encryption/crypto_factory_test.go b/parquet/encryption/crypto_factory_test.go index 0e2086ece..0f8b01925 100644 --- a/parquet/encryption/crypto_factory_test.go +++ b/parquet/encryption/crypto_factory_test.go @@ -466,6 +466,52 @@ func TestKekCacheEntryLifetime(t *testing.T) { "the second file carries a fresh KEK") } +// Refreshing an expired KEK races with writers that are still reading the +// previous one if the cache hands out entries it later mutates in place. +// With a 1ns lifetime every access refreshes, so concurrent writers force +// the window open; each produced key material must stay self-consistent +// (the retriever recovers the footer key from the metadata alone), and the +// run must be clean under -race. +func TestKekRefreshFromConcurrentWriters(t *testing.T) { + kms := &countingBase64KMS{} + factory := encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }, encryption.WithKekCacheLifetime(time.Nanosecond)) + retriever := encryption.NewKmsKeyRetriever(kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + const workers = 8 + errs := make(chan error, workers) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 25; i++ { + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + if err != nil { + errs <- err + return + } + key, err := retriever.UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + if err != nil { + errs <- fmt.Errorf("key material is not self-consistent: %w", err) + return + } + if string(key) != props.FooterKey() { + errs <- fmt.Errorf("recovered footer key does not match the encrypted one") + return + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + // KEK generation locks per cache entry: files under different master keys // wrap concurrently, while files sharing a master key wait for one wrap. func TestKekWrappingLocksPerMasterKey(t *testing.T) {