diff --git a/parquet/encryption/crypto_factory.go b/parquet/encryption/crypto_factory.go new file mode 100644 index 000000000..5c80d4fa8 --- /dev/null +++ b/parquet/encryption/crypto_factory.go @@ -0,0 +1,432 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/apache/arrow-go/v18/parquet" +) + +// Key sizes for generated keys, matching the parquet-cpp/pyarrow +// CryptoFactory. Data key length is configurable via +// EncryptionConfiguration.DataKeyLengthBits; the KEK is always AES-128, +// as in parquet-cpp (kKekLength = 16). +const ( + kekSize = 16 + kekIDSize = 16 + defaultBits = 128 +) + +// DefaultKekCacheLifetime is how long a cached key encryption key is +// reused before it is wrapped again, matching parquet-cpp's +// cache_lifetime_seconds default of 600. +const DefaultKekCacheLifetime = 10 * time.Minute + +// KmsClientFactory creates KmsClients for a connection configuration, +// mirroring parquet-cpp's KmsClientFactory. +type KmsClientFactory func(KmsConnectionConfig) (KmsClient, error) + +// KmsConnectionConfig identifies the KMS instance and access credentials +// used for a file's key material. It is passed to the client factory and +// serialized into the footer key material, mirroring parquet-cpp/pyarrow. +type KmsConnectionConfig struct { + KmsInstanceID string + KmsInstanceURL string + KeyAccessToken string +} + +// EncryptionConfiguration describes which master keys encrypt what in a +// file, mirroring parquet-cpp/pyarrow's EncryptionConfiguration. +type EncryptionConfiguration struct { + // FooterKeyID is the master key id that wraps the footer key (and, in + // uniform encryption, the single key encrypting every column). Required. + FooterKeyID string + + // UniformEncryption encrypts every column with the footer key. Either + // it or ColumnKeys must be set, never both — an empty ColumnKeys map + // is a configuration error, not an implicit uniform layout (the + // parquet-cpp rule). + UniformEncryption bool + + // ColumnKeys maps a master key id to the column paths it encrypts; every + // listed column gets its own data encryption key wrapped by that master + // key. Columns not listed stay in plaintext. + ColumnKeys map[string][]string + + // DataKeyLengthBits is the data encryption key length in bits: 128, + // 192 or 256. Zero means 128, the parquet-cpp default. + DataKeyLengthBits int + + // Algorithm picks the Parquet cipher: parquet.AesGcm (the zero value + // and the parquet-cpp default) or parquet.AesCtr. + Algorithm parquet.Cipher + + // SingleWrapping wraps each data encryption key directly with its master + // key. The default (false) is double wrapping (the parquet-cpp/pyarrow + // default), where a per-master-key key encryption key wraps the data + // keys locally and only it is wrapped by the KMS. + SingleWrapping bool + + // AadPrefix is the AAD prefix identifier stored in the file (unless + // DisableAadPrefixStorage is set) so readers can verify file identity. + AadPrefix string + + // DisableAadPrefixStorage keeps the AadPrefix out of the file; readers + // must then supply it themselves. + DisableAadPrefixStorage bool + + // PlaintextFooter leaves the footer unencrypted (still authenticated + // with a signature). + PlaintextFooter bool +} + +// CryptoFactory builds parquet.FileEncryptionProperties whose keys are +// wrapped by a key management service, producing the PKMT1 key material +// that KmsKeyRetriever (and parquet-cpp/pyarrow readers) resolve. +// +// In double wrapping mode the factory keeps one key encryption key per +// (KMS instance, key access token, master key) and reuses it across +// files, so writing many files costs one KEK wrap per master key per +// token — parquet-cpp's per-token KEK write cache, additionally scoped +// to the KMS instance so wraps are never shared across deployments. +// Entries expire after the cache lifetime (DefaultKekCacheLifetime, +// tunable with WithKekCacheLifetime) and are then wrapped again; a new +// entry joining the map sweeps out expired ones (parquet-cpp's +// CheckCacheForExpiredTokens), so rotating access tokens don't keep +// stale plaintext KEKs in memory. +// Generation is serialized per cache entry, so files under different +// master keys wrap concurrently; only writers sharing a KEK wait for +// one another. +type CryptoFactory struct { + clientFactory KmsClientFactory + kekLifetime time.Duration + + mu sync.Mutex // guards the keks map only + keks map[kekReuseKey]*kekSlot +} + +// kekReuseKey identifies the KEK cached for a master key under one KMS +// instance and key access token: the token separates tenants the KMS +// authenticates, the instance keeps wraps from being shared across KMS +// deployments that may bind the same master key id to different keys. +type kekReuseKey struct { + kmsInstanceID string + accessToken string + masterKeyID string +} + +// cachedKek is a generated key encryption key together with its KMS-wrapped +// form, ready to be stamped into key material. Entries are immutable once +// published: a refresh installs a brand-new value instead of mutating the +// old one, so a KEK handed to a writer can never change — let alone tear — +// underneath the unlocked reads in wrapDataKey. +type cachedKek struct { + createdAt time.Time + + id []byte // raw KEK id; also the AAD for local data key wrapping + kek []byte + wrapped string // KmsClient-wrapped KEK +} + +// kekSlot owns the KEK cached for one kekReuseKey. Its mutex serializes +// (re)generation so the KMS call is made once per refresh and guards the +// cur pointer; the cachedKek it holds is immutable once published. +type kekSlot struct { + mu sync.Mutex + cur *cachedKek +} + +// A CryptoFactoryOption configures a CryptoFactory. +type CryptoFactoryOption func(*CryptoFactory) + +// WithKekCacheLifetime sets how long cached key encryption keys are +// reused before being wrapped again. A zero or negative duration falls +// back to DefaultKekCacheLifetime — caching cannot be disabled by +// passing 0. +func WithKekCacheLifetime(d time.Duration) CryptoFactoryOption { + return func(f *CryptoFactory) { f.kekLifetime = d } +} + +// NewCryptoFactory returns a factory that creates KmsClients with +// clientFactory. A single factory can encrypt many files; call +// FileEncryptionProperties once per file, since the returned properties are +// single use (the parquet writer wipes their keys). +func NewCryptoFactory(clientFactory KmsClientFactory, opts ...CryptoFactoryOption) *CryptoFactory { + f := &CryptoFactory{clientFactory: clientFactory} + for _, opt := range opts { + opt(f) + } + return f +} + +// FileEncryptionProperties generates fresh data encryption keys for the +// footer and the configured columns, wraps them through the KMS client, and +// returns file encryption properties carrying the PKMT1 key material in the +// footer and column key metadata. +func (f *CryptoFactory) FileEncryptionProperties(kmsCfg KmsConnectionConfig, + cfg EncryptionConfiguration) (*parquet.FileEncryptionProperties, error) { + if cfg.FooterKeyID == "" { + return nil, fmt.Errorf("parquet: encryption configuration needs a footer key id") + } + if cfg.UniformEncryption && len(cfg.ColumnKeys) > 0 { + return nil, fmt.Errorf("parquet: cannot set both uniform encryption and column keys") + } + if !cfg.UniformEncryption && len(cfg.ColumnKeys) == 0 { + return nil, fmt.Errorf("parquet: either column keys or uniform encryption must be set") + } + bits := cfg.DataKeyLengthBits + if bits == 0 { + bits = defaultBits + } + if bits != 128 && bits != 192 && bits != 256 { + return nil, fmt.Errorf("parquet: data key length must be 128, 192 or 256 bits, got %d", bits) + } + if cfg.Algorithm != parquet.AesGcm && cfg.Algorithm != parquet.AesCtr { + return nil, fmt.Errorf("parquet: encryption algorithm must be parquet.AesGcm or parquet.AesCtr, got %d", cfg.Algorithm) + } + keyBytes := bits / 8 + if f.clientFactory == nil { + return nil, fmt.Errorf("parquet: CryptoFactory needs a client factory, construct it with NewCryptoFactory") + } + if cfg.DisableAadPrefixStorage && cfg.AadPrefix == "" { + return nil, fmt.Errorf("parquet: DisableAadPrefixStorage is set without an AadPrefix") + } + client, err := f.clientFactory(kmsCfg) + if err != nil { + return nil, fmt.Errorf("parquet: create kms client: %w", err) + } + + footerDEK, err := randomKey(keyBytes) + if err != nil { + return nil, err + } + material, err := f.wrapDataKey(client, kmsCfg, cfg, footerDEK, cfg.FooterKeyID) + if err != nil { + return nil, err + } + material.IsFooterKey = true + material.KmsInstanceID = kmsCfg.KmsInstanceID + material.KmsInstanceURL = kmsCfg.KmsInstanceURL + footerJSON, err := json.Marshal(material) + if err != nil { + return nil, err + } + + opts := []parquet.EncryptOption{ + parquet.WithFooterKeyMetadata(string(footerJSON)), + parquet.WithAlg(cfg.Algorithm), + } + + cols := parquet.ColumnPathToEncryptionPropsMap{} + for masterKeyID, paths := range cfg.ColumnKeys { + if masterKeyID == "" { + return nil, fmt.Errorf("parquet: column keys map needs a master key id") + } + for _, path := range paths { + if path == "" { + return nil, fmt.Errorf("parquet: column keys for master key %q contain an empty column path", masterKeyID) + } + dek, err := randomKey(keyBytes) + if err != nil { + return nil, err + } + material, err := f.wrapDataKey(client, kmsCfg, cfg, dek, masterKeyID) + if err != nil { + return nil, err + } + colJSON, err := json.Marshal(material) + if err != nil { + return nil, err + } + if _, exists := cols[path]; exists { + return nil, fmt.Errorf("parquet: column %q is listed under more than one master key", path) + } + cols[path] = parquet.NewColumnEncryptionProperties(path, + parquet.WithKey(string(dek)), parquet.WithKeyMetadata(string(colJSON))) + } + } + if len(cols) > 0 { + opts = append(opts, parquet.WithEncryptedColumns(cols)) + } + + if cfg.AadPrefix != "" { + opts = append(opts, parquet.WithAadPrefix(cfg.AadPrefix)) + if cfg.DisableAadPrefixStorage { + opts = append(opts, parquet.DisableAadPrefixStorage()) + } + } + if cfg.PlaintextFooter { + opts = append(opts, parquet.WithPlaintextFooter()) + } + return parquet.NewFileEncryptionProperties(string(footerDEK), opts...), nil +} + +// randomKey returns n cryptographically random bytes. +func randomKey(n int) ([]byte, error) { + key := make([]byte, n) + if _, err := rand.Read(key); err != nil { + return nil, fmt.Errorf("parquet: generate key: %w", err) + } + return key, nil +} + +// wrapDataKey wraps dek with the master key masterKeyID and returns the +// PKMT1 key material for it. +func (f *CryptoFactory) wrapDataKey(client KmsClient, kmsCfg KmsConnectionConfig, + cfg EncryptionConfiguration, dek []byte, masterKeyID string) (KeyMaterial, error) { + + material := KeyMaterial{ + KeyMaterialType: keyMaterialType1, + InternalStorage: true, + MasterKeyID: masterKeyID, + } + if cfg.SingleWrapping { + wrapped, err := client.WrapKey(dek, masterKeyID) + if err != nil { + return KeyMaterial{}, fmt.Errorf("parquet: wrap data encryption key with master key %q: %w", masterKeyID, err) + } + material.WrappedDEK = wrapped + return material, nil + } + + // double wrapping: a key encryption key wraps data keys locally and only + // it is wrapped by the KMS + kek, err := f.kekFor(client, kmsCfg, masterKeyID) + if err != nil { + return KeyMaterial{}, err + } + wrappedDEK, err := wrapKeyLocally(dek, kek.kek, kek.id) + if err != nil { + return KeyMaterial{}, err + } + material.DoubleWrapping = true + material.WrappedDEK = base64.StdEncoding.EncodeToString(wrappedDEK) + material.KeyEncryptionKeyID = base64.StdEncoding.EncodeToString(kek.id) + material.WrappedKEK = kek.wrapped + return material, nil +} + +// kekFor returns the cached key encryption key for masterKeyID under the +// config's key access token, generating and wrapping a fresh one on first +// use or once the cache lifetime has passed. The returned entry is +// immutable: refreshes replace it wholesale, so the unlocked reads in +// wrapDataKey can never race a refresh. Each cache key has its own lock, +// so only writers sharing a KEK wait for its KMS wrap; the map lock is +// never held across a KMS call. +func (f *CryptoFactory) kekFor(client KmsClient, kmsCfg KmsConnectionConfig, + masterKeyID string) (*cachedKek, error) { + key := kekReuseKey{ + kmsInstanceID: kmsCfg.KmsInstanceID, + accessToken: kmsCfg.KeyAccessToken, + masterKeyID: masterKeyID, + } + + f.mu.Lock() + slot, ok := f.keks[key] + if !ok { + slot = &kekSlot{} + if f.keks == nil { + f.keks = make(map[kekReuseKey]*kekSlot) + } + f.keks[key] = slot + f.removeExpiredLocked(key) + } + f.mu.Unlock() + + slot.mu.Lock() + defer slot.mu.Unlock() + if slot.cur != nil && time.Since(slot.cur.createdAt) <= f.kekLifetimeOrDefault() { + return slot.cur, nil + } + id, err := randomKey(kekIDSize) + if err != nil { + return nil, err + } + kekKey, err := randomKey(kekSize) + if err != nil { + return nil, err + } + wrapped, err := client.WrapKey(kekKey, masterKeyID) + if err != nil { + return nil, fmt.Errorf("parquet: wrap key encryption key with master key %q: %w", masterKeyID, err) + } + fresh := &cachedKek{ + createdAt: time.Now(), + id: id, + kek: kekKey, + wrapped: wrapped, + } + slot.cur = fresh + return fresh, nil +} + +// removeExpiredLocked deletes cache entries whose lifetime has passed, so +// rotating access tokens don't keep every stale plaintext KEK in memory — +// parquet-cpp's CheckCacheForExpiredTokens, triggered when a new entry +// joins the cache. The caller must hold f.mu; keep is the entry the caller +// is inserting. Slots busy with a regeneration are left alone (their next +// value is fresh by construction). +func (f *CryptoFactory) removeExpiredLocked(keep kekReuseKey) { + lifetime := f.kekLifetimeOrDefault() + now := time.Now() + for k, slot := range f.keks { + if k == keep || !slot.mu.TryLock() { + continue + } + if slot.cur == nil || now.Sub(slot.cur.createdAt) > lifetime { + delete(f.keks, k) + } + slot.mu.Unlock() + } +} + +// kekLifetimeOrDefault is the configured cache lifetime, falling back to +// DefaultKekCacheLifetime for zero or negative values. +func (f *CryptoFactory) kekLifetimeOrDefault() time.Duration { + if f.kekLifetime <= 0 { + return DefaultKekCacheLifetime + } + return f.kekLifetime +} + +// wrapKeyLocally is parquet-cpp's EncryptKeyLocally: AES-GCM with a +// 12-byte nonce prefix and no length framing (nonce || ciphertext || tag), +// producing the format decryptKeyLocally unwinds. +func wrapKeyLocally(key, kek, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(kek) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("parquet: generate nonce: %w", err) + } + return gcm.Seal(nonce, nonce, key, aad), nil +} diff --git a/parquet/encryption/crypto_factory_internal_test.go b/parquet/encryption/crypto_factory_internal_test.go new file mode 100644 index 000000000..b07b5b1ff --- /dev/null +++ b/parquet/encryption/crypto_factory_internal_test.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// evictKMS is the smallest working KmsClient: symmetric base64 wrapping. +type evictKMS struct{} + +func (evictKMS) WrapKey(key []byte, _ string) (string, error) { + return base64.StdEncoding.EncodeToString(key), nil +} + +func (evictKMS) UnwrapKey(wrappedKey string, _ string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// Expired cache entries must leave the map when a new entry joins it +// (parquet-cpp's CheckCacheForExpiredTokens): with rotating access tokens +// the cache must not keep every stale plaintext KEK alive forever. +func TestKekCacheEvictsExpiredEntries(t *testing.T) { + f := NewCryptoFactory(func(KmsConnectionConfig) (KmsClient, error) { + return evictKMS{}, nil + }, WithKekCacheLifetime(time.Nanosecond)) + cfg := EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + tokenA := KmsConnectionConfig{KmsInstanceID: "in-memory", KeyAccessToken: "token-a"} + _, err := f.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + + // the entry expires; a different token adds a new cache entry, which + // is the sweep trigger + time.Sleep(2 * time.Millisecond) + tokenB := KmsConnectionConfig{KmsInstanceID: "in-memory", KeyAccessToken: "token-b"} + _, err = f.FileEncryptionProperties(tokenB, cfg) + require.NoError(t, err) + + f.mu.Lock() + defer f.mu.Unlock() + assert.Len(t, f.keks, 1, "the expired token-a entry must be swept") + _, ok := f.keks[kekReuseKey{kmsInstanceID: "in-memory", accessToken: "token-b", masterKeyID: "footer_master"}] + assert.True(t, ok, "the fresh token-b entry must survive the sweep") +} diff --git a/parquet/encryption/crypto_factory_test.go b/parquet/encryption/crypto_factory_test.go new file mode 100644 index 000000000..0f8b01925 --- /dev/null +++ b/parquet/encryption/crypto_factory_test.go @@ -0,0 +1,609 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption_test + +import ( + "encoding/base64" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/encryption" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingBase64KMS mirrors the test KMS client of the interop harness: +// wrapping is plain base64. It counts calls so tests can assert how many +// KMS round trips key material generation costs. +type countingBase64KMS struct { + wraps atomic.Int32 + unwraps atomic.Int32 +} + +func (k *countingBase64KMS) WrapKey(key []byte, masterKeyID string) (string, error) { + k.wraps.Add(1) + return base64.StdEncoding.EncodeToString(key), nil +} + +func (k *countingBase64KMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + k.unwraps.Add(1) + return base64.StdEncoding.DecodeString(wrappedKey) +} + +func countingFactory(t *testing.T, kms *countingBase64KMS) *encryption.CryptoFactory { + t.Helper() + return encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) +} + +var testKmsConfig = encryption.KmsConnectionConfig{ + KmsInstanceID: "in-memory", + KmsInstanceURL: "local", +} + +// In uniform encryption every column is encrypted with the footer key, so +// only the footer key material is generated. Single wrapping hands each data +// encryption key straight to the KMS client. +func TestFileEncryptionPropertiesUniformSingleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + SingleWrapping: true, + }) + require.NoError(t, err) + + // the footer key is a fresh 16-byte data key + assert.Len(t, props.FooterKey(), 16) + + material, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, "footer_master", material.MasterKeyID) + assert.True(t, material.IsFooterKey) + assert.False(t, material.DoubleWrapping) + assert.Equal(t, "in-memory", material.KmsInstanceID) + assert.Equal(t, "local", material.KmsInstanceURL) + + // the retriever must recover the footer key from the metadata alone + key, err := encryption.NewKmsKeyRetriever(kms).UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, props.FooterKey(), string(key)) + + // uniform encryption: no column properties, columns fall back to the + // footer key + assert.Empty(t, props.EncryptedColumns()) + assert.True(t, props.ColumnEncryptionProperties("a").IsEncryptedWithFooterKey()) + + // one DEK wrapped directly by the KMS + assert.Equal(t, int32(1), kms.wraps.Load()) +} + +// Double wrapping wraps only a per-master-key key encryption key with the +// KMS; data keys are wrapped locally. The KEK is reused across files, so a +// second properties object costs no extra KMS call (the pyarrow behavior). +func TestFileEncryptionPropertiesDoubleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Len(t, props.FooterKey(), 16) + + material, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.True(t, material.DoubleWrapping) + assert.True(t, material.IsFooterKey) + assert.Equal(t, "footer_master", material.MasterKeyID) + + // the wrapped DEK carries the AES-GCM framing: 12 byte nonce + 16 byte + // key + 16 byte tag + wrappedDEK, err := base64.StdEncoding.DecodeString(material.WrappedDEK) + require.NoError(t, err) + assert.Len(t, wrappedDEK, 44) + kekID, err := base64.StdEncoding.DecodeString(material.KeyEncryptionKeyID) + require.NoError(t, err) + assert.Len(t, kekID, 16) + + // the retriever must recover the footer key from the metadata alone, + // unwrapping the KEK with the KMS and the DEK locally + key, err := encryption.NewKmsKeyRetriever(kms).UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.Equal(t, props.FooterKey(), string(key)) + assert.Equal(t, int32(1), kms.unwraps.Load(), "one KMS call to unwrap the KEK") + + // only the KEK is wrapped by the KMS, and it is reused for the next file + _, err = factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load(), "KEK must be reused across files") + + // a fresh factory wraps a fresh KEK + other := &countingBase64KMS{} + _, err = countingFactory(t, other).FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), other.wraps.Load()) +} + +// With column keys every listed column gets its own data encryption key +// wrapped by its master key; columns under the same master key share the +// key encryption key, and unlisted columns stay in plaintext. +func TestFileEncryptionPropertiesColumnKeys(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + retriever := encryption.NewKmsKeyRetriever(kms) + + cfg := encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "float_field"}}, + } + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + + for _, path := range []string{"double_field", "float_field"} { + col := props.ColumnEncryptionProperties(path) + require.NotNil(t, col, path) + assert.True(t, col.IsEncrypted(), path) + assert.False(t, col.IsEncryptedWithFooterKey(), path) + assert.Len(t, col.Key(), 16, path) + + material, err := encryption.ParseKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err, path) + assert.Equal(t, "column_master", material.MasterKeyID, path) + assert.False(t, material.IsFooterKey, path) + + // column key material carries no KMS instance fields, matching + // parquet-cpp + assert.NotContains(t, col.KeyMetadata(), "kmsInstanceID", path) + + key, err := retriever.UnwrapKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err, path) + assert.Equal(t, col.Key(), string(key), path) + } + + // every column gets a fresh data key, but both share the column master + // key's KEK, distinct from the footer KEK + a := props.ColumnEncryptionProperties("double_field") + b := props.ColumnEncryptionProperties("float_field") + assert.NotEqual(t, a.Key(), b.Key()) + am, err := encryption.ParseKeyMetadata([]byte(a.KeyMetadata())) + require.NoError(t, err) + bm, err := encryption.ParseKeyMetadata([]byte(b.KeyMetadata())) + require.NoError(t, err) + assert.Equal(t, am.KeyEncryptionKeyID, bm.KeyEncryptionKeyID) + fm, err := encryption.ParseKeyMetadata([]byte(props.FooterKeyMetadata())) + require.NoError(t, err) + assert.NotEqual(t, am.KeyEncryptionKeyID, fm.KeyEncryptionKeyID) + + // unlisted columns stay in plaintext + assert.Nil(t, props.ColumnEncryptionProperties("boolean_field")) + + // two master keys wrapped once each: the footer KEK and the column KEK + assert.Equal(t, int32(2), kms.wraps.Load()) +} + +// Single wrapping with column keys wraps every data key with the KMS. +func TestFileEncryptionPropertiesColumnKeysSingleWrapping(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + retriever := encryption.NewKmsKeyRetriever(kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "float_field"}}, + SingleWrapping: true, + }) + require.NoError(t, err) + + // footer + one data key per column: three KMS wraps + assert.Equal(t, int32(3), kms.wraps.Load()) + + col := props.ColumnEncryptionProperties("double_field") + key, err := retriever.UnwrapKeyMetadata([]byte(col.KeyMetadata())) + require.NoError(t, err) + assert.Equal(t, col.Key(), string(key)) +} + +func TestFileEncryptionPropertiesErrors(t *testing.T) { + factory := countingFactory(t, &countingBase64KMS{}) + + _, err := factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{FooterKeyID: ""}) + assert.ErrorContains(t, err, "footer key id") + + // an empty ColumnKeys map is a configuration error, not an implicit + // uniform layout (the parquet-cpp rule) + _, err = factory.FileEncryptionProperties(testKmsConfig, + encryption.EncryptionConfiguration{FooterKeyID: "footer_master"}) + assert.ErrorContains(t, err, "either column keys or uniform encryption") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + ColumnKeys: map[string][]string{"column_master": {"a"}}, + }) + assert.ErrorContains(t, err, "cannot set both") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + DataKeyLengthBits: 100, + }) + assert.ErrorContains(t, err, "128, 192 or 256 bits") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"": {"a"}}, + }) + assert.ErrorContains(t, err, "master key id") + + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", ""}}, + }) + assert.ErrorContains(t, err, "column path") + + // a column listed under two master keys would otherwise win + // nondeterministically (map iteration order) + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{ + "column_master": {"a"}, + "other_master": {"a"}, + }, + }) + assert.ErrorContains(t, err, "more than one master key") + + // parquet-cpp rejects the combination too + _, err = factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + UniformEncryption: true, + DisableAadPrefixStorage: true, + }) + assert.ErrorContains(t, err, "DisableAadPrefixStorage") +} + +// writeKmsEncryptedFile writes a file with an encrypted float64 column and a +// plaintext boolean column, using keys from the CryptoFactory. +func writeKmsEncryptedFile(t *testing.T, path string, + props *parquet.FileEncryptionProperties) { + t.Helper() + + fields := schema.FieldList{ + schema.Must(schema.NewPrimitiveNode("double_field", parquet.Repetitions.Required, + parquet.Types.Double, 0, -1)), + schema.NewBooleanNode("boolean_field", parquet.Repetitions.Required, -1), + } + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Repeated, fields, -1) + require.NoError(t, err) + + out, err := os.Create(path) + require.NoError(t, err) + + writerProps := parquet.NewWriterProperties(parquet.WithEncryptionProperties(props)) + w := file.NewParquetWriter(out, root, file.WithWriterProps(writerProps)) + + rgr, err := w.AppendBufferedRowGroupChecked() + require.NoError(t, err) + + dw, err := rgr.Column(0) + require.NoError(t, err) + doubleWriter := dw.(*file.Float64ColumnChunkWriter) + for i := 0; i < 3; i++ { + _, err := doubleWriter.WriteBatch([]float64{float64(i) * 10}, nil, nil) + require.NoError(t, err) + } + + bw, err := rgr.Column(1) + require.NoError(t, err) + boolWriter := bw.(*file.BooleanColumnChunkWriter) + for i := 0; i < 3; i++ { + _, err := boolWriter.WriteBatch([]bool{i%2 == 0}, nil, nil) + require.NoError(t, err) + } + + require.NoError(t, w.Close()) +} + +// A file written with CryptoFactory keys must read back through the KMS key +// retriever alone, with no explicit keys anywhere. +func TestCryptoFactoryWriteReadRoundTrip(t *testing.T) { + configs := []struct { + name string + singleWrapping bool + expectedUnwraps int32 + }{ + {"double_wrapping", false, 2}, + {"single_wrapping", true, 3}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field", "boolean_field"}}, + SingleWrapping: config.singleWrapping, + }) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "kms_written.parquet") + writeKmsEncryptedFile(t, path, props) + + readProps := parquet.NewReaderProperties(memory.DefaultAllocator) + readProps.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) + + fr, err := file.OpenParquetFile(path, false, file.WithReadProps(readProps)) + require.NoError(t, err) + defer fr.Close() + + assert.Equal(t, int64(3), fr.MetaData().NumRows) + + rg := fr.RowGroup(0) + d, err := rg.Column(0) + require.NoError(t, err) + doubleReader := d.(*file.Float64ColumnChunkReader) + vals := make([]float64, 8) + n, _, err := doubleReader.ReadBatch(8, vals, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []float64{0, 10, 20}, vals[:3]) + + b, err := rg.Column(1) + require.NoError(t, err) + boolReader := b.(*file.BooleanColumnChunkReader) + bools := make([]bool, 8) + n, _, err = boolReader.ReadBatch(8, bools, nil, nil) + require.NoError(t, err) + assert.Equal(t, int64(3), n) + assert.Equal(t, []bool{true, false, true}, bools[:3]) + + assert.Equal(t, config.expectedUnwraps, kms.unwraps.Load(), + "KMS unwrap calls while reading") + }) + } +} + +// slowKMS counts wraps like countingBase64KMS but takes delay per call, to +// observe which KMS wraps the factory actually serializes. +type slowKMS struct { + wraps atomic.Int32 + delay time.Duration +} + +func (k *slowKMS) WrapKey(key []byte, masterKeyID string) (string, error) { + k.wraps.Add(1) + time.Sleep(k.delay) + return base64.StdEncoding.EncodeToString(key), nil +} + +func (k *slowKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) { + return base64.StdEncoding.DecodeString(wrappedKey) +} + +// The KEK write cache is per (key access token, master key), as in +// parquet-cpp's two-level cache: two tokens over one master key get distinct +// KEKs, and revisiting a token costs nothing new. +func TestKekCacheIsPerAccessToken(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + tokenA := testKmsConfig + tokenA.KeyAccessToken = "token-a" + tokenB := testKmsConfig + tokenB.KeyAccessToken = "token-b" + + _, err := factory.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load()) + + _, err = factory.FileEncryptionProperties(tokenB, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "second access token must get its own KEK") + + _, err = factory.FileEncryptionProperties(tokenA, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "revisiting a token reuses its cached KEK") + + // a different KMS instance never shares wraps, even with the same + // token: the same master key id may resolve to a different key there + otherInstance := tokenA + otherInstance.KmsInstanceID = "other-instance" + _, err = factory.FileEncryptionProperties(otherInstance, cfg) + require.NoError(t, err) + assert.Equal(t, int32(3), kms.wraps.Load(), "another KMS instance gets its own KEK") +} + +// Cached KEKs expire after the cache lifetime and are wrapped again. +func TestKekCacheEntryLifetime(t *testing.T) { + kms := &countingBase64KMS{} + factory := encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }, encryption.WithKekCacheLifetime(50*time.Millisecond)) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + first, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(1), kms.wraps.Load()) + + time.Sleep(80 * time.Millisecond) + + second, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + require.NoError(t, err) + assert.Equal(t, int32(2), kms.wraps.Load(), "expired KEK must be wrapped again") + assert.NotEqual(t, first.FooterKeyMetadata(), second.FooterKeyMetadata(), + "the second file carries a fresh KEK") +} + +// Refreshing an expired KEK races with writers that are still reading the +// previous one if the cache hands out entries it later mutates in place. +// With a 1ns lifetime every access refreshes, so concurrent writers force +// the window open; each produced key material must stay self-consistent +// (the retriever recovers the footer key from the metadata alone), and the +// run must be clean under -race. +func TestKekRefreshFromConcurrentWriters(t *testing.T) { + kms := &countingBase64KMS{} + factory := encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }, encryption.WithKekCacheLifetime(time.Nanosecond)) + retriever := encryption.NewKmsKeyRetriever(kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + const workers = 8 + errs := make(chan error, workers) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 25; i++ { + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + if err != nil { + errs <- err + return + } + key, err := retriever.UnwrapKeyMetadata([]byte(props.FooterKeyMetadata())) + if err != nil { + errs <- fmt.Errorf("key material is not self-consistent: %w", err) + return + } + if string(key) != props.FooterKey() { + errs <- fmt.Errorf("recovered footer key does not match the encrypted one") + return + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + +// KEK generation locks per cache entry: files under different master keys +// wrap concurrently, while files sharing a master key wait for one wrap. +func TestKekWrappingLocksPerMasterKey(t *testing.T) { + const delay = 100 * time.Millisecond + newFactory := func(kms *slowKMS) *encryption.CryptoFactory { + return encryption.NewCryptoFactory(func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) + } + + t.Run("same master key serializes to one wrap", func(t *testing.T) { + kms := &slowKMS{delay: delay} + factory := newFactory(kms) + cfg := encryption.EncryptionConfiguration{FooterKeyID: "footer_master", UniformEncryption: true} + + var wg sync.WaitGroup + metas := make([]string, 4) + for i := range metas { + wg.Add(1) + go func(i int) { + defer wg.Done() + props, err := factory.FileEncryptionProperties(testKmsConfig, cfg) + if err == nil { + metas[i] = props.FooterKeyMetadata() + } + }(i) + } + wg.Wait() + + assert.Equal(t, int32(1), kms.wraps.Load(), "concurrent files sharing a master key get one KEK wrap") + firstMaterial, err := encryption.ParseKeyMetadata([]byte(metas[0])) + require.NoError(t, err) + for _, meta := range metas[1:] { + material, err := encryption.ParseKeyMetadata([]byte(meta)) + require.NoError(t, err) + assert.Equal(t, firstMaterial.KeyEncryptionKeyID, material.KeyEncryptionKeyID, + "all files carry the same KEK") + } + }) + + t.Run("different master keys wrap concurrently", func(t *testing.T) { + kms := &slowKMS{delay: delay} + factory := newFactory(kms) + + var wg sync.WaitGroup + start := time.Now() + for i := 0; i < 4; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: fmt.Sprintf("footer_master_%d", i), + UniformEncryption: true, + }) + assert.NoError(t, err) + }(i) + } + wg.Wait() + + assert.Equal(t, int32(4), kms.wraps.Load()) + elapsed := time.Since(start) + assert.Less(t, elapsed, 3*delay, + "wraps under different master keys must not serialize (took %v)", elapsed) + }) +} + +// DataKeyLengthBits and Algorithm reach the generated properties: keys get +// the configured length, the cipher is passed through, and the file still +// reads back through the key retriever. +func TestDataKeyLengthAndAlgorithm(t *testing.T) { + kms := &countingBase64KMS{} + factory := countingFactory(t, kms) + + props, err := factory.FileEncryptionProperties(testKmsConfig, encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"double_field"}}, + DataKeyLengthBits: 256, + Algorithm: parquet.AesCtr, + }) + require.NoError(t, err) + assert.Len(t, props.FooterKey(), 32) + + assert.Equal(t, parquet.AesCtr, props.Algorithm().Algo) + + path := filepath.Join(t.TempDir(), "kms_ctr256.parquet") + writeKmsEncryptedFile(t, path, props) + + readProps := parquet.NewReaderProperties(memory.DefaultAllocator) + readProps.FileDecryptProps = parquet.NewFileDecryptionProperties( + parquet.WithKeyRetriever(encryption.NewKmsKeyRetriever(kms))) + fr, err := file.OpenParquetFile(path, false, file.WithReadProps(readProps)) + require.NoError(t, err) + defer fr.Close() + assert.Equal(t, int64(3), fr.MetaData().NumRows) +} diff --git a/parquet/encryption/doc.go b/parquet/encryption/doc.go index 2959010ff..8417addf4 100644 --- a/parquet/encryption/doc.go +++ b/parquet/encryption/doc.go @@ -14,14 +14,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package encryption provides optional KMS integration for reading parquet -// files encrypted with PKMT1 key material, the format written by the +// Package encryption provides optional KMS integration for parquet +// encryption with PKMT1 key material, the format written by the // CryptoFactory of parquet-cpp and pyarrow (both single and double key // wrapping). // -// It plugs into the existing low level API as a +// Reading plugs into the existing low level API as a // parquet.DecryptionKeyRetriever: pass a KmsClient implementation to // NewKmsKeyRetriever and set the retriever with // parquet.WithKeyRetriever. Files using explicit keys or custom key // metadata continue to work without this package. +// +// Writing uses NewCryptoFactory: FileEncryptionProperties generates the +// data encryption keys and their PKMT1 key material and returns +// parquet.FileEncryptionProperties for parquet.WithEncryptionProperties, +// producing files readable by the same factory on the parquet-cpp/pyarrow +// side. Writing with explicit keys via parquet.NewFileEncryptionProperties +// continues to work without this package. package encryption diff --git a/parquet/encryption/example_test.go b/parquet/encryption/example_test.go index fe1417287..e1dcb78c3 100644 --- a/parquet/encryption/example_test.go +++ b/parquet/encryption/example_test.go @@ -70,3 +70,41 @@ func ExampleKmsKeyRetriever() { // Output: "secret-dek" } + +// ExampleCryptoFactory shows how to write files whose keys are wrapped by a +// key management service: implement KmsClient, wrap a client factory in a +// CryptoFactory, and pass the properties it builds to the writer with +// parquet.WithEncryptionProperties. The generated PKMT1 key material is read +// back by KmsKeyRetriever and by parquet-cpp/pyarrow readers. +func ExampleCryptoFactory() { + factory := encryption.NewCryptoFactory( + func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return toyKMS{}, nil + }) + + props, err := factory.FileEncryptionProperties( + encryption.KmsConnectionConfig{ + KmsInstanceID: "kms-instance", + KmsInstanceURL: "https://kms.example", + }, + encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + // every listed column gets its own data key wrapped by this + // master key; columns left out stay in plaintext + ColumnKeys: map[string][]string{"column_master": {"sensitive_column"}}, + // SingleWrapping: true wraps each data key directly with its + // master key instead of a per-master-key key encryption key + }) + if err != nil { + panic(err) + } + fmt.Printf("footer key is %d bytes\n", len(props.FooterKey())) + + // The properties are single use: build fresh ones for every file. + // + // writerProps := parquet.NewWriterProperties( + // parquet.WithEncryptionProperties(props)) + // w := file.NewParquetWriter(out, schema, file.WithWriterProps(writerProps)) + + // Output: footer key is 16 bytes +} diff --git a/parquet/encryption/key_material.go b/parquet/encryption/key_material.go index 74d485dfb..78257bafa 100644 --- a/parquet/encryption/key_material.go +++ b/parquet/encryption/key_material.go @@ -37,9 +37,12 @@ type KeyMaterial struct { InternalStorage bool `json:"internalStorage"` // IsFooterKey reports whether this material belongs to the footer key. // Only footer key material carries KmsInstanceID and KmsInstanceURL. - IsFooterKey bool `json:"isFooterKey"` - KmsInstanceID string `json:"kmsInstanceID"` - KmsInstanceURL string `json:"kmsInstanceURL"` + IsFooterKey bool `json:"isFooterKey"` + // KmsInstanceID and KmsInstanceURL identify the KMS instance that wrote + // the footer key material; they are omitted from column key material, + // matching parquet-cpp. + KmsInstanceID string `json:"kmsInstanceID,omitempty"` + KmsInstanceURL string `json:"kmsInstanceURL,omitempty"` // MasterKeyID identifies the master key used to wrap the key. MasterKeyID string `json:"masterKeyID"` // WrappedDEK is the base64 encoded wrapped data encryption key. @@ -49,10 +52,10 @@ type KeyMaterial struct { DoubleWrapping bool `json:"doubleWrapping"` // KeyEncryptionKeyID is the base64 encoded key encryption key ID, set // only in double wrapping mode. - KeyEncryptionKeyID string `json:"keyEncryptionKeyID"` + KeyEncryptionKeyID string `json:"keyEncryptionKeyID,omitempty"` // WrappedKEK is the base64 encoded wrapped key encryption key, set only // in double wrapping mode. - WrappedKEK string `json:"wrappedKEK"` + WrappedKEK string `json:"wrappedKEK,omitempty"` } // ParseKeyMetadata parses the key metadata JSON of a footer or column key. diff --git a/parquet/encryption_interop_pyarrow.py b/parquet/encryption_interop_pyarrow.py index 0339cc2ff..1f41a69d0 100644 --- a/parquet/encryption_interop_pyarrow.py +++ b/parquet/encryption_interop_pyarrow.py @@ -22,6 +22,7 @@ python3 encryption_interop_pyarrow.py gen python3 encryption_interop_pyarrow.py gen-kms python3 encryption_interop_pyarrow.py read + python3 encryption_interop_pyarrow.py read-kms """ import sys @@ -57,14 +58,12 @@ def write(name, fep): create_encryption_properties(footer_key, plaintext_footer=True)) -def gen_kms(outdir): - """Write PKMT1 key material files with a CryptoFactory and a test-only - in-memory KMS client whose wrapping is plain base64 (no cryptography - dependency). The Go side mirrors the client to unwrap the keys.""" +def _kms_crypto_factory(): + """A CryptoFactory over a test-only in-memory KMS client whose wrapping + is plain base64 (no cryptography dependency), shared by gen-kms and + read-kms.""" import base64 as b64 - import os - import pyarrow.parquet as pq import pyarrow.parquet.encryption as pe class Base64KmsClient(pe.KmsClient): @@ -80,11 +79,18 @@ def unwrap_key(self, wrapped_key, master_key_identifier): config = pe.KmsConnectionConfig( kms_instance_id="in-memory", kms_instance_url="local", key_access_token="token") + return pe.CryptoFactory(lambda cfg: Base64KmsClient(cfg)), config + - def client_factory(kms_connection_config): - return Base64KmsClient(kms_connection_config) +def gen_kms(outdir): + """Write PKMT1 key material files with a CryptoFactory. The Go side + mirrors the client to unwrap the keys.""" + import os - factory = pe.CryptoFactory(client_factory) + import pyarrow.parquet as pq + import pyarrow.parquet.encryption as pe + + factory, config = _kms_crypto_factory() table = build_table() for name, double_wrapping in (("kms_double", True), ("kms_single", False)): @@ -97,6 +103,18 @@ def client_factory(kms_connection_config): print(name) +def read_kms(path): + """Read a file whose keys were wrapped by the test KMS client, resolving + them through the CryptoFactory.""" + import pyarrow.parquet as pq + + factory, config = _kms_crypto_factory() + table = pq.read_table(path, + decryption_properties=factory.file_decryption_properties(config)) + for name, col in zip(table.column_names, table.columns): + print(name + "\t" + " ".join(str(v) for v in col.to_pylist())) + + def read(path, footer_key): import pyarrow.parquet as pq from pyarrow.parquet.encryption import create_decryption_properties @@ -126,6 +144,9 @@ def main(): if mode == "gen-kms": gen_kms(sys.argv[2]) return 0 + if mode == "read-kms": + read_kms(sys.argv[2]) + return 0 if mode == "read": read(sys.argv[2], sys.argv[3].encode()) return 0 diff --git a/parquet/encryption_interop_pyarrow_test.go b/parquet/encryption_interop_pyarrow_test.go index ff222b635..30327e128 100644 --- a/parquet/encryption_interop_pyarrow_test.go +++ b/parquet/encryption_interop_pyarrow_test.go @@ -325,3 +325,101 @@ func writeInteropEncryptedFile(t *testing.T, path string) { require.NoError(t, w.Close()) } + +// writeGoKmsFile writes an encrypted file whose keys come from the Go +// CryptoFactory, mirroring the pyarrow gen-kms layout: a float64 and a +// float32 column under a column master key, plus the footer key. +func writeGoKmsFile(t *testing.T, path string, cfg encryption.EncryptionConfiguration) { + t.Helper() + + kms := &interopBase64KMS{} + factory := encryption.NewCryptoFactory( + func(encryption.KmsConnectionConfig) (encryption.KmsClient, error) { + return kms, nil + }) + props, err := factory.FileEncryptionProperties( + encryption.KmsConnectionConfig{KmsInstanceID: "in-memory", KmsInstanceURL: "local"}, cfg) + require.NoError(t, err) + + fields := schema.FieldList{ + schema.Must(schema.NewPrimitiveNode("a", parquet.Repetitions.Required, + parquet.Types.Double, 0, -1)), + schema.Must(schema.NewPrimitiveNode("b", parquet.Repetitions.Required, + parquet.Types.Float, 0, -1)), + } + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Repeated, fields, -1) + require.NoError(t, err) + + out, err := os.Create(path) + require.NoError(t, err) + + writerProps := parquet.NewWriterProperties(parquet.WithEncryptionProperties(props)) + w := file.NewParquetWriter(out, root, file.WithWriterProps(writerProps)) + + rgr, err := w.AppendBufferedRowGroupChecked() + require.NoError(t, err) + + aw, err := rgr.Column(0) + require.NoError(t, err) + doubleWriter := aw.(*file.Float64ColumnChunkWriter) + _, err = doubleWriter.WriteBatch([]float64{1, 2, 3}, nil, nil) + require.NoError(t, err) + + bw, err := rgr.Column(1) + require.NoError(t, err) + floatWriter := bw.(*file.Float32ColumnChunkWriter) + _, err = floatWriter.WriteBatch([]float32{4, 5, 6}, nil, nil) + require.NoError(t, err) + + require.NoError(t, w.Close()) +} + +// TestGoKmsWrittenReadByPyarrow writes files with the Go CryptoFactory and +// reads them back with pyarrow's, resolving every key through the test KMS. +func TestGoKmsWrittenReadByPyarrow(t *testing.T) { + requirePyarrow(t) + + configs := []struct { + name string + cfg encryption.EncryptionConfiguration + }{ + {"double", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + }}, + {"single", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + SingleWrapping: true, + }}, + {"plaintext_footer", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + PlaintextFooter: true, + }}, + {"ctr_256", encryption.EncryptionConfiguration{ + FooterKeyID: "footer_master", + ColumnKeys: map[string][]string{"column_master": {"a", "b"}}, + DataKeyLengthBits: 256, + Algorithm: parquet.AesCtr, + }}, + } + + for _, config := range configs { + t.Run(config.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "go_kms.parquet") + writeGoKmsFile(t, path, config.cfg) + + out := interopPython(t, "read-kms", path) + got := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + parts := strings.SplitN(line, " ", 2) + if len(parts) == 2 { + got[parts[0]] = parts[1] + } + } + assert.Equal(t, "1.0 2.0 3.0", got["a"]) + assert.Equal(t, "4.0 5.0 6.0", got["b"]) + }) + } +}