Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions parquet/encryption/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package encryption provides optional KMS integration for reading parquet
// files encrypted with PKMT1 key material, the format written by the
// CryptoFactory of parquet-cpp and pyarrow (both single and double key
// wrapping).
//
// It plugs into the existing low level API as a
// parquet.DecryptionKeyRetriever: pass a KmsClient implementation to
// NewKmsKeyRetriever and set the retriever with
// parquet.WithKeyRetriever. Files using explicit keys or custom key
// metadata continue to work without this package.
package encryption
72 changes: 72 additions & 0 deletions parquet/encryption/example_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package encryption_test

import (
"encoding/base64"
"fmt"

"github.com/apache/arrow-go/v18/parquet/encryption"
)

// toyKMS is a stand-in for a real key management service client: the wrapped
// key representation is opaque to the library, so anything reversible works
// for demonstration.
type toyKMS struct{}

func (toyKMS) WrapKey(key []byte, masterKeyID string) (string, error) {
return base64.StdEncoding.EncodeToString(key), nil
}

func (toyKMS) UnwrapKey(wrappedKey string, masterKeyID string) ([]byte, error) {
return base64.StdEncoding.DecodeString(wrappedKey)
}

// ExampleKmsKeyRetriever shows how to read files written by the
// parquet-cpp/pyarrow CryptoFactory: implement KmsClient, wrap it in a
// KmsKeyRetriever and set it on the decryption properties with
// parquet.WithKeyRetriever. This package handles the PKMT1 parsing and
// unwrapping that would otherwise need a custom retriever.
func ExampleKmsKeyRetriever() {
retriever := encryption.NewKmsKeyRetriever(toyKMS{})

// key metadata as found in the footer of a CryptoFactory-written file
// (single wrapping mode)
metadata := []byte(`{
"keyMaterialType": "PKMT1",
"internalStorage": true,
"isFooterKey": true,
"kmsInstanceID": "kms-instance",
"kmsInstanceURL": "https://kms.example",
"masterKeyID": "footer_master",
"wrappedDEK": "c2VjcmV0LWRlaw==",
"doubleWrapping": false
}`)

fmt.Printf("%q\n", retriever.GetKey(metadata))

// To open an encrypted file, pass the retriever to the reader:
//
// props := parquet.NewReaderProperties(memory.DefaultAllocator)
// props.FileDecryptProps = parquet.NewFileDecryptionProperties(
// parquet.WithKeyRetriever(retriever))
// fr, err := file.OpenParquetFile(path, false, file.WithReadProps(props))
//
// Code that does not import this package is unaffected.

// Output: "secret-dek"
}
80 changes: 80 additions & 0 deletions parquet/encryption/key_material.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package encryption

import (
"encoding/json"
"fmt"
)

// keyMaterialType1 is the only key material type defined by the parquet key
// management tools specification ("parquet key management tools, version 1").
const keyMaterialType1 = "PKMT1"

// KeyMaterial is the PKMT1 key material carried in the key metadata of a
// parquet file written with envelope encryption. With internal key storage,
// the material is stored as a flat JSON object in the footer and column key
// metadata of the file.
type KeyMaterial struct {
// KeyMaterialType must be "PKMT1".
KeyMaterialType string `json:"keyMaterialType"`
// InternalStorage reports whether the key material is stored inside the
// parquet file. External key material stores are not supported.
InternalStorage bool `json:"internalStorage"`
// IsFooterKey reports whether this material belongs to the footer key.
// Only footer key material carries KmsInstanceID and KmsInstanceURL.
IsFooterKey bool `json:"isFooterKey"`
KmsInstanceID string `json:"kmsInstanceID"`
KmsInstanceURL string `json:"kmsInstanceURL"`
// MasterKeyID identifies the master key used to wrap the key.
MasterKeyID string `json:"masterKeyID"`
// WrappedDEK is the base64 encoded wrapped data encryption key.
WrappedDEK string `json:"wrappedDEK"`
// DoubleWrapping reports whether the DEK is wrapped by a key encryption
// key that is itself wrapped by the master key.
DoubleWrapping bool `json:"doubleWrapping"`
// KeyEncryptionKeyID is the base64 encoded key encryption key ID, set
// only in double wrapping mode.
KeyEncryptionKeyID string `json:"keyEncryptionKeyID"`
// WrappedKEK is the base64 encoded wrapped key encryption key, set only
// in double wrapping mode.
WrappedKEK string `json:"wrappedKEK"`
}

// ParseKeyMetadata parses the key metadata JSON of a footer or column key.
// Only internal key storage (the parquet-cpp/pyarrow CryptoFactory default)
// is supported; external key material stores are rejected.
func ParseKeyMetadata(metadata []byte) (KeyMaterial, error) {
var material KeyMaterial
if err := json.Unmarshal(metadata, &material); err != nil {
return KeyMaterial{}, fmt.Errorf("parquet: parse key metadata: %w", err)
}
if material.KeyMaterialType != keyMaterialType1 {
return KeyMaterial{}, fmt.Errorf("parquet: unsupported key material type %q (want %q)",
material.KeyMaterialType, keyMaterialType1)
}
if !material.InternalStorage {
return KeyMaterial{}, fmt.Errorf("parquet: external key material storage is not supported")
}
if material.MasterKeyID == "" || material.WrappedDEK == "" {
return KeyMaterial{}, fmt.Errorf("parquet: key metadata missing masterKeyID or wrappedDEK")
}
if material.DoubleWrapping && (material.KeyEncryptionKeyID == "" || material.WrappedKEK == "") {
return KeyMaterial{}, fmt.Errorf("parquet: double wrapped key material missing keyEncryptionKeyID or wrappedKEK")
}
return material, nil
}
Loading
Loading