From d9382e0ddb05c40b4b319d832a333aab05a6f31c Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 11:32:49 +0500 Subject: [PATCH 1/8] feat(kms): add AppRole authentication to the Vault Transit KMS The Vault KMS previously accepted only a static token. Route the X-Vault-Token through a token source so it can authenticate via AppRole: it logs in against the configured AppRole mount and re-authenticates on demand when the cached token nears expiry or Vault rejects a call with 403. A static token stays the default when no AppRole credentials are set. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- internal/kms/vault.go | 203 +++++++++++++++++++++++++++++++++---- internal/kms/vault_test.go | 171 +++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+), 21 deletions(-) diff --git a/internal/kms/vault.go b/internal/kms/vault.go index c887b66..88ed58b 100644 --- a/internal/kms/vault.go +++ b/internal/kms/vault.go @@ -9,22 +9,40 @@ import ( "fmt" "io" "net/http" + "sync" "time" ) const ( defaultTransitMount = "transit" + defaultAppRoleMount = "approle" defaultVaultTimeout = 10 * time.Second + // renewSkew re-authenticates this long before the cached token's TTL + // expires, so an in-flight Wrap/Unwrap never races the expiry. + renewSkew = 30 * time.Second + // maxPostAttempts bounds Vault calls to one transparent re-auth: if the + // first attempt hits 403 (an AppRole token expired since the last call), + // the token is dropped and the request retried once with a fresh login. + maxPostAttempts = 2 ) // VaultConfig configures a VaultKMS against a Vault Transit secrets engine. -// The KEK is the Transit key named KeyName. +// The KEK is the Transit key named KeyName. Exactly one authentication mode +// must be supplied: a static Token, or an AppRole (RoleID + SecretID). type VaultConfig struct { Address string // Vault base address, e.g. "https://vault:8200". - Token string // Vault token with encrypt/decrypt on the Transit key. + Token string // Static Vault token with encrypt/decrypt on the KEK. KeyName string // Transit key name (the KEK). Mount string // Transit mount path; defaults to "transit". HTTPClient *http.Client + + // AppRoleMount is the AppRole auth mount path; defaults to "approle". + AppRoleMount string + // RoleID/SecretID select AppRole authentication instead of a static + // Token. When set, the KMS logs in against the AppRole mount and + // re-authenticates on demand when the issued token expires. + RoleID string + SecretID string } // VaultKMS wraps and unwraps the DEK using Vault Transit's encrypt/decrypt @@ -33,34 +51,37 @@ type VaultConfig struct { type VaultKMS struct { encryptURL string decryptURL string - token string + auth *vaultAuth client *http.Client } // NewVaultKMS builds a VaultKMS from cfg. func NewVaultKMS(cfg VaultConfig) (*VaultKMS, error) { - switch { - case cfg.Address == "": + if cfg.Address == "" { return nil, errors.New("kms: vault address is required") - case cfg.Token == "": - return nil, errors.New("kms: vault token is required") - case cfg.KeyName == "": + } + if cfg.KeyName == "" { return nil, errors.New("kms: vault transit key name is required") } - mount := cfg.Mount - if mount == "" { - mount = defaultTransitMount - } client := cfg.HTTPClient if client == nil { client = &http.Client{Timeout: defaultVaultTimeout} } + auth, err := newVaultAuth(cfg, client) + if err != nil { + return nil, err + } + + mount := cfg.Mount + if mount == "" { + mount = defaultTransitMount + } return &VaultKMS{ encryptURL: fmt.Sprintf("%s/v1/%s/encrypt/%s", cfg.Address, mount, cfg.KeyName), decryptURL: fmt.Sprintf("%s/v1/%s/decrypt/%s", cfg.Address, mount, cfg.KeyName), - token: cfg.Token, + auth: auth, client: client, }, nil } @@ -100,33 +121,173 @@ func (v *VaultKMS) Unwrap(ctx context.Context, wrappedDEK []byte) ([]byte, error return plaintext, nil } +// post sends a Transit request, authenticating with the current token and +// re-authenticating once if Vault rejects the token with 403. func (v *VaultKMS) post(ctx context.Context, url string, reqBody, respOut any) error { buf, err := json.Marshal(reqBody) if err != nil { return fmt.Errorf("kms: marshal request: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) + for attempt := 1; ; attempt++ { + token, terr := v.auth.currentToken(ctx) + if terr != nil { + return terr + } + status, body, derr := v.do(ctx, url, buf, token) + if derr != nil { + return derr + } + if status == http.StatusForbidden && attempt < maxPostAttempts && v.auth.reauth() { + continue + } + if status < http.StatusOK || status >= http.StatusMultipleChoices { + return fmt.Errorf("kms: vault returned status %d: %s", status, bytes.TrimSpace(body)) + } + if err := json.Unmarshal(body, respOut); err != nil { + return fmt.Errorf("kms: decode vault response: %w", err) + } + return nil + } +} + +// do performs a single authenticated POST and returns the status and body. +func (v *VaultKMS) do(ctx context.Context, url string, body []byte, token string) (int, []byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) if err != nil { - return fmt.Errorf("kms: build request: %w", err) + return 0, nil, fmt.Errorf("kms: build request: %w", err) } - req.Header.Set("X-Vault-Token", v.token) + req.Header.Set("X-Vault-Token", token) req.Header.Set("Content-Type", "application/json") resp, err := v.client.Do(req) if err != nil { - return fmt.Errorf("kms: vault request: %w", err) + return 0, nil, fmt.Errorf("kms: vault request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return 0, nil, fmt.Errorf("kms: read vault response: %w", err) + } + return resp.StatusCode, respBody, nil +} + +// vaultAuth supplies the X-Vault-Token for Transit calls. It is either a fixed +// static token or an AppRole login that mints (and, on demand, re-mints) a +// short-lived token. +type vaultAuth struct { + client *http.Client + static string // set for static-token auth; empty for AppRole. + loginURL string // set for AppRole auth. + roleID string + secretID string + + mu sync.Mutex + token string + expiry time.Time // zero once logged in => the token does not expire. +} + +// newVaultAuth builds the auth strategy from cfg, enforcing exactly one mode. +func newVaultAuth(cfg VaultConfig, client *http.Client) (*vaultAuth, error) { + approle := cfg.RoleID != "" || cfg.SecretID != "" + switch { + case approle && cfg.Token != "": + return nil, errors.New("kms: set either a static token or AppRole credentials, not both") + case approle && (cfg.RoleID == "" || cfg.SecretID == ""): + return nil, errors.New("kms: AppRole auth requires both a role id and a secret id") + case !approle && cfg.Token == "": + return nil, errors.New("kms: a vault token or AppRole credentials are required") + } + + if !approle { + return &vaultAuth{client: client, static: cfg.Token}, nil + } + mount := cfg.AppRoleMount + if mount == "" { + mount = defaultAppRoleMount + } + return &vaultAuth{ + client: client, + loginURL: fmt.Sprintf("%s/v1/auth/%s/login", cfg.Address, mount), + roleID: cfg.RoleID, + secretID: cfg.SecretID, + }, nil +} + +// currentToken returns a currently valid Vault token, logging in via AppRole +// when the cached token is absent or within renewSkew of expiry. +func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { + if a.loginURL == "" { + return a.static, nil + } + a.mu.Lock() + defer a.mu.Unlock() + if a.token != "" && (a.expiry.IsZero() || time.Until(a.expiry) > renewSkew) { + return a.token, nil + } + if err := a.login(ctx); err != nil { + return "", err + } + return a.token, nil +} + +// reauth drops the cached AppRole token so the next token call logs in again. +// It reports whether re-authentication is possible (false for static tokens). +func (a *vaultAuth) reauth() bool { + if a.loginURL == "" { + return false + } + a.mu.Lock() + a.token = "" + a.mu.Unlock() + return true +} + +// login performs an AppRole login and caches the issued token and its TTL. +// The caller must hold a.mu. +func (a *vaultAuth) login(ctx context.Context) error { + reqBody, err := json.Marshal(map[string]string{"role_id": a.roleID, "secret_id": a.secretID}) + if err != nil { + return fmt.Errorf("kms: marshal approle login: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.loginURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("kms: build approle login: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("kms: approle login request: %w", err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("kms: read vault response: %w", err) + return fmt.Errorf("kms: read approle login response: %w", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("kms: vault returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) + return fmt.Errorf("kms: approle login returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) + } + + var lr struct { + Auth struct { + ClientToken string `json:"client_token"` + LeaseDuration int `json:"lease_duration"` + } `json:"auth"` } - if err := json.Unmarshal(body, respOut); err != nil { - return fmt.Errorf("kms: decode vault response: %w", err) + if err := json.Unmarshal(body, &lr); err != nil { + return fmt.Errorf("kms: decode approle login response: %w", err) + } + if lr.Auth.ClientToken == "" { + return errors.New("kms: approle login returned an empty client token") + } + + a.token = lr.Auth.ClientToken + if lr.Auth.LeaseDuration > 0 { + a.expiry = time.Now().Add(time.Duration(lr.Auth.LeaseDuration) * time.Second) + } else { + a.expiry = time.Time{} } return nil } diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index 6dccc9f..64ef387 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" + "sync" "testing" "github.com/cozystack/keycloak-kms-proxy/internal/crypto" @@ -152,5 +154,174 @@ func TestVaultKMSEnvelopeRoundTrip(t *testing.T) { } } +const ( + approleRoleID = "role-abc" + approleSecretID = "secret-xyz" +) + +// fakeVaultAppRole is an in-memory Vault Transit stand-in that requires AppRole +// login. Tokens issued before acceptFromLogin are treated as already expired, +// so encrypt/decrypt reject them with 403 — this drives the re-auth path. The +// returned func reports how many logins happened. +func fakeVaultAppRole(t *testing.T, acceptFromLogin int) (*httptest.Server, func() int) { + t.Helper() + + const ( + key = "kms-proxy" + prefix = "vault:v1:" + ) + var mu sync.Mutex + logins := 0 + valid := map[string]bool{} + + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/approle/login", func(w http.ResponseWriter, r *http.Request) { + var req struct { + RoleID string `json:"role_id"` + SecretID string `json:"secret_id"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if req.RoleID != approleRoleID || req.SecretID != approleSecretID { + http.Error(w, `{"errors":["invalid role or secret id"]}`, http.StatusBadRequest) + return + } + mu.Lock() + logins++ + tok := fmt.Sprintf("approle-tok-%d", logins) + if logins >= acceptFromLogin { + valid[tok] = true + } + mu.Unlock() + writeJSON(w, map[string]any{"auth": map[string]any{ + "client_token": tok, "lease_duration": 3600, "renewable": true, + }}) + }) + authOK := func(w http.ResponseWriter, r *http.Request) bool { + mu.Lock() + ok := valid[r.Header.Get("X-Vault-Token")] + mu.Unlock() + if !ok { + http.Error(w, `{"errors":["permission denied"]}`, http.StatusForbidden) + } + return ok + } + mux.HandleFunc("/v1/transit/encrypt/"+key, func(w http.ResponseWriter, r *http.Request) { + if !authOK(w, r) { + return + } + var req struct { + Plaintext string `json:"plaintext"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + writeJSON(w, map[string]any{"data": map[string]string{"ciphertext": prefix + req.Plaintext}}) + }) + mux.HandleFunc("/v1/transit/decrypt/"+key, func(w http.ResponseWriter, r *http.Request) { + if !authOK(w, r) { + return + } + var req struct { + Ciphertext string `json:"ciphertext"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + writeJSON(w, map[string]any{"data": map[string]string{"plaintext": strings.TrimPrefix(req.Ciphertext, prefix)}}) + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv, func() int { + mu.Lock() + defer mu.Unlock() + return logins + } +} + +func newAppRoleVault(t *testing.T, srv *httptest.Server) *VaultKMS { + t.Helper() + + v, err := NewVaultKMS(VaultConfig{ + Address: srv.URL, + KeyName: "kms-proxy", + RoleID: approleRoleID, + SecretID: approleSecretID, + HTTPClient: srv.Client(), + }) + if err != nil { + t.Fatalf("NewVaultKMS: %v", err) + } + return v +} + +func TestVaultKMSAppRoleRoundTrip(t *testing.T) { + t.Parallel() + + srv, logins := fakeVaultAppRole(t, 1) + v := newAppRoleVault(t, srv) + + wrapped, err := v.Wrap(context.Background(), []byte("dek")) + if err != nil { + t.Fatalf("Wrap: %v", err) + } + got, err := v.Unwrap(context.Background(), wrapped) + if err != nil { + t.Fatalf("Unwrap: %v", err) + } + if string(got) != "dek" { + t.Fatalf("round-trip mismatch: %q", got) + } + if n := logins(); n != 1 { + t.Fatalf("expected exactly one AppRole login (token cached), got %d", n) + } +} + +func TestVaultKMSAppRoleReauthOn403(t *testing.T) { + t.Parallel() + + // The first issued token is treated as expired, so the first encrypt gets + // 403; the KMS must drop it, log in again, and retry successfully. + srv, logins := fakeVaultAppRole(t, 2) + v := newAppRoleVault(t, srv) + + if _, err := v.Wrap(context.Background(), []byte("dek")); err != nil { + t.Fatalf("Wrap should recover via re-auth: %v", err) + } + if n := logins(); n != 2 { + t.Fatalf("expected a re-auth login, got %d logins", n) + } +} + +func TestVaultKMSAppRoleBadCredentials(t *testing.T) { + t.Parallel() + + srv, _ := fakeVaultAppRole(t, 1) + v, err := NewVaultKMS(VaultConfig{ + Address: srv.URL, + KeyName: "kms-proxy", + RoleID: approleRoleID, + SecretID: "wrong-secret", + HTTPClient: srv.Client(), + }) + if err != nil { + t.Fatalf("NewVaultKMS: %v", err) + } + if _, err := v.Wrap(context.Background(), []byte("dek")); err == nil { + t.Fatal("Wrap succeeded with bad AppRole credentials") + } +} + +func TestVaultKMSAuthModeValidation(t *testing.T) { + t.Parallel() + + cases := map[string]VaultConfig{ + "token and approle": {Address: "http://v", KeyName: "k", Token: "t", RoleID: "r", SecretID: "s"}, + "approle missing secret": {Address: "http://v", KeyName: "k", RoleID: "r"}, + "neither token nor approle": {Address: "http://v", KeyName: "k"}, + } + for name, cfg := range cases { + if _, err := NewVaultKMS(cfg); err == nil { + t.Errorf("%s: NewVaultKMS accepted an invalid auth config", name) + } + } +} + // VaultKMS must satisfy the KMS interface. var _ KMS = (*VaultKMS)(nil) From b02b21bb4fb2f36096497b93a877fcd5b326b66d Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 11:32:57 +0500 Subject: [PATCH 2/8] feat(config): wire Vault AppRole settings into the proxy Add KKP_VAULT_ROLE_ID / KKP_VAULT_SECRET_ID / KKP_VAULT_APPROLE_MOUNT and validate that Vault auth is exactly one of a static token or an AppRole (both a role id and a secret id). Pass the settings through to the KMS. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- cmd/proxy/main.go | 11 ++++++---- internal/config/proxy.go | 36 ++++++++++++++++++++++++++++++-- internal/config/proxy_test.go | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index dfb4d51..8fd7740 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -177,10 +177,13 @@ func buildCipher(ctx context.Context, cfg *config.ProxyConfig) (*crypto.Cipher, func buildKMS(cfg *config.ProxyConfig) (kms.KMS, error) { if cfg.VaultAddr != "" { return kms.NewVaultKMS(kms.VaultConfig{ - Address: cfg.VaultAddr, - Token: cfg.VaultToken, - KeyName: cfg.VaultKeyName, - Mount: cfg.VaultMount, + Address: cfg.VaultAddr, + Token: cfg.VaultToken, + KeyName: cfg.VaultKeyName, + Mount: cfg.VaultMount, + AppRoleMount: cfg.VaultAppRoleMount, + RoleID: cfg.VaultRoleID, + SecretID: cfg.VaultSecretID, }) } return kms.NewStaticKMS(cfg.KEK) diff --git a/internal/config/proxy.go b/internal/config/proxy.go index 5e75842..1c9d819 100644 --- a/internal/config/proxy.go +++ b/internal/config/proxy.go @@ -52,6 +52,13 @@ type ProxyConfig struct { VaultToken string VaultKeyName string VaultMount string + // VaultRoleID/VaultSecretID select AppRole authentication against the + // Vault Transit KMS instead of a static VaultToken. When set, the proxy + // logs in via AppRole and re-authenticates on demand as the token expires. + // VaultAppRoleMount is the AppRole auth mount path; defaults to "approle". + VaultAppRoleMount string + VaultRoleID string + VaultSecretID string // TLSCertFile/TLSKeyFile, when both set, make the proxy terminate TLS on // the Keycloak-facing listener. Both must be set or both // must be empty. @@ -98,6 +105,9 @@ const ( envVaultToken = "KKP_VAULT_TOKEN" //nolint:gosec // env var name, not a credential envVaultKeyName = "KKP_VAULT_KEY_NAME" envVaultMount = "KKP_VAULT_MOUNT" + envVaultAppRoleMount = "KKP_VAULT_APPROLE_MOUNT" + envVaultRoleID = "KKP_VAULT_ROLE_ID" + envVaultSecretID = "KKP_VAULT_SECRET_ID" //nolint:gosec // env var name, not a credential envTLSCertFile = "KKP_TLS_CERT_FILE" envTLSKeyFile = "KKP_TLS_KEY_FILE" envMetricsAddr = "KKP_METRICS_ADDR" @@ -154,6 +164,9 @@ func LoadProxyConfig() (*ProxyConfig, error) { VaultToken: os.Getenv(envVaultToken), VaultKeyName: os.Getenv(envVaultKeyName), VaultMount: os.Getenv(envVaultMount), + VaultAppRoleMount: os.Getenv(envVaultAppRoleMount), + VaultRoleID: os.Getenv(envVaultRoleID), + VaultSecretID: os.Getenv(envVaultSecretID), TLSCertFile: os.Getenv(envTLSCertFile), TLSKeyFile: os.Getenv(envTLSKeyFile), MetricsAddr: os.Getenv(envMetricsAddr), @@ -226,8 +239,27 @@ func (c *ProxyConfig) validateKMS() error { return fmt.Errorf("config: one of %s or %s is required", envKEK, envVaultAddr) case hasStatic && len(c.KEK) != kekBytes: return fmt.Errorf("config: %s must decode to %d bytes (got %d)", envKEK, kekBytes, len(c.KEK)) - case hasVault && (c.VaultToken == "" || c.VaultKeyName == ""): - return fmt.Errorf("config: %s requires %s and %s", envVaultAddr, envVaultToken, envVaultKeyName) + case hasVault: + return c.validateVaultAuth() + } + return nil +} + +// validateVaultAuth checks the Vault Transit settings: a KEK name plus exactly +// one auth mode — a static token, or an AppRole (both role id and secret id). +func (c *ProxyConfig) validateVaultAuth() error { + if c.VaultKeyName == "" { + return fmt.Errorf("config: %s requires %s", envVaultAddr, envVaultKeyName) + } + hasToken := c.VaultToken != "" + hasAppRole := c.VaultRoleID != "" || c.VaultSecretID != "" + switch { + case hasToken && hasAppRole: + return fmt.Errorf("config: set %s or AppRole (%s + %s), not both", envVaultToken, envVaultRoleID, envVaultSecretID) + case hasAppRole && (c.VaultRoleID == "" || c.VaultSecretID == ""): + return fmt.Errorf("config: AppRole auth requires both %s and %s", envVaultRoleID, envVaultSecretID) + case !hasToken && !hasAppRole: + return fmt.Errorf("config: %s requires %s or AppRole (%s + %s)", envVaultAddr, envVaultToken, envVaultRoleID, envVaultSecretID) } return nil } diff --git a/internal/config/proxy_test.go b/internal/config/proxy_test.go index 98868b6..3b78748 100644 --- a/internal/config/proxy_test.go +++ b/internal/config/proxy_test.go @@ -52,6 +52,45 @@ func TestProxyConfigValidateKEKLength(t *testing.T) { } } +// vaultProxyConfig returns a valid config using Vault Transit (no static KEK). +func vaultProxyConfig() *ProxyConfig { + c := validProxyConfig() + c.KEK = nil + c.VaultAddr = "http://vault:8200" + c.VaultKeyName = "keycloak-kek" + return c +} + +func TestProxyConfigValidateVaultAppRole(t *testing.T) { + t.Parallel() + + c := vaultProxyConfig() + c.VaultRoleID = "role-abc" + c.VaultSecretID = "secret-xyz" + if err := c.Validate(); err != nil { + t.Fatalf("Validate rejected a valid AppRole config: %v", err) + } +} + +func TestProxyConfigValidateVaultAuthMisconfig(t *testing.T) { + t.Parallel() + + cases := map[string]func(*ProxyConfig){ + "token and approle": func(c *ProxyConfig) { + c.VaultToken, c.VaultRoleID, c.VaultSecretID = "t", "r", "s" + }, + "approle missing secret id": func(c *ProxyConfig) { c.VaultRoleID = "r" }, + "no auth at all": func(c *ProxyConfig) {}, + } + for name, mutate := range cases { + c := vaultProxyConfig() + mutate(c) + if err := c.Validate(); err == nil { + t.Errorf("%s: Validate accepted an invalid Vault auth config", name) + } + } +} + func TestLoadProxyConfigFromEnv(t *testing.T) { kek := base64.StdEncoding.EncodeToString(make([]byte, kekBytes)) t.Setenv(envListenAddr, ":5432") From 9464febde3ea9e3601e710ce11db238fc31ad11a Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 11:32:57 +0500 Subject: [PATCH 3/8] docs: document Vault AppRole authentication Describe the AppRole option in the README and operator guide, and correct the guide's stale reference to a Kubernetes-projected ServiceAccount token path that the proxy does not implement. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- README.md | 5 ++++- docs/operator-guide.md | 15 ++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8d6d05e..0225ace 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ Tagged releases publish a container image to `ghcr.io/cozystack/keycloak-kms-pro | `KKP_UPSTREAM_USER` / `KKP_UPSTREAM_PASSWORD` | credential the proxy verifies on the upstream (Keycloak) SCRAM leg | | `KKP_BACKEND_USER` / `KKP_BACKEND_PASSWORD` | credential the proxy uses on the downstream (CNPG) SCRAM leg | | `KKP_KEK` | base64 32-byte KEK for the static KMS (mutually exclusive with the Vault settings below) | -| `KKP_VAULT_ADDR` / `KKP_VAULT_TOKEN` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS settings | +| `KKP_VAULT_ADDR` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS settings | +| `KKP_VAULT_TOKEN` | static Vault token auth (mutually exclusive with AppRole below) | +| `KKP_VAULT_ROLE_ID` / `KKP_VAULT_SECRET_ID` / `KKP_VAULT_APPROLE_MOUNT` | Vault AppRole auth: the proxy logs in and re-authenticates on demand (mount defaults to `approle`) | | `KKP_DEKSET_FILE` | path to a JSON-encoded wrapped DEK set (lets the proxy and the backfill tool share a key) | | `KKP_FIELDS` | `disabled`, `email-only`, or empty for the full default set | | `KKP_LENIENT` | `true` downgrades fail-loud rules to passthrough — required for the Keycloak Liquibase bootstrap window only, must be off in steady state | @@ -64,6 +66,7 @@ The operator-facing guides live under [`docs/`](./docs): [operator-guide](./docs - **Static KMS** (`KKP_KEK`): a 32-byte AES-256 KEK provided in-cluster as a `Secret`. Suitable for tests and bootstrap. - **Vault Transit** (`KKP_VAULT_*`): the KEK lives in Vault. KEK rotation (`vault write -f transit/keys//rotate`) is transparent to the proxy — the `vault:vN:` version tag on each wrapped DEK lets Vault decrypt across rotations without re-encrypting any column data. Optionally run `vault write transit/rewrap/ ciphertext=…` later to bring the stored wrap up to the latest KEK version. +- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. ## Search support diff --git a/docs/operator-guide.md b/docs/operator-guide.md index 1ab710b..75bf24a 100644 --- a/docs/operator-guide.md +++ b/docs/operator-guide.md @@ -40,9 +40,12 @@ Keycloak ── kkp-proxy Service ──┬── replica 1 ──┐ Required for this shape: - `replicas: 2` on the proxy Deployment. -- `KKP_VAULT_ADDR`, `KKP_VAULT_TOKEN` (or projected ServiceAccount - token), `KKP_VAULT_KEY_NAME`, `KKP_VAULT_MOUNT` — `KKP_KEK` MUST be - unset. +- `KKP_VAULT_ADDR`, `KKP_VAULT_KEY_NAME`, `KKP_VAULT_MOUNT`, and Vault + auth — either `KKP_VAULT_TOKEN` (a static token) or AppRole + (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via + `KKP_VAULT_APPROLE_MOUNT`, default `approle`). With AppRole the proxy + logs in itself and re-authenticates on demand as the token expires, so + no long-lived token sits in a Secret. `KKP_KEK` MUST be unset. - `KKP_BACKEND_CA_FILE` + `KKP_BACKEND_SERVER_NAME` pointing at the CNPG `*-ca` Secret + the read-write Service DNS. - `KKP_TLS_CERT_FILE`/`KKP_TLS_KEY_FILE` for the listener side @@ -74,7 +77,7 @@ ciphertext. | `kc-db-ca` (CNPG-managed name varies) | CNPG operator | CA bundle for `KKP_BACKEND_CA_FILE` so the downstream TLS leg is verified | | `kc-db-app` (CNPG default) | CNPG operator | `KKP_BACKEND_PASSWORD` (the credential the proxy uses to authenticate to CNPG) | | `keycloak-creds` | operator | Keycloak's own admin password + `KC_DB_PASSWORD` (must match `KKP_UPSTREAM_PASSWORD` because the proxy verifies it on the upstream leg) | -| Vault token | platform | Either a Kubernetes-projected ServiceAccount token (preferred) or a long-lived Vault token in a Secret | +| Vault auth | platform | AppRole `role_id`/`secret_id` in a Secret (`KKP_VAULT_ROLE_ID`/`KKP_VAULT_SECRET_ID`, recommended — short-lived tokens minted on demand) or a static Vault token (`KKP_VAULT_TOKEN`) | ## Required network policies @@ -143,7 +146,9 @@ The list, with a brief intent for each: | `KKP_LENIENT` | `true` for bootstrap; **must be `false` in production** | | `KKP_KEK` | Static KMS — dev only | | `KKP_DEKSET_FILE` | Path to the wrapped DEK set (required so DEK survives restarts) | -| `KKP_VAULT_ADDR` / `KKP_VAULT_TOKEN` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS — production default | +| `KKP_VAULT_ADDR` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS — production default | +| `KKP_VAULT_TOKEN` | Static Vault token auth (mutually exclusive with AppRole) | +| `KKP_VAULT_ROLE_ID` / `KKP_VAULT_SECRET_ID` / `KKP_VAULT_APPROLE_MOUNT` | Vault AppRole auth — the proxy logs in and re-authenticates on demand (mount default `approle`) | | `KKP_TLS_CERT_FILE` / `KKP_TLS_KEY_FILE` | Listener TLS termination | | `KKP_BACKEND_CA_FILE` / `KKP_BACKEND_SERVER_NAME` | TLS re-origination to CNPG | | `KKP_HEALTH_ADDR` | `/healthz` + `/readyz` endpoint (k8s probes) | From 2716269c12ffc1dcb0fa52ce299c5b23e2b7b7b2 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 11:44:20 +0500 Subject: [PATCH 4/8] fix(kms): fall back to a valid token on proactive re-login failure Address review: when the cached AppRole token is still valid but within renewSkew of expiry, a transient login failure now falls back to the still-usable token instead of failing the request; a hard 403 (which clears the token first) still surfaces the error. Document that the AppRole secret_id must stay reusable for the on-demand re-login. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- README.md | 2 +- docs/operator-guide.md | 5 +++- internal/kms/vault.go | 8 +++++++ internal/kms/vault_test.go | 49 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0225ace..b418aa8 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The operator-facing guides live under [`docs/`](./docs): [operator-guide](./docs - **Static KMS** (`KKP_KEK`): a 32-byte AES-256 KEK provided in-cluster as a `Secret`. Suitable for tests and bootstrap. - **Vault Transit** (`KKP_VAULT_*`): the KEK lives in Vault. KEK rotation (`vault write -f transit/keys//rotate`) is transparent to the proxy — the `vault:vN:` version tag on each wrapped DEK lets Vault decrypt across rotations without re-encrypting any column data. Optionally run `vault write transit/rewrap/ ciphertext=…` later to bring the stored wrap up to the latest KEK version. -- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. +- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. The `secret_id` must stay reusable, though — provision the role without `secret_id_num_uses=1` and with a `secret_id_ttl` no shorter than the token TTL, otherwise the on-demand re-login will fail once the secret id is consumed or expires. ## Search support diff --git a/docs/operator-guide.md b/docs/operator-guide.md index 75bf24a..53f7dcd 100644 --- a/docs/operator-guide.md +++ b/docs/operator-guide.md @@ -45,7 +45,10 @@ Required for this shape: (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`, default `approle`). With AppRole the proxy logs in itself and re-authenticates on demand as the token expires, so - no long-lived token sits in a Secret. `KKP_KEK` MUST be unset. + no long-lived token sits in a Secret. Provision the AppRole with a + reusable `secret_id` (avoid `secret_id_num_uses=1` and keep + `secret_id_ttl` no shorter than the token TTL) — the proxy reuses the + secret id for every on-demand re-login. `KKP_KEK` MUST be unset. - `KKP_BACKEND_CA_FILE` + `KKP_BACKEND_SERVER_NAME` pointing at the CNPG `*-ca` Secret + the read-write Service DNS. - `KKP_TLS_CERT_FILE`/`KKP_TLS_KEY_FILE` for the listener side diff --git a/internal/kms/vault.go b/internal/kms/vault.go index 88ed58b..f9a7d7c 100644 --- a/internal/kms/vault.go +++ b/internal/kms/vault.go @@ -226,6 +226,14 @@ func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { return a.token, nil } if err := a.login(ctx); err != nil { + // A proactive re-login (the cached token is still valid but within + // renewSkew of expiry) may fail on a transient Vault error. login + // leaves the old token in place, so fall back to it while it lasts + // rather than failing the request. A hard 403 clears the token via + // reauth first, so that path still surfaces the error. + if a.token != "" && (a.expiry.IsZero() || time.Now().Before(a.expiry)) { + return a.token, nil + } return "", err } return a.token, nil diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index 64ef387..d362ca9 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -10,6 +10,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/cozystack/keycloak-kms-proxy/internal/crypto" ) @@ -323,5 +324,53 @@ func TestVaultKMSAuthModeValidation(t *testing.T) { } } +func TestVaultAuthProactiveRenewFallback(t *testing.T) { + t.Parallel() + + // A login endpoint that always fails, to simulate a transient Vault error + // during a proactive re-login. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"errors":["service unavailable"]}`, http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + + a := &vaultAuth{ + client: srv.Client(), + loginURL: srv.URL + "/v1/auth/approle/login", + roleID: "r", + secretID: "s", + token: "still-valid", + expiry: time.Now().Add(renewSkew / 2), // inside renewSkew, but not expired. + } + got, err := a.currentToken(context.Background()) + if err != nil { + t.Fatalf("currentToken should fall back to the still-valid cached token: %v", err) + } + if got != "still-valid" { + t.Fatalf("got %q, want the cached token", got) + } +} + +func TestVaultAuthReauthNoStaleToken(t *testing.T) { + t.Parallel() + + // After a hard 403 the token is cleared (reauth); a failing re-login must + // surface the error rather than returning a stale token. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"errors":["permission denied"]}`, http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + a := &vaultAuth{ + client: srv.Client(), + loginURL: srv.URL + "/v1/auth/approle/login", + roleID: "r", + secretID: "s", + } + if _, err := a.currentToken(context.Background()); err == nil { + t.Fatal("currentToken should fail when login fails and no token is cached") + } +} + // VaultKMS must satisfy the KMS interface. var _ KMS = (*VaultKMS)(nil) From b555a950c173b3dcc74912702475b269ba3d1676 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 11:49:06 +0500 Subject: [PATCH 5/8] refactor(kms): don't hold the token mutex across the AppRole login Address review: serialize logins with a dedicated loginMu and make login a pure function returning the token and expiry, so mu is never held during the network call and a caller holding a valid token is not blocked by an in-flight login. A concurrent login uses TryLock and falls back to the still-usable token. Add a concurrent-Wrap race test. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- internal/kms/vault.go | 103 ++++++++++++++++++++++++++----------- internal/kms/vault_test.go | 28 ++++++++++ 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/internal/kms/vault.go b/internal/kms/vault.go index f9a7d7c..045b39b 100644 --- a/internal/kms/vault.go +++ b/internal/kms/vault.go @@ -182,9 +182,11 @@ type vaultAuth struct { roleID string secretID string - mu sync.Mutex + mu sync.Mutex // guards token/expiry; never held across the login I/O. token string expiry time.Time // zero once logged in => the token does not expire. + + loginMu sync.Mutex // serializes AppRole logins. } // newVaultAuth builds the auth strategy from cfg, enforcing exactly one mode. @@ -214,29 +216,68 @@ func newVaultAuth(cfg VaultConfig, client *http.Client) (*vaultAuth, error) { }, nil } -// currentToken returns a currently valid Vault token, logging in via AppRole -// when the cached token is absent or within renewSkew of expiry. +// currentToken returns a valid Vault token, logging in via AppRole when the +// cached token is missing or within renewSkew of expiry. The login network +// call is serialized by loginMu and never holds mu, so a caller whose token is +// still valid is never blocked by an in-flight login. func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { if a.loginURL == "" { return a.static, nil } + if tok, ok := a.freshToken(); ok { + return tok, nil + } + if !a.loginMu.TryLock() { + // A login is already in flight; keep serving the current token while + // it is still usable rather than waiting on the login. + if tok, ok := a.usableToken(); ok { + return tok, nil + } + a.loginMu.Lock() + } + defer a.loginMu.Unlock() + + // Re-check under the login lock: a concurrent login may have just renewed. + if tok, ok := a.freshToken(); ok { + return tok, nil + } + tok, expiry, err := a.login(ctx) + if err != nil { + // A proactive re-login (token still valid but within renewSkew) may + // fail transiently; fall back to the still-usable token rather than + // failing the request. A hard 403 clears the token via reauth first, + // so usableToken returns nothing there and the error surfaces. + if fallback, ok := a.usableToken(); ok { + return fallback, nil + } + return "", err + } + a.mu.Lock() + a.token, a.expiry = tok, expiry + a.mu.Unlock() + return tok, nil +} + +// freshToken returns the cached token when present and not within renewSkew of +// expiry. +func (a *vaultAuth) freshToken() (string, bool) { a.mu.Lock() defer a.mu.Unlock() if a.token != "" && (a.expiry.IsZero() || time.Until(a.expiry) > renewSkew) { - return a.token, nil - } - if err := a.login(ctx); err != nil { - // A proactive re-login (the cached token is still valid but within - // renewSkew of expiry) may fail on a transient Vault error. login - // leaves the old token in place, so fall back to it while it lasts - // rather than failing the request. A hard 403 clears the token via - // reauth first, so that path still surfaces the error. - if a.token != "" && (a.expiry.IsZero() || time.Now().Before(a.expiry)) { - return a.token, nil - } - return "", err + return a.token, true + } + return "", false +} + +// usableToken returns the cached token when present and not yet expired (even +// if within renewSkew) — the fallback when a re-login fails or is in flight. +func (a *vaultAuth) usableToken() (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() + if a.token != "" && (a.expiry.IsZero() || time.Now().Before(a.expiry)) { + return a.token, true } - return a.token, nil + return "", false } // reauth drops the cached AppRole token so the next token call logs in again. @@ -247,35 +288,37 @@ func (a *vaultAuth) reauth() bool { } a.mu.Lock() a.token = "" + a.expiry = time.Time{} a.mu.Unlock() return true } -// login performs an AppRole login and caches the issued token and its TTL. -// The caller must hold a.mu. -func (a *vaultAuth) login(ctx context.Context) error { +// login performs an AppRole login and returns the issued token and its expiry +// (zero expiry meaning the token does not expire). It does not mutate a, so it +// runs safely without holding mu during the network call. +func (a *vaultAuth) login(ctx context.Context) (string, time.Time, error) { reqBody, err := json.Marshal(map[string]string{"role_id": a.roleID, "secret_id": a.secretID}) if err != nil { - return fmt.Errorf("kms: marshal approle login: %w", err) + return "", time.Time{}, fmt.Errorf("kms: marshal approle login: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.loginURL, bytes.NewReader(reqBody)) if err != nil { - return fmt.Errorf("kms: build approle login: %w", err) + return "", time.Time{}, fmt.Errorf("kms: build approle login: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := a.client.Do(req) if err != nil { - return fmt.Errorf("kms: approle login request: %w", err) + return "", time.Time{}, fmt.Errorf("kms: approle login request: %w", err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("kms: read approle login response: %w", err) + return "", time.Time{}, fmt.Errorf("kms: read approle login response: %w", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("kms: approle login returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) + return "", time.Time{}, fmt.Errorf("kms: approle login returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) } var lr struct { @@ -285,17 +328,15 @@ func (a *vaultAuth) login(ctx context.Context) error { } `json:"auth"` } if err := json.Unmarshal(body, &lr); err != nil { - return fmt.Errorf("kms: decode approle login response: %w", err) + return "", time.Time{}, fmt.Errorf("kms: decode approle login response: %w", err) } if lr.Auth.ClientToken == "" { - return errors.New("kms: approle login returned an empty client token") + return "", time.Time{}, errors.New("kms: approle login returned an empty client token") } - a.token = lr.Auth.ClientToken + var expiry time.Time if lr.Auth.LeaseDuration > 0 { - a.expiry = time.Now().Add(time.Duration(lr.Auth.LeaseDuration) * time.Second) - } else { - a.expiry = time.Time{} + expiry = time.Now().Add(time.Duration(lr.Auth.LeaseDuration) * time.Second) } - return nil + return lr.Auth.ClientToken, expiry, nil } diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index d362ca9..4d42662 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -372,5 +372,33 @@ func TestVaultAuthReauthNoStaleToken(t *testing.T) { } } +func TestVaultKMSAppRoleConcurrent(t *testing.T) { + t.Parallel() + + srv, logins := fakeVaultAppRole(t, 1) + v := newAppRoleVault(t, srv) + + const n = 20 + var wg sync.WaitGroup + errs := make(chan error, n) + for range n { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := v.Wrap(context.Background(), []byte("dek")); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatalf("concurrent Wrap: %v", err) + } + if got := logins(); got < 1 { + t.Fatalf("expected at least one login, got %d", got) + } +} + // VaultKMS must satisfy the KMS interface. var _ KMS = (*VaultKMS)(nil) From 7829546c0b56120b0d5350925c3c319ce9d0c367 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 13:48:54 +0500 Subject: [PATCH 6/8] fix(kms): warn on cached-token fallback and scope re-auth invalidation Address review: - Log a warning when a re-login fails and the still-valid cached token is served, so a permanently broken login (expired secret_id, revoked policy) is visible before it bites at DEK-rotation time. - reauth now clears the cache only when it still holds the rejected token, so a concurrent goroutine's freshly obtained token is not wiped. - Tighten the concurrent test to assert exactly one login (serialization). - Docs: recommend a token_ttl above the renew skew and require an HTTPS Vault address, since the secret_id travels on every re-login. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- README.md | 2 +- docs/operator-guide.md | 4 +++- internal/kms/vault.go | 24 +++++++++++++++++------- internal/kms/vault_test.go | 4 ++-- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b418aa8..27846b5 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The operator-facing guides live under [`docs/`](./docs): [operator-guide](./docs - **Static KMS** (`KKP_KEK`): a 32-byte AES-256 KEK provided in-cluster as a `Secret`. Suitable for tests and bootstrap. - **Vault Transit** (`KKP_VAULT_*`): the KEK lives in Vault. KEK rotation (`vault write -f transit/keys//rotate`) is transparent to the proxy — the `vault:vN:` version tag on each wrapped DEK lets Vault decrypt across rotations without re-encrypting any column data. Optionally run `vault write transit/rewrap/ ciphertext=…` later to bring the stored wrap up to the latest KEK version. -- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. The `secret_id` must stay reusable, though — provision the role without `secret_id_num_uses=1` and with a `secret_id_ttl` no shorter than the token TTL, otherwise the on-demand re-login will fail once the secret id is consumed or expires. +- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. The `secret_id` must stay reusable, though — provision the role without `secret_id_num_uses=1` and with a `secret_id_ttl` no shorter than the token TTL, otherwise the on-demand re-login will fail once the secret id is consumed or expires. Give the role a `token_ttl` comfortably above the 30s renew skew, and point `KKP_VAULT_ADDR` at an HTTPS endpoint — the `secret_id` is sent on every re-login. ## Search support diff --git a/docs/operator-guide.md b/docs/operator-guide.md index 53f7dcd..3cf421e 100644 --- a/docs/operator-guide.md +++ b/docs/operator-guide.md @@ -48,7 +48,9 @@ Required for this shape: no long-lived token sits in a Secret. Provision the AppRole with a reusable `secret_id` (avoid `secret_id_num_uses=1` and keep `secret_id_ttl` no shorter than the token TTL) — the proxy reuses the - secret id for every on-demand re-login. `KKP_KEK` MUST be unset. + secret id for every on-demand re-login, so `KKP_VAULT_ADDR` MUST be + HTTPS. Keep `token_ttl` above the proxy's 30s renew skew. `KKP_KEK` + MUST be unset. - `KKP_BACKEND_CA_FILE` + `KKP_BACKEND_SERVER_NAME` pointing at the CNPG `*-ca` Secret + the read-write Service DNS. - `KKP_TLS_CERT_FILE`/`KKP_TLS_KEY_FILE` for the listener side diff --git a/internal/kms/vault.go b/internal/kms/vault.go index 045b39b..5b4c69d 100644 --- a/internal/kms/vault.go +++ b/internal/kms/vault.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "sync" "time" @@ -137,7 +138,7 @@ func (v *VaultKMS) post(ctx context.Context, url string, reqBody, respOut any) e if derr != nil { return derr } - if status == http.StatusForbidden && attempt < maxPostAttempts && v.auth.reauth() { + if status == http.StatusForbidden && attempt < maxPostAttempts && v.auth.reauth(token) { continue } if status < http.StatusOK || status >= http.StatusMultipleChoices { @@ -246,8 +247,12 @@ func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { // A proactive re-login (token still valid but within renewSkew) may // fail transiently; fall back to the still-usable token rather than // failing the request. A hard 403 clears the token via reauth first, - // so usableToken returns nothing there and the error surfaces. + // so usableToken returns nothing there and the error surfaces. Warn + // loudly: if the login is permanently broken (expired secret_id, + // revoked policy) the next DEK operation after the cached token dies + // will fail, so an operator needs to see this before rotation time. if fallback, ok := a.usableToken(); ok { + log.Printf("kkp: WARN vault re-login failed; serving the cached token until it expires: %v", err) return fallback, nil } return "", err @@ -280,15 +285,20 @@ func (a *vaultAuth) usableToken() (string, bool) { return "", false } -// reauth drops the cached AppRole token so the next token call logs in again. -// It reports whether re-authentication is possible (false for static tokens). -func (a *vaultAuth) reauth() bool { +// reauth drops the cached AppRole token so the next token call logs in again, +// but only if the cache still holds stale — the token that was just rejected. +// A concurrent goroutine may have already refreshed it, and wiping the fresh +// token would force a needless extra login. It reports whether re-auth is +// possible (false for static tokens). +func (a *vaultAuth) reauth(stale string) bool { if a.loginURL == "" { return false } a.mu.Lock() - a.token = "" - a.expiry = time.Time{} + if a.token == stale { + a.token = "" + a.expiry = time.Time{} + } a.mu.Unlock() return true } diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index 4d42662..6fb5ed6 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -395,8 +395,8 @@ func TestVaultKMSAppRoleConcurrent(t *testing.T) { for err := range errs { t.Fatalf("concurrent Wrap: %v", err) } - if got := logins(); got < 1 { - t.Fatalf("expected at least one login, got %d", got) + if got := logins(); got != 1 { + t.Fatalf("concurrent cold-start Wraps must serialize to exactly one login, got %d", got) } } From ea66f0c4411103f6efc4f047ce53316c7617b293 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 13:59:40 +0500 Subject: [PATCH 7/8] feat(kms): add Vault Kubernetes auth (preferred) alongside AppRole Kubernetes auth is the cleaner target for this proxy, so support it as a first-class mode. The auth strategy is generalized into a tokenLogin, so the proxy can log in with its projected ServiceAccount token (KKP_VAULT_KUBERNETES_ROLE / _MOUNT / _JWT_FILE), re-reading the token from disk on each login so the kubelet's rotation is picked up without a restart. AppRole and the static token remain; the config enforces exactly one auth mode. Docs mark Kubernetes as the recommended method. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- README.md | 10 ++- cmd/proxy/main.go | 4 + docs/operator-guide.md | 35 +++++--- internal/config/proxy.go | 85 ++++++++++++------ internal/config/proxy_test.go | 16 ++++ internal/kms/vault.go | 159 +++++++++++++++++++++++----------- internal/kms/vault_test.go | 139 ++++++++++++++++++++++++----- 7 files changed, 330 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 27846b5..92b4aa8 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ Tagged releases publish a container image to `ghcr.io/cozystack/keycloak-kms-pro | `KKP_BACKEND_USER` / `KKP_BACKEND_PASSWORD` | credential the proxy uses on the downstream (CNPG) SCRAM leg | | `KKP_KEK` | base64 32-byte KEK for the static KMS (mutually exclusive with the Vault settings below) | | `KKP_VAULT_ADDR` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS settings | -| `KKP_VAULT_TOKEN` | static Vault token auth (mutually exclusive with AppRole below) | +| `KKP_VAULT_KUBERNETES_ROLE` / `KKP_VAULT_KUBERNETES_MOUNT` / `KKP_VAULT_KUBERNETES_JWT_FILE` | Vault Kubernetes auth (**recommended**): the proxy logs in with its projected ServiceAccount token (mount defaults to `kubernetes`, JWT file to the standard in-pod path) | | `KKP_VAULT_ROLE_ID` / `KKP_VAULT_SECRET_ID` / `KKP_VAULT_APPROLE_MOUNT` | Vault AppRole auth: the proxy logs in and re-authenticates on demand (mount defaults to `approle`) | +| `KKP_VAULT_TOKEN` | static Vault token auth (mutually exclusive with the AppRole / Kubernetes options) | | `KKP_DEKSET_FILE` | path to a JSON-encoded wrapped DEK set (lets the proxy and the backfill tool share a key) | | `KKP_FIELDS` | `disabled`, `email-only`, or empty for the full default set | | `KKP_LENIENT` | `true` downgrades fail-loud rules to passthrough — required for the Keycloak Liquibase bootstrap window only, must be off in steady state | @@ -66,7 +67,12 @@ The operator-facing guides live under [`docs/`](./docs): [operator-guide](./docs - **Static KMS** (`KKP_KEK`): a 32-byte AES-256 KEK provided in-cluster as a `Secret`. Suitable for tests and bootstrap. - **Vault Transit** (`KKP_VAULT_*`): the KEK lives in Vault. KEK rotation (`vault write -f transit/keys//rotate`) is transparent to the proxy — the `vault:vN:` version tag on each wrapped DEK lets Vault decrypt across rotations without re-encrypting any column data. Optionally run `vault write transit/rewrap/ ciphertext=…` later to bring the stored wrap up to the latest KEK version. -- **Vault auth**: either a static token (`KKP_VAULT_TOKEN`) or **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). With AppRole the proxy performs the login itself and transparently re-authenticates when the issued token expires, so no long-lived token has to sit in a Secret. The proxy touches Vault only at startup and on DEK rotation, so a short-lived AppRole token is sufficient. The `secret_id` must stay reusable, though — provision the role without `secret_id_num_uses=1` and with a `secret_id_ttl` no shorter than the token TTL, otherwise the on-demand re-login will fail once the secret id is consumed or expires. Give the role a `token_ttl` comfortably above the 30s renew skew, and point `KKP_VAULT_ADDR` at an HTTPS endpoint — the `secret_id` is sent on every re-login. +- **Vault auth** — exactly one of, all logging in on demand and re-authenticating as the issued token expires (the proxy touches Vault only at startup and on DEK rotation, so a short-lived token is fine): + - **Kubernetes** (`KKP_VAULT_KUBERNETES_ROLE`; mount via `KKP_VAULT_KUBERNETES_MOUNT`, default `kubernetes`; JWT via `KKP_VAULT_KUBERNETES_JWT_FILE`, default the standard in-pod ServiceAccount token path) — **recommended**. The proxy logs in with its projected ServiceAccount token, re-read from disk each login so the kubelet's rotation is picked up. No long-lived credential is stored. + - **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via `KKP_VAULT_APPROLE_MOUNT`). The `secret_id` must stay reusable — provision the role without `secret_id_num_uses=1` and with a `secret_id_ttl` no shorter than the token TTL, otherwise the on-demand re-login fails once the secret id is consumed or expires. + - **Static token** (`KKP_VAULT_TOKEN`) — simplest; a long-lived token in a Secret. + + For all methods give a `token_ttl` comfortably above the 30s renew skew, and point `KKP_VAULT_ADDR` at an HTTPS endpoint — credentials travel on every login. ## Search support diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 8fd7740..bc66fcd 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -184,6 +184,10 @@ func buildKMS(cfg *config.ProxyConfig) (kms.KMS, error) { AppRoleMount: cfg.VaultAppRoleMount, RoleID: cfg.VaultRoleID, SecretID: cfg.VaultSecretID, + + KubernetesRole: cfg.VaultKubernetesRole, + KubernetesMount: cfg.VaultKubernetesMount, + KubernetesJWTFile: cfg.VaultKubernetesJWTFile, }) } return kms.NewStaticKMS(cfg.KEK) diff --git a/docs/operator-guide.md b/docs/operator-guide.md index 3cf421e..5a21028 100644 --- a/docs/operator-guide.md +++ b/docs/operator-guide.md @@ -40,17 +40,25 @@ Keycloak ── kkp-proxy Service ──┬── replica 1 ──┐ Required for this shape: - `replicas: 2` on the proxy Deployment. -- `KKP_VAULT_ADDR`, `KKP_VAULT_KEY_NAME`, `KKP_VAULT_MOUNT`, and Vault - auth — either `KKP_VAULT_TOKEN` (a static token) or AppRole - (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via - `KKP_VAULT_APPROLE_MOUNT`, default `approle`). With AppRole the proxy - logs in itself and re-authenticates on demand as the token expires, so - no long-lived token sits in a Secret. Provision the AppRole with a - reusable `secret_id` (avoid `secret_id_num_uses=1` and keep - `secret_id_ttl` no shorter than the token TTL) — the proxy reuses the - secret id for every on-demand re-login, so `KKP_VAULT_ADDR` MUST be - HTTPS. Keep `token_ttl` above the proxy's 30s renew skew. `KKP_KEK` - MUST be unset. +- `KKP_VAULT_ADDR`, `KKP_VAULT_KEY_NAME`, `KKP_VAULT_MOUNT`, and exactly + one Vault auth mode (`KKP_KEK` MUST be unset): + - **Kubernetes** (`KKP_VAULT_KUBERNETES_ROLE`; mount via + `KKP_VAULT_KUBERNETES_MOUNT`, default `kubernetes`; token via + `KKP_VAULT_KUBERNETES_JWT_FILE`, default the in-pod path) — + **recommended**. The proxy logs in with its projected ServiceAccount + token, which the kubelet rotates, so no long-lived credential is + stored. Bind the Vault role to the proxy's ServiceAccount + namespace. + - **AppRole** (`KKP_VAULT_ROLE_ID` + `KKP_VAULT_SECRET_ID`, mount via + `KKP_VAULT_APPROLE_MOUNT`, default `approle`). Provision a reusable + `secret_id` (avoid `secret_id_num_uses=1`, keep `secret_id_ttl` no + shorter than the token TTL) — the proxy reuses it for every on-demand + re-login. + - **Static token** (`KKP_VAULT_TOKEN`) — simplest, a long-lived token in + a Secret. + + For the login methods point `KKP_VAULT_ADDR` at HTTPS (credentials + travel on every login) and keep `token_ttl` above the proxy's 30s renew + skew. - `KKP_BACKEND_CA_FILE` + `KKP_BACKEND_SERVER_NAME` pointing at the CNPG `*-ca` Secret + the read-write Service DNS. - `KKP_TLS_CERT_FILE`/`KKP_TLS_KEY_FILE` for the listener side @@ -82,7 +90,7 @@ ciphertext. | `kc-db-ca` (CNPG-managed name varies) | CNPG operator | CA bundle for `KKP_BACKEND_CA_FILE` so the downstream TLS leg is verified | | `kc-db-app` (CNPG default) | CNPG operator | `KKP_BACKEND_PASSWORD` (the credential the proxy uses to authenticate to CNPG) | | `keycloak-creds` | operator | Keycloak's own admin password + `KC_DB_PASSWORD` (must match `KKP_UPSTREAM_PASSWORD` because the proxy verifies it on the upstream leg) | -| Vault auth | platform | AppRole `role_id`/`secret_id` in a Secret (`KKP_VAULT_ROLE_ID`/`KKP_VAULT_SECRET_ID`, recommended — short-lived tokens minted on demand) or a static Vault token (`KKP_VAULT_TOKEN`) | +| Vault auth | platform | Kubernetes auth bound to the proxy ServiceAccount (recommended — no stored credential), AppRole `role_id`/`secret_id` in a Secret, or a static Vault token (`KKP_VAULT_TOKEN`) | ## Required network policies @@ -152,8 +160,9 @@ The list, with a brief intent for each: | `KKP_KEK` | Static KMS — dev only | | `KKP_DEKSET_FILE` | Path to the wrapped DEK set (required so DEK survives restarts) | | `KKP_VAULT_ADDR` / `KKP_VAULT_KEY_NAME` / `KKP_VAULT_MOUNT` | Vault Transit KMS — production default | -| `KKP_VAULT_TOKEN` | Static Vault token auth (mutually exclusive with AppRole) | +| `KKP_VAULT_KUBERNETES_ROLE` / `KKP_VAULT_KUBERNETES_MOUNT` / `KKP_VAULT_KUBERNETES_JWT_FILE` | Vault Kubernetes auth (recommended) — login with the pod ServiceAccount token (mount default `kubernetes`) | | `KKP_VAULT_ROLE_ID` / `KKP_VAULT_SECRET_ID` / `KKP_VAULT_APPROLE_MOUNT` | Vault AppRole auth — the proxy logs in and re-authenticates on demand (mount default `approle`) | +| `KKP_VAULT_TOKEN` | Static Vault token auth (exactly one auth mode) | | `KKP_TLS_CERT_FILE` / `KKP_TLS_KEY_FILE` | Listener TLS termination | | `KKP_BACKEND_CA_FILE` / `KKP_BACKEND_SERVER_NAME` | TLS re-origination to CNPG | | `KKP_HEALTH_ADDR` | `/healthz` + `/readyz` endpoint (k8s probes) | diff --git a/internal/config/proxy.go b/internal/config/proxy.go index 1c9d819..c2d047e 100644 --- a/internal/config/proxy.go +++ b/internal/config/proxy.go @@ -59,6 +59,14 @@ type ProxyConfig struct { VaultAppRoleMount string VaultRoleID string VaultSecretID string + // VaultKubernetesRole selects Vault Kubernetes authentication (preferred in + // production): the proxy logs in with its projected ServiceAccount token, + // which the kubelet rotates, so no long-lived credential is stored. + // VaultKubernetesMount defaults to "kubernetes"; VaultKubernetesJWTFile + // defaults to the standard in-pod ServiceAccount token path. + VaultKubernetesRole string + VaultKubernetesMount string + VaultKubernetesJWTFile string // TLSCertFile/TLSKeyFile, when both set, make the proxy terminate TLS on // the Keycloak-facing listener. Both must be set or both // must be empty. @@ -108,6 +116,9 @@ const ( envVaultAppRoleMount = "KKP_VAULT_APPROLE_MOUNT" envVaultRoleID = "KKP_VAULT_ROLE_ID" envVaultSecretID = "KKP_VAULT_SECRET_ID" //nolint:gosec // env var name, not a credential + envVaultK8sRole = "KKP_VAULT_KUBERNETES_ROLE" + envVaultK8sMount = "KKP_VAULT_KUBERNETES_MOUNT" + envVaultK8sJWTFile = "KKP_VAULT_KUBERNETES_JWT_FILE" //nolint:gosec // env var name, not a credential envTLSCertFile = "KKP_TLS_CERT_FILE" envTLSKeyFile = "KKP_TLS_KEY_FILE" envMetricsAddr = "KKP_METRICS_ADDR" @@ -150,29 +161,32 @@ func LoadProxyConfig() (*ProxyConfig, error) { return nil, err } cfg := &ProxyConfig{ - ListenAddr: os.Getenv(envListenAddr), - BackendAddr: os.Getenv(envBackendAddr), - KEK: kek, - UpstreamUser: os.Getenv(envUpstreamUser), - UpstreamPassword: os.Getenv(envUpstreamPassword), - BackendUser: os.Getenv(envBackendUser), - BackendPassword: os.Getenv(envBackendPassword), - Fields: fields, - Lenient: os.Getenv(envLenient) == "true", - DEKSetFile: os.Getenv(envDEKSetFile), - VaultAddr: os.Getenv(envVaultAddr), - VaultToken: os.Getenv(envVaultToken), - VaultKeyName: os.Getenv(envVaultKeyName), - VaultMount: os.Getenv(envVaultMount), - VaultAppRoleMount: os.Getenv(envVaultAppRoleMount), - VaultRoleID: os.Getenv(envVaultRoleID), - VaultSecretID: os.Getenv(envVaultSecretID), - TLSCertFile: os.Getenv(envTLSCertFile), - TLSKeyFile: os.Getenv(envTLSKeyFile), - MetricsAddr: os.Getenv(envMetricsAddr), - HealthAddr: os.Getenv(envHealthAddr), - BackendCAFile: os.Getenv(envBackendCAFile), - BackendServerName: os.Getenv(envBackendServerName), + ListenAddr: os.Getenv(envListenAddr), + BackendAddr: os.Getenv(envBackendAddr), + KEK: kek, + UpstreamUser: os.Getenv(envUpstreamUser), + UpstreamPassword: os.Getenv(envUpstreamPassword), + BackendUser: os.Getenv(envBackendUser), + BackendPassword: os.Getenv(envBackendPassword), + Fields: fields, + Lenient: os.Getenv(envLenient) == "true", + DEKSetFile: os.Getenv(envDEKSetFile), + VaultAddr: os.Getenv(envVaultAddr), + VaultToken: os.Getenv(envVaultToken), + VaultKeyName: os.Getenv(envVaultKeyName), + VaultMount: os.Getenv(envVaultMount), + VaultAppRoleMount: os.Getenv(envVaultAppRoleMount), + VaultRoleID: os.Getenv(envVaultRoleID), + VaultSecretID: os.Getenv(envVaultSecretID), + VaultKubernetesRole: os.Getenv(envVaultK8sRole), + VaultKubernetesMount: os.Getenv(envVaultK8sMount), + VaultKubernetesJWTFile: os.Getenv(envVaultK8sJWTFile), + TLSCertFile: os.Getenv(envTLSCertFile), + TLSKeyFile: os.Getenv(envTLSKeyFile), + MetricsAddr: os.Getenv(envMetricsAddr), + HealthAddr: os.Getenv(envHealthAddr), + BackendCAFile: os.Getenv(envBackendCAFile), + BackendServerName: os.Getenv(envBackendServerName), } if v := os.Getenv(envMaxConnections); v != "" { n, perr := strconv.Atoi(v) @@ -246,24 +260,39 @@ func (c *ProxyConfig) validateKMS() error { } // validateVaultAuth checks the Vault Transit settings: a KEK name plus exactly -// one auth mode — a static token, or an AppRole (both role id and secret id). +// one auth mode — a static token, an AppRole (both role id and secret id), or +// Kubernetes (a role, logging in with the pod ServiceAccount token). func (c *ProxyConfig) validateVaultAuth() error { if c.VaultKeyName == "" { return fmt.Errorf("config: %s requires %s", envVaultAddr, envVaultKeyName) } hasToken := c.VaultToken != "" hasAppRole := c.VaultRoleID != "" || c.VaultSecretID != "" + hasK8s := c.VaultKubernetesRole != "" switch { - case hasToken && hasAppRole: - return fmt.Errorf("config: set %s or AppRole (%s + %s), not both", envVaultToken, envVaultRoleID, envVaultSecretID) + case boolCount(hasToken, hasAppRole, hasK8s) > 1: + return fmt.Errorf("config: configure exactly one Vault auth mode — %s, AppRole (%s), or Kubernetes (%s)", + envVaultToken, envVaultRoleID, envVaultK8sRole) case hasAppRole && (c.VaultRoleID == "" || c.VaultSecretID == ""): return fmt.Errorf("config: AppRole auth requires both %s and %s", envVaultRoleID, envVaultSecretID) - case !hasToken && !hasAppRole: - return fmt.Errorf("config: %s requires %s or AppRole (%s + %s)", envVaultAddr, envVaultToken, envVaultRoleID, envVaultSecretID) + case !hasToken && !hasAppRole && !hasK8s: + return fmt.Errorf("config: %s requires one of %s, AppRole (%s), or Kubernetes (%s)", + envVaultAddr, envVaultToken, envVaultRoleID, envVaultK8sRole) } return nil } +// boolCount counts how many of the given conditions are true. +func boolCount(bs ...bool) int { + n := 0 + for _, b := range bs { + if b { + n++ + } + } + return n +} + // selectFields builds the FieldSet matching the KKP_FIELDS env value. func selectFields(value string) (*FieldSet, error) { switch value { diff --git a/internal/config/proxy_test.go b/internal/config/proxy_test.go index 3b78748..ca14454 100644 --- a/internal/config/proxy_test.go +++ b/internal/config/proxy_test.go @@ -72,6 +72,16 @@ func TestProxyConfigValidateVaultAppRole(t *testing.T) { } } +func TestProxyConfigValidateVaultKubernetes(t *testing.T) { + t.Parallel() + + c := vaultProxyConfig() + c.VaultKubernetesRole = "keycloak-proxy" + if err := c.Validate(); err != nil { + t.Fatalf("Validate rejected a valid Kubernetes-auth config: %v", err) + } +} + func TestProxyConfigValidateVaultAuthMisconfig(t *testing.T) { t.Parallel() @@ -79,6 +89,12 @@ func TestProxyConfigValidateVaultAuthMisconfig(t *testing.T) { "token and approle": func(c *ProxyConfig) { c.VaultToken, c.VaultRoleID, c.VaultSecretID = "t", "r", "s" }, + "token and kubernetes": func(c *ProxyConfig) { + c.VaultToken, c.VaultKubernetesRole = "t", "r" + }, + "approle and kubernetes": func(c *ProxyConfig) { + c.VaultRoleID, c.VaultSecretID, c.VaultKubernetesRole = "r", "s", "r" + }, "approle missing secret id": func(c *ProxyConfig) { c.VaultRoleID = "r" }, "no auth at all": func(c *ProxyConfig) {}, } diff --git a/internal/kms/vault.go b/internal/kms/vault.go index 5b4c69d..93f0a91 100644 --- a/internal/kms/vault.go +++ b/internal/kms/vault.go @@ -10,14 +10,20 @@ import ( "io" "log" "net/http" + "os" + "strings" "sync" "time" ) const ( - defaultTransitMount = "transit" - defaultAppRoleMount = "approle" - defaultVaultTimeout = 10 * time.Second + defaultTransitMount = "transit" + defaultAppRoleMount = "approle" + defaultKubernetesMount = "kubernetes" + // defaultKubernetesJWTFile is the projected ServiceAccount token mounted + // into every pod; Vault Kubernetes auth logs in with it. + defaultKubernetesJWTFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint:gosec // well-known path, not a credential + defaultVaultTimeout = 10 * time.Second // renewSkew re-authenticates this long before the cached token's TTL // expires, so an in-flight Wrap/Unwrap never races the expiry. renewSkew = 30 * time.Second @@ -44,6 +50,15 @@ type VaultConfig struct { // re-authenticates on demand when the issued token expires. RoleID string SecretID string + + // KubernetesRole selects Vault Kubernetes authentication: the KMS logs in + // with the pod's projected ServiceAccount token. This is the preferred + // production method — the token is short-lived and rotated by the kubelet, + // so no long-lived credential is stored. KubernetesMount defaults to + // "kubernetes" and KubernetesJWTFile to the standard in-pod token path. + KubernetesRole string + KubernetesMount string + KubernetesJWTFile string } // VaultKMS wraps and unwraps the DEK using Vault Transit's encrypt/decrypt @@ -173,56 +188,61 @@ func (v *VaultKMS) do(ctx context.Context, url string, body []byte, token string return resp.StatusCode, respBody, nil } +// tokenLogin authenticates to Vault and returns a fresh token and its expiry +// (zero expiry meaning the token does not expire). +type tokenLogin func(ctx context.Context) (string, time.Time, error) + // vaultAuth supplies the X-Vault-Token for Transit calls. It is either a fixed -// static token or an AppRole login that mints (and, on demand, re-mints) a -// short-lived token. +// static token or a login strategy (AppRole / Kubernetes) that mints (and, on +// demand, re-mints) a short-lived token. type vaultAuth struct { - client *http.Client - static string // set for static-token auth; empty for AppRole. - loginURL string // set for AppRole auth. - roleID string - secretID string + static string // set for static-token auth; loginFn is nil. + loginFn tokenLogin // set for AppRole / Kubernetes auth. mu sync.Mutex // guards token/expiry; never held across the login I/O. token string expiry time.Time // zero once logged in => the token does not expire. - loginMu sync.Mutex // serializes AppRole logins. + loginMu sync.Mutex // serializes logins. } -// newVaultAuth builds the auth strategy from cfg, enforcing exactly one mode. +// newVaultAuth builds the auth strategy from cfg, enforcing exactly one mode: +// a static token, AppRole, or Kubernetes. func newVaultAuth(cfg VaultConfig, client *http.Client) (*vaultAuth, error) { - approle := cfg.RoleID != "" || cfg.SecretID != "" + hasToken := cfg.Token != "" + hasAppRole := cfg.RoleID != "" || cfg.SecretID != "" + hasK8s := cfg.KubernetesRole != "" + switch { - case approle && cfg.Token != "": - return nil, errors.New("kms: set either a static token or AppRole credentials, not both") - case approle && (cfg.RoleID == "" || cfg.SecretID == ""): + case btoi(hasToken)+btoi(hasAppRole)+btoi(hasK8s) > 1: + return nil, errors.New("kms: configure exactly one Vault auth mode (token, AppRole, or Kubernetes)") + case hasAppRole && (cfg.RoleID == "" || cfg.SecretID == ""): return nil, errors.New("kms: AppRole auth requires both a role id and a secret id") - case !approle && cfg.Token == "": - return nil, errors.New("kms: a vault token or AppRole credentials are required") + case !hasToken && !hasAppRole && !hasK8s: + return nil, errors.New("kms: a vault token, AppRole, or Kubernetes auth is required") } - if !approle { - return &vaultAuth{client: client, static: cfg.Token}, nil - } - mount := cfg.AppRoleMount - if mount == "" { - mount = defaultAppRoleMount + switch { + case hasAppRole: + mount := orDefault(cfg.AppRoleMount, defaultAppRoleMount) + url := fmt.Sprintf("%s/v1/auth/%s/login", cfg.Address, mount) + return &vaultAuth{loginFn: approleLogin(client, url, cfg.RoleID, cfg.SecretID)}, nil + case hasK8s: + mount := orDefault(cfg.KubernetesMount, defaultKubernetesMount) + jwtFile := orDefault(cfg.KubernetesJWTFile, defaultKubernetesJWTFile) + url := fmt.Sprintf("%s/v1/auth/%s/login", cfg.Address, mount) + return &vaultAuth{loginFn: kubernetesLogin(client, url, cfg.KubernetesRole, jwtFile)}, nil + default: + return &vaultAuth{static: cfg.Token}, nil } - return &vaultAuth{ - client: client, - loginURL: fmt.Sprintf("%s/v1/auth/%s/login", cfg.Address, mount), - roleID: cfg.RoleID, - secretID: cfg.SecretID, - }, nil } -// currentToken returns a valid Vault token, logging in via AppRole when the -// cached token is missing or within renewSkew of expiry. The login network -// call is serialized by loginMu and never holds mu, so a caller whose token is -// still valid is never blocked by an in-flight login. +// currentToken returns a valid Vault token, logging in when the cached token is +// missing or within renewSkew of expiry. The login network call is serialized +// by loginMu and never holds mu, so a caller whose token is still valid is +// never blocked by an in-flight login. func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { - if a.loginURL == "" { + if a.loginFn == nil { return a.static, nil } if tok, ok := a.freshToken(); ok { @@ -242,7 +262,7 @@ func (a *vaultAuth) currentToken(ctx context.Context) (string, error) { if tok, ok := a.freshToken(); ok { return tok, nil } - tok, expiry, err := a.login(ctx) + tok, expiry, err := a.loginFn(ctx) if err != nil { // A proactive re-login (token still valid but within renewSkew) may // fail transiently; fall back to the still-usable token rather than @@ -291,7 +311,7 @@ func (a *vaultAuth) usableToken() (string, bool) { // token would force a needless extra login. It reports whether re-auth is // possible (false for static tokens). func (a *vaultAuth) reauth(stale string) bool { - if a.loginURL == "" { + if a.loginFn == nil { return false } a.mu.Lock() @@ -303,32 +323,69 @@ func (a *vaultAuth) reauth(stale string) bool { return true } -// login performs an AppRole login and returns the issued token and its expiry -// (zero expiry meaning the token does not expire). It does not mutate a, so it -// runs safely without holding mu during the network call. -func (a *vaultAuth) login(ctx context.Context) (string, time.Time, error) { - reqBody, err := json.Marshal(map[string]string{"role_id": a.roleID, "secret_id": a.secretID}) +// btoi returns 1 for true and 0 for false, to count how many auth modes are set. +func btoi(b bool) int { + if b { + return 1 + } + return 0 +} + +// orDefault returns v when non-empty, otherwise def. +func orDefault(v, def string) string { + if v == "" { + return def + } + return v +} + +// approleLogin returns a tokenLogin that authenticates with an AppRole +// role id / secret id. +func approleLogin(client *http.Client, url, roleID, secretID string) tokenLogin { + return func(ctx context.Context) (string, time.Time, error) { + return vaultLogin(ctx, client, url, map[string]string{"role_id": roleID, "secret_id": secretID}) + } +} + +// kubernetesLogin returns a tokenLogin that authenticates with the pod's +// projected ServiceAccount token. The token is re-read from disk on every +// login, so the kubelet's rotation is picked up without a restart. +func kubernetesLogin(client *http.Client, url, role, jwtFile string) tokenLogin { + return func(ctx context.Context) (string, time.Time, error) { + jwt, err := os.ReadFile(jwtFile) //nolint:gosec // path is operator-controlled config. + if err != nil { + return "", time.Time{}, fmt.Errorf("kms: read serviceaccount token: %w", err) + } + return vaultLogin(ctx, client, url, map[string]string{"role": role, "jwt": strings.TrimSpace(string(jwt))}) + } +} + +// vaultLogin posts an auth login request and returns the issued token and its +// expiry (zero expiry meaning the token does not expire). It touches no shared +// state, so it runs safely without holding a lock during the network call. +func vaultLogin(ctx context.Context, client *http.Client, url string, params map[string]string) (string, time.Time, error) { + reqBody, err := json.Marshal(params) if err != nil { - return "", time.Time{}, fmt.Errorf("kms: marshal approle login: %w", err) + return "", time.Time{}, fmt.Errorf("kms: marshal vault login: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.loginURL, bytes.NewReader(reqBody)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBody)) if err != nil { - return "", time.Time{}, fmt.Errorf("kms: build approle login: %w", err) + return "", time.Time{}, fmt.Errorf("kms: build vault login: %w", err) } req.Header.Set("Content-Type", "application/json") - resp, err := a.client.Do(req) + resp, err := client.Do(req) if err != nil { - return "", time.Time{}, fmt.Errorf("kms: approle login request: %w", err) + return "", time.Time{}, fmt.Errorf("kms: vault login request: %w", err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { - return "", time.Time{}, fmt.Errorf("kms: read approle login response: %w", err) + return "", time.Time{}, fmt.Errorf("kms: read vault login response: %w", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return "", time.Time{}, fmt.Errorf("kms: approle login returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) + return "", time.Time{}, fmt.Errorf("kms: vault login returned status %d: %s", resp.StatusCode, bytes.TrimSpace(body)) } var lr struct { @@ -338,10 +395,10 @@ func (a *vaultAuth) login(ctx context.Context) (string, time.Time, error) { } `json:"auth"` } if err := json.Unmarshal(body, &lr); err != nil { - return "", time.Time{}, fmt.Errorf("kms: decode approle login response: %w", err) + return "", time.Time{}, fmt.Errorf("kms: decode vault login response: %w", err) } if lr.Auth.ClientToken == "" { - return "", time.Time{}, errors.New("kms: approle login returned an empty client token") + return "", time.Time{}, errors.New("kms: vault login returned an empty client token") } var expiry time.Time diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index 6fb5ed6..04b175e 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -7,6 +7,8 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync" "testing" @@ -313,9 +315,11 @@ func TestVaultKMSAuthModeValidation(t *testing.T) { t.Parallel() cases := map[string]VaultConfig{ - "token and approle": {Address: "http://v", KeyName: "k", Token: "t", RoleID: "r", SecretID: "s"}, - "approle missing secret": {Address: "http://v", KeyName: "k", RoleID: "r"}, - "neither token nor approle": {Address: "http://v", KeyName: "k"}, + "token and approle": {Address: "http://v", KeyName: "k", Token: "t", RoleID: "r", SecretID: "s"}, + "token and kubernetes": {Address: "http://v", KeyName: "k", Token: "t", KubernetesRole: "r"}, + "approle and kubernetes": {Address: "http://v", KeyName: "k", RoleID: "r", SecretID: "s", KubernetesRole: "r"}, + "approle missing secret": {Address: "http://v", KeyName: "k", RoleID: "r"}, + "no auth mode": {Address: "http://v", KeyName: "k"}, } for name, cfg := range cases { if _, err := NewVaultKMS(cfg); err == nil { @@ -324,23 +328,116 @@ func TestVaultKMSAuthModeValidation(t *testing.T) { } } -func TestVaultAuthProactiveRenewFallback(t *testing.T) { +func TestVaultKMSKubernetesRoundTrip(t *testing.T) { t.Parallel() - // A login endpoint that always fails, to simulate a transient Vault error - // during a proactive re-login. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, `{"errors":["service unavailable"]}`, http.StatusServiceUnavailable) - })) + const ( + role = "keycloak-proxy" + jwt = "sa-jwt-token" + key = "kms-proxy" + prefix = "vault:v1:" + issued = "k8s-issued-token" + ) + // Write the JWT with a trailing newline to exercise the TrimSpace. + jwtFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(jwtFile, []byte(jwt+"\n"), 0o600); err != nil { + t.Fatalf("write jwt: %v", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/kubernetes/login", func(w http.ResponseWriter, r *http.Request) { + var req struct { + Role string `json:"role"` + JWT string `json:"jwt"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Role != role || req.JWT != jwt { + http.Error(w, `{"errors":["invalid role or jwt"]}`, http.StatusBadRequest) + return + } + writeJSON(w, map[string]any{"auth": map[string]any{"client_token": issued, "lease_duration": 3600}}) + }) + guard := func(w http.ResponseWriter, r *http.Request) bool { + if r.Header.Get("X-Vault-Token") != issued { + http.Error(w, `{"errors":["permission denied"]}`, http.StatusForbidden) + return false + } + return true + } + mux.HandleFunc("/v1/transit/encrypt/"+key, func(w http.ResponseWriter, r *http.Request) { + if !guard(w, r) { + return + } + var req struct { + Plaintext string `json:"plaintext"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + writeJSON(w, map[string]any{"data": map[string]string{"ciphertext": prefix + req.Plaintext}}) + }) + mux.HandleFunc("/v1/transit/decrypt/"+key, func(w http.ResponseWriter, r *http.Request) { + if !guard(w, r) { + return + } + var req struct { + Ciphertext string `json:"ciphertext"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + writeJSON(w, map[string]any{"data": map[string]string{"plaintext": strings.TrimPrefix(req.Ciphertext, prefix)}}) + }) + srv := httptest.NewServer(mux) t.Cleanup(srv.Close) + v, err := NewVaultKMS(VaultConfig{ + Address: srv.URL, + KeyName: key, + KubernetesRole: role, + KubernetesJWTFile: jwtFile, + HTTPClient: srv.Client(), + }) + if err != nil { + t.Fatalf("NewVaultKMS: %v", err) + } + wrapped, err := v.Wrap(context.Background(), []byte("dek")) + if err != nil { + t.Fatalf("Wrap: %v", err) + } + got, err := v.Unwrap(context.Background(), wrapped) + if err != nil { + t.Fatalf("Unwrap: %v", err) + } + if string(got) != "dek" { + t.Fatalf("round-trip mismatch: %q", got) + } +} + +func TestVaultKMSKubernetesMissingJWT(t *testing.T) { + t.Parallel() + + v, err := NewVaultKMS(VaultConfig{ + Address: "http://vault.invalid", + KeyName: "kms-proxy", + KubernetesRole: "keycloak-proxy", + KubernetesJWTFile: filepath.Join(t.TempDir(), "does-not-exist"), + }) + if err != nil { + t.Fatalf("NewVaultKMS: %v", err) + } + if _, err := v.Wrap(context.Background(), []byte("dek")); err == nil { + t.Fatal("Wrap succeeded with a missing ServiceAccount token file") + } +} + +func TestVaultAuthProactiveRenewFallback(t *testing.T) { + t.Parallel() + + // login always fails, simulating a transient Vault error during a + // proactive re-login while the cached token is still valid. a := &vaultAuth{ - client: srv.Client(), - loginURL: srv.URL + "/v1/auth/approle/login", - roleID: "r", - secretID: "s", - token: "still-valid", - expiry: time.Now().Add(renewSkew / 2), // inside renewSkew, but not expired. + loginFn: func(context.Context) (string, time.Time, error) { + return "", time.Time{}, fmt.Errorf("transient vault error") + }, + token: "still-valid", + expiry: time.Now().Add(renewSkew / 2), // inside renewSkew, but not expired. } got, err := a.currentToken(context.Background()) if err != nil { @@ -356,16 +453,10 @@ func TestVaultAuthReauthNoStaleToken(t *testing.T) { // After a hard 403 the token is cleared (reauth); a failing re-login must // surface the error rather than returning a stale token. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, `{"errors":["permission denied"]}`, http.StatusForbidden) - })) - t.Cleanup(srv.Close) - a := &vaultAuth{ - client: srv.Client(), - loginURL: srv.URL + "/v1/auth/approle/login", - roleID: "r", - secretID: "s", + loginFn: func(context.Context) (string, time.Time, error) { + return "", time.Time{}, fmt.Errorf("login denied") + }, } if _, err := a.currentToken(context.Background()); err == nil { t.Fatal("currentToken should fail when login fails and no token is cached") From fc05cc8d0c5689365938bd1dd7aae8c02e151d60 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Wed, 8 Jul 2026 14:05:31 +0500 Subject: [PATCH 8/8] test(kms): prove the Kubernetes SA token is re-read on each login Lock in the advertised kubelet-rotation behavior: rotate the on-disk JWT and revoke the cached token between two Wraps, then assert the forced re-login sends the rotated token rather than a cached one. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- internal/kms/vault_test.go | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go index 04b175e..225c969 100644 --- a/internal/kms/vault_test.go +++ b/internal/kms/vault_test.go @@ -491,5 +491,95 @@ func TestVaultKMSAppRoleConcurrent(t *testing.T) { } } +func TestVaultKMSKubernetesRereadsToken(t *testing.T) { + t.Parallel() + + const ( + role = "kc" + key = "kms-proxy" + prefix = "vault:v1:" + ) + jwtFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(jwtFile, []byte("jwt-v1"), 0o600); err != nil { + t.Fatalf("write jwt: %v", err) + } + + var mu sync.Mutex + var seenJWTs []string + issued := 0 + live := map[string]bool{} // tokens the transit endpoints currently accept + + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/kubernetes/login", func(w http.ResponseWriter, r *http.Request) { + var req struct { + Role string `json:"role"` + JWT string `json:"jwt"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Role != role { + http.Error(w, `{"errors":["invalid role"]}`, http.StatusBadRequest) + return + } + mu.Lock() + seenJWTs = append(seenJWTs, req.JWT) + issued++ + tok := fmt.Sprintf("k8s-tok-%d", issued) + live[tok] = true + mu.Unlock() + writeJSON(w, map[string]any{"auth": map[string]any{"client_token": tok, "lease_duration": 3600}}) + }) + guard := func(w http.ResponseWriter, r *http.Request) bool { + mu.Lock() + ok := live[r.Header.Get("X-Vault-Token")] + mu.Unlock() + if !ok { + http.Error(w, `{"errors":["permission denied"]}`, http.StatusForbidden) + } + return ok + } + mux.HandleFunc("/v1/transit/encrypt/"+key, func(w http.ResponseWriter, r *http.Request) { + if !guard(w, r) { + return + } + var req struct { + Plaintext string `json:"plaintext"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + writeJSON(w, map[string]any{"data": map[string]string{"ciphertext": prefix + req.Plaintext}}) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + v, err := NewVaultKMS(VaultConfig{ + Address: srv.URL, KeyName: key, KubernetesRole: role, + KubernetesJWTFile: jwtFile, HTTPClient: srv.Client(), + }) + if err != nil { + t.Fatalf("NewVaultKMS: %v", err) + } + + // First Wrap logs in with jwt-v1. + if _, err := v.Wrap(context.Background(), []byte("dek")); err != nil { + t.Fatalf("first Wrap: %v", err) + } + // Revoke the issued token and rotate the on-disk JWT, as the kubelet would. + mu.Lock() + live = map[string]bool{} + mu.Unlock() + if err := os.WriteFile(jwtFile, []byte("jwt-v2"), 0o600); err != nil { + t.Fatalf("rotate jwt: %v", err) + } + // The cached token now 403s; the re-login must re-read the rotated JWT. + if _, err := v.Wrap(context.Background(), []byte("dek")); err != nil { + t.Fatalf("second Wrap should re-login with the rotated token: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(seenJWTs) < 2 || seenJWTs[0] != "jwt-v1" || seenJWTs[len(seenJWTs)-1] != "jwt-v2" { + t.Fatalf("expected logins to re-read jwt-v1 then jwt-v2, saw %v", seenJWTs) + } +} + // VaultKMS must satisfy the KMS interface. var _ KMS = (*VaultKMS)(nil)