diff --git a/.gitignore b/.gitignore index e2baf11..8b3fdd5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,12 @@ docker/dev/tls/ # were live minutes ago. docker/dev/.recovery-keys.json* +# vault-unseal's own unseal key and root token. It is the root of +# trust for the local profile — the Transit key every node unseals +# against — so this file is the most sensitive one the bootstrap +# writes, and the one whose loss costs the whole cluster. +docker/dev/.unseal-keys.json* + # Vault Agent credentials, written by scripts/bootstrap-agent.sh. # role_id is not a secret; secret_id is, and Agent deletes it after # reading it anyway. Neither belongs in the repository. diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..8d75728 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,42 @@ +# gitleaks configuration. +# +# Until this file existed gitleaks ran on its defaults, which is what you +# want almost all of the time — this exists to record one accepted +# finding, not to relax the ruleset. `useDefault` keeps every default +# rule; nothing below removes one. +# +# The same rule applies here as in .trivyignore.yaml: an allowlist entry +# states what it permits and why. A suppression without a justification +# is indistinguishable from never having run the scanner, and gitleaks +# scans full history, so anything muted here stays muted for every commit +# that follows. + +title = "vault-reference-platform" + +[extend] +useDefault = true + +[allowlist] +description = """ +The names of two JSON fields, which are not secrets. + +Vault returns key shares under `unseal_keys_b64` (Shamir) or +`recovery_keys_b64` (auto-unsealed), and scripts/rotate-keys.sh selects +between them with a variable: + + KEY_FIELD="unseal_keys_b64" + +generic-api-key reads that as KEY = and reports it, +with an entropy of 3.5 for a string that is documented API vocabulary and +appears verbatim in Vault's own documentation. + +The allowlist is anchored to those two exact strings rather than to the +file or the rule, so a real credential on the same line, in the same +file, or assigned to the same variable is still reported. Shares +themselves are ~44 characters of base64 and match nothing below. +""" +regexTarget = "secret" +regexes = [ + '''^unseal_keys_b64$''', + '''^recovery_keys_b64$''', +] diff --git a/CLAUDE.md b/CLAUDE.md index 959d38e..8e79af9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,11 +122,15 @@ Log output goes to stderr; the root token is the only thing on stdout, so `ROOT_TOKEN=$(./scripts/bootstrap-dev-cluster.sh)` works. Preserve that split when editing. -The recovery keys go to `docker/dev/.recovery-keys.json` (0600, -gitignored) rather than stdout. With a seal stanza `operator init` -returns those instead of unseal keys, and they are what `generate-root` -and `rekey` need — the script used to discard them, which made revoking -the root token a one-way door. +Two key files go to `docker/dev/` (0600, gitignored) rather than stdout: +`.recovery-keys.json` for the cluster, and `.unseal-keys.json` for +`vault-unseal` itself — that one is the root of trust for the profile, +and without it a restart of that single container ends the cluster. + +With a seal stanza `operator init` returns recovery keys instead of +unseal keys, and they are what `generate-root` and `rekey` need. The +script used to discard both sets, which made revoking the root token a +one-way door and a restart of vault-unseal an unrecoverable one. ## The four profiles @@ -383,7 +387,13 @@ CI enforces several invariants worth knowing before you push: go in `.trivyignore.yaml` *with the reason* — an unjustified suppression is indistinguishable from never having run the scanner. - **gitleaks scans full history** (`fetch-depth: 0`). A secret committed - and later removed is still leaked. + and later removed is still leaked. Accepted findings go in + `.gitleaks.toml`, with the reason, on the same terms as + `.trivyignore.yaml` — and anchored to the exact string rather than to a + file or a rule, because anything muted there stays muted for every + commit after it. The one entry today is two JSON field names, + `unseal_keys_b64` and `recovery_keys_b64`, which `generic-api-key` + reads as high-entropy values assigned to something called KEY. - **Every alert rule needs a severity route of its own** and every freshness alert needs a paired `absent()` alert; the alerting and alert-routing suites fail if a new one arrives without its partner. A diff --git a/docs/roadmap.md b/docs/roadmap.md index 307bea0..392803b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -373,11 +373,26 @@ not a property the architecture has to have on the day it is stood up. will not auto-unseal even with a working seal stanza, which makes the obvious way to verify the migration the thing that breaks it. - What remains needs something this repository does not have. Migrating - between two *cloud* KMS providers — the case where an organisation - changes cloud — has the same shape and no coverage, and neither does a - Shamir **rekey**: the local profile is Transit-sealed, so its shares - are recovery keys and `rotate-keys.sh` exercises that path only. + The Shamir rekey came off this list too. `rotate-keys.sh + --unseal-keys` runs the same ceremony against `sys/rekey` rather than + `sys/rekey-recovery-key`, and `tests/key-rotation` exercises it against + `vault-unseal` — rekeying it from the 1-of-1 the bootstrap creates to + 5-of-3, then again, and requiring that a full quorum of the superseded + generation no longer opens it. One share proves nothing there: Vault + accepts shares and only validates the combination at the threshold, so + a lone stale share returns success. + + That work needed a prior fix, the same shape as the recovery keys in + v0.15. `vault-unseal` is the root of trust for the whole local profile + and its unseal key lived only in a shell variable, so a restart of that + one container ended the cluster — it came back sealed with nobody + holding the key, and cluster nodes restarted afterwards failed to start + rather than coming back sealed. The keys are kept now, and the suite + restarts `vault-unseal` on every run to prove it. + + What remains needs something this repository does not have: migrating + between two *cloud* KMS providers, the case where an organisation + changes cloud, which has the same shape and no coverage. - **Restore verification at a real cloud destination.** The mechanism came off this list in v0.16. `tests/restore-from-object-store` puts a real snapshot of a real cluster through an S3 API, reads the object diff --git a/docs/security.md b/docs/security.md index a7f53e4..6ae376e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -314,6 +314,57 @@ shares are dead, and if the new ones were not captured, nobody can generate a root token or unseal by recovery again — discovered in the emergency where you needed them. +### Recovery keys and unseal keys are the same ceremony + +Which kind a Vault has depends only on how it is sealed. An auto-unsealed +cluster has recovery keys; a Shamir-sealed one has unseal keys; +`scripts/migrate-seal.sh` turns each into the other without changing +their values. So one script covers both, against two endpoints: + +| Flag | Endpoint | CLI | +|---|---|---| +| `--recovery-keys` | `sys/rekey-recovery-key/*` | `vault operator rekey -target=recovery` | +| `--unseal-keys` | `sys/rekey/*` | `vault operator rekey` | + +The CLI calls the second one *barrier* and makes it the default target, +which is worth knowing: `vault operator rekey` with no arguments, run +against an auto-unsealed cluster, addresses a set of keys that cluster +does not use. + +```bash +export VAULT_ADDR=https://127.0.0.1:8300 # vault-unseal +export VAULT_TOKEN=$(jq -r .root_token docker/dev/.unseal-keys.json) + +./scripts/rotate-keys.sh --unseal-keys \ + --keys-file docker/dev/.unseal-keys.json --shares 5 --threshold 3 +``` + +### The local root of trust keeps its own key now + +`vault-unseal` is a Shamir-sealed Vault holding the Transit key every +cluster node auto-unseals against. Its unseal key used to live in a shell +variable inside `bootstrap-dev-cluster.sh` and nowhere else. + +That made a single `docker compose restart vault-unseal` unrecoverable, +which is not hypothetical — it is what a Docker Desktop restart or a host +reboot does. And the failure is not graceful. `vault-unseal` comes back +sealed, and a cluster node restarted after that does not come back +sealed; it fails to start: + +```text +error parsing Seal configuration: ... 503 * Vault is sealed +``` + +with no key anywhere to fix it. The only way back was `make destroy`. + +The bootstrap writes `docker/dev/.unseal-keys.json` (0600, gitignored) +now, holding the shares and the root token, and `tests/key-rotation` +restarts `vault-unseal` on every run to prove the kept keys open it and +that a cluster node auto-unseals against it afterwards. + +It is also what makes a Shamir rekey testable at all: rekeying needs a +quorum of the current shares, and there were none. + ### Why the rekey is scripted rather than documented Vault has a verification phase for exactly this risk: the new shares are diff --git a/scripts/bootstrap-dev-cluster.sh b/scripts/bootstrap-dev-cluster.sh index 09e8c96..7f05a59 100755 --- a/scripts/bootstrap-dev-cluster.sh +++ b/scripts/bootstrap-dev-cluster.sh @@ -257,6 +257,34 @@ INIT_JSON="$(compose exec -T vault-unseal vault operator init -key-shares=1 -key UNSEAL_KEY="$(jq -r '.unseal_keys_b64[0]' <<< "$INIT_JSON")" UNSEAL_ROOT_TOKEN="$(jq -r '.root_token' <<< "$INIT_JSON")" +# Kept, for the same reason the cluster's recovery keys are kept a few +# hundred lines below, and with more at stake. +# +# vault-unseal is the root of trust for the whole local profile: it holds +# the Transit key every cluster node auto-unseals against. Until now its +# unseal key lived in this shell variable and nowhere else, which made a +# single `docker compose restart vault-unseal` unrecoverable — and that +# is not a hypothetical, it is what a Docker Desktop restart or a host +# reboot does. +# +# The failure is not graceful. vault-unseal comes back sealed, and a +# cluster node restarted after that does not come back sealed, it fails +# to start at all: +# +# error parsing Seal configuration: ... 503 * Vault is sealed +# +# with no key anywhere to fix it. The only route back was `make destroy`, +# which is to say: losing everything in the cluster because the container +# providing its seal was restarted. +UNSEAL_KEYS_FILE="${COMPOSE_DIR}/.unseal-keys.json" +rm -f "$UNSEAL_KEYS_FILE" +( umask 077; jq '{unseal_keys_b64, unseal_keys_shares: .unseal_shares, + unseal_threshold: .unseal_threshold, root_token}' \ + <<< "$INIT_JSON" > "$UNSEAL_KEYS_FILE" ) +chmod 0600 "$UNSEAL_KEYS_FILE" +log "vault-unseal keys written to docker/dev/.unseal-keys.json (mode 0600)." +log " Without them a restart of vault-unseal takes the cluster with it." + log "Unsealing vault-unseal..." compose exec -T vault-unseal vault operator unseal "$UNSEAL_KEY" >/dev/null diff --git a/scripts/rotate-keys.sh b/scripts/rotate-keys.sh index 32df851..60eb335 100755 --- a/scripts/rotate-keys.sh +++ b/scripts/rotate-keys.sh @@ -10,12 +10,17 @@ # Options: # --barrier Rotate the barrier encryption key. Online, no # shares needed, no downtime. -# --recovery-keys Re-issue the recovery key shares. Needs a -# quorum of the current ones. -# --keys-file JSON holding recovery_keys_b64. Read for the -# current shares; replaced with the new ones -# once they are verified. The previous file is -# kept as .superseded. +# --recovery-keys Re-issue the recovery key shares of an +# auto-unsealed cluster. Needs a quorum of the +# current ones. +# --unseal-keys Re-issue the unseal key shares of a +# Shamir-sealed Vault. Same ceremony, different +# endpoint; see below. +# --keys-file JSON holding the current shares — +# recovery_keys_b64 with --recovery-keys, +# unseal_keys_b64 with --unseal-keys. Replaced +# with the new ones once they are verified; the +# previous file is kept as .superseded. # --shares New share count (default: keep the current). # --threshold New threshold (default: keep the current). # --no-verify Skip Vault's rekey verification. Refuses @@ -94,8 +99,23 @@ # Shares go to files, never to stdout. A recovery share in a CI log or # a scrollback buffer is a compromised share. # +# RECOVERY KEYS AND UNSEAL KEYS ARE THE SAME CEREMONY +# +# Which one a Vault has depends only on how it is sealed. An +# auto-unsealed cluster has recovery keys; a Shamir-sealed one has unseal +# keys; scripts/migrate-seal.sh turns each into the other without +# changing their values. So this runs one ceremony against two endpoints: +# +# --recovery-keys sys/rekey-recovery-key/* vault operator rekey -target=recovery +# --unseal-keys sys/rekey/* vault operator rekey +# +# The CLI calls the second one "barrier" and makes it the default target, +# which is worth knowing because `vault operator rekey` with no arguments +# on an auto-unsealed cluster addresses a set of keys that cluster does +# not use. +# # Requirements: vault, jq. VAULT_ADDR and a token with sudo on -# sys/rekey-recovery-key and sys/rotate. +# sys/rekey-recovery-key or sys/rekey, and on sys/rotate. set -euo pipefail @@ -119,6 +139,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --barrier) MODE="barrier"; shift ;; --recovery-keys) MODE="recovery"; shift ;; + --unseal-keys) MODE="unseal"; shift ;; --keys-file) KEYS_FILE="$2"; shift 2 ;; --shares) SHARES="$2"; shift 2 ;; --threshold) THRESHOLD="$2"; shift 2 ;; @@ -129,7 +150,8 @@ while [[ $# -gt 0 ]]; do esac done -[[ -n "$MODE" ]] || die "one of --barrier or --recovery-keys is required" +[[ -n "$MODE" ]] \ + || die "one of --barrier, --recovery-keys or --unseal-keys is required" for dep in vault jq; do command -v "$dep" >/dev/null 2>&1 || die "${dep} not found on PATH" done @@ -164,7 +186,7 @@ fi # The recovery key shares # --------------------------------------------------------------------------- -[[ -n "$KEYS_FILE" ]] || die "--recovery-keys needs --keys-file" +[[ -n "$KEYS_FILE" ]] || die "--${MODE}-keys needs --keys-file" [[ -f "$KEYS_FILE" ]] || die "keys file not found: ${KEYS_FILE}" if [[ "$VERIFY" == false && "$ACKNOWLEDGED" == false ]]; then @@ -176,14 +198,37 @@ if [[ "$VERIFY" == false && "$ACKNOWLEDGED" == false ]]; then well if that is genuinely what you want." fi -mapfile -t OLD_KEYS < <(jq -r '.recovery_keys_b64[]? // empty' "$KEYS_FILE") -[[ ${#OLD_KEYS[@]} -gt 0 ]] || die "no recovery_keys_b64 found in ${KEYS_FILE}" +# The only differences between the two ceremonies, in one place. Getting +# these crossed addresses a set of keys the Vault in front of you does +# not use, and the error it produces says nothing about which set. +if [[ "$MODE" == "recovery" ]]; then + REKEY_PATH="sys/rekey-recovery-key" + TARGET_ARGS=(-target=recovery) + KEY_FIELD="recovery_keys_b64" + SHARES_FIELD="recovery_keys_shares" + THRESHOLD_FIELD="recovery_keys_threshold" + KIND="recovery" +else + REKEY_PATH="sys/rekey" + TARGET_ARGS=() + KEY_FIELD="unseal_keys_b64" + SHARES_FIELD="unseal_keys_shares" + THRESHOLD_FIELD="unseal_threshold" + KIND="unseal" +fi + +mapfile -t OLD_KEYS < <(jq -r ".${KEY_FIELD}[]? // empty" "$KEYS_FILE") +[[ ${#OLD_KEYS[@]} -gt 0 ]] || die "no ${KEY_FIELD} found in ${KEYS_FILE}. + + That file holds the other kind of key. An auto-unsealed cluster + has recovery keys and a Shamir-sealed one has unseal keys; check + which this Vault is with 'vault status'." # Default to the shape already in use rather than Vault's defaults. # Silently turning a 5-of-3 into Vault's default hands back a different # number of shares than the holders expect, and nobody counts them. -CUR_SHARES="$(jq -r '.recovery_keys_shares // empty' "$KEYS_FILE")" -CUR_THRESHOLD="$(jq -r '.recovery_keys_threshold // empty' "$KEYS_FILE")" +CUR_SHARES="$(jq -r ".${SHARES_FIELD} // empty" "$KEYS_FILE")" +CUR_THRESHOLD="$(jq -r ".${THRESHOLD_FIELD} // empty" "$KEYS_FILE")" SHARES="${SHARES:-${CUR_SHARES:-${#OLD_KEYS[@]}}}" THRESHOLD="${THRESHOLD:-${CUR_THRESHOLD:-3}}" @@ -194,14 +239,14 @@ THRESHOLD="${THRESHOLD:-${CUR_THRESHOLD:-3}}" NEW_FILE="${KEYS_FILE}.new" -vault operator rekey -target=recovery -cancel >/dev/null 2>&1 || true +vault operator rekey "${TARGET_ARGS[@]+"${TARGET_ARGS[@]}"}" -cancel >/dev/null 2>&1 || true INIT_BODY="$(jq -n --argjson shares "$SHARES" --argjson threshold "$THRESHOLD" \ --argjson verify "$VERIFY" \ '{secret_shares: $shares, secret_threshold: $threshold, require_verification: $verify}')" INIT_JSON="$(printf '%s' "$INIT_BODY" \ - | vault write -format=json sys/rekey-recovery-key/init - 2>&1)" \ + | vault write -format=json "${REKEY_PATH}/init" - 2>&1)" \ || die "could not start a rekey: ${INIT_JSON}" NONCE="$(jq -r '.data.nonce // .nonce // empty' <<< "$INIT_JSON")" @@ -216,22 +261,22 @@ if [[ "$VERIFY" == true ]]; then || die "asked for verification and Vault did not enable it: ${INIT_JSON}" fi -log "Started a recovery rekey (nonce ${NONCE}), ${THRESHOLD} of ${SHARES}." +log "Started a ${KIND}-key rekey (nonce ${NONCE}), ${THRESHOLD} of ${SHARES}." OUT="" USED=0 for KEY in "${OLD_KEYS[@]}"; do [[ -n "$KEY" ]] || continue USED=$((USED + 1)) - OUT="$(vault operator rekey -target=recovery -nonce="$NONCE" -format=json "$KEY" 2>&1)" || { - vault operator rekey -target=recovery -cancel >/dev/null 2>&1 || true + OUT="$(vault operator rekey "${TARGET_ARGS[@]+"${TARGET_ARGS[@]}"}" -nonce="$NONCE" -format=json "$KEY" 2>&1)" || { + vault operator rekey "${TARGET_ARGS[@]+"${TARGET_ARGS[@]}"}" -cancel >/dev/null 2>&1 || true die "share ${USED} was rejected: ${OUT}" } [[ "$(jq -r '.complete // false' <<< "$OUT" 2>/dev/null)" == "true" ]] && break done if [[ "$(jq -r '.complete // false' <<< "$OUT" 2>/dev/null)" != "true" ]]; then - vault operator rekey -target=recovery -cancel >/dev/null 2>&1 || true + vault operator rekey "${TARGET_ARGS[@]+"${TARGET_ARGS[@]}"}" -cancel >/dev/null 2>&1 || true die "ran out of shares after ${USED} without reaching the threshold" fi @@ -246,7 +291,7 @@ mapfile -t NEW_KEYS < <(jq -r '.keys_base64[]? // empty' <<< "$OUT") ( umask 077; jq -n \ --argjson keys "$(jq -n '$ARGS.positional' --args "${NEW_KEYS[@]}")" \ --argjson shares "$SHARES" --argjson threshold "$THRESHOLD" \ - '{recovery_keys_b64: $keys, recovery_keys_shares: $shares, recovery_keys_threshold: $threshold}' \ + --arg kf "$KEY_FIELD" --arg sf "$SHARES_FIELD" --arg tf "$THRESHOLD_FIELD" '{($kf): $keys, ($sf): $shares, ($tf): $threshold}' \ > "$NEW_FILE" ) || die "could not write the new shares to ${NEW_FILE}" chmod 0600 "$NEW_FILE" log "New shares written to ${NEW_FILE} (0600), before verification." @@ -267,7 +312,7 @@ if [[ "$VERIFY" == true ]]; then [[ "$V_USED" -ge "$THRESHOLD" ]] && break V_USED=$((V_USED + 1)) - V_OUT="$(vault operator rekey -target=recovery -verify -nonce="$V_NONCE" \ + V_OUT="$(vault operator rekey "${TARGET_ARGS[@]+"${TARGET_ARGS[@]}"}" -verify -nonce="$V_NONCE" \ -format=json "$KEY" 2>&1)" || { die "the new share ${V_USED} was rejected during verification: ${V_OUT} @@ -300,6 +345,6 @@ mv "$NEW_FILE" "$KEYS_FILE" \ || die "could not install the new keys file; they are in ${NEW_FILE}" chmod 0600 "$KEYS_FILE" -log "Wrote ${SHARES} new shares to ${KEYS_FILE} (0600)." +log "Wrote ${SHARES} new ${KIND} shares to ${KEYS_FILE} (0600)." log "The previous shares are at ${KEYS_FILE}.superseded and no longer work." log "Distribute the new shares and delete both copies from this host." diff --git a/tests/key-rotation/README.md b/tests/key-rotation/README.md index 4b9ff2f..9cd2db7 100644 --- a/tests/key-rotation/README.md +++ b/tests/key-rotation/README.md @@ -1,6 +1,7 @@ # tests/key-rotation -Rotating the barrier key, and re-issuing the recovery key shares. +Rotating the barrier key, and re-issuing both kinds of key share — +the cluster's recovery keys and `vault-unseal`'s unseal keys. ```bash ./tests/key-rotation/run-tests.sh @@ -18,6 +19,19 @@ the shares Vault issued actually replaced the ones it had. Only Vault can answer that, so the suite asks it the only way that means anything: by requiring the superseded shares to stop working. +## Three things, not two + +| | | +|---|---| +| The barrier key | Online, no shares, cannot lock anyone out | +| Recovery keys | The cluster's, because it is auto-unsealed | +| Unseal keys | `vault-unseal`'s, because it is Shamir-sealed | + +The last two are the same ceremony against different endpoints — +`sys/rekey-recovery-key` and `sys/rekey` — and which kind a Vault has +depends only on how it is sealed. `scripts/migrate-seal.sh` turns each +into the other without changing their values. + ## The two halves are not the same risk **The barrier key** is online, needs no shares and cannot lock anyone @@ -31,13 +45,47 @@ shares are dead, and if the new ones were not captured, nobody can generate a root token again — discovered in the emergency where you needed them. +## The root of trust keeps its own key now + +`vault-unseal` holds the Transit key every cluster node auto-unseals +against, and its unseal key used to live in a shell variable inside +`bootstrap-dev-cluster.sh` and nowhere else. A single +`docker compose restart vault-unseal` was therefore unrecoverable — and +that is what a Docker Desktop restart or a host reboot does. + +The failure is not graceful. `vault-unseal` comes back sealed, and a +cluster node restarted afterwards does not come back sealed; it fails to +start, with `error parsing Seal configuration: ... 503 Vault is sealed`, +and no key anywhere to fix it. + +This suite restarts `vault-unseal` on every run and requires that the +kept keys open it, and that a cluster node auto-unseals against it +afterwards. It is also what makes the Shamir rekey testable: rekeying +needs a quorum of the current shares, and before this there were none. + +## One stale share proves nothing + +The suite rekeys `vault-unseal` **twice** — 1-of-1 to 5-of-3, then again +— so that a full quorum of the superseded generation exists to try. + +That is not thoroughness for its own sake. Vault accepts unseal shares +and only validates the combination once the threshold is reached, so +submitting a single stale share returns success and 1/3 progress. An +assertion built on one old share would pass whether or not the rekey did +anything, and would also leave that progress behind to break the next +unseal. + ## What a green run does not mean -Seal migration is not covered. Moving a cluster between seal types with -`-migrate` is the operation most likely to produce a cluster that will -not unseal, and nothing here exercises it. Neither is a Shamir rekey: -this cluster uses a Transit seal, so its shares are recovery keys and the -unseal-key path has no coverage. +Seal migration is not covered here. Moving a cluster between seal types +with `-migrate` is the operation most likely to produce a cluster that +will not unseal, and it has a suite of its own in +`tests/seal-migration`. + +The Shamir path this suite does exercise is `vault-unseal`'s, which is +a single node. A rekey of a Shamir-sealed *cluster* — several nodes, +each needing the new shares before any of them can start — is not +covered by anything here. ## Three things Vault does that a script has to know @@ -69,9 +117,20 @@ Every row was watched to fail. | K2 | Only a JSON `.complete` counts as verified — the bug above, restored | and verified the new shares before committing them; and the new shares can; the previous shares were kept alongside (8 in total) | | K3 | New shares read from `keys_b64` instead of `keys_base64` | rotate-keys.sh --recovery-keys succeeds; the shares in the keys file changed | | K4 | Verification is never requested at init | and verified the new shares before committing them; the shares in the keys file changed | +| U1 | `vault-unseal`'s keys are not kept, as before this release | the bootstrap kept vault-unseal's own keys; and the kept keys open it again | +| U2 | `--unseal-keys` addresses the recovery endpoint | rotate-keys.sh --unseal-keys rekeys 1-of-1 to 5-of-3 (5 in total) | +| U3 | Unseal mode reads `recovery_keys_b64` | rotate-keys.sh --unseal-keys rekeys 1-of-1 to 5-of-3 (5 in total) | K2 is the one worth reading. It breaks nothing in the code the assertions name — the rekey still runs, Vault still accepts every share, the cluster stays healthy — and it takes eight assertions with it, because the shares the operator is left holding are not the shares the cluster now wants. That is the shape of the failure this suite exists for. + +U3 is a row worth reading for what it did *not* catch. It was aimed at +the two assertions about handing the wrong key file to the wrong mode, +and neither of them moved — the run was refused anyway, for a different +reason, and the refusal happened to satisfy them. What caught it was the +rekey failing outright. The row records what actually went red rather +than what the mutation was aimed at, because the second is a guess and +the first is an observation. diff --git a/tests/key-rotation/run-tests.sh b/tests/key-rotation/run-tests.sh index f70a7c7..3ff5be0 100755 --- a/tests/key-rotation/run-tests.sh +++ b/tests/key-rotation/run-tests.sh @@ -74,7 +74,9 @@ cleanup() { info "Tearing down..." "${COMPOSE[@]}" --profile spare down -v >/dev/null 2>&1 fi - rm -f "${KEYS_FILE}.superseded" "${KEYS_FILE}.new" + rm -f "${KEYS_FILE}.superseded" "${KEYS_FILE}.new" \ + "${REPO_ROOT}/docker/dev/.unseal-keys.json.superseded" \ + "${REPO_ROOT}/docker/dev/.unseal-keys.json.new" rm -rf "$WORK" exit "$rc" } @@ -333,6 +335,203 @@ for f in ".recovery-keys.json" ".recovery-keys.json.superseded" ".recovery-keys. fi done +# --------------------------------------------------------------------------- +info "" +info "=== vault-unseal keeps its own key, and can rekey it ===" +# --------------------------------------------------------------------------- +# The cluster above is auto-unsealed, so its shares are recovery keys and +# everything so far exercised that path only. vault-unseal is the +# Shamir-sealed Vault underneath it — the root of trust the whole local +# profile leans on — and it has a different endpoint, different field +# names and, until this release, no persisted key at all. + +UNSEAL_KEYS="${REPO_ROOT}/docker/dev/.unseal-keys.json" + +if [[ -f "$UNSEAL_KEYS" ]]; then + ok "the bootstrap kept vault-unseal's own keys" +else + bad "the bootstrap kept vault-unseal's own keys" \ + "without them a restart of vault-unseal is unrecoverable" +fi + +if [[ "$(stat -c '%a' "$UNSEAL_KEYS" 2>/dev/null)" == "600" ]]; then + ok "and wrote them 0600" +else + bad "and wrote them 0600" "$(stat -c '%a' "$UNSEAL_KEYS" 2>/dev/null)" +fi + +if [[ "$(jq -r '.unseal_keys_b64 | length' "$UNSEAL_KEYS" 2>/dev/null)" -ge 1 ]] \ + && [[ "$(jq -r '.root_token // empty' "$UNSEAL_KEYS" 2>/dev/null)" == hvs.* ]]; then + ok "and both the shares and its root token are there" +else + bad "and both the shares and its root token are there" \ + "a share without the token cannot rekey; a token without the share cannot unseal" +fi + +# The failure this fixes, demonstrated rather than described. Before the +# keys were kept, this restart ended the cluster: vault-unseal came back +# sealed with nobody holding its key, and a cluster node restarted after +# it did not come back sealed — it failed to start, with +# "error parsing Seal configuration: ... 503 Vault is sealed". +info " restarting vault-unseal (previously unrecoverable)..." +"${COMPOSE[@]}" restart vault-unseal >/dev/null 2>&1 +for _ in $(seq 1 30); do + curl -sk --cacert "$VAULT_CACERT" --max-time 2 \ + https://127.0.0.1:8300/v1/sys/seal-status >/dev/null 2>&1 && break + sleep 2 +done + +UNSEAL_SEALED="$(curl -sk --cacert "$VAULT_CACERT" --max-time 4 \ + https://127.0.0.1:8300/v1/sys/seal-status 2>/dev/null | jq -r '.sealed')" +if [[ "$UNSEAL_SEALED" == "true" ]]; then + ok "a restarted vault-unseal comes back sealed" +else + bad "a restarted vault-unseal comes back sealed" \ + "sealed=${UNSEAL_SEALED}; if it never seals, the keys below prove nothing" +fi + +mapfile -t U_KEYS < <(jq -r '.unseal_keys_b64[]' "$UNSEAL_KEYS") +U_THRESHOLD="$(jq -r '.unseal_threshold // 1' "$UNSEAL_KEYS")" +for ((i = 0; i < U_THRESHOLD; i++)); do + "${COMPOSE[@]}" exec -T vault-unseal vault operator unseal "${U_KEYS[$i]}" >/dev/null 2>&1 +done +UNSEAL_SEALED="$(curl -sk --cacert "$VAULT_CACERT" --max-time 4 \ + https://127.0.0.1:8300/v1/sys/seal-status 2>/dev/null | jq -r '.sealed')" +if [[ "$UNSEAL_SEALED" == "false" ]]; then + ok "and the kept keys open it again" +else + bad "and the kept keys open it again" "sealed=${UNSEAL_SEALED}" +fi + +# And the reason it matters: the cluster leans on it. +info " restarting vault-0 to prove auto-unseal still works..." +"${COMPOSE[@]}" restart vault-0 >/dev/null 2>&1 +for _ in $(seq 1 40); do + curl -sk --cacert "$VAULT_CACERT" --max-time 2 \ + "${VAULT_ADDR}/v1/sys/seal-status" >/dev/null 2>&1 && break + sleep 2 +done +sleep 6 +if [[ "$(vault status -format=json 2>/dev/null | jq -r '.sealed')" == "false" ]]; then + ok "and a cluster node auto-unseals against it afterwards" +else + bad "and a cluster node auto-unseals against it afterwards" \ + "this is the failure the kept keys exist to prevent" +fi + +# --------------------------------------------------------------------------- +info "" +info "=== Rekeying the unseal shares ===" +# --------------------------------------------------------------------------- +# Same ceremony as the recovery rekey above, different endpoint: +# sys/rekey rather than sys/rekey-recovery-key, unseal_keys_b64 rather +# than recovery_keys_b64. + +U_ROOT="$(jq -r '.root_token' "$UNSEAL_KEYS")" +GEN1_FIRST="$(jq -r '.unseal_keys_b64[0]' "$UNSEAL_KEYS")" + +# The bootstrap makes vault-unseal 1-of-1, which is fine for a dev root of +# trust and useless for demonstrating a quorum. Rekeying to 5-of-3 is +# both the realistic operation and what gives the next step enough old +# shares to form a quorum with. +RK1="$(VAULT_ADDR="https://127.0.0.1:8300" VAULT_TOKEN="$U_ROOT" \ + bash "$ROTATE" --unseal-keys --keys-file "$UNSEAL_KEYS" --shares 5 --threshold 3 2>&1)"; RC=$? +if [[ "$RC" -eq 0 ]]; then + ok "rotate-keys.sh --unseal-keys rekeys 1-of-1 to 5-of-3" +else + bad "rotate-keys.sh --unseal-keys rekeys 1-of-1 to 5-of-3" "$(tail -4 <<< "$RK1")" +fi +if [[ "$RK1" == *"Verified."* ]]; then + ok "and verified the new shares before committing them" +else + bad "and verified the new shares before committing them" +fi +if [[ "$(jq -r '.unseal_keys_b64 | length' "$UNSEAL_KEYS" 2>/dev/null)" == "5" ]]; then + ok "and the file now holds five shares" +else + bad "and the file now holds five shares" \ + "$(jq -r '.unseal_keys_b64 | length' "$UNSEAL_KEYS" 2>/dev/null)" +fi +if [[ "$(jq -r '.unseal_keys_b64[0]' "$UNSEAL_KEYS")" != "$GEN1_FIRST" ]]; then + ok "and they are not the shares it started with" +else + bad "and they are not the shares it started with" +fi + +mapfile -t GEN2 < <(jq -r '.unseal_keys_b64[]' "$UNSEAL_KEYS") + +# A second rekey, so there is a full quorum of superseded shares to try. +# One share proves nothing: Vault accepts shares and only validates the +# combination once the threshold is reached, so a lone stale share +# returns success and 1/3 progress. +RK2="$(VAULT_ADDR="https://127.0.0.1:8300" VAULT_TOKEN="$U_ROOT" \ + bash "$ROTATE" --unseal-keys --keys-file "$UNSEAL_KEYS" 2>&1)"; RC=$? +if [[ "$RC" -eq 0 ]]; then + ok "a second rekey succeeds, keeping 5-of-3" +else + bad "a second rekey succeeds, keeping 5-of-3" "$(tail -4 <<< "$RK2")" +fi + +mapfile -t GEN3 < <(jq -r '.unseal_keys_b64[]' "$UNSEAL_KEYS") + +info " restarting vault-unseal to test both generations..." +"${COMPOSE[@]}" restart vault-unseal >/dev/null 2>&1 +for _ in $(seq 1 30); do + curl -sk --cacert "$VAULT_CACERT" --max-time 2 \ + https://127.0.0.1:8300/v1/sys/seal-status >/dev/null 2>&1 && break + sleep 2 +done + +# A full quorum of the superseded generation, which must not open it. +"${COMPOSE[@]}" exec -T vault-unseal vault operator unseal -reset >/dev/null 2>&1 +for i in 0 1 2; do + "${COMPOSE[@]}" exec -T vault-unseal vault operator unseal "${GEN2[$i]}" >/dev/null 2>&1 +done +STILL="$(curl -sk --cacert "$VAULT_CACERT" --max-time 4 \ + https://127.0.0.1:8300/v1/sys/seal-status 2>/dev/null | jq -r '.sealed')" +if [[ "$STILL" == "true" ]]; then + ok "a quorum of superseded shares does not open it" +else + bad "a quorum of superseded shares does not open it" \ + "the file changed and the barrier did not" +fi + +"${COMPOSE[@]}" exec -T vault-unseal vault operator unseal -reset >/dev/null 2>&1 +for i in 0 1 2; do + "${COMPOSE[@]}" exec -T vault-unseal vault operator unseal "${GEN3[$i]}" >/dev/null 2>&1 +done +NOW="$(curl -sk --cacert "$VAULT_CACERT" --max-time 4 \ + https://127.0.0.1:8300/v1/sys/seal-status 2>/dev/null | jq -r '.sealed')" +if [[ "$NOW" == "false" ]]; then + ok "and the current ones do" +else + bad "and the current ones do" "sealed=${NOW}" +fi + +# Handing the wrong kind of key file to the wrong mode is the mistake +# these two paths invite, and the endpoint's own error says nothing about +# which set of keys it wanted. +WRONG="$(VAULT_ADDR="https://127.0.0.1:8300" VAULT_TOKEN="$U_ROOT" \ + bash "$ROTATE" --unseal-keys --keys-file "$KEYS_FILE" 2>&1)"; RC=$? +if [[ "$RC" -ne 0 ]]; then + ok "--unseal-keys against a recovery-key file is refused" +else + bad "--unseal-keys against a recovery-key file is refused" +fi +if [[ "$WRONG" == *"unseal_keys_b64"* ]]; then + ok "and names the field it wanted" +else + bad "and names the field it wanted" "$(tail -2 <<< "$WRONG")" +fi + +for f in ".unseal-keys.json" ".unseal-keys.json.superseded" ".unseal-keys.json.new"; do + if git -C "$REPO_ROOT" check-ignore -q "docker/dev/${f}"; then + ok "docker/dev/${f} is gitignored" + else + bad "docker/dev/${f} is gitignored" + fi +done + # --------------------------------------------------------------------------- printf '\n=== Results ===\npassed: %d\nfailed: %d\n' "$PASS" "$FAIL" if [[ "$FAIL" -eq 0 ]]; then