diff --git a/ansible/inventory/aws.yml b/ansible/inventory/aws.yml index d97a340..1f95832 100644 --- a/ansible/inventory/aws.yml +++ b/ansible/inventory/aws.yml @@ -46,9 +46,29 @@ groups: # Nodes sit in private subnets with no public address, so connections go # to the private IP. Reaching them needs SSM, a bastion, or a VPN — see # docs/deployment.md. +# +# The instance id, and deliberately not tag:Name. +# +# An autoscaling group tags every instance it launches identically — +# there is no per-instance interpolation in a launch template — so +# compute.tf gives all three nodes Name=-vault. This list is an +# order of *preference*: aws_ec2 takes the first entry that resolves and +# stops, so tag:Name here named all three the same thing, and an Ansible +# inventory is keyed by host name. add_host() returns the existing host, +# set_variable() overwrites its vars, and three nodes collapsed into one +# — the last instance the paginator returned. site.yml then configured a +# single node and exited 0, which is the shape of failure this repository +# exists to refuse: snapshots, audit devices and PKI certificates landing +# on one machine out of three, reported as success. +# +# instance-id rather than private-ip-address because it is the identity +# the node already has: user-data.sh.tftpl derives Raft's node_id from +# the same value, so a host in this inventory and a voter in `vault +# operator raft list-peers` carry one name between them. Azure needs no +# equivalent line — azure_rm defaults to the VM name, which is per +# instance and is also what its cloud-init uses for node_id. hostnames: - - tag:Name - - private-ip-address + - instance-id compose: ansible_host: private_ip_address diff --git a/docs/deployment.md b/docs/deployment.md index 662f41e..f8df7db 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -212,6 +212,26 @@ healthy cluster is no evidence the inventory works. Nodes sit in private subnets with no public address, so reaching them needs SSM, a bastion, or a VPN. +**What a host is called matters as much as which hosts are found.** An +Ansible inventory is keyed by host name, so two hosts with one name are +one host. `aws_ec2`'s `hostnames` is a list of preferences and it stops +at the first that resolves — and an autoscaling group tags every instance +it launches identically, because a launch template has no per-instance +interpolation. So preferring `tag:Name`, which this file did, named all +three nodes `-vault`: they collapsed into whichever instance the +paginator returned last, and `site.yml` configured one node out of three +and exited 0. + +It names them by instance id now, which is unique and is also what +`user-data.sh.tftpl` gives Raft as `node_id` — so a host here and a voter +in `vault operator raft list-peers` carry the same name. Azure needed no +change: `azure_rm` defaults to the VM name, which is per instance and is +what its cloud-init uses for `node_id` too. + +`tests/preflight-static` asserts both halves, because this is a string +one layer produces and another consumes with nothing validating it in +between — the seam that suite exists for. + ### Certificates The role expects to find certificates on the control machine and copies diff --git a/tests/preflight-static/run-tests.sh b/tests/preflight-static/run-tests.sh index 3604bea..f781563 100755 --- a/tests/preflight-static/run-tests.sh +++ b/tests/preflight-static/run-tests.sh @@ -367,6 +367,79 @@ else "templates read '${CA_TPL}', Ansible writes '${TLS_DIR}/ca.crt'" fi +# --------------------------------------------------------------------------- +printf '\n=== The inventory can tell the nodes apart ===\n' +# --------------------------------------------------------------------------- +# An Ansible inventory is keyed by host name, so two hosts with one name +# are one host. aws_ec2's `hostnames` is a list of *preferences*: it takes +# the first entry that resolves and stops. +# +# This profile is an autoscaling group, and a launch template has no +# per-instance interpolation — every instance it launches carries the same +# tags. So any `tag:` entry names every node identically, add_host() +# returns the host that already exists, and three instances collapse into +# the last one the paginator returned. site.yml then configures one node +# and exits 0, with snapshots, audit devices and PKI certificates on one +# machine out of three. +# +# Nothing else here can see it. terraform validate reads one file, the +# mocks read the configuration's shape, and tests/ansible checks this file +# parses as YAML. It is the Terraform/Ansible seam this suite is for. +INV="${REPO_ROOT}/ansible/inventory/aws.yml" +HOSTNAMES="$(python3 -c ' +import sys, yaml +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) or {} +for entry in doc.get("hostnames") or []: + print(entry if isinstance(entry, str) else entry.get("name", "")) +' "$INV" 2>/dev/null)" + +if [[ -n "$HOSTNAMES" ]]; then + ok "aws: the inventory declares how it names a host" +else + bad "aws: the inventory declares how it names a host" \ + "no hostnames list — aws_ec2 then defaults to dns-name, which a private instance does not have" +fi + +# Pin the values rather than forbidding the one spelling that bit us. +# tag:Name is not the only constant an ASG propagates; VaultCluster is +# another, and it is in this very file. The question is whether a value +# distinguishes instances at all, so allow only the ones that do. +UNIQUE_PER_INSTANCE=" instance-id private-ip-address private-dns-name network-interface.addresses.private-ip-address " +NOT_UNIQUE="" +while IFS= read -r pref; do + [[ -z "$pref" ]] && continue + [[ "$UNIQUE_PER_INSTANCE" == *" ${pref} "* ]] && continue + NOT_UNIQUE="${NOT_UNIQUE}${pref} " +done <<< "$HOSTNAMES" + +if [[ -z "$NOT_UNIQUE" ]]; then + ok "aws: every hostname preference is unique per instance" +else + bad "aws: every hostname preference is unique per instance" \ + "'${NOT_UNIQUE% }' is the same on every instance the ASG launches; the nodes merge into one host" +fi + +# And the name it picks should be the name the cluster already uses, so an +# inventory host and a Raft voter can be matched up by eye. Both sides of +# the seam, each read from the file that owns it. +FIRST_PREF="$(head -1 <<< "$HOSTNAMES")" +NODE_ID_SRC="$(grep -oE 'INSTANCE_ID="\$\(imds [a-z-]+\)"' "$AWS_TPL" \ + | grep -oE 'imds [a-z-]+' | cut -d' ' -f2)" + +if [[ -n "$NODE_ID_SRC" ]] && grep -qE 'node_id *= *"\$+\{INSTANCE_ID\}"' "$AWS_TPL"; then + ok "aws: user-data derives Raft's node_id from ${NODE_ID_SRC}" +else + bad "aws: user-data derives Raft's node_id from the instance id" \ + "the pattern stopped matching in $(basename "$AWS_TPL") — this assertion is no longer reading anything" +fi + +if [[ "$FIRST_PREF" == "$NODE_ID_SRC" ]]; then + ok "and the inventory names hosts by the same value (${FIRST_PREF})" +else + bad "and the inventory names hosts by the same value" \ + "inventory prefers '${FIRST_PREF:-}', node_id is '${NODE_ID_SRC:-}'" +fi + # --------------------------------------------------------------------------- printf '\n=== Results ===\n' # ---------------------------------------------------------------------------