Stop three AWS nodes collapsing into one inventory host

ansible/inventory/aws.yml

@@ -46,9 +46,29 @@ groups:
 # Nodes sit in private subnets with no public address, so connections go
 # to the private IP. Reaching them needs SSM, a bastion, or a VPN — see
 # docs/deployment.md.
+#
+# The instance id, and deliberately not tag:Name.
+#
+# An autoscaling group tags every instance it launches identically —
+# there is no per-instance interpolation in a launch template — so
+# compute.tf gives all three nodes Name=<cluster>-vault. This list is an
+# order of *preference*: aws_ec2 takes the first entry that resolves and
+# stops, so tag:Name here named all three the same thing, and an Ansible
+# inventory is keyed by host name. add_host() returns the existing host,
+# set_variable() overwrites its vars, and three nodes collapsed into one
+# — the last instance the paginator returned. site.yml then configured a
+# single node and exited 0, which is the shape of failure this repository
+# exists to refuse: snapshots, audit devices and PKI certificates landing
+# on one machine out of three, reported as success.
+#
+# instance-id rather than private-ip-address because it is the identity
+# the node already has: user-data.sh.tftpl derives Raft's node_id from
+# the same value, so a host in this inventory and a voter in `vault
+# operator raft list-peers` carry one name between them. Azure needs no
+# equivalent line — azure_rm defaults to the VM name, which is per
+# instance and is also what its cloud-init uses for node_id.
 hostnames:
-  - tag:Name
-  - private-ip-address
+  - instance-id
 
 compose:
   ansible_host: private_ip_address

docs/deployment.md

@@ -212,6 +212,26 @@ healthy cluster is no evidence the inventory works.
 Nodes sit in private subnets with no public address, so reaching them
 needs SSM, a bastion, or a VPN.
 
+**What a host is called matters as much as which hosts are found.** An
+Ansible inventory is keyed by host name, so two hosts with one name are
+one host. `aws_ec2`'s `hostnames` is a list of preferences and it stops
+at the first that resolves — and an autoscaling group tags every instance
+it launches identically, because a launch template has no per-instance
+interpolation. So preferring `tag:Name`, which this file did, named all
+three nodes `<cluster>-vault`: they collapsed into whichever instance the
+paginator returned last, and `site.yml` configured one node out of three
+and exited 0.
+
+It names them by instance id now, which is unique and is also what
+`user-data.sh.tftpl` gives Raft as `node_id` — so a host here and a voter
+in `vault operator raft list-peers` carry the same name. Azure needed no
+change: `azure_rm` defaults to the VM name, which is per instance and is
+what its cloud-init uses for `node_id` too.
+
+`tests/preflight-static` asserts both halves, because this is a string
+one layer produces and another consumes with nothing validating it in
+between — the seam that suite exists for.
+
 ### Certificates
 
 The role expects to find certificates on the control machine and copies

tests/preflight-static/run-tests.sh

@@ -367,6 +367,79 @@ else
         "templates read '${CA_TPL}', Ansible writes '${TLS_DIR}/ca.crt'"
 fi
 
+# ---------------------------------------------------------------------------
+printf '\n=== The inventory can tell the nodes apart ===\n'
+# ---------------------------------------------------------------------------
+# An Ansible inventory is keyed by host name, so two hosts with one name
+# are one host. aws_ec2's `hostnames` is a list of *preferences*: it takes
+# the first entry that resolves and stops.
+#
+# This profile is an autoscaling group, and a launch template has no
+# per-instance interpolation — every instance it launches carries the same
+# tags. So any `tag:` entry names every node identically, add_host()
+# returns the host that already exists, and three instances collapse into
+# the last one the paginator returned. site.yml then configures one node
+# and exits 0, with snapshots, audit devices and PKI certificates on one
+# machine out of three.
+#
+# Nothing else here can see it. terraform validate reads one file, the
+# mocks read the configuration's shape, and tests/ansible checks this file
+# parses as YAML. It is the Terraform/Ansible seam this suite is for.
+INV="${REPO_ROOT}/ansible/inventory/aws.yml"
+HOSTNAMES="$(python3 -c '
+import sys, yaml
+doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) or {}
+for entry in doc.get("hostnames") or []:
+    print(entry if isinstance(entry, str) else entry.get("name", ""))
+' "$INV" 2>/dev/null)"
+
+if [[ -n "$HOSTNAMES" ]]; then
+    ok "aws: the inventory declares how it names a host"
+else
+    bad "aws: the inventory declares how it names a host" \
+        "no hostnames list — aws_ec2 then defaults to dns-name, which a private instance does not have"
+fi
+
+# Pin the values rather than forbidding the one spelling that bit us.
+# tag:Name is not the only constant an ASG propagates; VaultCluster is
+# another, and it is in this very file. The question is whether a value
+# distinguishes instances at all, so allow only the ones that do.
+UNIQUE_PER_INSTANCE=" instance-id private-ip-address private-dns-name network-interface.addresses.private-ip-address "
+NOT_UNIQUE=""
+while IFS= read -r pref; do
+    [[ -z "$pref" ]] && continue
+    [[ "$UNIQUE_PER_INSTANCE" == *" ${pref} "* ]] && continue
+    NOT_UNIQUE="${NOT_UNIQUE}${pref} "
+done <<< "$HOSTNAMES"
+
+if [[ -z "$NOT_UNIQUE" ]]; then
+    ok "aws: every hostname preference is unique per instance"
+else
+    bad "aws: every hostname preference is unique per instance" \
+        "'${NOT_UNIQUE% }' is the same on every instance the ASG launches; the nodes merge into one host"
+fi
+
+# And the name it picks should be the name the cluster already uses, so an
+# inventory host and a Raft voter can be matched up by eye. Both sides of
+# the seam, each read from the file that owns it.
+FIRST_PREF="$(head -1 <<< "$HOSTNAMES")"
+NODE_ID_SRC="$(grep -oE 'INSTANCE_ID="\$\(imds [a-z-]+\)"' "$AWS_TPL" \
+    | grep -oE 'imds [a-z-]+' | cut -d' ' -f2)"
+
+if [[ -n "$NODE_ID_SRC" ]] && grep -qE 'node_id *= *"\$+\{INSTANCE_ID\}"' "$AWS_TPL"; then
+    ok "aws: user-data derives Raft's node_id from ${NODE_ID_SRC}"
+else
+    bad "aws: user-data derives Raft's node_id from the instance id" \
+        "the pattern stopped matching in $(basename "$AWS_TPL") — this assertion is no longer reading anything"
+fi
+
+if [[ "$FIRST_PREF" == "$NODE_ID_SRC" ]]; then
+    ok "and the inventory names hosts by the same value (${FIRST_PREF})"
+else
+    bad "and the inventory names hosts by the same value" \
+        "inventory prefers '${FIRST_PREF:-<none>}', node_id is '${NODE_ID_SRC:-<none>}'"
+fi
+
 # ---------------------------------------------------------------------------
 printf '\n=== Results ===\n'
 # ---------------------------------------------------------------------------