diff --git a/ansible/inventory/aws.yml b/ansible/inventory/aws.yml index 1f95832..d332a09 100644 --- a/ansible/inventory/aws.yml +++ b/ansible/inventory/aws.yml @@ -70,5 +70,58 @@ groups: hostnames: - instance-id +# Reaching a node: SSH tunnelled through SSM Session Manager. +# +# Nodes have no public address and security.tf opens no port 22 from +# anywhere, so there is nothing to SSH *to* from outside the VPC. Session +# Manager solves that without changing either fact: the agent on the +# instance holds an outbound connection to the SSM service, and +# AWS-StartSSHSession carries an ordinary SSH session back down it. No +# inbound rule, no public IP, no bastion to patch, and the session is +# recorded against the caller's IAM identity rather than against whoever +# holds a key. +# +# `ansible_host` is therefore the instance id and not the private IP: +# `--target` names an instance to SSM, and ProxyCommand's %h is whatever +# Ansible puts in ansible_host. The private IP is no longer routable from +# where the playbook runs, so it would only look correct. +# +# THREE THINGS THIS STILL NEEDS, none of which Terraform can supply: +# +# - session-manager-plugin on the control machine. The AWS CLI shells +# out to it and fails with a message about the plugin, not about +# permissions, which is a confusing way to learn this at apply time. +# scripts/preflight-cloud.sh checks for it. +# - ssm:StartSession on YOUR identity, for the AWS-StartSSHSession +# document. The instance side is already covered — iam.tf attaches +# AmazonSSMManagedInstanceCore to the node role. +# - ssh_key_name set on the profile, so EC2 puts your public key in +# ec2-user's authorized_keys at boot. SSM carries the session; it does +# not authenticate you to sshd. An empty ssh_key_name still produces a +# cluster nobody can log into. +# +# Running from *inside* the VPC — a bastion, a VPN, a CI runner in a +# private subnet — wants neither of these: set ansible_host back to +# private_ip_address and drop ansible_ssh_common_args. compose: - ansible_host: private_ip_address + ansible_host: instance_id + + # AL2023's default user (compute.tf pins the AMI filter to al2023-*). + # Quoted twice deliberately: compose values are Jinja expressions, so a + # bare ec2-user would be read as an undefined variable. + ansible_user: "'ec2-user'" + + # accept-new, not no. + # + # It accepts a host key it has not seen and refuses one that changed. + # That reads as weak until you notice what a host is here: the id is + # per instance, so a replaced node is a new name with a new key and is + # correctly unknown, while a *changed* key under an id we have already + # seen is the case worth refusing. StrictHostKeyChecking=no would + # accept that too, and pairing accept-new with UserKnownHostsFile + # /dev/null would throw the refusal away just as thoroughly. + # + # It is still trust-on-first-use. Verifying properly means reading the + # key out of `aws ec2 get-console-output` before the first connection; + # see docs/deployment.md. + ansible_ssh_common_args: "'-o StrictHostKeyChecking=accept-new -o ProxyCommand=\"aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p\"'" diff --git a/docs/cloud-apply.md b/docs/cloud-apply.md index 2bd02d2..8fbbf15 100644 --- a/docs/cloud-apply.md +++ b/docs/cloud-apply.md @@ -82,7 +82,8 @@ otherwise: | Check | Why it is worth catching early | |---|---| -| `ssh_key_name` is empty | **The apply succeeds** and produces instances nobody can log into. Every item in the checklist below needs a shell on a node. | +| `ssh_key_name` is empty | **The apply succeeds** and produces instances nobody can log into. Every item in the checklist below needs a shell on a node, and the Ansible layer needs SSH specifically — the tunnel carries it, it does not replace it. | +| `session-manager-plugin` is missing | The AWS CLI execs it to open a Session Manager tunnel, which is how the playbook reaches a node with no public address. Every connection fails naming the plugin rather than the thing you were doing. | | The key pair does not exist in this region | The apply fails at instance launch — after the VPC and NAT gateways are already billing. | | Elastic IP quota | One EIP per NAT gateway, one NAT gateway per AZ, default limit 5. Three zones plus anything already in the account can exceed it. | | Azure role assignment permission | The profile creates a role assignment, which needs Owner or User Access Administrator. Contributor applies most of the profile and *then* fails. | @@ -90,6 +91,13 @@ otherwise: The `ssh_key_name` one is not hypothetical: `terraform/aws/variables.tf` ships it empty, so the default AWS apply produces an unreachable cluster. +Reaching the nodes at all is worth reading before the session rather than +during it — the nodes are in private subnets with no inbound 22, and the +AWS inventory tunnels SSH through Session Manager to get to them. See +[deployment.md](deployment.md#reaching-the-nodes) for what that needs. +Azure's inventory does not tunnel, so reaching those nodes is still +unsolved. + --- ## What it costs diff --git a/docs/deployment.md b/docs/deployment.md index f8df7db..c284714 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -209,8 +209,64 @@ set by resource group and name and never looks at tags. An empty inventory there says nothing about whether the cluster formed, and a healthy cluster is no evidence the inventory works. -Nodes sit in private subnets with no public address, so reaching them -needs SSM, a bastion, or a VPN. +### Reaching the nodes + +Nodes sit in private subnets with no public address, and `security.tf` +opens port 22 from nowhere. There is nothing outside the VPC to SSH *to*. + +On AWS the inventory resolves that by tunnelling SSH through SSM Session +Manager, which changes neither fact: the agent on the instance holds an +outbound connection to the SSM service, and the `AWS-StartSSHSession` +document carries an ordinary SSH session back down it. No inbound rule, +no public address, no bastion to patch and pay for, and the session is +recorded against the caller's IAM identity rather than against whoever +holds a key. + +It is configured in `ansible/inventory/aws.yml` and needs nothing from +the playbook. Three things it does need, none of which Terraform can +supply: + +| Requirement | Where it comes from | +|---|---| +| `session-manager-plugin` on the control machine | Installed separately; the AWS CLI execs it. `scripts/preflight-cloud.sh` warns when it is missing | +| `ssm:StartSession` on *your* identity | Your own IAM. The instance side is already covered — `iam.tf` attaches `AmazonSSMManagedInstanceCore` | +| `ssh_key_name` set on the profile | EC2 puts the public key in `ec2-user`'s `authorized_keys` at boot | + +**The third is the one that surprises people.** Session Manager replaces +the network path, not the authentication: what answers at the far end of +the tunnel is still `sshd`, still reading `authorized_keys`. An empty +`ssh_key_name` leaves `aws ssm start-session` — a shell, enough to +inspect a node — and no way to run the playbooks at all. + +The tunnel also decides what a host is *called*. `--target` names an +instance to SSM and `ProxyCommand`'s `%h` is whatever `ansible_host` +holds, so `ansible_host` is the instance id. The private IP is not +routable from where the playbook runs; using it would only look correct. + +Host keys are `accept-new`, which accepts an unseen key and refuses a +changed one. That reads as weak until you notice what a host is here: the +id is per instance, so a replaced node is a new name with a new key and +is correctly unknown, while a changed key under an id already seen is the +case worth refusing. `StrictHostKeyChecking=no` would accept that too. +It is still trust-on-first-use — verifying properly means reading the key +out of `aws ec2 get-console-output` before the first connection, which +this repository does not automate. + +Running from *inside* the VPC — a bastion, a VPN, a CI runner in a +private subnet — wants none of it: set `ansible_host` back to +`private_ip_address` and drop `ansible_ssh_common_args`. + +**Azure has no equivalent here.** Its nodes are equally private and its +inventory sets no connection arguments, so reaching them is still the +reader's problem. That is not an oversight being deferred quietly: the +Azure profile has never been applied, and adding an untested tunnel to an +untested profile would make the gap harder to see rather than smaller. + +None of this has run against a real account either. It is configuration +with reasoning attached, and `tests/ansible` asserts the values it +produces — that the expressions evaluate at all, that the target is the +instance id, that the document is `AWS-StartSSHSession` and not a plain +shell. What no test here can show is that the tunnel opens. **What a host is called matters as much as which hosts are found.** An Ansible inventory is keyed by host name, so two hosts with one name are diff --git a/scripts/preflight-cloud.sh b/scripts/preflight-cloud.sh index ee80d05..cfe4d0b 100755 --- a/scripts/preflight-cloud.sh +++ b/scripts/preflight-cloud.sh @@ -120,6 +120,23 @@ else bad "${CLI} CLI is not on PATH" "needed to check identity and quota, and by teardown-cloud.sh" fi +# The nodes have no public address and no inbound port 22, so Ansible +# reaches them by tunnelling SSH through SSM Session Manager -- see +# ansible/inventory/aws.yml. The AWS CLI does not implement that itself; +# it shells out to session-manager-plugin, and without it every +# connection fails naming the plugin rather than the thing you were +# doing, which is a slow way to learn this with the meter running. +# +# A warning rather than a failure: running the playbook from inside the +# VPC is a legitimate arrangement and needs none of this. +if [[ "$CLOUD" == "aws" ]]; then + if command -v session-manager-plugin >/dev/null 2>&1; then + ok "session-manager-plugin" + else + warn "session-manager-plugin is not on PATH" "ansible-playbook cannot reach the nodes without it, unless you are running from inside the VPC" + fi +fi + # --------------------------------------------------------------------------- info "" info "=== Credentials ===" diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index fdbedc8..6877545 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -66,9 +66,24 @@ variable "instance_type" { default = "t3.small" } +# SSM carries the session; it does not authenticate you to sshd. +# +# This said that leaving it empty "disables SSH entirely and uses SSM +# Session Manager instead", which read as a supported arrangement and was +# not one. Session Manager replaces the *network path* -- no public +# address, no inbound 22 -- and ansible/inventory/aws.yml tunnels SSH +# through it on that basis. What reaches the node at the far end is still +# sshd, still checking authorized_keys, which EC2 populates from this key +# pair at boot. Empty means no key, so nothing authenticates, and the +# apply succeeds into a cluster nobody can log into. +# +# Kept optional rather than made required: `aws ssm start-session` with no +# document gives a shell without SSH at all, which is enough to inspect a +# node and is the one case where empty is deliberate. It is not enough for +# Ansible. scripts/preflight-cloud.sh warns when it is empty. variable "ssh_key_name" { type = string - description = "Optional EC2 key pair for SSH. Leave empty to disable SSH entirely and use SSM Session Manager instead, which leaves an auditable trail and needs no open port 22." + description = "EC2 key pair whose public key EC2 puts in ec2-user's authorized_keys. Required for the Ansible layer, which tunnels SSH over SSM. Empty leaves only `aws ssm start-session` shell access, and no way to run the playbooks." default = "" } diff --git a/tests/ansible/README.md b/tests/ansible/README.md index af55deb..a6f60ce 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -85,3 +85,29 @@ the Ansible and cloud-init paths, renaming a Terraform output, and removing `tls_client_ca_file`. Each turned exactly one assertion red. Restore the file afterwards; a mutation left in the working tree is indistinguishable from a real regression. + +### The inventory's compose block + +`eval-compose.py` renders `ansible/inventory/aws.yml`'s `compose` values +the way `aws_ec2` would, against a synthetic instance. They are Jinja +expressions rather than strings, and the failure that motivated it is +invisible to every other check here: a literal written bare — +`ansible_user: ec2-user` — is an undefined variable, which composes to +nothing rather than erroring. Ansible then drops the setting and connects +as the local user, and the file is valid YAML either way. + +Four mutations, each watched to fail: + +| Mutation | What goes red | +|---|---| +| `ansible_user: ec2-user`, unquoted | every compose assertion — the render raises, which is the point | +| `ansible_host: private_ip_address` | the SSM target assertion alone | +| drop `--document-name AWS-StartSSHSession` | the SSH-document assertion alone | +| `StrictHostKeyChecking=no` | both host-key assertions, positive and exclusion | + +The first is worth its own note. The exclusion assertion — that nothing +throws the host-key check away — stayed green under it, because a render +that raised left no string to search. It is paired with a positive +assertion on the same value, which went red, and that pairing is the only +reason the mutation was caught. An exclusion with nothing to exclude +passes. diff --git a/tests/ansible/eval-compose.py b/tests/ansible/eval-compose.py new file mode 100644 index 0000000..3dcdfb1 --- /dev/null +++ b/tests/ansible/eval-compose.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Evaluate a dynamic inventory's `compose` block the way Ansible does. + +`compose` values are Jinja2 expressions, not strings. A literal has to be +quoted inside the expression -- `ansible_user: ec2-user` is an undefined +variable, and an undefined variable composes to nothing rather than to an +error, so the setting silently does not exist and Ansible falls back to +the local username. + +Nothing offline catches that. `tests/ansible` checked the file was valid +YAML, which it is either way, and the plugin itself only runs with +credentials in front of a real EC2 API. + +So render each expression against a host the way aws_ec2 would, and print +`namevalue` for the caller to assert on. The hostvars below are the +subset of a describe_instances entry these expressions read; add to them +rather than reaching for a real API. + +Usage: + eval-compose.py +""" + +import io +import sys + +import yaml +from jinja2 import Environment + +# Shaped like amazon.aws.aws_ec2's per-host vars: boto3 keys converted to +# snake_case, tags as a dict. +HOSTVARS = { + "instance_id": "i-0123456789abcdef0", + "private_ip_address": "10.0.2.15", + "private_dns_name": "ip-10-0-2-15.ec2.internal", + "placement": {"availability_zone": "us-east-1a"}, + "tags": {"Name": "vault-reference-vault", "VaultCluster": "vault-reference"}, +} + + +def main(): + if len(sys.argv) != 2: + print(__doc__.strip().split('Usage:')[-1].strip(), file=sys.stderr) + return 2 + + with io.open(sys.argv[1], encoding='utf-8') as handle: + inventory = yaml.safe_load(handle) or {} + + compose = inventory.get('compose') or {} + if not compose: + print('no compose block', file=sys.stderr) + return 1 + + env = Environment() + for name, expression in compose.items(): + # undefined_to_none=False makes an undefined name raise here + # rather than render as empty, which is the whole point: Ansible + # would drop the variable, and a test that accepted an empty + # string would agree with the bug. + value = env.compile_expression(expression, undefined_to_none=False)(**HOSTVARS) + print('%s\t%s' % (name, value)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/ansible/run-tests.sh b/tests/ansible/run-tests.sh index abc5801..a9574c1 100755 --- a/tests/ansible/run-tests.sh +++ b/tests/ansible/run-tests.sh @@ -50,6 +50,14 @@ assert_not_contains() { if [[ "$2" != *"$3"* ]]; then ok "$1"; else bad "$1" "expected NOT to find: $3"; fi } +# assert_eq