From 895dad98118900580b43d88c62a5a63bcea37eda Mon Sep 17 00:00:00 2001 From: 0x48core Date: Fri, 9 Oct 2026 16:40:45 +0700 Subject: [PATCH] feat(lab-09): Ansible playbook to rebuild the server in one command - site.yml: detects fresh (root:22) vs hardened (admin:ssh_port) by a real SSH login, then runs roles common, users, firewall, ssh, fail2ban, docker, opsforge; idempotent (second run changed=0) - ssh role: validate with sshd -t, restart (socket-aware), carry the host key to the new port in known_hosts, reconnect, close port 22 - opsforge role: deploy user, proxy network, Traefik from stacks/ with Let's Encrypt merged in when traefik_acme_email is set, hello-api files and .env for deploy.sh - Inventories: test (fake VPS, test values) and production templates; real hosts.yml and vault.yml git-ignored - test/fake-vps.sh: disposable Ubuntu 24.04 systemd container with SSH and Docker-in-Docker - Stacks: router hostnames use ${DOMAIN:-opsforge.localhost} - Lab 09 guide and knowledge base; roadmap updated Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + README.md | 2 +- ansible/README.md | 64 ++++++- ansible/ansible.cfg | 11 ++ ansible/group_vars/all.yml | 30 +++ .../production/group_vars/servers/main.yml | 4 + .../group_vars/servers/vault.yml.example | 5 + .../inventory/production/hosts.yml.example | 5 + ansible/inventory/test/group_vars/servers.yml | 11 ++ ansible/inventory/test/hosts.yml | 7 + ansible/requirements.yml | 6 + ansible/roles/common/defaults/main.yml | 12 ++ ansible/roles/common/tasks/main.yml | 27 +++ ansible/roles/docker/defaults/main.yml | 5 + ansible/roles/docker/handlers/main.yml | 4 + ansible/roles/docker/tasks/main.yml | 41 ++++ ansible/roles/fail2ban/defaults/main.yml | 3 + ansible/roles/fail2ban/handlers/main.yml | 4 + ansible/roles/fail2ban/tasks/main.yml | 17 ++ .../roles/fail2ban/templates/jail.local.j2 | 11 ++ ansible/roles/firewall/tasks/main.yml | 32 ++++ ansible/roles/opsforge/defaults/main.yml | 2 + ansible/roles/opsforge/handlers/main.yml | 4 + ansible/roles/opsforge/tasks/main.yml | 139 ++++++++++++++ ansible/roles/ssh/handlers/main.yml | 11 ++ ansible/roles/ssh/tasks/main.yml | 49 +++++ .../roles/ssh/templates/00-hardening.conf.j2 | 11 ++ ansible/roles/users/tasks/main.yml | 23 +++ ansible/site.yml | 43 +++++ ansible/test/Dockerfile | 17 ++ ansible/test/fake-vps.sh | 51 +++++ labs/09-ansible/README.md | 180 +++++++++++++++++- labs/09-ansible/knowledge.md | 179 +++++++++++++++++ stacks/README.md | 2 + stacks/gitea/compose.yaml | 6 +- stacks/hello-api/compose.traefik.yaml | 2 +- stacks/portainer/compose.yaml | 2 +- stacks/traefik/compose.yaml | 2 +- stacks/uptime-kuma/compose.yaml | 2 +- stacks/whoami/compose.yaml | 2 +- 40 files changed, 1016 insertions(+), 16 deletions(-) create mode 100644 ansible/ansible.cfg create mode 100644 ansible/group_vars/all.yml create mode 100644 ansible/inventory/production/group_vars/servers/main.yml create mode 100644 ansible/inventory/production/group_vars/servers/vault.yml.example create mode 100644 ansible/inventory/production/hosts.yml.example create mode 100644 ansible/inventory/test/group_vars/servers.yml create mode 100644 ansible/inventory/test/hosts.yml create mode 100644 ansible/requirements.yml create mode 100644 ansible/roles/common/defaults/main.yml create mode 100644 ansible/roles/common/tasks/main.yml create mode 100644 ansible/roles/docker/defaults/main.yml create mode 100644 ansible/roles/docker/handlers/main.yml create mode 100644 ansible/roles/docker/tasks/main.yml create mode 100644 ansible/roles/fail2ban/defaults/main.yml create mode 100644 ansible/roles/fail2ban/handlers/main.yml create mode 100644 ansible/roles/fail2ban/tasks/main.yml create mode 100644 ansible/roles/fail2ban/templates/jail.local.j2 create mode 100644 ansible/roles/firewall/tasks/main.yml create mode 100644 ansible/roles/opsforge/defaults/main.yml create mode 100644 ansible/roles/opsforge/handlers/main.yml create mode 100644 ansible/roles/opsforge/tasks/main.yml create mode 100644 ansible/roles/ssh/handlers/main.yml create mode 100644 ansible/roles/ssh/tasks/main.yml create mode 100644 ansible/roles/ssh/templates/00-hardening.conf.j2 create mode 100644 ansible/roles/users/tasks/main.yml create mode 100644 ansible/site.yml create mode 100644 ansible/test/Dockerfile create mode 100755 ansible/test/fake-vps.sh create mode 100644 labs/09-ansible/knowledge.md diff --git a/.gitignore b/.gitignore index b7caadc..f93c991 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,10 @@ crash.log # Ansible *.retry +# Real inventory and vault stay local (commit vault.yml only encrypted: git add -f) +ansible/inventory/production/hosts.yml +ansible/inventory/production/group_vars/servers/vault.yml +ansible/collections/ # Backups / dumps *.sql diff --git a/README.md b/README.md index 32ae450..11662e1 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Status: ⬜ todo Β· 🟨 in progress Β· βœ… done ### Phase 4 β€” Infrastructure as Code | # | Lab | Status | |---|---|---| -| 09 | [Ansible: rebuild the server in one command](labs/09-ansible/) | ⬜ | +| 09 | [Ansible: rebuild the server in one command](labs/09-ansible/) | 🟨 | | 10 | [Terraform: provision VPS, DNS, firewall](labs/10-terraform/) | ⬜ | ### Phase 5 β€” Observability & Kubernetes diff --git a/ansible/README.md b/ansible/README.md index 0b47b44..7c2d4d1 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -1,3 +1,65 @@ # ansible -Server configuration as code: `inventory/`, `roles//`, `site.yml`. Built in lab 09 β€” goal: a fresh VPS becomes the full server with one command. +Turns a fresh Ubuntu 24.04 server into the OpsForge server with one command (lab 09). + +```bash +ansible-playbook site.yml # fake VPS (inventory/test) +ansible-playbook -i inventory/production site.yml --ask-vault-pass +``` + +Safe to re-run: a second run reports `changed=0`. The first play detects whether the server is fresh (root on port 22) or already hardened (admin user on `ssh_port`), so the same command works for both. + +## Roles + +Run in this order by [site.yml](site.yml): + +| Role | Does | From lab | +|---|---|---| +| [common](roles/common/) | apt upgrade, base packages, timezone, hostname, automatic security updates | 01 | +| [users](roles/users/) | Admin user with SSH key only, passwordless sudo | 01 | +| [firewall](roles/firewall/) | ufw: deny incoming; allow SSH, 80, 443 | 01 | +| [ssh](roles/ssh/) | Hardened sshd on `ssh_port`, reconnects, closes port 22 | 01 | +| [fail2ban](roles/fail2ban/) | sshd jail, systemd backend | 01 | +| [docker](roles/docker/) | Docker Engine + Compose from Docker's repo, log rotation | 04 | +| [opsforge](roles/opsforge/) | `deploy` user, `proxy` network, Traefik (+ Let's Encrypt), hello-api files and `.env` for CI deploys | 05, 07, 08 | + +Run a subset with tags: `ansible-playbook site.yml --tags ssh,firewall`. + +## Layout + +``` +ansible/ +β”œβ”€β”€ site.yml # the playbook +β”œβ”€β”€ ansible.cfg # defaults to inventory/test +β”œβ”€β”€ group_vars/all.yml # defaults for every server +β”œβ”€β”€ inventory/ +β”‚ β”œβ”€β”€ test/ # fake VPS: hosts.yml + group_vars/servers.yml (test values) +β”‚ └── production/ # your VPS: hosts.yml + group_vars/servers/{main,vault}.yml +β”œβ”€β”€ roles//{tasks,handlers,templates,defaults}/ +└── test/ # fake VPS: Dockerfile + fake-vps.sh +``` + +Overrides go in `inventory//group_vars/servers…`, not `group_vars/all`: a playbook's `group_vars/all` takes precedence over an inventory's `group_vars/all`. + +## Test locally (no VPS needed) + +```bash +ansible/test/fake-vps.sh up # fresh Ubuntu 24.04 with systemd + sshd, as root on 127.0.0.1:2201 +cd ansible && ansible-playbook site.yml +ansible/test/fake-vps.sh recreate # destroy and start fresh +ansible/test/fake-vps.sh down +``` + +The fake VPS is a privileged container. Fine for testing on your Mac; never a real server. + +## Production + +```bash +cp inventory/production/hosts.yml.example inventory/production/hosts.yml +cp inventory/production/group_vars/servers/vault.yml.example inventory/production/group_vars/servers/vault.yml +ansible-vault encrypt inventory/production/group_vars/servers/vault.yml +# edit inventory/production/group_vars/servers/main.yml (domain, email, deploy key) +ansible-playbook -i inventory/production site.yml --ask-vault-pass +``` + +`hosts.yml` and `vault.yml` are git-ignored. Commit `vault.yml` only if it's encrypted (`git add -f`). diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..efe54f4 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = inventory/test/hosts.yml +roles_path = roles +interpreter_python = auto_silent +callback_result_format = yaml +retry_files_enabled = False +# Never accept unknown SSH host keys silently (see lab 01: known_hosts). +host_key_checking = True + +[ssh_connection] +pipelining = True diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000..f845608 --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,30 @@ +# Defaults for every server. Override per inventory in inventory//group_vars/servers/ +# (a named group beats "all"; inventory group_vars/all would NOT override these). + +# --- Connection ------------------------------------------------------------- +# A fresh server: root on port 22. After the ssh role: admin_user on ssh_port. +# site.yml detects which one works, so the same command runs on both. +bootstrap_user: root +bootstrap_port: 22 # how to reach the fresh server's sshd +ssh_initial_port: 22 # the port sshd listens on before hardening + +# --- Users ------------------------------------------------------------------ +admin_user: ops +admin_ssh_pubkey: "{{ lookup('ansible.builtin.file', '~/.ssh/id_ed25519.pub') }}" +deploy_user: deploy +deploy_ssh_pubkey: "" # public half of the CI deploy key (lab 07); empty = none + +# --- SSH and firewall ------------------------------------------------------- +ssh_port: 2222 +firewall_allowed_tcp_ports: [80, 443] + +# --- System ----------------------------------------------------------------- +server_hostname: opsforge +server_timezone: Asia/Ho_Chi_Minh + +# --- OpsForge stacks -------------------------------------------------------- +domain: opsforge.localhost +traefik_acme_email: "" # set to enable Let's Encrypt (needs a public domain) +# Secrets: define these in the inventory's vault (see inventory/production). +traefik_dashboard_users: "{{ vault_traefik_dashboard_users }}" +hello_api_postgres_password: "{{ vault_hello_api_postgres_password }}" diff --git a/ansible/inventory/production/group_vars/servers/main.yml b/ansible/inventory/production/group_vars/servers/main.yml new file mode 100644 index 0000000..f723db6 --- /dev/null +++ b/ansible/inventory/production/group_vars/servers/main.yml @@ -0,0 +1,4 @@ +# Production settings (non-secret). Secrets go in vault.yml next to this file. +domain: +traefik_acme_email: +# deploy_ssh_pubkey: "ssh-ed25519 AAAA... github-actions-deploy" diff --git a/ansible/inventory/production/group_vars/servers/vault.yml.example b/ansible/inventory/production/group_vars/servers/vault.yml.example new file mode 100644 index 0000000..134b82b --- /dev/null +++ b/ansible/inventory/production/group_vars/servers/vault.yml.example @@ -0,0 +1,5 @@ +# Copy to vault.yml, fill in, then ENCRYPT before doing anything else: +# ansible-vault encrypt inventory/production/group_vars/servers/vault.yml +# vault.yml is git-ignored. Commit it only encrypted, with `git add -f`. +vault_traefik_dashboard_users: "admin:" +vault_hello_api_postgres_password: "" diff --git a/ansible/inventory/production/hosts.yml.example b/ansible/inventory/production/hosts.yml.example new file mode 100644 index 0000000..105ad42 --- /dev/null +++ b/ansible/inventory/production/hosts.yml.example @@ -0,0 +1,5 @@ +# Copy to hosts.yml (git-ignored) and fill in your VPS. +servers: + hosts: + opsforge: + ansible_host: diff --git a/ansible/inventory/test/group_vars/servers.yml b/ansible/inventory/test/group_vars/servers.yml new file mode 100644 index 0000000..0079cd5 --- /dev/null +++ b/ansible/inventory/test/group_vars/servers.yml @@ -0,0 +1,11 @@ +# Test overrides for the fake VPS. Its published ports differ from a real server: +# 127.0.0.1:2201 -> container :22 (fresh sshd), 127.0.0.1:2202 -> :2202 (hardened) +bootstrap_port: 2201 +ssh_port: 2202 + +# Containers can't change their hostname or kernel settings like a VM can. +common_set_hostname: false + +# Test-only values, NOT secrets: a real server gets these from an encrypted vault. +vault_traefik_dashboard_users: "admin:$2y$05$1v9HS6HsMDzluqBjEk77oeKjenPLeRbd2T0oZQrJO8O/E0e1Lb/we" # password: test +vault_hello_api_postgres_password: test-only-password diff --git a/ansible/inventory/test/hosts.yml b/ansible/inventory/test/hosts.yml new file mode 100644 index 0000000..defd5fb --- /dev/null +++ b/ansible/inventory/test/hosts.yml @@ -0,0 +1,7 @@ +# The disposable fake VPS from test/fake-vps.sh (a local container). +# Hosts live in the "servers" group: inventory group_vars/servers/ override the +# defaults in ../../group_vars/all.yml (a plain group beats "all"). +servers: + hosts: + fake-vps: + ansible_host: 127.0.0.1 diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..266c453 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,6 @@ +# Collections used by the roles. Included in `brew install ansible`; +# with ansible-core only: ansible-galaxy collection install -r requirements.yml +collections: + - name: ansible.posix + - name: community.general + - name: community.docker diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000..afc439f --- /dev/null +++ b/ansible/roles/common/defaults/main.yml @@ -0,0 +1,12 @@ +common_upgrade: true # apt dist-upgrade on every run +common_set_hostname: true +common_packages: + - ca-certificates + - curl + - git + - gnupg + - htop + - python3-debian # used by deb822_repository (docker role) + - python3-requests # used by the community.docker modules + - rsync + - unattended-upgrades diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000..f895026 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Update the apt cache and upgrade all packages + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + upgrade: "{{ 'dist' if common_upgrade else 'no' }}" + +- name: Install base packages + ansible.builtin.apt: + name: "{{ common_packages }}" + state: present + +- name: Set the timezone + community.general.timezone: + name: "{{ server_timezone }}" + +- name: Set the hostname + ansible.builtin.hostname: + name: "{{ server_hostname }}" + when: common_set_hostname + +- name: Enable automatic security updates + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + mode: "0644" diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 0000000..b8a0d4c --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,5 @@ +docker_daemon_config: + log-driver: json-file + log-opts: + max-size: 10m + max-file: "3" diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml new file mode 100644 index 0000000..0964d40 --- /dev/null +++ b/ansible/roles/docker/handlers/main.yml @@ -0,0 +1,4 @@ +- name: Restart docker + ansible.builtin.service: + name: docker + state: restarted diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100644 index 0000000..6ac741b --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,41 @@ +# Docker Engine from Docker's own apt repository (newer than Ubuntu's docker.io). +- name: Add Docker's apt signing key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + +- name: Add Docker's apt repository + ansible.builtin.deb822_repository: + name: docker + types: deb + uris: https://download.docker.com/linux/ubuntu + suites: "{{ ansible_facts['distribution_release'] }}" + components: stable + architectures: "{{ 'arm64' if ansible_facts['architecture'] == 'aarch64' else 'amd64' }}" + signed_by: /etc/apt/keyrings/docker.asc + +- name: Install Docker Engine and the Compose plugin + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: true + +# Without limits, container logs grow until the disk is full. +- name: Configure log rotation for containers + ansible.builtin.copy: + dest: /etc/docker/daemon.json + content: "{{ docker_daemon_config | to_nice_json }}\n" + mode: "0644" + notify: Restart docker + +- name: Start Docker at boot + ansible.builtin.service: + name: docker + state: started + enabled: true diff --git a/ansible/roles/fail2ban/defaults/main.yml b/ansible/roles/fail2ban/defaults/main.yml new file mode 100644 index 0000000..53f285a --- /dev/null +++ b/ansible/roles/fail2ban/defaults/main.yml @@ -0,0 +1,3 @@ +fail2ban_bantime: 1h +fail2ban_findtime: 10m +fail2ban_maxretry: 5 diff --git a/ansible/roles/fail2ban/handlers/main.yml b/ansible/roles/fail2ban/handlers/main.yml new file mode 100644 index 0000000..0643fba --- /dev/null +++ b/ansible/roles/fail2ban/handlers/main.yml @@ -0,0 +1,4 @@ +- name: Restart fail2ban + ansible.builtin.service: + name: fail2ban + state: restarted diff --git a/ansible/roles/fail2ban/tasks/main.yml b/ansible/roles/fail2ban/tasks/main.yml new file mode 100644 index 0000000..b6cd3a9 --- /dev/null +++ b/ansible/roles/fail2ban/tasks/main.yml @@ -0,0 +1,17 @@ +- name: Install fail2ban + ansible.builtin.apt: + name: fail2ban + state: present + +- name: Configure the sshd jail + ansible.builtin.template: + src: jail.local.j2 + dest: /etc/fail2ban/jail.local + mode: "0644" + notify: Restart fail2ban + +- name: Start fail2ban at boot + ansible.builtin.service: + name: fail2ban + state: started + enabled: true diff --git a/ansible/roles/fail2ban/templates/jail.local.j2 b/ansible/roles/fail2ban/templates/jail.local.j2 new file mode 100644 index 0000000..dd5b053 --- /dev/null +++ b/ansible/roles/fail2ban/templates/jail.local.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} +[DEFAULT] +bantime = {{ fail2ban_bantime }} +findtime = {{ fail2ban_findtime }} +maxretry = {{ fail2ban_maxretry }} +# Ubuntu 24.04 has no /var/log/auth.log by default: read the journal. +backend = systemd + +[sshd] +enabled = true +port = {{ ssh_port }} diff --git a/ansible/roles/firewall/tasks/main.yml b/ansible/roles/firewall/tasks/main.yml new file mode 100644 index 0000000..09264f3 --- /dev/null +++ b/ansible/roles/firewall/tasks/main.yml @@ -0,0 +1,32 @@ +- name: Install ufw + ansible.builtin.apt: + name: ufw + state: present + +- name: Deny incoming and allow outgoing by default + community.general.ufw: + direction: "{{ item.direction }}" + default: "{{ item.policy }}" + loop: + - { direction: incoming, policy: deny } + - { direction: outgoing, policy: allow } + +# On a fresh server we are still connected through the initial SSH port: +# keep it open until the ssh role has moved sshd and reconnected. +- name: Keep the initial SSH port open while bootstrapping + community.general.ufw: + rule: allow + port: "{{ ssh_initial_port | string }}" + proto: tcp + when: ansible_user == bootstrap_user + +- name: Allow the hardened SSH port and web ports + community.general.ufw: + rule: allow + port: "{{ item | string }}" + proto: tcp + loop: "{{ [ssh_port] + firewall_allowed_tcp_ports }}" + +- name: Enable the firewall + community.general.ufw: + state: enabled diff --git a/ansible/roles/opsforge/defaults/main.yml b/ansible/roles/opsforge/defaults/main.yml new file mode 100644 index 0000000..2387d3d --- /dev/null +++ b/ansible/roles/opsforge/defaults/main.yml @@ -0,0 +1,2 @@ +opsforge_dir: "/home/{{ deploy_user }}/opsforge" +opsforge_stacks_src: "{{ playbook_dir }}/../stacks" diff --git a/ansible/roles/opsforge/handlers/main.yml b/ansible/roles/opsforge/handlers/main.yml new file mode 100644 index 0000000..7958267 --- /dev/null +++ b/ansible/roles/opsforge/handlers/main.yml @@ -0,0 +1,4 @@ +- name: Restart traefik + community.docker.docker_compose_v2: + project_src: "{{ opsforge_dir }}/stacks/traefik" + state: restarted diff --git a/ansible/roles/opsforge/tasks/main.yml b/ansible/roles/opsforge/tasks/main.yml new file mode 100644 index 0000000..751779d --- /dev/null +++ b/ansible/roles/opsforge/tasks/main.yml @@ -0,0 +1,139 @@ +# Prepares the server for the OpsForge stacks. App releases are NOT done here: +# CI (lab 07) deploys hello-api with deploy.sh. Ansible = server, CI = releases. + +- name: Create the deploy user (runs deploys from CI) + ansible.builtin.user: + name: "{{ deploy_user }}" + shell: /bin/bash + groups: docker # root-equivalent: see labs/07 knowledge.md + append: true + password: "!" + +- name: Install the CI deploy key + ansible.posix.authorized_key: + user: "{{ deploy_user }}" + key: "{{ deploy_ssh_pubkey }}" + exclusive: true + when: deploy_ssh_pubkey | length > 0 + +- name: Create the stack directories + ansible.builtin.file: + path: "{{ opsforge_dir }}/stacks/{{ item }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0750" + loop: [traefik, traefik/dynamic, traefik/secrets, traefik/certs, traefik/acme, hello-api] + +- name: Create the shared proxy network + community.docker.docker_network: + name: proxy + +# --- Traefik ------------------------------------------------------------------- + +- name: Copy the Traefik stack from this repo + ansible.builtin.copy: + src: "{{ opsforge_stacks_src }}/traefik/{{ item }}" + dest: "{{ opsforge_dir }}/stacks/traefik/{{ item }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0640" + loop: [compose.yaml, dynamic/middlewares.yml] + notify: Restart traefik + +# Same static config as stacks/traefik/traefik.yml, plus Let's Encrypt when +# traefik_acme_email is set: one source of truth, merged here. +- name: Write traefik.yml + ansible.builtin.copy: + dest: "{{ opsforge_dir }}/stacks/traefik/traefik.yml" + content: "{{ traefik_static | to_nice_yaml(indent=2) }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0640" + vars: + traefik_base: "{{ lookup('ansible.builtin.file', opsforge_stacks_src + '/traefik/traefik.yml') | from_yaml }}" + traefik_acme: + entryPoints: + websecure: + http: + tls: + certResolver: letsencrypt + certificatesResolvers: + letsencrypt: + acme: + email: "{{ traefik_acme_email }}" + storage: /etc/traefik/acme/acme.json + httpChallenge: + entryPoint: web + traefik_static: "{{ traefik_base | combine(traefik_acme if traefik_acme_email else {}, recursive=true) }}" + notify: Restart traefik + +- name: Write the dashboard users file + ansible.builtin.copy: + dest: "{{ opsforge_dir }}/stacks/traefik/secrets/dashboard-users" + content: "{{ traefik_dashboard_users }}\n" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + no_log: true + +- name: Write the Traefik server overrides + ansible.builtin.copy: + dest: "{{ opsforge_dir }}/stacks/traefik/{{ item.name }}" + content: "{{ item.content }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0640" + loop: + - name: .env + content: "DOMAIN={{ domain }}\n" + - name: compose.override.yaml # picked up automatically by docker compose + content: | + # Managed by Ansible (roles/opsforge): local edits are overwritten. + # On a server Traefik listens on all interfaces, and keeps ACME certificates. + services: + traefik: + ports: !override + - "80:80" + - "443:443" + volumes: + - ./acme:/etc/traefik/acme + loop_control: + label: "{{ item.name }}" + notify: Restart traefik + +- name: Start Traefik + community.docker.docker_compose_v2: + project_src: "{{ opsforge_dir }}/stacks/traefik" + state: present + +# --- hello-api: files for deploy.sh; CI deploys the actual release --------------- + +- name: Copy the hello-api stack from this repo + ansible.builtin.copy: + src: "{{ opsforge_stacks_src }}/hello-api/{{ item.name }}" + dest: "{{ opsforge_dir }}/stacks/hello-api/{{ item.name }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "{{ item.mode }}" + loop: + - { name: compose.yaml, mode: "0640" } + - { name: compose.traefik.yaml, mode: "0640" } + - { name: compose.deploy.yaml, mode: "0640" } + - { name: deploy.sh, mode: "0750" } + - { name: rollback.sh, mode: "0750" } + loop_control: + label: "{{ item.name }}" + +- name: Write the hello-api .env + ansible.builtin.copy: + dest: "{{ opsforge_dir }}/stacks/hello-api/.env" + content: | + DOMAIN={{ domain }} + POSTGRES_USER=hello + POSTGRES_PASSWORD={{ hello_api_postgres_password }} + POSTGRES_DB=hello + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + no_log: true diff --git a/ansible/roles/ssh/handlers/main.yml b/ansible/roles/ssh/handlers/main.yml new file mode 100644 index 0000000..8398205 --- /dev/null +++ b/ansible/roles/ssh/handlers/main.yml @@ -0,0 +1,11 @@ +# Ubuntu 24.04 starts sshd through ssh.socket: the port lives in the socket, +# which systemd regenerates from sshd_config on daemon-reload (lab 01). +- name: Restart sshd + ansible.builtin.shell: | + set -e + systemctl daemon-reload + if systemctl is-enabled --quiet ssh.socket 2>/dev/null; then + systemctl restart ssh.socket + fi + systemctl restart ssh.service + changed_when: true diff --git a/ansible/roles/ssh/tasks/main.yml b/ansible/roles/ssh/tasks/main.yml new file mode 100644 index 0000000..4859e80 --- /dev/null +++ b/ansible/roles/ssh/tasks/main.yml @@ -0,0 +1,49 @@ +- name: Install the SSH hardening config + ansible.builtin.template: + src: 00-hardening.conf.j2 + dest: /etc/ssh/sshd_config.d/00-hardening.conf + mode: "0644" + notify: Restart sshd + +- name: Check the full sshd config is valid + ansible.builtin.command: sshd -t + changed_when: false + +# known_hosts stores keys per host AND port. Carry the trust we already have +# for this server over to the new port, instead of disabling host key checks. +- name: Read the server's SSH host key + ansible.builtin.slurp: + src: /etc/ssh/ssh_host_ed25519_key.pub + register: ssh_host_key + +- name: Trust the same host key on the hardened port + ansible.builtin.known_hosts: + name: "[{{ ansible_host }}]:{{ ssh_public_port | default(ssh_port) }}" + key: "[{{ ansible_host }}]:{{ ssh_public_port | default(ssh_port) }} {{ (ssh_host_key.content | b64decode).split()[:2] | join(' ') }}" + delegate_to: localhost + become: false + +# Restart now (not at the end of the play), then reconnect on the new port. +- name: Apply the SSH changes now + ansible.builtin.meta: flush_handlers + +- name: Switch the connection to the hardened port and admin user + ansible.builtin.set_fact: + ansible_port: "{{ ssh_public_port | default(ssh_port) }}" + ansible_user: "{{ admin_user }}" + when: ansible_user == bootstrap_user + +- name: Drop the old SSH connection + ansible.builtin.meta: reset_connection + +- name: Reconnect + ansible.builtin.wait_for_connection: + timeout: 60 + +- name: Close the initial SSH port in the firewall + community.general.ufw: + rule: allow + port: "{{ ssh_initial_port | string }}" + proto: tcp + delete: true + when: ssh_initial_port != ssh_port diff --git a/ansible/roles/ssh/templates/00-hardening.conf.j2 b/ansible/roles/ssh/templates/00-hardening.conf.j2 new file mode 100644 index 0000000..2977d08 --- /dev/null +++ b/ansible/roles/ssh/templates/00-hardening.conf.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} +# SSH hardening from lab 01. Loads first (00-), and in sshd the first value wins. +Port {{ ssh_port }} +PermitRootLogin no +PasswordAuthentication no +KbdInteractiveAuthentication no +PubkeyAuthentication yes +AllowUsers {{ ([admin_user] + ([deploy_user] if deploy_ssh_pubkey else [])) | join(' ') }} +MaxAuthTries 3 +LoginGraceTime 30 +X11Forwarding no diff --git a/ansible/roles/users/tasks/main.yml b/ansible/roles/users/tasks/main.yml new file mode 100644 index 0000000..1d857bd --- /dev/null +++ b/ansible/roles/users/tasks/main.yml @@ -0,0 +1,23 @@ +- name: Create the admin user + ansible.builtin.user: + name: "{{ admin_user }}" + shell: /bin/bash + groups: sudo + append: true + password: "!" # locked: no password login, SSH keys only + update_password: on_create + +- name: Install the admin SSH key + ansible.posix.authorized_key: + user: "{{ admin_user }}" + key: "{{ admin_ssh_pubkey }}" + exclusive: true # remove keys that aren't managed here + +# Ansible needs sudo without a password prompt. Trade-off: whoever has the +# admin SSH key has root. Alternative: set a password and run with -K. +- name: Allow the admin user to sudo without a password + ansible.builtin.copy: + dest: "/etc/sudoers.d/90-{{ admin_user }}" + content: "{{ admin_user }} ALL=(ALL) NOPASSWD:ALL\n" + mode: "0440" + validate: visudo -cf %s diff --git a/ansible/site.yml b/ansible/site.yml new file mode 100644 index 0000000..2915661 --- /dev/null +++ b/ansible/site.yml @@ -0,0 +1,43 @@ +# Turn a fresh Ubuntu 24.04 server into the OpsForge server (lab 09). +# ansible-playbook site.yml # fake VPS (inventory/test) +# ansible-playbook -i inventory/production site.yml --ask-vault-pass +# Safe to re-run: a second run should report changed=0. + +- name: Work out how to connect + hosts: all + gather_facts: false + tags: always + tasks: + # A fresh server only lets root in on port 22; a hardened one only lets the + # admin user in on ssh_port. Try a real login (an open port isn't enough: + # some port forwarders accept TCP even when nothing listens behind them). + - name: Try to log in as the admin user on the hardened port + ansible.builtin.command: >- + ssh -o BatchMode=yes -o ConnectTimeout=5 + -p {{ ssh_public_port | default(ssh_port) }} {{ admin_user }}@{{ ansible_host }} true + delegate_to: localhost + register: hardened_login + changed_when: false + failed_when: false + check_mode: false # read-only: must also run in --check, or detection is skipped + + - name: Connect as the admin user, or bootstrap as root + ansible.builtin.set_fact: + ansible_port: "{{ (ssh_public_port | default(ssh_port)) if (hardened_login.rc | default(1)) == 0 else bootstrap_port }}" + ansible_user: "{{ admin_user if (hardened_login.rc | default(1)) == 0 else bootstrap_user }}" + + - name: Show the connection in use + ansible.builtin.debug: + msg: "{{ ansible_user }}@{{ ansible_host }}:{{ ansible_port }} ({{ 'hardened' if (hardened_login.rc | default(1)) == 0 else 'fresh server' }})" + +- name: Configure the OpsForge server + hosts: all + become: true + roles: + - { role: common, tags: [common] } + - { role: users, tags: [users] } + - { role: firewall, tags: [firewall] } # before ssh: open the new port first + - { role: ssh, tags: [ssh] } + - { role: fail2ban, tags: [fail2ban] } + - { role: docker, tags: [docker] } + - { role: opsforge, tags: [opsforge] } diff --git a/ansible/test/Dockerfile b/ansible/test/Dockerfile new file mode 100644 index 0000000..fee1667 --- /dev/null +++ b/ansible/test/Dockerfile @@ -0,0 +1,17 @@ +# A disposable "fake VPS" for testing the playbook locally (lab 09): +# Ubuntu 24.04 with systemd and OpenSSH, like a fresh cloud server. +# Runs privileged so systemd, ufw (iptables), and Docker-in-Docker work; +# fake-vps.sh mounts volumes for /var/lib/docker (overlay can't nest on overlay). +# Never use this as a real server. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends systemd systemd-sysv openssh-server python3 sudo ca-certificates iproute2 \ + && rm -rf /var/lib/apt/lists/* \ + # Like a cloud image: root logs in with an SSH key on port 22. + && mkdir -p /root/.ssh && chmod 700 /root/.ssh \ + && systemctl enable ssh + +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/ansible/test/fake-vps.sh b/ansible/test/fake-vps.sh new file mode 100755 index 0000000..c9c4494 --- /dev/null +++ b/ansible/test/fake-vps.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Manage a disposable fake VPS (a privileged systemd container) for lab 09. +# +# Usage: ansible/test/fake-vps.sh up | down | recreate | ssh | status +# +# SSH is published on 127.0.0.1:2201 (port 22 inside, the "fresh server") +# and 127.0.0.1:2202 (the hardened port the playbook switches to). +set -euo pipefail + +NAME=opsforge-fake-vps +IMAGE=opsforge-fake-vps:24.04 +KEY="${SSH_PUBKEY:-$HOME/.ssh/id_ed25519.pub}" +DIR="$(cd "$(dirname "$0")" && pwd)" + +up() { + [[ -f "$KEY" ]] || { echo "no public key at $KEY (set SSH_PUBKEY)" >&2; exit 1; } + docker build -q -t "$IMAGE" "$DIR" >/dev/null + docker run -d --name "$NAME" --hostname fake-vps --privileged --cgroupns=host \ + -v /sys/fs/cgroup:/sys/fs/cgroup:rw \ + --tmpfs /run --tmpfs /run/lock \ + -v "$NAME-docker:/var/lib/docker" -v "$NAME-containerd:/var/lib/containerd" \ + -p 127.0.0.1:2201:22 -p 127.0.0.1:2202:2202 \ + -p 127.0.0.1:8080:80 -p 127.0.0.1:8443:443 \ + "$IMAGE" >/dev/null + docker exec -i "$NAME" sh -c 'cat > /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys' < "$KEY" + for _ in $(seq 1 30); do + docker exec "$NAME" systemctl is-active --quiet ssh && break + sleep 1 + done + # A fresh server has a new host key: forget the old one for these ports. + ssh-keygen -R "[127.0.0.1]:2201" >/dev/null 2>&1 || true + ssh-keygen -R "[127.0.0.1]:2202" >/dev/null 2>&1 || true + # Trust the new host key on first use (fine for a local throwaway container; + # on a real VPS, compare the fingerprint with the provider's console). + ssh-keyscan -p 2201 -t ed25519 127.0.0.1 2>/dev/null >> "$HOME/.ssh/known_hosts" + echo "fake VPS up: ssh -p 2201 root@127.0.0.1" +} + +down() { + docker rm -f "$NAME" >/dev/null 2>&1 && echo "fake VPS removed" || echo "not running" + docker volume rm "$NAME-docker" "$NAME-containerd" >/dev/null 2>&1 || true +} + +case "${1:-}" in + up) up ;; + down) down ;; + recreate) down; up ;; + ssh) exec docker exec -it "$NAME" bash ;; + status) docker ps --filter "name=$NAME" --format '{{.Names}} {{.Status}} {{.Ports}}' ;; + *) echo "usage: $0 up | down | recreate | ssh | status" >&2; exit 1 ;; +esac diff --git a/labs/09-ansible/README.md b/labs/09-ansible/README.md index 69d8550..4da061b 100644 --- a/labs/09-ansible/README.md +++ b/labs/09-ansible/README.md @@ -1,26 +1,194 @@ # Lab 09 β€” Ansible: rebuild the server in one command - **Phase:** 4 β€” Infrastructure as Code -- **Status:** ⬜ todo +- **Status:** 🟨 in progress - **Started / finished:** β€” / β€” +- **Environment:** a disposable "fake VPS" container on the Mac; the same playbook runs on a real VPS + +> πŸ“– **Learn first:** read [knowledge.md](knowledge.md) before starting the steps below. ## Goal -Automate everything from phase 1 and 2 so the server is disposable. +Everything done by hand in labs 01 and 05–08 becomes code. The server becomes **disposable**: destroy it, create a new one, run one command, and it's back. ## Done when -- [ ] Roles for hardening, nginx/proxy, docker, backup -- [ ] Playbook is idempotent (second run reports no changes) -- [ ] Secrets in `ansible-vault` -- [ ] **Destroyed the VPS, created a new one, ran one command, everything came back** +- [ ] Roles for hardening (common, users, firewall, ssh, fail2ban), Docker, and the OpsForge stacks +- [ ] One command takes a **fresh** server to the full setup +- [ ] Idempotent: a second run reports `changed=0` +- [ ] Destroyed the (fake) VPS, created a new one, ran one command, and everything came back +- [ ] Secrets in `ansible-vault` for production (test values only in `inventory/test`) +- [ ] An app deploy (`deploy.sh`) works on the Ansible-built server +- [ ] On a real VPS: the same command, with Let's Encrypt via `traefik_acme_email` + +## Measured while preparing this lab + +On the fake VPS (Docker Desktop, Apple Silicon): + +| Run | Detected as | Result | Time | +|---|---|---|---| +| Fresh server | `root@…:2201` (fresh) | ok=45, **changed=35**, failed=0 | 97 s | +| Same command again | `ops@…:2202` (hardened) | ok=39, **changed=0** | 20 s | + +Then verified on the server: +- root login denied, the old port closed, only `ops` allowed +- ufw allows only 2202, 80, 443 +- fail2ban counting failures +- Docker 29.9, with `deploy` in the docker group +- Traefik: `401` without auth, `200` with it, HTTP β†’ `301` +- `deploy.sh main` as `deploy` pulled CI's GHCR image `sha-db2a6db`, and `/ready` reported Postgres and Redis ok + +## Files + +See [ansible/README.md](../../ansible/README.md) for the layout and roles. Key files: + +| File | Role | +|---|---| +| [site.yml](../../ansible/site.yml) | Connection detection + all roles in order | +| [group_vars/all.yml](../../ansible/group_vars/all.yml) | Defaults: users, ports, domain, secret variable names | +| [inventory/test/](../../ansible/inventory/test/) | The fake VPS, with test values | +| [inventory/production/](../../ansible/inventory/production/) | Templates for your VPS and vault | +| [roles/ssh/tasks/main.yml](../../ansible/roles/ssh/tasks/main.yml) | The trickiest role: changes the port it's connected through | +| [test/fake-vps.sh](../../ansible/test/fake-vps.sh) | Create and destroy the fake VPS | + +The stacks now take their domain from `DOMAIN` (default `opsforge.localhost`), so the same compose files work locally and on a server. ## Steps +### 0. Install Ansible + +```bash +brew install ansible # includes the community.* and ansible.posix collections +ansible --version +``` + +### 1. Start a fresh fake VPS + +```bash +ansible/test/fake-vps.sh up +ssh -p 2201 root@127.0.0.1 'hostnamectl; whoami' # a "fresh cloud server": root, port 22 +``` + +It's Ubuntu 24.04 with systemd and OpenSSH in a privileged container. Port mapping: `127.0.0.1:2201` β†’ its port 22, `:2202` β†’ the hardened SSH port, `:8080`/`:8443` β†’ its 80/443. + +### 2. Read before you run + +```bash +cd ansible +ansible-inventory --graph # which hosts, which groups +ansible-inventory --host fake-vps --yaml # variables from the inventory +``` + +Read [site.yml](../../ansible/site.yml), then each role in order. For each task, ask: **what does this do the second time it runs?** + +### 3. Dry run + +```bash +ansible-playbook site.yml --check --diff +``` + +`--check` predicts changes without making them. On a **fresh** server it stops at the first `apt` task (`python3-apt must be installed to use check mode`): check mode can't predict steps that depend on things earlier steps would install. On a **hardened** server, `--check` works fully, and `changed=0` means no drift. Try it again after step 4. + +### 4. The real run + +```bash +time ansible-playbook site.yml +``` + +Watch the first play print `root@127.0.0.1:2201 (fresh server)`, then the `ssh` role switch to `ops@…:2202` partway through. + +### 5. Verify like an attacker, then like an admin + +```bash +ssh -p 2202 root@127.0.0.1 # denied +ssh -p 2201 root@127.0.0.1 # times out: port 22 is closed +ssh -p 2202 ops@127.0.0.1 'sudo sshd -T | grep -Ei "^(port|permitroot|passwordauth|allowusers)"; sudo ufw status; sudo fail2ban-client status sshd; sudo docker ps' +curl -sk -u admin:test -H 'Host: traefik.opsforge.localhost' -o /dev/null -w '%{http_code}\n' https://127.0.0.1:8443/dashboard/ +``` + +### 6. Idempotency + +```bash +ansible-playbook site.yml # look at the PLAY RECAP: changed=0 +``` + +If anything shows `changed`, find out why. It's a bug: a task that doesn't really converge. + +### 7. Deploy the app on the new server + +As the `deploy` user, like CI would (lab 07). The GHCR image is public: + +```bash +ssh -p 2202 ops@127.0.0.1 'sudo -iu deploy ~/opsforge/stacks/hello-api/deploy.sh main' +curl -sk -H 'Host: api.opsforge.localhost' https://127.0.0.1:8443/ready +``` + +### 8. Destroy and rebuild, the real test + +```bash +cd .. +ansible/test/fake-vps.sh recreate # the server is gone; a fresh one replaces it +cd ansible && time ansible-playbook site.yml +``` + +Everything comes back with the same command. That's the point of the lab: **the server is cattle, not a pet.** Note the time in "Lessons learned". + +### 9. Change something the Ansible way + +Pick one, and make the change **only in the repo**, never on the server: +- add port `8000` to `firewall_allowed_tcp_ports` in `inventory/test/group_vars/servers.yml` +- change `fail2ban_maxretry` to `3` + +Run the playbook, check that only the related tasks show `changed`, then revert. Also try `--tags firewall`. + +Then try **drift**: change something by hand on the server (e.g. `sudo ufw allow 9999`), and re-run. Does Ansible remove it? Why not? (Hint: Ansible enforces what's *declared*, and doesn't remove what isn't.) + +### 10. Production, when you have a VPS + +```bash +cd ansible +cp inventory/production/hosts.yml.example inventory/production/hosts.yml # set +cp inventory/production/group_vars/servers/vault.yml.example inventory/production/group_vars/servers/vault.yml +# fill in: htpasswd -nB admin and openssl rand -hex 24 +ansible-vault encrypt inventory/production/group_vars/servers/vault.yml +head -1 inventory/production/group_vars/servers/vault.yml # $ANSIBLE_VAULT;1.1;AES256 +# edit inventory/production/group_vars/servers/main.yml: domain, traefik_acme_email, deploy_ssh_pubkey +ssh root@ true # accept the host key once (compare with the provider's console) +ansible-playbook -i inventory/production site.yml --ask-vault-pass +``` + +DNS `A` records for `` subdomains must point to the VPS **before** this run, or Let's Encrypt can't validate them. + +### 11. Clean up + +```bash +ansible/test/fake-vps.sh down +``` + ## Code +- [ansible/](../../ansible/) +- Stack labels now use `${DOMAIN:-opsforge.localhost}` (all files in [stacks/](../../stacks/)) + ## What broke +_Write down errors and fixes here as you go._ + +Found while preparing this lab: +- **Test overrides ignored** (Ansible tried ports 2222 and 22): the playbook's `group_vars/all.yml` takes precedence over the inventory's `group_vars/all`. Fixed by putting hosts in a `servers` group and overriding in `group_vars/servers`. +- **"Hardened" detected on a fresh server:** Docker Desktop's port forwarding accepts TCP on `127.0.0.1:2202` even with nothing listening behind it. Fixed by testing an actual SSH login instead of an open port. +- **New SSH port = unknown host:** `known_hosts` is per host *and* port. Fixed by copying the server's host key to the new port entry during the switch (no `StrictHostKeyChecking=no`). +- **`--check` on a fresh server connected as `ops@:2202`:** check mode *skips* `command` tasks, so the login test never ran, and its missing result counted as "hardened". Fixed with `check_mode: false` on that read-only task, and `rc | default(1)`. +- **`python3-debian is not installed`** for `deb822_repository`: added to the base packages. +- **`'ansible_managed' is undefined`** in a `copy` task: that variable only exists in templates. +- **Docker-in-Docker** `overlay … invalid argument` in the fake VPS: overlay can't nest on overlay. Fixed with volumes for `/var/lib/docker` and `/var/lib/containerd`. + ## Lessons learned ## References + +- [Ansible: getting started](https://docs.ansible.com/ansible/latest/getting_started/index.html) +- [Ansible: variable precedence](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html#understanding-variable-precedence) +- [Ansible: roles](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_reuse_roles.html) +- [Ansible Vault](https://docs.ansible.com/ansible/latest/vault_guide/index.html) +- [ansible-lint](https://ansible.readthedocs.io/projects/lint/) diff --git a/labs/09-ansible/knowledge.md b/labs/09-ansible/knowledge.md new file mode 100644 index 0000000..736efac --- /dev/null +++ b/labs/09-ansible/knowledge.md @@ -0,0 +1,179 @@ +# Lab 09 β€” Knowledge base + +Study this **before** the hands-on part in [README.md](README.md). Add your own notes as you learn. + +## Why this matters + +Your lab 01 server lives in your memory and your shell history. If it dies, or you need a second one, you redo everything by hand and probably miss a step. **Configuration management** writes the desired state of a server as code: + +- **Reproducible:** a new server is one command away. +- **Reviewable:** changes go through git and PRs, like application code. +- **Documented:** the playbook *is* the documentation, and it can't go out of date. +- **Recoverable:** a broken server is replaced, not repaired. + +This is the "**cattle, not pets**" idea. Pets get names and nursing; cattle are numbered and replaced. Your VPS should be cattle. + +## Core concepts + +### How Ansible works + +``` +Your Mac (control node) Server (managed node) +ansible-playbook site.yml ──── SSH ────────────▢ runs small Python modules, returns JSON + inventory: which hosts nothing to install except Python + sshd + playbook: what state they should be in +``` + +- **Agentless:** it only needs SSH and Python on the server. Compare Puppet or Chef, which need an agent running. +- **Push-based:** you run it from your machine (or CI). Nothing on the server pulls config. +- Every task calls a **module** (`apt`, `user`, `template`, `ufw`…) that checks the current state and changes it only if needed. + +### Vocabulary + +| Term | Meaning | Here | +|---|---|---| +| **Inventory** | The hosts, grouped | `inventory/test/hosts.yml`: group `servers` | +| **Playbook** | Ordered plays: "these hosts β†’ these tasks/roles" | `site.yml` | +| **Play** | One `hosts:` + tasks block | "Work out how to connect", "Configure the server" | +| **Task** | One module call | `ansible.builtin.apt: name=ufw` | +| **Module** | The code a task runs | `apt`, `template`, `community.general.ufw` | +| **Role** | A reusable bundle: tasks, handlers, templates, defaults | `roles/ssh/` | +| **Handler** | A task that runs only when notified, once, at the end (or at `flush_handlers`) | "Restart sshd" | +| **Facts** | Info gathered from the host | `ansible_facts['distribution_release']` = `noble` | +| **Collection** | A package of modules and roles | `community.docker`, `ansible.posix` | + +### Declarative and idempotent + +You describe **what should be true**, not the steps to get there: + +```yaml +- ansible.builtin.user: + name: ops + groups: sudo +``` + +means "user `ops` exists and is in `sudo`". The first run creates it (`changed`); every later run finds it already true (`ok`). That's **idempotency**: running it once or a hundred times gives the same result. A second run reporting `changed=0` is your test that every task really converges. + +Watch out for: +- `command:` / `shell:` always report `changed`, unless you set `changed_when:` (as the `sshd -t` check does). +- Tasks that generate random values or timestamps. +- "Converges but flips": two tasks that keep undoing each other. + +### Templates and handlers + +- **Templates** (`*.j2`, Jinja2) render files with variables: `Port {{ ssh_port }}`. +- A changed file usually needs a service restart. `notify: Restart sshd` runs the handler **once**, **only if** something changed. +- Handlers normally run at the end of the play. `meta: flush_handlers` runs them *now*. The ssh role needs that, because it must reconnect on the new port before continuing. + +### Variables and precedence + +Variables come from many places, and **the higher one wins**. The ones used here, simplified, from lowest to highest: + +``` +role defaults (roles/x/defaults/main.yml) ← safest place for role defaults +inventory group_vars/all +playbook group_vars/all (ansible/group_vars/all.yml) +inventory group_vars/ (inventory/test/group_vars/servers.yml) ← per-environment overrides +host vars +set_fact / registered vars ← what the ssh role uses to switch port and user +extra vars (-e) ← beats everything, even set_fact +``` + +This lab hit that trap: test overrides in the inventory's `group_vars/all` lost to the playbook's `group_vars/all`. Moving them to a named group (`servers`) fixed it. + +### Changing the connection mid-play + +The ssh role moves sshd from port 22 to `ssh_port`, while Ansible is connected through port 22. The safe order: + +1. The firewall allows **both** ports. +2. Write the sshd config; validate it with `sshd -t`, so a typo can't lock you out. +3. `flush_handlers`: restart sshd now. +4. Copy the host key to the new port in `known_hosts`. +5. `set_fact` the new `ansible_port`/`ansible_user`, `reset_connection`, `wait_for_connection`. +6. Only then close port 22. + +The first play detects which state the server is in by **really logging in** as the admin user. So the same command works on a fresh server and on a hardened one. + +### Secrets: Ansible Vault + +- `ansible-vault encrypt file.yml` encrypts it with AES-256. `--ask-vault-pass` (or `--vault-password-file`) decrypts it at run time. +- Convention: real values are `vault_*` in `vault.yml`, referenced from readable variables (`traefik_dashboard_users: "{{ vault_traefik_dashboard_users }}"`). Then you can grep where a secret is used without decrypting anything. +- `no_log: true` stops secrets from being printed in task output. +- An encrypted vault *can* be committed. A weak vault password, or an unencrypted file committed by mistake, can't be undone. Rotate the secret if that happens. + +### Ansible vs Terraform vs Docker + +| Tool | Manages | Question it answers | +|---|---|---| +| **Terraform** (lab 10) | Cloud resources: VPS, DNS, firewall | "Which machines exist?" | +| **Ansible** (this lab) | What's on a machine: users, packages, configs, services | "How is each machine configured?" | +| **Docker / Compose** | The apps, packaged | "What runs, in which version?" | +| **CI/CD** (lab 07) | Releases | "When does a new version go out?" | + +In this repo: Terraform creates the VPS β†’ Ansible configures it β†’ CI deploys app releases onto it. Ansible deliberately **doesn't** deploy hello-api releases: it prepares the files and `.env`, and `deploy.sh` does the rollout. + +### Drift + +**Drift** is when a server no longer matches its code, e.g. someone ran `ufw allow 9999` by hand. Ansible fixes drift in what it *manages*, but doesn't remove what it doesn't know about. The cure is discipline (never change servers by hand) or rebuilding often from code, which makes drift impossible. + +### Testing infrastructure code + +- `ansible-playbook --syntax-check`, `ansible-lint` (style and common mistakes). +- `--check --diff`: a dry run that predicts changes. +- A **throwaway target**: this lab's fake VPS. The bigger version of that idea is **Molecule**, which creates containers or VMs, runs the role, checks idempotency, and destroys everything, often in CI. + +## Key terms + +| Term | Meaning | +|---|---| +| Idempotent | Same result whether run once or many times | +| Declarative | Describe the end state, not the steps | +| Agentless | No software installed on managed hosts | +| Convergence | Repeated runs bring the system to the declared state | +| Drift | Differences between real state and declared state | +| Cattle vs pets | Replaceable, identical servers vs hand-cared-for unique ones | +| Handler | Runs once, only when notified by a change | +| Vault | Encrypted variables file | + +## Commands to know + +| Command | What it does | +|---|---| +| `ansible-inventory --graph` / `--host --yaml` | Show hosts and groups / a host's variables | +| `ansible servers -m ping` | Check Ansible can reach and run Python on hosts | +| `ansible-playbook site.yml --syntax-check` | Parse only | +| `ansible-playbook site.yml --check --diff` | Dry run, show file diffs | +| `ansible-playbook site.yml --tags ssh` / `--skip-tags docker` | Run part of a playbook | +| `ansible-playbook site.yml --start-at-task "Install fail2ban"` | Resume from a task | +| `ansible-playbook site.yml -v` / `-vvv` | More output, down to SSH details | +| `ansible-vault encrypt / edit / view ` | Manage encrypted files | +| `ansible-lint` | Lint playbooks and roles | + +## Before you start, can you answer these? + +1. What does "idempotent" mean, and how do you prove a playbook is? +2. Why must the firewall role run **before** the ssh role? +3. Why doesn't Ansible need anything installed on the server except Python and sshd? +4. When does a handler run, and why does the ssh role call `flush_handlers`? +5. Which tool creates the VPS, which configures it, and which deploys the app? + +## After the lab, check yourself + +1. How long did a full rebuild from a fresh server take? +2. What did the first play detect on the first run, and on the second? +3. Which tasks changed on the second run, if any, and why? +4. What happened to your hand-made `ufw allow 9999` after re-running the playbook? +5. Why are the test secrets in `inventory/test` acceptable, and the production ones not? +6. Name two things on the server Ansible doesn't manage yet (hint: lab 03, lab 06). + +## My notes + +_Your own explanations, analogies, and "aha" moments._ + +## Further reading + +- [Ansible: playbook guide](https://docs.ansible.com/ansible/latest/playbook_guide/index.html) +- [Ansible: best practices / tips and tricks](https://docs.ansible.com/ansible/latest/tips_tricks/index.html) +- [Variable precedence](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html#understanding-variable-precedence) +- [Molecule: testing Ansible roles](https://ansible.readthedocs.io/projects/molecule/) +- [The history of pets vs cattle](https://cloudscaling.com/blog/cloud-computing/the-history-of-pets-vs-cattle/) diff --git a/stacks/README.md b/stacks/README.md index 501e8db..a75df03 100644 --- a/stacks/README.md +++ b/stacks/README.md @@ -12,3 +12,5 @@ Docker Compose stacks, one folder per stack (`stacks//compose.yaml`, plus | [portainer](portainer/) | Docker management UI (`portainer.`); read-write Docker socket | 06 | Web-facing stacks join the shared external `proxy` network (`docker network create proxy`) and are routed by Traefik labels. + +Router hostnames use `${DOMAIN:-opsforge.localhost}`: unset locally, set to your real domain on a server (written to each stack's `.env` by [Ansible](../ansible/), lab 09). diff --git a/stacks/gitea/compose.yaml b/stacks/gitea/compose.yaml index 9b73732..2041ffc 100644 --- a/stacks/gitea/compose.yaml +++ b/stacks/gitea/compose.yaml @@ -9,8 +9,8 @@ services: image: gitea/gitea:1.25.5-rootless environment: # Gitea reads GITEA__
__ into app.ini on startup. - GITEA__server__DOMAIN: git.opsforge.localhost - GITEA__server__ROOT_URL: https://git.opsforge.localhost/ + GITEA__server__DOMAIN: git.${DOMAIN:-opsforge.localhost} + GITEA__server__ROOT_URL: https://git.${DOMAIN:-opsforge.localhost}/ GITEA__server__SSH_DOMAIN: localhost GITEA__server__SSH_PORT: "2222" GITEA__database__DB_TYPE: sqlite3 @@ -27,7 +27,7 @@ services: - proxy labels: - traefik.enable=true - - traefik.http.routers.gitea.rule=Host(`git.opsforge.localhost`) + - traefik.http.routers.gitea.rule=Host(`git.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.gitea.entrypoints=websecure - traefik.http.services.gitea.loadbalancer.server.port=3000 restart: unless-stopped diff --git a/stacks/hello-api/compose.traefik.yaml b/stacks/hello-api/compose.traefik.yaml index 4515480..04773ee 100644 --- a/stacks/hello-api/compose.traefik.yaml +++ b/stacks/hello-api/compose.traefik.yaml @@ -12,7 +12,7 @@ services: - proxy # reachable by Traefik labels: - traefik.enable=true - - traefik.http.routers.hello-api.rule=Host(`api.opsforge.localhost`) + - traefik.http.routers.hello-api.rule=Host(`api.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.hello-api.entrypoints=websecure - traefik.http.services.hello-api.loadbalancer.server.port=8000 # Zero-downtime deploys (lab 08): diff --git a/stacks/portainer/compose.yaml b/stacks/portainer/compose.yaml index 1f9aea8..e538304 100644 --- a/stacks/portainer/compose.yaml +++ b/stacks/portainer/compose.yaml @@ -15,7 +15,7 @@ services: - proxy labels: - traefik.enable=true - - traefik.http.routers.portainer.rule=Host(`portainer.opsforge.localhost`) + - traefik.http.routers.portainer.rule=Host(`portainer.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.portainer.entrypoints=websecure - traefik.http.services.portainer.loadbalancer.server.port=9000 restart: unless-stopped diff --git a/stacks/traefik/compose.yaml b/stacks/traefik/compose.yaml index 61c6587..05ad2ee 100644 --- a/stacks/traefik/compose.yaml +++ b/stacks/traefik/compose.yaml @@ -21,7 +21,7 @@ services: - proxy labels: - traefik.enable=true - - traefik.http.routers.dashboard.rule=Host(`traefik.opsforge.localhost`) + - traefik.http.routers.dashboard.rule=Host(`traefik.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.dashboard.entrypoints=websecure - traefik.http.routers.dashboard.service=api@internal - traefik.http.routers.dashboard.middlewares=dashboard-auth@file diff --git a/stacks/uptime-kuma/compose.yaml b/stacks/uptime-kuma/compose.yaml index 6b5342f..37addd1 100644 --- a/stacks/uptime-kuma/compose.yaml +++ b/stacks/uptime-kuma/compose.yaml @@ -14,7 +14,7 @@ services: - proxy labels: - traefik.enable=true - - traefik.http.routers.uptime-kuma.rule=Host(`status.opsforge.localhost`) + - traefik.http.routers.uptime-kuma.rule=Host(`status.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.uptime-kuma.entrypoints=websecure - traefik.http.services.uptime-kuma.loadbalancer.server.port=3001 restart: unless-stopped diff --git a/stacks/whoami/compose.yaml b/stacks/whoami/compose.yaml index 515f2bd..a0ff154 100644 --- a/stacks/whoami/compose.yaml +++ b/stacks/whoami/compose.yaml @@ -10,7 +10,7 @@ services: - proxy labels: - traefik.enable=true - - traefik.http.routers.whoami.rule=Host(`whoami.opsforge.localhost`) + - traefik.http.routers.whoami.rule=Host(`whoami.${DOMAIN:-opsforge.localhost}`) - traefik.http.routers.whoami.entrypoints=websecure restart: unless-stopped # No ports: Traefik reaches it over the proxy network.