From 64bcfd7305cb69c59ac156d1b968d3815f65bb2c Mon Sep 17 00:00:00 2001 From: Julien D <150942337+jdaln@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:11:24 +0200 Subject: [PATCH] add AWS providers: aws-eks and aws-rke2, with IAM deploy tooling Introduce two AWS cluster stacks and the IAM plumbing to deploy them: - aws-eks: managed Kubernetes on EKS, with EFS storage, a haproxy-edge ingress path, and the mip-contract delivered as a local Helm chart. Deployed end to end with mip-infra; this is the tested path so far. - aws-rke2: self-managed RKE2 on EC2, with data disk, guest agent and node-name knobs, plus cluster-power and kubeconfig helper scripts. Not yet exercised end to end. - scripts/aws-iam: create-deploy-identity.sh and policy documents that generate the exact rights each stack needs, with the deploy credential locked to the operator's source IP. - upcloud-uks: pair MetalLB pools with an L2Advertisement. - docs: register the AWS stacks with region carbon and egress columns. --- .github/workflows/validate.yml | 2 + README.md | 16 +- docs/access.md | 5 +- docs/architecture.md | 40 +- docs/aws-credentials.md | 259 ++++++++++++ docs/cost-comparison.md | 161 ++++++-- docs/firewall.md | 43 ++ docs/maintenance.md | 24 +- docs/providers.md | 12 +- docs/why-self-managed.md | 35 +- modules/rke2-bootstrap/README.md | 7 +- .../rke2-bootstrap/cloudinit/agent.yaml.tftpl | 73 +++- .../cloudinit/server.yaml.tftpl | 54 ++- modules/rke2-bootstrap/main.tf | 29 +- modules/rke2-bootstrap/variables.tf | 51 +++ providers/aws-eks/.terraform.lock.hcl | 128 ++++++ providers/aws-eks/README.md | 378 ++++++++++++++++++ providers/aws-eks/addons.tf | 161 ++++++++ .../aws-eks/charts/mip-contract/Chart.yaml | 12 + .../templates/clusterissuers.yaml | 43 ++ .../templates/haproxy-edge-configmap.yaml | 46 +++ .../mip-contract/templates/haproxy-edge.yaml | 58 +++ .../mip-contract/templates/metallb.yaml | 30 ++ .../aws-eks/charts/mip-contract/values.yaml | 33 ++ providers/aws-eks/edge.tf | 183 +++++++++ providers/aws-eks/main.tf | 285 +++++++++++++ providers/aws-eks/outputs.tf | 83 ++++ providers/aws-eks/storage.tf | 194 +++++++++ providers/aws-eks/terraform.tfvars.example | 52 +++ providers/aws-eks/variables.tf | 268 +++++++++++++ providers/aws-eks/versions.tf | 69 ++++ providers/aws-rke2/.terraform.lock.hcl | 76 ++++ providers/aws-rke2/Makefile | 36 ++ providers/aws-rke2/README.md | 242 +++++++++++ providers/aws-rke2/locals.tf | 42 ++ providers/aws-rke2/network.tf | 52 +++ providers/aws-rke2/outputs.tf | 84 ++++ providers/aws-rke2/scripts/cluster-power.sh | 73 ++++ providers/aws-rke2/scripts/contract.sh | 38 ++ .../aws-rke2/scripts/fetch-kubeconfig.sh | 19 + .../aws-rke2/scripts/setup-fork-git-server.sh | 52 +++ providers/aws-rke2/security.tf | 115 ++++++ providers/aws-rke2/servers.tf | 158 ++++++++ providers/aws-rke2/terraform.tfvars.example | 42 ++ providers/aws-rke2/variables.tf | 182 +++++++++ providers/aws-rke2/versions.tf | 33 ++ providers/upcloud-uks/addons.tf | 23 ++ renovate.json | 4 +- scripts/aws-iam/create-deploy-identity.sh | 287 +++++++++++++ scripts/aws-iam/mip-ec2-base.json | 135 +++++++ scripts/aws-iam/mip-eks.json | 242 +++++++++++ scripts/aws-iam/mip-rke2.json | 41 ++ 52 files changed, 4718 insertions(+), 92 deletions(-) create mode 100644 docs/aws-credentials.md create mode 100644 providers/aws-eks/.terraform.lock.hcl create mode 100644 providers/aws-eks/README.md create mode 100644 providers/aws-eks/addons.tf create mode 100644 providers/aws-eks/charts/mip-contract/Chart.yaml create mode 100644 providers/aws-eks/charts/mip-contract/templates/clusterissuers.yaml create mode 100644 providers/aws-eks/charts/mip-contract/templates/haproxy-edge-configmap.yaml create mode 100644 providers/aws-eks/charts/mip-contract/templates/haproxy-edge.yaml create mode 100644 providers/aws-eks/charts/mip-contract/templates/metallb.yaml create mode 100644 providers/aws-eks/charts/mip-contract/values.yaml create mode 100644 providers/aws-eks/edge.tf create mode 100644 providers/aws-eks/main.tf create mode 100644 providers/aws-eks/outputs.tf create mode 100644 providers/aws-eks/storage.tf create mode 100644 providers/aws-eks/terraform.tfvars.example create mode 100644 providers/aws-eks/variables.tf create mode 100644 providers/aws-eks/versions.tf create mode 100644 providers/aws-rke2/.terraform.lock.hcl create mode 100644 providers/aws-rke2/Makefile create mode 100644 providers/aws-rke2/README.md create mode 100644 providers/aws-rke2/locals.tf create mode 100644 providers/aws-rke2/network.tf create mode 100644 providers/aws-rke2/outputs.tf create mode 100755 providers/aws-rke2/scripts/cluster-power.sh create mode 100755 providers/aws-rke2/scripts/contract.sh create mode 100755 providers/aws-rke2/scripts/fetch-kubeconfig.sh create mode 100755 providers/aws-rke2/scripts/setup-fork-git-server.sh create mode 100644 providers/aws-rke2/security.tf create mode 100644 providers/aws-rke2/servers.tf create mode 100644 providers/aws-rke2/terraform.tfvars.example create mode 100644 providers/aws-rke2/variables.tf create mode 100644 providers/aws-rke2/versions.tf create mode 100755 scripts/aws-iam/create-deploy-identity.sh create mode 100644 scripts/aws-iam/mip-ec2-base.json create mode 100644 scripts/aws-iam/mip-eks.json create mode 100644 scripts/aws-iam/mip-rke2.json diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b9a01ca..38244f6 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -28,6 +28,8 @@ jobs: stack: - providers/upcloud-rke2 - providers/upcloud-uks + - providers/aws-rke2 + - providers/aws-eks - providers/exoscale-rke2 - providers/exoscale-sks - providers/hetzner-rke2 diff --git a/README.md b/README.md index dfeb2dc..3709fab 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ Kubernetes flavour are **swappable options**, not baked-in assumptions. ├── providers/ # one self-contained stack (root module) per option │ ├── upcloud-rke2/ # self-managed RKE2 on UpCloud VMs — TESTED ✅ │ ├── upcloud-uks/ # UpCloud Managed Kubernetes (UKS) — TESTED +│ ├── aws-rke2/ # self-managed RKE2 on AWS EC2 — UNTESTED ⚠️ +│ ├── aws-eks/ # Amazon EKS (managed) — TESTED ⚠️ │ ├── exoscale-rke2/ # self-managed RKE2 on Exoscale VMs — UNTESTED ⚠️ │ ├── exoscale-sks/ # Exoscale SKS (managed) — UNTESTED ⚠️ │ ├── hetzner-rke2/ # self-managed RKE2 on Hetzner VMs — UNTESTED ⚠️ @@ -27,12 +29,14 @@ Kubernetes flavour are **swappable options**, not baked-in assumptions. │ ├── lima-rke2/ # local Lima VMs (macOS/Linux), free — TESTED ✅ (macOS) │ └── openshift/ # OpenShift — STUB ├── scripts/ # shared: prepare-mip-infra-fork.sh (works on any stack) +│ └── aws-iam/ # least-privilege deploy policies + identity generator └── docs/ ├── providers.md # the support matrix + what "tested" means ├── architecture.md # provider-agnostic architecture + the mip-infra contract - ├── cost-comparison.md # all providers × options, with totals - ├── firewall.md # the cloud-firewall model (UpCloud specifics noted) + ├── cost-comparison.md # all providers × options, with totals, grid CO₂ and egress + ├── firewall.md # the cloud-firewall model (stateless vs stateful noted) ├── access.md # consoles, dashboards, credentials + ├── aws-credentials.md # exactly which AWS rights the stacks need, and why └── why-self-managed.md # self-managed RKE2 vs managed Kubernetes, provider-agnostic ``` @@ -82,6 +86,10 @@ Provider and module versions, the RKE2 version, Helm chart versions, container images and GitHub Actions are all kept current by **Renovate** ([renovate.json](renovate.json)) — see [docs/maintenance.md](docs/maintenance.md). -## License +## License & acknowledgements -[Apache License 2.0](LICENSE). +Apache License 2.0 — see [Apache License 2.0](LICENSE). + +This project has received funding from the European Union’s Horizon 2020 Framework Partnership Agreement No. 650003 (Human Brain Project), and from the European Union’s Horizon Europe research and innovation programme under Grant Agreement No. 101147319 (EBRAINS 2.0). This work was also supported by the Swiss State Secretariat for Education, Research and Innovation (SERI) under contract No. 23.00638 (EBRAINS 2.0). + +The authors acknowledge the support of Amazon Web Services through the AWS Cloud Credit for Research program. Specifically, Amazon EC2 and Amazon EKS were used to develop this test environment's Amazon Terraform provider and advance further the testing of this repository. Any opinions, findings, and conclusions expressed in this material are those of the authors and do not necessarily reflect the views of Amazon Web Services. diff --git a/docs/access.md b/docs/access.md index 668f3dc..89452a8 100644 --- a/docs/access.md +++ b/docs/access.md @@ -13,6 +13,7 @@ mentioned are gitignored. | **MIP frontend** (federation-A portal) | `https://federation-a..sslip.io` | EBRAINS Keycloak login — **dummy creds in dev, login fails by design** until real client credentials are provisioned via `scripts/gen_secrets.sh` | | **Data Catalogue** | `https://datacatalogue..sslip.io` | public pages work; authenticated features need the same Keycloak fix | | **UpCloud Hub** (billing, servers, console) | `https://hub.upcloud.com` | your UpCloud account; API token via `UPCLOUD_TOKEN` env for tofu/upctl | +| **AWS Console** (billing, EC2/EKS, serial console) | `https://console.aws.amazon.com` | your AWS account; credentials via `AWS_PROFILE` (or access keys) for tofu and the `aws` CLI | ## Dashboards without ingress (kubectl port-forward, on purpose) @@ -28,9 +29,11 @@ mentioned are gitignored. |---|---| | **Kubernetes (RKE2)** | `export KUBECONFIG=/kubeconfig` — cluster-admin client cert, API `https://:6443`. Refetch anytime: `SSH_KEY=~/.ssh/mip-dev-cluster ./scripts/fetch-kubeconfig.sh` | | **Argo CD CLI** | `argocd login argocd..sslip.io --grpc-web --username admin --password "$(cat argocd-admin-password)"` | -| **Node SSH** | `ssh -i ~/.ssh/mip-dev-cluster root@` (all four nodes; IPs in `tofu output`) | +| **Node SSH** | `ssh -i ~/.ssh/mip-dev-cluster root@` (all four nodes; IPs in `tofu output`). **On the AWS stacks the login user is `ubuntu`** (Canonical AMIs have no root login) or `ec2-user` on the EKS node group's AL2023 image — so `ssh ubuntu@` then `sudo` | | **Fork git server** | push/clone `ssh://git@/home/git/mip-infra.git` with the same key; Argo CD pulls `ssh://git@10.100.1.10/...` with `~/.ssh/argocd-deploy-key` | | **UpCloud API** | `UPCLOUD_TOKEN` env (never in files) for `tofu` and `upctl` | +| **AWS API** | `AWS_PROFILE` (or `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`) in the env, never in files, for `tofu` and the `aws` CLI. Which rights that principal needs, and a generator for a least-privilege one: [aws-credentials.md](aws-credentials.md) | +| **Kubernetes on `aws-eks`** | `export KUBECONFIG=/kubeconfig`, or `aws eks update-kubeconfig --name mip-eks --region eu-north-1`. Auth is the `aws eks get-token` exec plugin — no long-lived token is stored | ## In-cluster secrets (owner: `scripts/gen_secrets.sh`; read with kubectl) diff --git a/docs/architecture.md b/docs/architecture.md index 11d239b..2acc1a4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,16 +59,38 @@ reads them from `STACK_DIR` to generate the fork diff. ## Per-provider divergences (self-managed) -| Knob | UpCloud | Exoscale | Hetzner | Lima (local) | -|---|---|---|---|---| -| `private_iface` (Canal/flannel bind) | `eth2` | `eth1` (verify) | `enp7s0`/`ens10` (verify) | `eth0` (the user-v2 net replaces Lima's SLIRP NIC) | -| public-IP first-boot lookup | UpCloud metadata JSON | Exoscale metadata | Hetzner metadata | `echo 127.0.0.1` (loopback forwards) | -| firewall/security groups | `upcloud_firewall_rules` | `exoscale_security_group_rule` | `hcloud_firewall` | none (nothing leaves the host) | -| private-net MTU | 1500 | 1500 | **1450** (set CNI MTU) | 1500 (usernet) | -| storage for Longhorn | dedicated block volume | block storage volume | `hcloud_volume` or Longhorn-on-disk | root disk, or `limactl disk` (`data_disk_gib`) | - -These are exactly the values the untested stubs are most likely to need +| Knob | UpCloud | AWS (EC2) | Exoscale | Hetzner | Lima (local) | +|---|---|---|---|---|---| +| `private_iface` (Canal/flannel bind) | `eth2` | `ens5` (verify: single ENI, also the default route) | `eth1` (verify) | `enp7s0`/`ens10` (verify) | `eth0` (the user-v2 net replaces Lima's SLIRP NIC) | +| public-IP first-boot lookup | UpCloud metadata JSON | IMDSv2 — token `PUT` then `meta-data/public-ipv4` | Exoscale metadata | Hetzner metadata | `echo 127.0.0.1` (loopback forwards) | +| firewall/security groups | `upcloud_firewall_rules` (**stateless**) | `aws_vpc_security_group_*_rule` (**stateful**, per role) | `exoscale_security_group_rule` | `hcloud_firewall` | none (nothing leaves the host) | +| private-net MTU | 1500 | 9001 (jumbo, intra-VPC; flannel auto-detects) | 1500 | **1450** (set CNI MTU) | 1500 (usernet) | +| storage for Longhorn | dedicated block volume | `aws_ebs_volume`, presented as `/dev/nvme1n1` (Nitro renames it) | block storage volume | `hcloud_volume` or Longhorn-on-disk | root disk, or `limactl disk` (`data_disk_gib`) | +| stable public-IP anchor | node public IPs (survive stop/start) | Elastic IPs (survive an EC2 stop) | node public IPs | node public IPs | host loopback | +| guest agent | `qemu-guest-agent` | none — Nitro has no equivalent (`guest_agent_package = ""`) | `qemu-guest-agent` | `qemu-guest-agent` | `qemu-guest-agent` | +| node naming | from the server resource | set via cloud-init + RKE2 `node-name` (EC2 derives hostnames from the private IP) | from the instance | from the server | from the Lima instance | + +These are exactly the values the untested stacks are most likely to need corrected on first real `apply`. The Lima stack additionally differs in *how* VMs come up: node IPs are DHCP leases unknown at render time, so the module gets a `__NODE_IP__` sentinel (resolved in-guest) and agents join the CP via its stable `lima-.internal` DNS name (`cp_join_host`). + +## Managed stacks: how the traffic path differs + +Managed clusters have no control-plane node to hang Traefik off and, on EKS, no +node whose public IP survives being recycled. `aws-eks` therefore replaces the +two node-IP anchors with **two Network Load Balancers, each holding an Elastic +IP** — the same two-anchor split, one address per entry point: + +``` +argocd..sslip.io ─► NLB :80/:443 ─► node :30080/:30443 Traefik (DaemonSet, class `traefik`) +*..sslip.io ───────► NLB :80/:443 ─► node :80/:443 haproxy-edge (hostNetwork) + └─► haproxy-public (via cluster DNS) ─► apps +``` + +The one structural difference from the self-managed path: haproxy-edge resolves +`haproxy-public` by its **Service DNS name** rather than a MetalLB VIP, so the +app path holds regardless of which controller ends up owning that Service on a +managed cluster. `upcloud-uks` and `exoscale-sks` leave both entry points to the +cloud's own LB integration and document the gap instead. diff --git a/docs/aws-credentials.md b/docs/aws-credentials.md new file mode 100644 index 0000000..2757660 --- /dev/null +++ b/docs/aws-credentials.md @@ -0,0 +1,259 @@ +# AWS credentials: what rights the stacks need, and why + +The two AWS stacks (`providers/aws-rke2`, `providers/aws-eks`) read their +credentials from the environment and never from a file. This document says +exactly which permissions those credentials need, why each one is there, and how +to create an identity that has those and nothing else. + +**There is no such thing as an AWS "API token".** AWS has three credential +shapes, and the choice matters more than the permission list: + +| Shape | Lifetime | Use it when | +|---|---|---| +| **IAM Identity Center (SSO)** permission set | minutes, auto-refreshed | your org already has Identity Center. Best option: nothing long-lived exists to leak | +| **IAM role** you assume (`--mode role`) | ≤ 1 h, auto-refreshed | you have any identity in the account already. Second best | +| **IAM user access key** (`--mode user --with-access-key`) | forever, until you delete it | quickest for a one-off test, and what "give me an API key" usually means. Rotate it, and delete it when the testing is done | + +## The easy way + +```bash +cd scripts/aws-iam +./create-deploy-identity.sh --print-only # see the policies, create nothing +``` + +Then, with an admin session for the target account: + +```bash +./create-deploy-identity.sh --stack both --region eu-north-1 --mode role \ + --trust-arn arn:aws:iam::111122223333:user/you +``` + +That creates three scoped policies and a role you assume, and prints the +`~/.aws/config` block to use it. For a plain access key instead: + +```bash +./create-deploy-identity.sh --stack rke2 --with-access-key +``` + +The script is idempotent — re-running updates the policies in place. Useful +flags: `--stack rke2|eks|both`, `--region`, `--role-prefix` (must match +`var.cluster_name` in `aws-eks`), `--identity-name`, `--source-ip`, +`--no-ip-lock`, `--print-only`. + +## The credential is locked to your IP + +By default the identity **only works from the public IP the script was run +from**, so a leaked key is useless anywhere else. This is the single most +valuable restriction available here, because it holds even if the permission +list is wrong: an attacker with the key still cannot call AWS. + +It is implemented as one `Deny` statement in `mip-ec2-base.json`, which is +always attached — and because an explicit `Deny` is evaluated across every +attached policy and always wins, that one statement pins the whole credential, +including the EKS and RKE2 policies: + +```json +{ + "Sid": "DenyEverythingFromAnyOtherIp", + "Effect": "Deny", + "Action": "*", + "Resource": "*", + "Condition": { + "NotIpAddress": { "aws:SourceIp": ["203.0.113.7/32"] }, + "BoolIfExists": { "aws:ViaAWSService": "false" } + } +} +``` + +`BoolIfExists` on `aws:ViaAWSService` is not decoration — it is what stops this +from being a footgun. When an AWS service makes a call on your behalf, +`aws:SourceIp` is absent from the request; `NotIpAddress` against an absent key +evaluates **true**, so without that second condition the Deny would fire on +legitimate service-initiated calls. `BoolIfExists` exempts exactly those. + +| Flag | Effect | +|---|---| +| *(default)* | discovers your public IPv4 via `checkip.amazonaws.com` and locks to `/32` | +| `--source-ip 203.0.113.7` | lock to that address (a bare address becomes `/32`) | +| `--source-ip 198.51.100.0/24 --source-ip 203.0.113.7/32` | repeatable — e.g. an office range plus a home address | +| `--no-ip-lock` | no restriction. For a roaming laptop or shared CI runners, where the egress address is not stable | + +**Use the same value for `operator_cidr`** in `terraform.tfvars`. It is the same +address the security groups and the EKS API endpoint allowlist, so one lookup +feeds both: + +```bash +curl -4 -s https://checkip.amazonaws.com # -> use as both --source-ip and operator_cidr +``` + +**The failure mode to recognise:** if your address changes, every AWS call +starts returning `AccessDenied` — including read-only ones — **except +`aws sts get-caller-identity`**, which AWS exempts from IAM policy evaluation +entirely (an explicit Deny cannot block it). That asymmetry is the signature: +`get-caller-identity` succeeding while everything else is denied means the +Deny statement is doing its job, not that the permission list is wrong. +Re-point it: + +```bash +./create-deploy-identity.sh --source-ip "$(curl -4 -s https://checkip.amazonaws.com)/32" +``` + +You will usually need to update `operator_cidr` and re-apply at the same time, +for the same reason. + +## What the rights are, and why + +Three policy documents in [`scripts/aws-iam/`](../scripts/aws-iam), so each +stack gets only what it uses: + +| Document | Needed by | Size | +|---|---|--:| +| `mip-ec2-base.json` | both stacks | ~1.9 kB | +| `mip-rke2.json` | `aws-rke2` only | ~0.6 kB | +| `mip-eks.json` | `aws-eks` only | ~4.4 kB | + +### `mip-ec2-base.json` — the VPC and its plumbing + +| Statement | Grants | Why it is needed | +|---|---|---| +| `DenyEverythingFromAnyOtherIp` | denies **all** actions from outside your IP | [the source-IP lock](#the-credential-is-locked-to-your-ip). Deny wins over every Allow in every attached policy, so this one statement pins the whole credential | +| `ReadOnlyDiscovery` | `ec2:Describe*`, `sts:GetCallerIdentity` | every `tofu plan`/refresh re-reads what it manages. Left as a wildcard on purpose: EC2 `Describe` actions do not support resource-level permissions, so enumerating them individually buys nothing and only risks a missing one failing a refresh. The provider calls `GetCallerIdentity` on every run | +| `UbuntuAmiFromPublicSsmParameter` | `ssm:GetParameter` on `parameter/aws/service/canonical/*` | `aws-rke2` resolves the current Ubuntu 24.04 AMI from Canonical's public SSM parameter instead of hardcoding a per-region AMI id. Scoped to that one public path | +| `VpcLifecycle` | create/delete VPC, internet gateway, subnets, route tables, routes | each stack builds its own VPC — `aws_vpc`, `aws_internet_gateway`, `aws_subnet`, `aws_route_table`, `aws_route_table_association` | +| `SecurityGroups` | create/delete groups and their rules | this is the firewall. `aws-rke2` has three role-based groups; `aws-eks` has groups for SSH, EFS and the load balancers | +| `ElasticIps` | allocate/release/associate addresses | the Elastic IPs that anchor the sslip.io hostnames — the whole no-DNS-to-manage design rests on them | +| `SshKeyPair` | `ImportKeyPair`, `DeleteKeyPair` | uploads `var.ssh_public_key`. Import only — it never asks AWS to *generate* a key, so no private key is ever created server-side | +| `Tagging` | `CreateTags`, `DeleteTags` | the provider's `default_tags` puts `project=mip-dev-cluster` on everything, and `scripts/cluster-power.sh` finds the cluster by that tag. Tag-on-create also requires it during `RunInstances`/`CreateVolume` | + +Every write statement is fenced with `aws:RequestedRegion` = your region, so +these credentials cannot touch any other region. + +### `mip-rke2.json` — the self-managed stack's own resources + +| Statement | Grants | Why | +|---|---|---| +| `Instances` | `RunInstances`, `TerminateInstances`, `StartInstances`, `StopInstances`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions` | the four nodes. Start/Stop is what `make down` / `make up` uses; `ModifyInstanceMetadataOptions` is the IMDSv2 enforcement | +| `LonghornDataVolumes` | create/delete/attach/detach/modify volumes | the per-worker Longhorn data volumes, which are separate resources so they survive a node replacement | + +**No `iam:*` at all.** The nodes deliberately carry no instance profile — +nothing running on them calls AWS — so this stack never needs to create or pass +a role. If you only ever deploy `aws-rke2`, your deploy credentials can create +zero IAM objects, which is a meaningful reduction in blast radius. + +### `mip-eks.json` — the managed stack + +| Statement | Grants | Why | +|---|---|---| +| `EksCluster` | create/delete/describe/update clusters, tags | `aws_eks_cluster`. `DescribeClusterVersions` backs the `aws eks describe-cluster-versions` check in that stack's README | +| `EksNodeGroupsAndAddons` | node group and add-on lifecycle | `aws_eks_node_group` plus the seven `aws_eks_addon` resources (VPC CNI, CoreDNS, kube-proxy, pod-identity-agent, metrics-server, EBS CSI, EFS CSI) | +| `EksAccessEntriesAndPodIdentity` | access entries, access policy associations, pod identity associations | the cluster uses `authentication_mode = "API"` (access entries, not the legacy `aws-auth` ConfigMap), and the CSI drivers get their IAM through EKS Pod Identity | +| `ClusterAndCsiRolesByNamePrefixOnly` | **IAM role creation**, scoped to `arn:aws:iam::*:role/-*` | the stack creates four roles itself: the cluster role, the node role, and one each for the EFS and EBS CSI drivers. **This is the one genuinely privileged grant** — see below | +| `PassOnlyThoseRolesAndOnlyToClusterServices` | `iam:PassRole` on the same prefix, conditioned on `iam:PassedToService` | handing a role to a service requires `PassRole`. Unconditioned `PassRole` is a privilege-escalation path, so it is pinned to those role names *and* to the three services the stack passes roles to: `eks.amazonaws.com` (cluster + node group), `ec2.amazonaws.com` (node role trust), and `pods.eks.amazonaws.com` (the CSI drivers' EKS Pod Identity associations) | +| `AwsManagedPoliciesOnlyReadable` | `GetPolicy` on `arn:aws:iam::aws:policy/*` | the provider reads the AWS-managed policies it attaches (`AmazonEKSClusterPolicy`, `AmazonEKS_CNI_Policy`, the CSI driver policies…). Read-only, and only AWS's own policies | +| `ServiceLinkedRolesForClusterServices` | `CreateServiceLinkedRole`, conditioned on `iam:AWSServiceName` | EKS, the EKS node group, ELB, **EFS** and Auto Scaling each need their service-linked role to exist on first use in an account. Conditioned so it cannot create one for any other service. (EFS was found missing on the first live apply: `CreateFileSystem` fails without it.) | +| `ReadServiceLinkedRoles` | `iam:GetRole` / `ListAttachedRolePolicies`, on `role/aws-service-role/*` **and** `role/AWSServiceRoleFor*` | `CreateNodegroup` checks whether `AWSServiceRoleForAmazonEKSNodegroup` already exists. Both ARN forms are required and the reason is subtle: a service-linked role's real ARN carries the `aws-service-role/` path, but the check addresses it **by bare name**, and when the role does not exist yet IAM cannot know its path — so it authorises against `role/AWSServiceRoleForAmazonEKSNodegroup` at path `/`. Scoping to the path alone therefore fails on a fresh account, which is exactly when you need it. Read-only, and only on roles named `AWSServiceRoleFor*` | +| `EfsForRwxStorageClasses` | EFS file system, mount targets, lifecycle, tags, and `DescribeAccessPoints` | the EFS file system behind the RWX `ceph-corbo-cephfs*` classes — the thing that makes `exareme2`'s ReadWriteMany PVC bindable on a managed cluster. `DescribeAccessPoints` is read-only but worth having: the CSI driver creates one access point per PVC and Terraform does not manage them, so it is how you check for leftovers before or after a teardown | +| `EfsMountTargetNetworkInterfaces` | `ec2:CreateNetworkInterface` / `DeleteNetworkInterface` / `ModifyNetworkInterfaceAttribute` | a mount target *is* an ENI in your subnet, and EFS creates it with **the caller's** credentials rather than a service-linked role — so `elasticfilesystem:CreateMountTarget` alone returns `AccessDeniedException`. Found on the second live apply | +| `NetworkLoadBalancers` | NLB, target group and listener lifecycle | the two NLBs that hold the Elastic IPs and give the sslip.io hostnames a stable address | +| `AttachNodeGroupAsgToTargetGroups` | `autoscaling:*LoadBalancerTargetGroups`, `DescribeAutoScalingGroups` | `aws_autoscaling_attachment` wires the node group's Auto Scaling group into the target groups, so replaced nodes re-register by themselves | + +### The one grant to think about + +`aws-eks` needs **`iam:CreateRole`**. That is the difference between "can build a +dev cluster" and "can create identities", so it is scoped three ways: + +- **By name**: only `arn:aws:iam::*:role/-*`, where the prefix must + equal `var.cluster_name` (default `mip-eks`). It cannot touch any other role. +- **`PassRole` is separately conditioned** on `iam:PassedToService` being + `eks.amazonaws.com`, `ec2.amazonaws.com` or `pods.eks.amazonaws.com` (the + last is what EKS Pod Identity associations pass roles to), so a created role + cannot be handed to an arbitrary service. +- **No `iam:*Policy` write actions at all** — it can attach AWS-managed policies + and read them, but it cannot author a policy. That closes the usual escalation + route of creating a role, writing an admin policy, attaching it, assuming it. + +If even that is too much, add a **permissions boundary** so any role this +identity creates is capped: + +```json +{ + "Sid": "ClusterAndCsiRolesByNamePrefixOnly", + "Condition": { + "StringEquals": { + "iam:PermissionsBoundary": "arn:aws:iam::111122223333:policy/your-boundary" + } + } +} +``` + +Note this makes the stack fail until you also set +`permissions_boundary` on the four `aws_iam_role` resources in +`providers/aws-eks`, which is currently unset. + +## What the cluster itself gets at runtime + +Distinct from the deploy credentials, and worth knowing: + +| Stack | Runtime AWS identity | Why | +|---|---|---| +| `aws-rke2` | **none** — no instance profile on any node | nothing on the nodes calls AWS. Longhorn and MetalLB handle storage and load balancing in-cluster, and the stack deliberately does not enable the AWS cloud provider | +| `aws-eks` | node role (`AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, `AmazonEC2ContainerRegistryReadOnly`) + one role each for the EBS and EFS CSI drivers via Pod Identity | the VPC CNI assigns pod addresses through the EC2 API; the CSI drivers create volumes and EFS access points. All AWS-managed policies, all created by the stack | + +## Verifying before you commit to an apply + +```bash +# 1. Who am I? +aws sts get-caller-identity + +# 2. The real test — plan exercises every Describe call the stacks make. +cd providers/aws-rke2 && tofu init && tofu plan +``` + +If **everything fails except `get-caller-identity`** (which no IAM policy can +deny, so it keeps working even when the lock is active), check the source-IP +lock before suspecting the permission list: + +```bash +curl -4 -s https://checkip.amazonaws.com # still the address you locked to? +``` + +`tofu plan` succeeding proves the read path. The write path only gets exercised +by `apply`, so expect the possibility of one `AccessDenied` mid-apply the first +time. When that happens: + +1. The error names the action, e.g. + `UnauthorizedOperation: ... ec2:CreateNatGateway`. +2. Add it to the matching statement in the relevant JSON. +3. Re-run `./create-deploy-identity.sh` — it updates the policy in place. +4. Re-run `tofu apply`; it resumes from where it stopped. + +Nothing is left half-created by a denial: Terraform records what it made, and +`tofu destroy` cleans up with the same permissions. + +To check a specific action without applying anything: + +```bash +aws iam simulate-principal-policy \ + --policy-source-arn arn:aws:iam::111122223333:user/mip-dev-cluster-deploy \ + --action-names ec2:RunInstances elasticfilesystem:CreateFileSystem \ + --query 'EvaluationResults[].[EvalActionName,EvalDecision]' --output table +``` + +## Hygiene + +- **Never put credentials in a file inside this repo.** `terraform.tfvars` is + gitignored, but it is still the wrong place — the stacks read no credential + variables at all, by design. Use `aws configure --profile mip-dev-cluster` + (which writes to `~/.aws/`, outside the repo) plus + `export AWS_PROFILE=mip-dev-cluster`. +- **State contains secrets.** `terraform.tfstate` holds the RKE2 join token, and + for `aws-eks` the EFS id and cluster CA. It is gitignored; keep it that way. + The kubeconfig `aws-eks` writes contains no token — it calls + `aws eks get-token` at use time. +- **Rotate or delete access keys.** `aws iam list-access-keys --user-name + mip-dev-cluster-deploy` shows age; + `aws iam delete-access-key --user-name mip-dev-cluster-deploy --access-key-id + AKIA...` removes one. +- **Tearing down for good**: `tofu destroy` in each stack, then delete the + identity and its three policies (`aws iam list-attached-user-policies` → + `detach-user-policy` → `delete-policy` → `delete-user`). diff --git a/docs/cost-comparison.md b/docs/cost-comparison.md index 6126eb6..cb5f03d 100644 --- a/docs/cost-comparison.md +++ b/docs/cost-comparison.md @@ -1,7 +1,10 @@ # Cost comparison — all providers × options Monthly cost of the **reference dev cluster** on every provider and Kubernetes -flavour in this repo, plus each provider's plan menu so you can re-size. +flavour in this repo, plus each provider's plan menu so you can re-size. Two +non-price columns are included because they change the decision as much as the +sticker price does: the **grid carbon intensity** of the region each stack +defaults to, and how each provider **charges for egress**. ## Method & caveats @@ -10,49 +13,132 @@ flavour in this repo, plus each provider's plan menu so you can re-size. + public IPs. 3 workers because Argo CD redis-ha needs 3 schedulable nodes. - **UpCloud** figures are **exact**, computed from the live price API (`GET /1.3/price`, zone `fi-hel2`): `credits/hr × €0.01 × 730 h`. +- **AWS** figures are **exact list prices** for `eu-north-1`, read from AWS's + own pricing endpoints (EC2 on-demand, EBS gp3 $0.0836/GB-mo, EFS One Zone + $0.16/GB-mo, NLB $0.02394/LB-hr + $0.0057/LCU-hr, public IPv4 $0.005/hr, EKS + control plane $0.10/hr) × 730 h. The stacks themselves are **not live-tested**, + so treat the *shape* of the bill as the estimate, not the unit prices. AWS + bills in **USD**: totals give the euro equivalent with the USD figure in + brackets, converted at **1 USD = 0.8664 EUR (2026-08-05)** — redo the + conversion before budgeting. - **Exoscale & Hetzner** figures are **estimates from vendor pricing pages** (mid-2026) — these stacks are **not live-tested**. Flagged ⚠️ where a specific number was uncertain at research time. - **VAT**: UpCloud/Exoscale quoted incl. the listed currency basis; **Hetzner - is EXCL. VAT — add 19%** for VAT-liable EU billing. + is EXCL. VAT — add 19%** for VAT-liable EU billing. AWS list prices are excl. + VAT. - **Billing model matters more than sticker price** (see notes): UpCloud Cloud - Native and all hourly plans stop billing compute when powered off; Hetzner - bills stopped-but-not-deleted servers; managed control planes have free dev - tiers. + Native, all UpCloud hourly plans and **EC2** stop billing compute when powered + off; Hetzner bills stopped-but-not-deleted servers; managed control planes + have free dev tiers *except* on AWS. +- **Grid gCO₂e/kWh** is the annual average carbon intensity of the electricity + grid in the country each stack's **default** region sits in — see + [about the carbon column](#about-the-carbon-column) for the source and for what + the number does and does not mean. It is a default, never enforced: every + stack takes its region as a plain variable. - OpenShift: no cost row — it's a stub (managed ROSA/ARO or self-hosted OKD would be priced when implemented). ## Reference dev cluster — totals (24/7, EUR/month) -| Provider × option | Control plane | 3× workers | Storage | IPs/other | **Total 24/7** | Tested | -|---|--:|--:|--:|--:|--:|:--:| -| **Lima (local)** self-managed RKE2 (your workstation) | 0.00 | 0.00 | 0.00 | 0.00 | **€0** — needs ≥ 4 free vCPU / ~16 GiB RAM for the default shape | ✅ | -| **Hetzner** self-managed RKE2 (cx33 + 3× cx43) | 8.49 | 47.97 | 13.73 | 2.40 | **≈ 72.6** ⚠️ excl VAT | ⚠️ | -| **Hetzner** self-managed, ARM (cax21 + 3× cax31) | 10.49 | 62.97 | 13.73 | 2.40 | **≈ 89.6** ⚠️ excl VAT | ⚠️ | -| **UpCloud** UKS managed (free dev CP + 3× 4xCPU-8GB) | 0.00 | 195.54 | incl | 3.51 | **≈ 199** | ✅ | -| **UpCloud** self-managed RKE2 (CN-4x8 + 3× CN-4x16) | 34.76 | 221.61 | 52.36 | 14.04 | **≈ 323** | ✅ | -| **Exoscale** SKS managed (free starter CP + 3× extra-large) | 0.00 | 409.53 | incl | — | **≈ 410** ⚠️ | ⚠️ | -| **Exoscale** self-managed RKE2 (large + 3× extra-large) | 67.89 | 409.53 | 24.00 | ~few | **≈ 501** ⚠️ | ⚠️ | +| Provider × option | Default region | Grid gCO₂e/kWh | Control plane | 3× workers | Storage | IPs/LB/other | **Total 24/7** | Egress | Tested | +|---|---|--:|--:|--:|--:|--:|--:|---|:--:| +| **Lima (local)** self-managed RKE2 (your workstation) | your desk | your own grid | 0.00 | 0.00 | 0.00 | 0.00 | **€0** — needs ≥ 4 free vCPU / ~16 GiB RAM for the default shape | n/a (nothing leaves the host) | ✅ | +| **Hetzner** self-managed RKE2 (cx33 + 3× cx43) | `hel1` (FI) | 57 | 8.49 | 47.97 | 13.73 | 2.40 | **≈ 72.6** ⚠️ excl VAT | 20 TB/server incl., then €1/TB | ⚠️ | +| **Hetzner** self-managed, ARM (cax21 + 3× cax31) | `hel1` (FI) | 57 | 10.49 | 62.97 | 13.73 | 2.40 | **≈ 89.6** ⚠️ excl VAT | as above | ⚠️ | +| **UpCloud** UKS managed (free dev CP + 3× 4xCPU-8GB) | `fi-hel2` (FI) | 57 | 0.00 | 195.54 | incl | 3.51 | **≈ 199** | zero-cost under the fair-transfer quota | ✅ | +| **UpCloud** self-managed RKE2 (CN-4x8 + 3× CN-4x16) | `fi-hel2` (FI) | 57 | 34.76 | 221.61 | 52.36 | 14.04 | **≈ 323** | zero-cost under the fair-transfer quota | ✅ | +| **Exoscale** SKS managed (free starter CP + 3× extra-large) | `ch-gva-2` (CH) | 39 | 0.00 | 409.53 | incl | — | **≈ 410** ⚠️ | ⚠️ check the calculator | ⚠️ | +| **Exoscale** self-managed RKE2 (large + 3× extra-large) | `ch-gva-2` (CH) | 39 | 67.89 | 409.53 | 24.00 | ~few | **≈ 501** ⚠️ | ⚠️ check the calculator | ⚠️ | +| **AWS** EKS managed (EKS CP + 3× m7i.xlarge) | `eu-north-1` (SE) | **35** | 63.25 | 406.42 | 17.80 | 53.30 | **≈ 541** (USD 624) | 100 GB/mo free, then ~$0.09/GB; **+ $0.01/GB each way cross-AZ** | ⚠️ | +| **AWS** self-managed RKE2 (c7i.xlarge + 3× m7i.xlarge) | `eu-north-1` (SE) | **35** | 120.86 | 406.42 | 27.53 | 12.65 | **≈ 567** (USD 655) | as above | ⚠️ | **Same cluster, dev usage with power-off** (~176 h/mo ≈ 8 h × 22 workdays), where the billing model allows it: | Provider × option | Total (dev usage) | Why | |---|--:|---| +| Hetzner self-managed | ≈ 73 (no saving) | Hetzner bills stopped servers; delete to save. Already cheapest 24/7. | | UpCloud self-managed RKE2 | **≈ 128** | Cloud Native compute billed only while on; storage + IPv4 bill 24/7. `make down` overnight. | | UpCloud UKS | ≈ 150–200 | scale node group to 0 to save (deletes/recreates nodes); free CP already | -| Hetzner self-managed | ≈ 73 (no saving) | Hetzner bills stopped servers; delete to save. Already cheapest 24/7. | +| AWS self-managed RKE2 | **≈ 167** (USD 193) | EC2 bills compute per second while running only, and an Elastic IP survives a stop — so `make down` keeps every public IP, hostname and certificate. EBS + 4 EIPs bill 24/7 (~€40). | +| AWS EKS | ≈ 217 (USD 250) | no clean power-off: the only lever is scaling the node group to 0, which **deletes and recreates** nodes. A ~€131/mo floor (control plane + 2 NLBs + EFS + 2 EIPs) bills regardless. | | Exoscale (either) | ≈ 410–500 (little saving) | premium instance pricing dominates | -**Ranking for a dev cluster:** Hetzner self-managed is ~4× cheaper than -UpCloud self-managed and ~7× cheaper than Exoscale, and its "workers" are -actually 8 vCPU. UpCloud wins on the **power-off workflow** (clean stop/resume -with preserved IPs) and is the **tested, production-faithful** option. UKS -(free control plane) roughly halves UpCloud self-managed at 24/7. Exoscale is -the premium option (Swiss data residency is its selling point). +**Ranking for a dev cluster:** Hetzner self-managed is the cheapest by a wide +margin (~4× under UpCloud self-managed, ~8× under AWS) and its "workers" are +actually 8 vCPU. UpCloud is the **tested, production-faithful** option and wins +on the power-off workflow; **AWS reproduces that workflow** (EC2 + Elastic IPs +stop and resume cleanly, keeping every hostname and certificate) but costs +~1.75× more at 24/7. AWS is the most expensive option in the matrix — the price +of being on the biggest cloud — while its `eu-north-1` default is the +**lowest-carbon region here**. Exoscale is the premium European option (Swiss +data residency is its selling point). + +## About the carbon column + +Figures are the **annual average carbon intensity of the national electricity +grid** where each region sits, from **Ember, *Yearly Electricity Data* (2025 +release)** — 2024 annual, location-based averages, in gCO₂e/kWh: + + +| Country | Region(s) used in this repo | gCO₂e/kWh (2024) | +|---|---|--:| +| Sweden | AWS `eu-north-1` (default for both AWS stacks) | 35 | +| Switzerland | Exoscale `ch-gva-2` | 39 | +| Finland | UpCloud `fi-hel2`, Hetzner `hel1` | 57 | +| Ireland | AWS `eu-west-1` (documented fallback) | 256 | +| Germany | AWS `eu-central-1` (documented fallback), Hetzner `nbg1`/`fsn1` | 330 | + +Two things this column is **not**: + +- It is **not** a provider claim. AWS reports matching 100 % of its electricity + consumption with renewable-energy purchases, and other providers make similar + market-based claims. That is a different measure: purchase matching is + accounting, grid intensity is the physical mix on the wire at a given + location, and only the latter varies between the regions above. +- It is **not** a full footprint — embodied carbon of the hardware, datacentre + PUE and water use are all excluded. + +What it is good for is the choice it actually informs: **which region to put a +cluster in**. Sweden, Switzerland and Finland are 5–9× cleaner per kWh than +Germany or Ireland, which is why every stack in this repo defaults to one of +them. Note that Hetzner's two German locations are ~6× their Finnish one for +identical hardware at an identical price — a free improvement if you use that +stack. + +The preference is expressed as **defaults and documentation only**. Every stack +takes its region/zone as an ordinary variable with no allow-list validation, so +moving to Ireland, Frankfurt or anywhere else is a one-line change (adjust the +zone/AZ to match). ## Per-provider plan menus +### AWS (eu-north-1, list prices; €/mo = $/hr × 730 × 0.8664) + +| Item | vCPU | RAM | $/mo | €/mo | Notes | +|---|--:|--:|--:|--:|---| +| m7i.large | 2 | 8 | 78.18 | 67.73 | cheaper control-plane option | +| c7i.xlarge | 4 | 8 | 139.50 | 120.86 | **default CP** (matches the tested shape) | +| t3.xlarge | 4 | 16 | 126.14 | 109.33 | cheapest 4/16 — burstable, may throttle Elasticsearch/exareme2 | +| m7i-flex.xlarge | 4 | 16 | 148.55 | 128.75 | 40 % sustained-CPU baseline, bursts above it | +| m7i.xlarge | 4 | 16 | 156.37 | 135.48 | **default worker** | +| m7i.2xlarge | 8 | 32 | 312.73 | 270.95 | | +| m8g.xlarge (Graviton) | 4 | 16 | 139.25 | 120.65 | **unusable here** — MIP images are amd64-only | +| EBS gp3 | | | 0.0836 /GB | 0.072 /GB | 3 000 IOPS + 125 MB/s included | +| EFS One Zone | | | 0.16 /GB | 0.139 /GB | RWX; single-AZ (what `aws-eks` uses) | +| EFS Standard | | | 0.30 /GB | 0.260 /GB | RWX; multi-AZ | +| Public IPv4 (incl. Elastic IP) | | | 3.65 | 3.16 | per address, attached or not, running or stopped | +| Network Load Balancer | | | 17.48 | 15.14 | + $0.0057/LCU-hr | +| NAT gateway | | | 32.85 | 28.46 | **not used** — nodes egress via the internet gateway | +| EKS control plane — standard support | | | **73.00** | **63.25** | no free dev tier | +| EKS control plane — extended support | | | 438.00 | 379.48 | what an un-upgraded cluster costs | + +Region deltas for the same instances: `eu-west-1` (Ireland) ≈ +2 %, +`eu-central-1` (Frankfurt) ≈ +11 %. `eu-north-1` carries no AMD (`*6a`) +families — check `aws ec2 describe-instance-type-offerings` before switching +either way. + ### UpCloud (fi-hel2, exact from price API; €/mo = credits/hr × €0.01 × 730) | Plan | vCPU | RAM | €/mo | Notes | @@ -108,15 +194,36 @@ the premium option (Swiss data residency is its selling point). managed *cheaper than self-managed on the same cloud* for dev, because you drop a whole control-plane node and its storage. The trade-off is the fidelity gap documented in [why-self-managed.md](why-self-managed.md) - (Cilium, no RKE2, managed-LB vs MetalLB). -- **The power-off workflow only pays off on UpCloud** in this repo — it's why - the tested stack uses Cloud Native plans. Elsewhere, delete-and-recreate is - the cost lever. + (different CNI, no RKE2, managed-LB vs MetalLB). +- **AWS has no free control plane.** EKS is $0.10/hr (~€63/mo) minimum, and + **6× that** once a cluster falls off standard support — so an EKS cluster + nobody upgrades quietly becomes the most expensive line in this document. + Managed still edges out self-managed on AWS, because a €63 control plane beats + a €121 control-plane node. +- **The power-off workflow pays off on UpCloud and AWS.** Both bill compute only + while running, and both preserve public IPs across a stop (UpCloud natively, + AWS via Elastic IPs) — which is what keeps the sslip.io hostnames and issued + certificates valid. Elsewhere, delete-and-recreate is the cost lever. A + managed node group has no equivalent at all. +- **AWS meters things the others bundle.** Every public IPv4 is $3.65/mo whether + attached or not; internet egress is free only for the first 100 GB/mo + account-wide; and cross-AZ traffic costs $0.01/GB **in each direction** — + which is why both AWS stacks default to a single availability zone, where the + alternative would tax every flannel VXLAN packet and every storage replica + write. Both stacks also skip a NAT gateway (~€28/mo) by giving nodes public + addresses behind locked-down security groups. - **Storage inclusion differs**: UpCloud Cloud Native plans include **no** disk (you add block storage — hence the storage column); general/UKS plans and Hetzner/Exoscale instances include a root disk, so only the extra - Longhorn/data volume is charged. + Longhorn/data volume is charged. On AWS every EBS volume is charged, root + included. +- **Only `aws-eks` gives a managed cluster real RWX storage.** EFS is a shared + filesystem, so the contract's `ceph-corbo-cephfs` classes behave as intended; + UKS and SKS ship block-only CSI drivers, where exareme2's ReadWriteMany PVC + cannot bind at all. That may be worth more than the price difference if the + goal is a working mip-infra on a managed control plane. - All estimates should be re-checked on the vendor calculators before - budgeting: [UpCloud](https://calc.upcloud.com/) · + budgeting: [AWS](https://calculator.aws/) · + [UpCloud](https://calc.upcloud.com/) · [Exoscale](https://calculator.exoscale.com/) · [Hetzner](https://www.hetzner.com/cloud/) (and apply VAT). diff --git a/docs/firewall.md b/docs/firewall.md index 589d09d..730dcbc 100644 --- a/docs/firewall.md +++ b/docs/firewall.md @@ -151,6 +151,49 @@ kubectl wait --for=condition=Ready certificate/fw-test --timeout=180s kubectl delete certificate fw-test; kubectl delete secret fw-test-tls ``` +## On AWS: the same model, most of the machinery gone + +Both AWS stacks enforce the identical reachability matrix, but **security groups +are stateful**, which deletes the most intricate part of this document. There +are no source-port accepts, no ephemeral-range scoping, and no source-port +bypass to defend against: one egress rule allows outbound, and the replies come +back automatically. The `net.ipv4.ip_local_port_range` sysctl still ships in +cloud-init (it is harmless and keeps the templates uniform), but nothing depends +on it here. + +What replaces `upcloud_firewall_rules` in +[providers/aws-rke2/security.tf](../providers/aws-rke2/security.tf) is three +security groups composed per node — the same base/api/edge split, expressed as +group membership instead of an ordered rule list: + +| Group | Attached to | Rules | +|---|---|---| +| `nodes` | every node | all traffic from itself (etcd, kubelet, supervisor, flannel VXLAN, MetalLB memberlist, Longhorn), SSH 22 + ICMP from `operator_cidr`, all egress | +| `api` | control plane | 6443 from `operator_cidr` | +| `web` | control plane + worker-0 | 80 world-open (ACME HTTP-01 only), 443 from `app_access_cidr` | + +Consequences worth knowing: + +- **Rule order is irrelevant.** Security groups are allow-only with no + precedence, so there is no "the deny must stay last" hazard — the default is + deny, and anything not allowed is dropped. +- **kubelet (10250), the RKE2 supervisor (9345), etcd and every NodePort** are + reachable only through the `nodes` self-reference, i.e. from inside the + cluster. They need no explicit blocking. +- **IMDSv2 is enforced** (`http_tokens = "required"`), so the instance metadata + service cannot be read by a plain unauthenticated GET — which is why the + first-boot public-IP lookup does a token `PUT` first. +- **The private zone is the VPC subnet**, not a separate SDN: nodes have one ENI + each, and node-to-node traffic is permitted by the group self-reference rather + than by being on an unfiltered network. +- **On `aws-eks`** the same split applies to + [providers/aws-eks/edge.tf](../providers/aws-eks/edge.tf): the two Network Load + Balancers carry the 80/443 rules (NLBs honour security groups), and the ports + they forward to are opened on EKS's own cluster security group. The API + endpoint is allowlisted with `public_access_cidrs` — the AWS analogue of + UKS's `control_plane_ip_filter` — which **must** include the IP `tofu apply` + runs from, since the in-cluster providers connect during the apply. + ## Known limitations - The allowlist is IPv4-only (nodes request no public IPv6; `operator_cidr` diff --git a/docs/maintenance.md b/docs/maintenance.md index ab5f3d2..0c3fd70 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -11,12 +11,14 @@ that live outside `.tf`). | Thing | Where | Manager | Auto-merge | |---|---|---|---| -| Terraform providers (`upcloud`, `random`, `exoscale`, `hcloud`, `kubernetes`, `helm`, `tls`, `local`, …) | every `providers/*/versions.tf` + `modules/*` | native `terraform` (globs `**/*.tf`, recurses all stacks) | minor/patch | +| Terraform providers (`upcloud`, `aws`, `random`, `exoscale`, `hcloud`, `kubernetes`, `helm`, `kubectl`, `tls`, `local`, …) | every `providers/*/versions.tf` + `modules/*` | native `terraform` (globs `**/*.tf`, recurses all stacks) | minor/patch | | `.terraform.lock.hcl` hashes | every stack | `terraform` + `lockFileMaintenance` (weekly) | yes | | Terraform module sources | `providers/*/` → `../../modules/*` | native `terraform` | — (local path, nothing to bump) | | **RKE2 version** (`rke2_version` default) | `providers/*/variables.tf` | customManager → GitHub releases `rancher/rke2` | minor/patch | | **Helm chart versions** (cert-manager, MetalLB, Longhorn, ECK) | `modules/rke2-bootstrap/cloudinit/manifests/*.tftpl` | customManager → `helm` datasource (reads the `repo:`/`chart:`/`version:` triplet) | grouped "kubernetes add-ons" | -| **Container images** (haproxy-edge) | manifest, flagged with a `# renovate:` hint | customManager → `docker` | minor/patch/digest | +| **Helm chart versions in the managed stacks** (`helm_release` blocks: cert-manager, ECK, MetalLB, Traefik) | `providers/{upcloud-uks,exoscale-sks,aws-eks}/addons.tf` | native `terraform` — it extracts `resource "helm_release"` with the `helm` datasource, no custom manager needed | grouped "kubernetes add-ons" | +| **Container images** (haproxy-edge) | the RKE2 manifest template, and `providers/aws-eks/charts/mip-contract/values.yaml`; both flagged with a `# renovate:` hint | customManager → `docker` | minor/patch/digest | +| **The local `mip-contract` chart** (`providers/aws-eks/charts/`) | versioned with the stack, not published | — (nothing to bump; its *contents* are covered by the image manager above) | — | | **GitHub Actions** (`opentofu/setup-opentofu`, `actions/checkout`) | `.github/workflows/*` | native `github-actions`, digest-pinned | yes | Everything is grouped (one "terraform providers" PR, one "kubernetes add-ons" @@ -28,6 +30,19 @@ for manual review. A Dependency Dashboard issue lists everything pending. - **Node OS image** (`os_template = "Ubuntu Server 24.04 LTS …"`): pinned to an LTS on purpose; bump deliberately (it triggers a rolling node replacement). + On `aws-rke2` the AMI is resolved from Canonical's SSM parameter and `ami` is + in `ignore_changes`, so a new image only lands on an explicit + `tofu apply -replace`. +- **Managed Kubernetes versions** (`k8s_version` in the UKS/SKS/EKS stacks): a + control-plane upgrade is a deliberate, one-way operation, and the vendors' + available versions do not track upstream Kubernetes releases. On EKS there is + a cost consequence too — a version that falls off standard support goes from + $0.10 to $0.60 per cluster-hour — so check + `aws eks describe-cluster-versions` on a calendar, not via a PR. +- **EKS add-on versions** (`aws_eks_addon`): `addon_version` is deliberately + *unset* in `providers/aws-eks`, so EKS installs the default version for the + cluster's Kubernetes version. Nothing is pinned, so nothing can drift and + there is nothing for Renovate to chase. - **The add-on charts' *app* versions** inside upstream charts (e.g. the Elasticsearch/Kibana version ECK deploys) — those follow the chart bump. - **mip-infra itself** — it's a separate repo with its own pinned upstream @@ -46,8 +61,9 @@ for manual review. A Dependency Dashboard issue lists everything pending. ## Notes / caveats - Providers resolve from the default registry (`registry.terraform.io`); all - providers used here (upcloud, random, exoscale, hcloud, kubernetes, helm, - tls, local) are mirrored there, so no OpenTofu-registry override is needed. + providers used here (upcloud, aws, random, exoscale, hcloud, kubernetes, helm, + kubectl, tls, local) are mirrored there, so no OpenTofu-registry override is + needed. If you ever pin a provider that is OpenTofu-registry-only, add a `registryUrls` rule for `matchDatasources: ["terraform-provider"]`. - Renovate updates lock hashes for providers **already present** in a stack's diff --git a/docs/providers.md b/docs/providers.md index f170504..073f353 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -8,6 +8,8 @@ plain VMs, using `modules/rke2-bootstrap`) or a **managed service**. |---|---|---|---|---|---| | `upcloud-rke2` | UpCloud | self-managed RKE2 | **production-ready for dev** | ✅ full A-Z, 23/23 mip-infra apps green, power-cycle verified | the reference stack | | `upcloud-uks` | UpCloud | managed (UKS, Cilium) | **working** | ✅ provisioned, add-ons applied, verified, then torn down | free dev control plane | +| `aws-rke2` | AWS (EC2) | self-managed RKE2 | code complete | ⚠️ **not tested** | defaults to `eu-north-1` (lowest-carbon EU region, also cheapest of the three); Elastic IPs are the stable sslip.io anchors; security groups are stateful, so no return-traffic rules | +| `aws-eks` | AWS | managed (EKS, VPC CNI) | code complete | ⚠️ **tested** | only managed stack with **real RWX** storage (EFS CSI); two NLB+EIP anchors; no free control plane ($73/mo); VPC CNI breaks the pod-CIDR assumption | | `exoscale-rke2` | Exoscale | self-managed RKE2 | code complete | ⚠️ **not tested** | resource/arg names from provider docs | | `exoscale-sks` | Exoscale | managed (SKS, Calico/Cilium) | code complete | ⚠️ **not tested** | free "starter" control plane | | `hetzner-rke2` | Hetzner | self-managed RKE2 | code complete | ⚠️ **not tested** | note MTU 1450; hccm/CSI optional | @@ -25,12 +27,18 @@ plain VMs, using `modules/rke2-bootstrap`) or a **managed service**. up, kubeconfig worked, the managed-cluster add-on layer applied and the mip-infra contract objects were verified; then `tofu destroy`. See its README for exactly how far the mip-infra deployment was taken. -- **⚠️ not tested** (Exoscale, Hetzner): the Terraform is written against the - providers' current documented resource/argument names but has **never been +- **⚠️ not tested** (AWS, Exoscale, Hetzner): the Terraform is written against + the providers' current documented resource/argument names but has **never been `apply`-ed**. Treat as a strong starting point, not a turnkey deploy — expect to fix cloud-init interface names, security-group/firewall port lists, CNI MTU, and image/plan identifiers on first real run. Each stub's README lists its specific unknowns. + The two AWS stacks additionally had every resource argument checked against + the provider's own schema (`tofu providers schema -json`), and every Helm + release in `aws-eks` rendered with `helm template` at its pinned version and + asserted on — Helm ignores unknown values silently, so an unverified values + key is a no-op rather than an error. That closes the class of bug where a + stack applies cleanly and still does not do what it says. ## Adding a provider diff --git a/docs/why-self-managed.md b/docs/why-self-managed.md index c4512da..d084fb9 100644 --- a/docs/why-self-managed.md +++ b/docs/why-self-managed.md @@ -13,17 +13,20 @@ Managed Kubernetes services rarely expose their underlying distribution and often ship a different default CNI (Cilium, for example). mip-infra encodes RKE2-specific assumptions upstream: -| Upstream assumption (file) | RKE2 (this cluster) | Typical managed service | -|---|---|---| -| Submariner `clusterCidr: 10.42.0.0/16` (`common/submariner/operator/values.yaml`) — RKE2's default pod CIDR | matches out of the box | different CIDR → fork edit needed | -| ECK operator pre-installed in `kube-system` (`common/monitoring/eck/README.md`) | replicated via an RKE2 `HelmChart` manifest | not provided → install & wire manually | -| Argo CD ingress expects class `traefik` (`argo-setup/patches/argocd-server-ingress.yaml`) | Traefik ships as RKE2's bundled default ingress | not provided | -| Canal/Calico-flavoured NetworkPolicy behaviour (`common/security/*`) | Canal, as in production | a different CNI's policy engine (e.g. Cilium) behaves differently | -| MetalLB pool `pool-no-auto` + fixed `loadBalancerIP`s hardcoded in Services | MetalLB deployed with the same pool name; only the two IP literals change | expects its own managed-LB integration instead | +| Upstream assumption (file) | RKE2 (this cluster) | Typical managed service | On EKS specifically | +|---|---|---|---| +| Submariner `clusterCidr: 10.42.0.0/16` (`common/submariner/operator/values.yaml`) — RKE2's default pod CIDR | matches out of the box | different CIDR → fork edit needed | **worse than different**: the VPC CNI gives pods VPC addresses, so there is no pod CIDR at all — set it to the VPC CIDR | +| ECK operator pre-installed in `kube-system` (`common/monitoring/eck/README.md`) | replicated via an RKE2 `HelmChart` manifest | not provided → install & wire manually | installed by `aws-eks`'s add-on layer | +| Argo CD ingress expects class `traefik` (`argo-setup/patches/argocd-server-ingress.yaml`) | Traefik ships as RKE2's bundled default ingress | not provided | installed by `aws-eks` (NodePort DaemonSet) — the one managed stack that closes this gap | +| Canal/Calico-flavoured NetworkPolicy behaviour (`common/security/*`) | Canal, as in production | a different CNI's policy engine (e.g. Cilium) behaves differently | VPC CNI enforces NetworkPolicy through its own agent, again not Canal | +| MetalLB pool `pool-no-auto` + fixed `loadBalancerIP`s hardcoded in Services | MetalLB deployed with the same pool name; only the two IP literals change | expects its own managed-LB integration instead | EKS's legacy in-tree Service Controller claims every unclassed `type: LoadBalancer` Service, so MetalLB needs an explicit `loadBalancerClass` on those Services — two more fork lines | +| RWX StorageClass for exareme2's PVC | Longhorn share-manager | usually block/RWO only → the workload cannot schedule | EFS CSI gives **real RWX** — the one place a managed stack matches the self-managed one | With self-managed RKE2 the mip-infra fork diff stays small (~10 files, mostly repo URLs). Each row above would otherwise be an additional diff or a -behavioural difference from production on a managed service. +behavioural difference from production on a managed service. `aws-eks` closes +two of those rows (Traefik, RWX storage) and makes one worse (the pod CIDR), +which is a fair summary of what a managed cluster costs you here. ## 2. Version and lifecycle control @@ -63,6 +66,22 @@ and recreates nodes. The modest cost difference buys the fidelity described above. See [cost-comparison.md](cost-comparison.md) for the full matrix across all providers. +The same comparison on AWS lands in the same place for a different reason — +there is no free EKS tier, but a managed control plane is still cheaper than a +control-plane *node*: + +| | Self-managed RKE2 (`aws-rke2`) | EKS equivalent (`aws-eks`) | +|---|---|---| +| Control plane | ~€121/mo (`c7i.xlarge` node) | ~€63/mo (EKS fee; **6× that** off standard support) | +| Workers ×3 | ~€406/mo | ~€406/mo | +| Storage | ~€28/mo (EBS root + Longhorn volumes) | ~€18/mo (EBS root + EFS) | +| IPs / load balancers | ~€13/mo (4 Elastic IPs) | ~€53/mo (2 NLBs + 5 public IPv4) | +| **24/7 total** | **~€567/mo** | **~€541/mo** | + +So on AWS the fidelity of self-managing costs about €26/month — but the managed +option gives up node access, the RKE2 version pin, and the pod CIDR, and gains +the two NLBs it needs just to have a stable public address. + ## 5. Trade-offs of self-managing You run the control plane (single node, no HA) and do upgrades yourself; diff --git a/modules/rke2-bootstrap/README.md b/modules/rke2-bootstrap/README.md index 5f0c2da..eb51b04 100644 --- a/modules/rke2-bootstrap/README.md +++ b/modules/rke2-bootstrap/README.md @@ -36,8 +36,11 @@ module "bootstrap" { | Variable | Why it varies | Examples | |---|---|---| -| `private_iface` | flannel/Canal must bind the private NIC, whose name differs per platform | UpCloud `eth2`, Hetzner `enp7s0`, Exoscale `eth1`, Lima `eth0` | -| `public_ipv4_lookup` | first-boot public-IP discovery for tls-san + ACME; default hits the UpCloud metadata service | override with the provider's metadata endpoint/jq; `echo 127.0.0.1` on Lima (loopback port forwards) | +| `private_iface` | flannel/Canal must bind the private NIC, whose name differs per platform | UpCloud `eth2`, Hetzner `enp7s0`, Exoscale `eth1`, Lima `eth0`, AWS `ens5` | +| `public_ipv4_lookup` | first-boot public-IP discovery for tls-san + ACME; default hits the UpCloud metadata service | override with the provider's metadata endpoint/jq; AWS needs the IMDSv2 token dance; `echo 127.0.0.1` on Lima (loopback port forwards) | +| `data_disk_device` (default `/dev/vdb`) | the per-worker Longhorn disk's device name | virtio guests `/dev/vdb`; AWS Nitro renames every EBS volume to `/dev/nvme1n1` whatever the block-device mapping says | +| `guest_agent_package` (default `qemu-guest-agent`) | KVM clouds want the agent so a soft/ACPI stop is a clean shutdown; AWS Nitro has no equivalent and the unit would fail to start | `""` on AWS | +| `cp_node_name` / `worker_node_names` (default `null`) | platforms that derive the hostname from the private IP need the node name set explicitly, or the cluster comes up as `ip-10-100-1-10…` instead of `mip-cp` | AWS `"mip-cp"` / `["mip-w1","mip-w2","mip-w3"]` | | `cp_join_host` (default `null` = use `cp_node_ip`) | platforms where node IPs are unknown at render time but a stable DNS name exists | Lima `lima-mip-cp.internal` | | `issuer_mode` (default `acme`) | `selfsigned` keeps the same ClusterIssuer names but self-signs, for clusters ACME can't reach | local Lima clusters | | `longhorn_replica_count` (default `2`) | compact local clusters with a single worker can't place 2 replicas | `min(2, worker_count)` | diff --git a/modules/rke2-bootstrap/cloudinit/agent.yaml.tftpl b/modules/rke2-bootstrap/cloudinit/agent.yaml.tftpl index 7cbbca2..6e18f9f 100644 --- a/modules/rke2-bootstrap/cloudinit/agent.yaml.tftpl +++ b/modules/rke2-bootstrap/cloudinit/agent.yaml.tftpl @@ -1,16 +1,24 @@ #cloud-config # RKE2 agent (worker) on Ubuntu 24.04 LTS. Rendered by OpenTofu, delivered -# via the UpCloud metadata service. +# via the platform's cloud-init user_data / metadata service. disable_root: false ssh_authorized_keys: - ${ssh_public_key} +%{ if node_name != null ~} +# Platforms that derive the hostname from the private IP (EC2's +# ip-10-0-0-1..compute.internal) need it set here instead. +hostname: ${node_name} +preserve_hostname: false +%{ endif ~} package_update: true packages: - jq - open-iscsi # Longhorn prerequisite - nfs-common # Longhorn RWX (share-manager) prerequisite - - qemu-guest-agent +%{ if guest_agent_package != "" ~} + - ${guest_agent_package} +%{ endif ~} write_files: - path: /etc/rancher/rke2/config.yaml @@ -20,6 +28,9 @@ write_files: token: "${rke2_token}" server: ${server_url} node-ip: ${node_ip} +%{ if node_name != null ~} + node-name: ${node_name} +%{ endif ~} node-label: # Longhorn creates its default storage disk only on labeled nodes # (see manifests/longhorn.yaml.tftpl) — workers only, not the CP. @@ -31,31 +42,67 @@ write_files: content: | net.ipv4.ip_forward = 1 net.ipv6.conf.all.forwarding = 1 - # Pin the ephemeral range to match the cloud firewall's return-traffic - # accepts (see firewall.tf / docs/firewall.md). Do not widen without - # widening the firewall rules too. + # Pin the ephemeral range so a stateless cloud firewall can scope its + # return-traffic accepts to exactly this range (see docs/firewall.md). + # Do not widen it without widening those rules too. Harmless where the + # firewall is stateful (AWS security groups, Hetzner). net.ipv4.ip_local_port_range = 32768 60999 runcmd: - | set -e - # Longhorn data disk (second virtio disk): format once, mount at - # /var/lib/longhorn. The blkid guard keeps re-provisioning non-destructive. - if [ -b /dev/vdb ]; then - if ! blkid /dev/vdb >/dev/null 2>&1; then - mkfs.ext4 -L longhorn /dev/vdb + # cloud-init's package phase can lose a race against the platform's egress + # path coming up (on AWS a public-subnet instance has no internet until its + # EIP associates, seconds after launch) — and a failed install there is + # silent, surfacing much later as broken Longhorn. Retry until the + # prerequisites are really present; on a healthy boot this is a no-op. + for i in $(seq 1 60); do + dpkg -s jq open-iscsi nfs-common >/dev/null 2>&1 && break + apt-get update -qq >/dev/null 2>&1 || true + DEBIAN_FRONTEND=noninteractive apt-get install -qqy jq open-iscsi nfs-common >/dev/null 2>&1 || sleep 5 + done + dpkg -s jq open-iscsi nfs-common >/dev/null 2>&1 + - | + set -e + # Longhorn data disk: format once, mount at /var/lib/longhorn. The device + # name is platform-specific (var.data_disk_device) — virtio guests get + # /dev/vdb, AWS Nitro renames every EBS volume to /dev/nvmeXn1. The blkid + # guard keeps re-provisioning non-destructive. + # + # The attachment is a separate API call that can land after boot reaches + # this point (on AWS it is made only once the instance is running), so + # poll for the device instead of sampling once. + for i in $(seq 1 60); do + [ -b ${data_disk_device} ] && break + sleep 2 + done + if [ -b ${data_disk_device} ]; then + if ! blkid ${data_disk_device} >/dev/null 2>&1; then + mkfs.ext4 -L longhorn ${data_disk_device} fi mkdir -p /var/lib/longhorn grep -q "LABEL=longhorn" /etc/fstab || echo "LABEL=longhorn /var/lib/longhorn ext4 defaults,nofail 0 2" >> /etc/fstab systemctl daemon-reload mount -a || true else - echo "WARNING: no /dev/vdb data disk found — Longhorn will use the OS disk" >&2 + echo "WARNING: no ${data_disk_device} data disk found — Longhorn will use the OS disk" >&2 fi - sysctl --system - - systemctl enable --now iscsid.service qemu-guest-agent.service + # iscsid gets its own command: a missing/unstartable guest agent must not take + # it down with it (Longhorn needs iscsid). + - systemctl enable --now iscsid.service +%{ if guest_agent_package != "" ~} + - systemctl enable --now ${guest_agent_package}.service +%{ endif ~} # multipathd grabs Longhorn's iSCSI devices and breaks volume attach — off for dev. - systemctl disable --now multipathd.socket multipathd.service 2>/dev/null || true - - curl -sfL https://get.rke2.io -o /tmp/rke2-install.sh + - | + set -e + # Retried: transient failures here (a still-settling egress path, a + # get.rke2.io hiccup) would otherwise abort the whole bootstrap. + for i in $(seq 1 30); do + curl -sfL https://get.rke2.io -o /tmp/rke2-install.sh && break || sleep 4 + done + [ -s /tmp/rke2-install.sh ] - INSTALL_RKE2_VERSION="${rke2_version}" INSTALL_RKE2_TYPE=agent sh /tmp/rke2-install.sh - systemctl enable --now rke2-agent.service diff --git a/modules/rke2-bootstrap/cloudinit/server.yaml.tftpl b/modules/rke2-bootstrap/cloudinit/server.yaml.tftpl index 24430d3..f873587 100644 --- a/modules/rke2-bootstrap/cloudinit/server.yaml.tftpl +++ b/modules/rke2-bootstrap/cloudinit/server.yaml.tftpl @@ -1,16 +1,24 @@ #cloud-config # RKE2 server (control plane) on Ubuntu 24.04 LTS. Rendered by OpenTofu, -# delivered via the UpCloud metadata service. +# delivered via the platform's cloud-init user_data / metadata service. disable_root: false ssh_authorized_keys: - ${ssh_public_key} +%{ if node_name != null ~} +# Platforms that derive the hostname from the private IP (EC2's +# ip-10-0-0-1..compute.internal) need it set here instead. +hostname: ${node_name} +preserve_hostname: false +%{ endif ~} package_update: true packages: - jq # metadata parsing below - open-iscsi # Longhorn prerequisite - nfs-common # Longhorn RWX (share-manager) prerequisite - - qemu-guest-agent +%{ if guest_agent_package != "" ~} + - ${guest_agent_package} +%{ endif ~} write_files: - path: /etc/rancher/rke2/config.yaml @@ -19,7 +27,10 @@ write_files: content: | token: "${rke2_token}" node-ip: ${node_ip} - # __PUBLIC_IP__ is resolved at first boot from the UpCloud metadata +%{ if node_name != null ~} + node-name: ${node_name} +%{ endif ~} + # __PUBLIC_IP__ is resolved at first boot from the platform's metadata # service (see runcmd) — a server cannot know its own public IP at # template-render time. tls-san: @@ -30,9 +41,10 @@ write_files: content: | net.ipv4.ip_forward = 1 net.ipv6.conf.all.forwarding = 1 - # Pin the ephemeral range to match the cloud firewall's return-traffic - # accepts (see firewall.tf / docs/firewall.md). Do not widen without - # widening the firewall rules too. + # Pin the ephemeral range so a stateless cloud firewall can scope its + # return-traffic accepts to exactly this range (see docs/firewall.md). + # Do not widen it without widening those rules too. Harmless where the + # firewall is stateful (AWS security groups, Hetzner). net.ipv4.ip_local_port_range = 32768 60999 %{ for name, content in manifests ~} - path: /var/lib/rancher/rke2/server/manifests/${name} @@ -43,6 +55,20 @@ write_files: %{ endfor ~} runcmd: + - | + set -e + # cloud-init's package phase can lose a race against the platform's egress + # path coming up (on AWS a public-subnet instance has no internet until its + # EIP associates, seconds after launch) — and a failed install there is + # silent. jq is also needed by the metadata lookup just below on some + # platforms. Retry until the packages are really present; on a healthy + # boot this is a no-op. + for i in $(seq 1 60); do + dpkg -s jq open-iscsi nfs-common >/dev/null 2>&1 && break + apt-get update -qq >/dev/null 2>&1 || true + DEBIAN_FRONTEND=noninteractive apt-get install -qqy jq open-iscsi nfs-common >/dev/null 2>&1 || sleep 5 + done + dpkg -s jq open-iscsi nfs-common >/dev/null 2>&1 - | set -e # Resolve this node's public IPv4 at first boot and substitute it where @@ -64,9 +90,21 @@ runcmd: /var/lib/rancher/rke2/server/manifests/10-cert-manager.yaml fi - sysctl --system - - systemctl enable --now iscsid.service qemu-guest-agent.service + # iscsid gets its own command: a missing/unstartable guest agent must not take + # it down with it (Longhorn needs iscsid). + - systemctl enable --now iscsid.service +%{ if guest_agent_package != "" ~} + - systemctl enable --now ${guest_agent_package}.service +%{ endif ~} # multipathd grabs Longhorn's iSCSI devices and breaks volume attach — off for dev. - systemctl disable --now multipathd.socket multipathd.service 2>/dev/null || true - - curl -sfL https://get.rke2.io -o /tmp/rke2-install.sh + - | + set -e + # Retried: transient failures here (a still-settling egress path, a + # get.rke2.io hiccup) would otherwise abort the whole bootstrap. + for i in $(seq 1 30); do + curl -sfL https://get.rke2.io -o /tmp/rke2-install.sh && break || sleep 4 + done + [ -s /tmp/rke2-install.sh ] - INSTALL_RKE2_VERSION="${rke2_version}" INSTALL_RKE2_TYPE=server sh /tmp/rke2-install.sh - systemctl enable --now rke2-server.service diff --git a/modules/rke2-bootstrap/main.tf b/modules/rke2-bootstrap/main.tf index 8ab3658..c039fc3 100644 --- a/modules/rke2-bootstrap/main.tf +++ b/modules/rke2-bootstrap/main.tf @@ -31,22 +31,27 @@ locals { } server_user_data = templatefile("${path.module}/cloudinit/server.yaml.tftpl", { - rke2_token = var.rke2_token - rke2_version = var.rke2_version - node_ip = var.cp_node_ip - ssh_public_key = var.ssh_public_key - manifests = local.addon_manifests - public_ipv4_lookup = var.public_ipv4_lookup + rke2_token = var.rke2_token + rke2_version = var.rke2_version + node_ip = var.cp_node_ip + node_name = var.cp_node_name + ssh_public_key = var.ssh_public_key + manifests = local.addon_manifests + public_ipv4_lookup = var.public_ipv4_lookup + guest_agent_package = var.guest_agent_package }) agent_user_data = [ for i, ip in var.worker_node_ips : templatefile("${path.module}/cloudinit/agent.yaml.tftpl", { - rke2_token = var.rke2_token - rke2_version = var.rke2_version - server_url = "https://${coalesce(var.cp_join_host, var.cp_node_ip)}:9345" - node_ip = ip - edge = i == var.edge_worker_index - ssh_public_key = var.ssh_public_key + rke2_token = var.rke2_token + rke2_version = var.rke2_version + server_url = "https://${coalesce(var.cp_join_host, var.cp_node_ip)}:9345" + node_ip = ip + node_name = try(var.worker_node_names[i], null) + edge = i == var.edge_worker_index + ssh_public_key = var.ssh_public_key + data_disk_device = var.data_disk_device + guest_agent_package = var.guest_agent_package }) ] } diff --git a/modules/rke2-bootstrap/variables.tf b/modules/rke2-bootstrap/variables.tf index d0519e2..c4dda4e 100644 --- a/modules/rke2-bootstrap/variables.tf +++ b/modules/rke2-bootstrap/variables.tf @@ -92,3 +92,54 @@ variable "public_ipv4_lookup" { type = string default = "curl -sf --max-time 5 http://169.254.169.254/metadata/v1.json | jq -r '[.network.interfaces[] | select(.type == \"public\") | .ip_addresses[] | select(.family == \"IPv4\") | .address][0] // empty'" } + +variable "data_disk_device" { + description = <<-EOT + Block device of the per-worker Longhorn data disk, formatted and mounted at + /var/lib/longhorn on first boot. Platform-specific: virtio guests expose + the second disk as /dev/vdb, while AWS Nitro renames every EBS volume to + an NVMe name (/dev/nvme1n1) regardless of the block-device mapping. If the + device is absent the agent falls back to the OS disk with a warning. + EOT + type = string + default = "/dev/vdb" +} + +variable "guest_agent_package" { + description = <<-EOT + Hypervisor guest-agent package to install and enable (empty string = none). + KVM-based clouds want qemu-guest-agent — it is what makes a soft/ACPI stop + a clean RKE2/Longhorn shutdown. On AWS Nitro there is no such agent: the + package installs but its unit cannot start, which would take the same + `systemctl enable --now` down with it and leave iscsid disabled — breaking + Longhorn. Set it to "" there. + EOT + type = string + default = "qemu-guest-agent" +} + +variable "cp_node_name" { + description = <<-EOT + Optional explicit name for the control-plane node: sets the OS hostname and + the RKE2 `node-name`, so the Kubernetes node is called this. Needed where + the platform derives a hostname from the private IP (EC2's + ip-10-0-0-1..compute.internal) instead of taking one from the + instance definition. null = leave the platform default. + EOT + type = string + default = null +} + +variable "worker_node_names" { + description = <<-EOT + Optional explicit names for the worker nodes, in worker_node_ips order (see + cp_node_name). null = leave the platform default. + EOT + type = list(string) + default = null + + validation { + condition = var.worker_node_names == null || length(var.worker_node_names) >= length(var.worker_node_ips) + error_message = "worker_node_names must have one entry per worker in worker_node_ips." + } +} diff --git a/providers/aws-eks/.terraform.lock.hcl b/providers/aws-eks/.terraform.lock.hcl new file mode 100644 index 0000000..4cbba85 --- /dev/null +++ b/providers/aws-eks/.terraform.lock.hcl @@ -0,0 +1,128 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "6.57.1" + constraints = "~> 6.0" + hashes = [ + "h1:8wbrCS3R5lunN2HMVfQ+oMB3uFVAO7gfUbItdZa+tdA=", + "h1:C4XEcAWUKXExKCF6NhvMxkSyZ/JqNWjtwtwwEqQtl70=", + "h1:Fah+p3BD+/E7wdgazQFp4fsjuvTh5/Q7fQFyMsv9juM=", + "h1:S6eLURiAh6L00nr+9s2zwuhHQOU04rEG60Z9xO1ORRM=", + "h1:UdllVJ4482tLgz5q8se3SPzpPoXuFXwvxiykpgdipWs=", + "h1:ZAkIIdmQryKnldenUI2fqThlAciNz2UYKVTZ5F/57ME=", + "h1:a/i6TVZX4jSNSvHIWWTqLIlaugg7SYfQMZNVtzDWlKM=", + "h1:gGtYen1GfQvynKT8lHguY+cvmxNww+TOqCaapWoNW1w=", + "h1:iFr4+F/b94MEadHCiaNm6o6TapyvLT6TrwmbPyDQs9E=", + "h1:k0OzZI0aBgeEhCUcoK00/6C3hBK59ETo7g5Jq62wRqQ=", + "h1:nlvu+qO8eMFdnMptYSUK1c5dOXoK6r3HG87jzE7EYvQ=", + "h1:rVP8Z9tc37nwsW+NLSvEQyUBoDG/zsMtPSMeHhBrR3k=", + "h1:rxnmHWTUE2Aou/otNocOI4Rvcm69LQsoNYssj0s7hIo=", + "h1:va0Q0eDSt7xyvkSX7GbXmAO/QUqugZb/05gMdcQlR0I=", + "h1:yXF1EzUTMepZv4SeQnBNUvjOj2HKEKSl+jVndIKksGU=", + "zh:053910cce06bba506fb7a3dcecdcd719280d7df8b8a7cce06ae3e2dd20b1cc0d", + "zh:14feabeefef742b528c978a78f6896240a7ca3464a4714a5b1e1b2d448db4fd5", + "zh:22dd549e3c479b2ff2121e00a005aa3b6b43a166e727ef3208a529a59feb1608", + "zh:3ca5eab63ac77da335d42ebe0af8569c5c617dc5b239750b111f5ba9f2f889cd", + "zh:3d686f9fca9752850958a353971cd7eb1f5a48e55c2c56fcd96d18cc6e54f8ae", + "zh:512e46d35abfec0ba85fc0f2da7dc39fece9106f02165e95096b6549f920453a", + "zh:5fe83903b4bef5b7318de4b61e0ff9c88aa8ac42b7abeae83d1599cb054b0908", + "zh:61ec8a18a7270e28f43c0f810c82a147af7a5de622eb1f243c3a773e234b2172", + "zh:6428cd62e615ccaaa3a26a22f68a84f80a0d497a04f0405648119ef69e709ff9", + "zh:776e57bb2c94716f452eea5803c90b97d304c9f343e6746780a255d8217b95f9", + "zh:99bacf79ce92568d086ed62b287d9b77fa0c4e86a6c25ad398bec04ea0e44897", + "zh:b4361d4ad5074df3e83dc0e40452b63f231ae5959b4007763739d3a02caffe29", + "zh:d96c0f8603a1f8ab61c633a7a707922313f6bdfa8d17fa21d9ccf103fade6af7", + "zh:edfd955636489371e5ad1f40fc77a023d7eb7d09140342b151f0ea15451ef41b", + "zh:eeb03674754bd059fafa62b58cf4015c2c07bc25477def3c64eef487d583a63c", + ] +} + +provider "registry.opentofu.org/hashicorp/helm" { + version = "2.17.0" + constraints = "~> 2.17" + hashes = [ + "h1:+NIiFaAqUKl8JBJNk11tfXE4reXce5e3D3V0MOl7/SI=", + "h1:/1qQSZBcGfGdSJ1PVDIIRLHF6cCNQDpFnZHA8Z1h5Hg=", + "h1:69PnHoYrrDrm7C8+8PiSvRGPI55taqL14SvQR/FGM+g=", + "h1:CaN+iT1/mgn062wIkmcZyqhIQEvUQYpGcBYZ7Y7sM84=", + "h1:RMCFME+dnIAfqWesdTwaktFf5TCa9shMDtlLWwpJjAw=", + "h1:ShIag7wqd5Rs+zYpVMpjAh+T0ozr4XGYfSTKWqceQBY=", + "h1:WX5D6JJke4iG8ecGoUt1Gml/ggLHTFhswesyQqxKKG0=", + "h1:gy5bFfc81+K/Mi5KRQ6LfRJmgyaTxJnLTzDK+OYJAQg=", + "h1:kzDwclZLK5tIKJo3ATWM7a5ODmeczfWkvQDZkr9dVro=", + "h1:ojHGbVqPy4ShrUnNL7jif6AnEwgc8vC8sP7f37/VBC8=", + "zh:02690815e35131a42cb9851f63a3369c216af30ad093d05b39001d43da04b56b", + "zh:27a62f12b29926387f4d71aeeee9f7ffa0ccb81a1b6066ee895716ad050d1b7a", + "zh:2d0a5babfa73604b3fefc9dab9c87f91c77fce756c2e32b294e9f1290aed26c0", + "zh:3976400ceba6dda4636e1d297e3097e1831de5628afa534a166de98a70d1dcbe", + "zh:54440ef14f342b41d75c1aded7487bfcc3f76322b75894235b47b7e89ac4bfa4", + "zh:6512e2ab9f2fa31cbb90d9249647b5c5798f62eb1215ec44da2cdaa24e38ad25", + "zh:795f327ca0b8c5368af0ed03d5d4f6da7260692b4b3ca0bd004ed542e683464d", + "zh:ba659e1d94f224bc3f1fd34cbb9d2663e3a8e734108e5a58eb49eda84b140978", + "zh:c5c8575c4458835c2acbc3d1ed5570589b14baa2525d8fbd04295c097caf41eb", + "zh:e0877a5dac3de138e61eefa26b2f5a13305a17259779465899880f70e11314e0", + ] +} + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.31" + hashes = [ + "h1:3VVgWmwdwXFT54fjrplnq+N4+4LZ3ZeLHAlr/0jhPiA=", + "h1:9LfHMXiMOboc6PhcEEelKjA3VL94l3MCj7RlbKO1PQM=", + "h1:AkW6iAcMGHS+P2BIc2nvQe3PZCHtDL4m6+80tEDLge0=", + "h1:HGkB9bCmUqMRcR5/bAUOSqPBsx6DAIEnbT1fZ8vzI78=", + "h1:eCV78xGlh9eay+62U4gAgCEMohuiBJXN9XTIZNn+rX4=", + "h1:ems+O2dA7atxMWpbtqIrsH7Oa+u+ERWSfpMaFnZPbh0=", + "h1:iEDf790HE0h3kz58zfj5IhTVODCDqWF1hISPHz210Uw=", + "h1:nY7J9jFXcsRINog0KYagiWZw1GVYF9D2JmtIB7Wnrao=", + "h1:yw6JHRONXmiTumaQfJBxl1ierFnNNT/Qio8+ttfMcG4=", + "zh:1096b41c4e5b2ee6c1980916fb9a8579bc1892071396f7a9432be058aabf3cbc", + "zh:2959fde9ae3d1deb5e317df0d7b02ea4977951ee6b9c4beb083c148ca8f3681c", + "zh:5082f98fcb3389c73339365f7df39fc6912bf2bd1a46d5f97778f441a67fd337", + "zh:620fd5d0fbc2d7a24ac6b420a4922e6093020358162a62fa8cbd37b2bac1d22e", + "zh:7f47c2de179bba35d759147c53082cad6c3449d19b0ec0c5a4ca8db5b06393e1", + "zh:89c3aa2a87e29febf100fd21cead34f9a4c0e6e7ae5f383b5cef815c677eb52a", + "zh:96eecc9f94938a0bc35b8a63d2c4a5f972395e44206620db06760b730d0471fc", + "zh:e15567c1095f898af173c281b66bffdc4f3068afdd9f84bb5b5b5521d9f29584", + "zh:ecc6b912629734a9a41a7cf1c4c73fb13b4b510afc9e7b2e0011d290bcd6d77f", + ] +} + +provider "registry.opentofu.org/hashicorp/local" { + version = "2.9.0" + constraints = "~> 2.5" + hashes = [ + "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", + "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", + "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", + "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", + "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", + "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", + "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", + "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", + "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", + "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", + "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", + "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", + "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", + "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", + "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", + "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", + "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", + "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", + "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", + "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", + "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", + "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", + "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", + "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", + "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", + "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", + "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", + "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", + "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", + "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", + ] +} diff --git a/providers/aws-eks/README.md b/providers/aws-eks/README.md new file mode 100644 index 0000000..c73ba07 --- /dev/null +++ b/providers/aws-eks/README.md @@ -0,0 +1,378 @@ +# provider: aws-eks (Amazon EKS, managed) — ⚠️ UNTESTED + +Managed Kubernetes on AWS: an EKS control plane plus one managed node group, +with the mip-infra contract add-ons applied via the `helm` and `kubernetes` +providers instead of `modules/rke2-bootstrap`. The AWS counterpart of +[`upcloud-uks`](../upcloud-uks). See +[docs/why-self-managed.md](../../docs/why-self-managed.md) for why the +self-managed stacks, not this one, are the reference for mirroring production. + +> **Status: LIVE-TESTED, WITH mip-infra DEPLOYED.** Applied from zero against a +> real AWS account in `eu-north-1`: 3 nodes Ready on EKS 1.35.6, both load +> balancers serving over their Elastic IPs with all 12 targets healthy, the RWX +> StorageClass **proven** (two pods on different nodes wrote to one file and +> each read the other's line), then Argo CD installed via `argo-setup` and +> mip-infra deployed from a prepared fork: +> +> * **21 / 23 Argo CD Applications Synced + Healthy**, zero failing pods +> cluster-wide (the other two are understood — see +> [Verification](#verification-what-a-good-run-looks-like)) +> * `haproxy-public` holds the contract VIP `10.101.255.200` +> * `https://argocd..sslip.io`, `https://federation-a..sslip.io` and +> `https://datacatalogue..sslip.io` all return **HTTP 200** +> +> Certificates were self-signed for that run (placeholder `acme_email`); put a +> real address in and `issuer_mode = "auto"` switches to Let's Encrypt. +> Written against the current +> `hashicorp/aws` v6 provider and AWS documentation, with every resource +> argument checked against the provider schema. Every Helm release was +> **rendered with `helm template` at its pinned version and asserted on** — +> Traefik really is a DaemonSet with a `NodePort` Service on 30080/30443 and an +> IngressClass literally named `traefik`; MetalLB really passes +> `--lb-class=metallb.io/metallb` to both its controller and its speaker; +> cert-manager's `crds.enabled` really is what installs the 6 CRDs (0 without +> it); the ECK operator really lands in `kube-system` with the Elastic CRDs. +> That matters because Helm ignores unknown values **silently**, so an untested +> values key is a no-op rather than an error. The [likely first-run +> fixes](#likely-first-run-fixes) are listed at the bottom. + +- **Control plane:** EKS, `$0.10/hour` (~$73/month) on standard support. + There is **no free dev tier** here, unlike UKS `dev-md` or SKS starter. +- **CNI:** AWS VPC CNI, fixed. Pods get **VPC addresses**, not a pod CIDR — the + one divergence that costs a real fork edit (see below). +- **Storage:** EFS via the managed `aws-efs-csi-driver` add-on — a real shared + filesystem, so the contract's `ceph-corbo-cephfs*` classes are **genuinely + RWX**. This is the gap `upcloud-uks` and `exoscale-sks` cannot close with + their block-only CSI drivers, and it is the closest analogue to the CephFS the + upstream names come from. EBS (block/RWO) is installed alongside. +- **Cost:** roughly **$620/month at 24/7** — see + [docs/cost-comparison.md](../../docs/cost-comparison.md). Unlike the + self-managed stack there is no clean power-off: a managed node group's lever + is scaling to zero, which deletes and recreates nodes. + +## Region: why `eu-north-1` is the default + +`eu-north-1` (Stockholm) has the **lowest-carbon grid of AWS's European +regions** — roughly 35 gCO₂e/kWh against ~256 in Ireland and ~330 in Germany +(sourced figures in +[docs/cost-comparison.md](../../docs/cost-comparison.md)) — and is also the +cheapest of the three for these instance families. EKS, EFS and three AZs are +all available there. + +This is a **default, not a constraint**: + +```hcl +region = "eu-west-1" +availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] +``` + +## Architecture + +``` +Internet + ├── argocd..sslip.io ──► NLB :80/:443 ──► node :30080/:30443 + │ Traefik (DaemonSet, class `traefik`) + └── *..sslip.io ─────────► NLB :80/:443 ──► node :80/:443 + haproxy-edge (hostNetwork) + │ resolves via cluster DNS + ▼ + mip-infra's haproxy-public ──► apps + +EKS control plane (AWS-managed, API allowlisted to operator_cidr) +Nodes: 3x m7i.xlarge, managed node group, one AZ by default +Storage: EFS (One Zone) -> ceph-corbo-cephfs / -retain, RWX +``` + +**Two load balancers with Elastic IPs** are the AWS answer to a problem the +self-managed stack solves with node EIPs: sslip.io hostnames need a **stable +IPv4**, and a managed node group's instances (and their auto-assigned public +IPs) are replaced whenever the ASG recycles one. An NLB with an EIP is fixed for +its lifetime, and the two of them reproduce the reference stack's two-anchor +split (Argo CD on one, the MIP apps on the other). + +## Why the contract objects ship as a local chart + +`charts/mip-contract/` holds the ClusterIssuers, the MetalLB pool and the +haproxy-edge forwarder — plain Kubernetes objects that the other managed stacks +apply one at a time with the `alekc/kubectl` provider. That approach **cannot +work here**, and the reason generalises: + +> Terraform will not accept provider configuration that is unknown at plan time. +> This stack *creates* the cluster, so the kubectl provider's `host` and +> `cluster_ca_certificate` are unknown until apply. The `kubernetes` and `helm` +> providers tolerate that and defer; `alekc/kubectl` refuses and fails the plan +> with `no configuration has been provided`. Pointing it at a `config_path` +> instead fails the same way — it stats the file during plan, before anything +> has written it. + +Delivering them through the `helm` provider sidesteps it entirely, and drops the +only third-party provider the stack had. Two incidental wins: Helm's own +`checksum/config` idiom rolls the haproxy-edge DaemonSet when its ConfigMap +changes, and `helm lint` / `helm template` make the objects testable without a +cluster: + +```sh +helm lint charts/mip-contract +helm template mip-contract ./charts/mip-contract -n kube-system \ + --set acmeEmail=email@example.com --set haproxyEdge.clusterDns=172.20.0.10 +``` + +`upcloud-uks` and `exoscale-sks` keep the kubectl provider. They get away with +it only because their `install_metallb` defaults to `false` and their +ClusterIssuers are applied against a cluster their own data source resolves — +if either is ever run from scratch in one apply, expect the same failure. + +## haproxy-edge cannot use ports 80/443 on EKS + +The self-managed stacks bind haproxy-edge to host ports 80/443 directly. On EKS +that is impossible, and the failure is opaque — a `CrashLoopBackOff` with: + +``` +cannot bind socket (Address in use) for [0.0.0.0:80] +``` + +The cause: **`eks-pod-identity-agent` listens on `169.254.170.23:80` on every +node**. A wildcard bind (`0.0.0.0:80`) collides with an existing bind on a +specific address on the same port, so no `hostNetwork` pod can take port 80 on +an EKS node — and the pod-identity agent is not optional here, it is what gives +the EFS and EBS CSI drivers their IAM. + +So haproxy-edge binds `haproxy_edge_host_ports` (default **8080/8443**) and the +apps load balancer listens publicly on 80/443 and forwards there. Same shape as +Traefik's pinned NodePorts. Nothing downstream notices; it only matters if you +go looking for a listener on port 80. + +## Known divergences from the contract + +Same spirit as the `upcloud-uks` notes — these are the things to know before +deploying mip-infra here. + +| Divergence | Impact | What to do | +|---|---|---| +| **VPC CNI gives pods VPC addresses**, so there is no `10.42.0.0/16` pod CIDR | submariner's `clusterCidr: 10.42.0.0/16` (`common/submariner/operator/values.yaml`) is simply wrong | set it to `vpc_cidr` in the fork. One line — but a real extra edit the RKE2 stacks do not need | +| **MetalLB does not own LoadBalancer Services by default** | EKS's legacy in-tree Service Controller still claims every *unclassed* `type: LoadBalancer` Service and gives it a Classic LB. Two controllers reconciling one Service flap its status, so MetalLB here is scoped to `loadBalancerClass: metallb.io/metallb` | add `loadBalancerClass: metallb.io/metallb` to the two Services that carry `metallb.io/address-pool` (haproxy-public and the submariner gateway) in the fork. Without it they still get addresses — from AWS, as Classic LBs, at ~$18/month each — and the fixed VIPs are unused | +| **The kube-API is on `:443`, not `:6443`**, at a DNS name rather than an IP | `prepare-mip-infra-fork.sh` rewrites the submariner broker endpoint as `:6443`, which has no meaning here | after running the fork script, replace that endpoint with the `kube_api` output (already `https://…:443`). One line | +| **No node/etcd/apiserver access** | no cloud-init, no per-node firewall rules, no RKE2 version pinning | expected on any managed control plane; it is the trade documented in `why-self-managed.md` | +| **EFS squashes root, so anything that chowns its data directory dies** | every stock Postgres image chowns `/var/lib/postgresql/...` at startup and CrashLoopBackOffs with `chown: Operation not permitted`. mip-infra runs two of them (`datacatalog/db`, `platform-backend-db`) | already handled: both StorageClasses set `uid = "0"` / `gid = "0"`, so the access point enforces root identity and the chown succeeds. Longhorn on the self-managed stacks is block storage and has no such restriction — **this is the single biggest behavioural difference between EFS and Longhorn here**, and it is invisible until a workload happens to chown | +| **No clean power-off** | the self-managed stack's `make down` has no equivalent; scaling a managed node group to zero deletes and recreates nodes | `tofu destroy` between sessions, or scale to zero **out-of-band**: `aws eks update-nodegroup-config --cluster-name mip-eks --nodegroup-name workers --scaling-config minSize=0,maxSize=0,desiredSize=0` (and back up the same way). Do **not** use `node_count = 0`: `desired_size` is deliberately in `ignore_changes`, so the update would send the old desired size with `maxSize=0` and be rejected by the EKS API | + +## Prerequisites + +- `tofu` (>= 1.9), the `aws` CLI **v2** (the providers authenticate through + `aws eks get-token`), `kubectl`, `jq`. +- AWS credentials in the environment (**never in files**): + `export AWS_PROFILE=...` or `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY`. + The principal needs EKS, EC2/VPC, EFS, ELB **and IAM role creation** (this + stack creates the cluster, node and CSI-driver roles itself), and it becomes + the cluster admin via `bootstrap_cluster_creator_admin_permissions`. Exact + permission list, why each one is there, how the IAM grant is scoped to + `-*`, and a generator: + **[docs/aws-credentials.md](../../docs/aws-credentials.md)**: + ```sh + ./scripts/aws-iam/create-deploy-identity.sh --stack eks --role-prefix mip-eks + ``` + The generated credential is **locked to the public IP you run that from** — + use the same address for `operator_cidr`, since it is what the EKS API + endpoint and the load-balancer security group allowlist too. +- An SSH keypair (worker-node debugging only). +- `operator_cidr` **must** include the IP `tofu apply` runs from: the EKS public + endpoint is allowlisted to it, and the `kubernetes`/`helm` providers connect + to the control plane during the same apply. + +## Quickstart + +```sh +cp terraform.tfvars.example terraform.tfvars # operator_cidr, ssh_public_key, acme_email +tofu init +tofu apply # ~15 min (EKS control plane is the slow part) + +export KUBECONFIG=$PWD/kubeconfig # written by the stack, gitignored +# or: aws eks update-kubeconfig --name mip-eks --region eu-north-1 + +kubectl get nodes # 3 Ready +kubectl get sc # ceph-corbo-cephfs (default), ceph-corbo-cephfs-retain +kubectl get clusterissuer # letsencrypt, letsencrypt-staging +kubectl get ingressclass # traefik +kubectl -n metallb-system get ipaddresspool # pool-no-auto +kubectl -n kube-system get ds haproxy-edge # one pod per node +tofu output # the sslip.io hostnames to use +``` + +Verify RWX actually works before trusting the storage layer — it is the whole +point of the EFS detour: + +```sh +kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: rwx-test} +spec: + accessModes: [ReadWriteMany] + storageClassName: ceph-corbo-cephfs + resources: {requests: {storage: 1Gi}} +EOF +kubectl wait --for=jsonpath='{.status.phase}'=Bound pvc/rwx-test --timeout=120s +kubectl delete pvc rwx-test +``` + +## Deploying mip-infra + +The deployment follows mip-infra's **own** docs (`docs/getting-started.md` + +`argo-setup/README.md`), as on every other stack. What differs on EKS is +collected here so you can follow upstream verbatim and apply these deltas. + +**Nothing in this stack installs Argo CD.** It provides the `traefik` ingress +class and the load balancer in front of it; Argo CD itself comes from +`argo-setup`, which is step 1 below. Until then +`https://argocd..sslip.io` correctly returns **404 from Traefik** — the +path works, there is simply no Ingress behind it yet. + +### 0. Fork hosting has no control-plane node here + +The RKE2 stacks run `scripts/setup-fork-git-server.sh`, which puts a bare repo +on the control-plane node and has Argo CD pull it over the private network. A +managed cluster has no such node, so that script does not exist here. Use +whichever applies: + +| Option | When | +|---|---| +| Real git hosting (GitHub/GitLab, private repo + deploy key) | the intended answer — `argocd repo add --ssh-private-key-path …` | +| A git server pod inside the cluster | no external hosting available; Argo CD pulls it over cluster DNS, you push through `kubectl port-forward` | + +Whatever you pick, that URL is the `` below. + +### 1. Prepare the fork + +```sh +STACK_DIR=providers/aws-eks ../../scripts/prepare-mip-infra-fork.sh /path/to/mip-infra +``` + +⚠️ **`cp_private_ip` is not emitted by this stack** (a managed control plane has +no private node IP), so the shared script cannot rewrite the Submariner broker +endpoint. Pass it explicitly, or fix it afterwards — see the divergences table. + +Then apply the EKS-specific edits from [Known +divergences](#known-divergences-from-the-contract): the Submariner +`clusterCidr`, the two `loadBalancerClass` lines, and the broker endpoint's +port. + +### 2. Install Argo CD (`argo-setup/README.md`) + +Follow it verbatim, with: + +| Upstream step | Value here | +|---|---| +| `ARGOCD_HOST` | `argocd..sslip.io` — from `tofu output argocd_hostname` | +| `kustomize build patches` | the overlay pulls the pinned Argo CD HA base from `raw.githubusercontent.com`. On a workstation with a **per-binary firewall**, `kustomize` (a Go binary) is blocked while `curl` is not — allow it, or vendor the base locally | +| `kubectl apply --server-side` | required, not optional: the `applicationsets` CRD exceeds the 256 KiB client-side annotation limit | +| HA prerequisites | satisfied — 3 schedulable nodes, which is why `node_count` defaults to 3 | + +After this, `https://argocd..sslip.io` serves the Argo CD UI. + +**Certificates: put a real address in `acme_email` and it just works.** With the +default `issuer_mode = "auto"` the stack self-signs when `acme_email` is a +placeholder and switches to real Let's Encrypt when it is a real address — +because Let's Encrypt refuses reserved domains outright +(`contact email has forbidden domain "example.com"`). That failure is +particularly unhelpful to debug: the Argo CD Ingress is annotated +`router.entrypoints: websecure`, so it is **HTTPS-only** — no certificate means +no TLS secret means Traefik answers **404 on both ports**, with nothing in the +Traefik log pointing at the email. Force either behaviour with +`issuer_mode = "acme"` / `"selfsigned"`. + +Port 80 returning 404 for the Argo CD host is therefore correct at all times; +80 exists only for ACME HTTP-01 challenges. + +### 3. Follow `docs/getting-started.md` + +Verbatim, except where noted: + +| Step | Value / delta here | +|---|---| +| §2 Register repositories | `argocd repo add --ssh-private-key-path …`. For an SSH URL also register the host key: `ssh-keyscan \| argocd cert add-ssh --batch --from -` | +| §3 Out-of-band RBAC | `kubectl create namespace ingress-nginx` **first** — `haproxy-public-rbac.yaml` contains namespaced objects and the namespace is otherwise only created when the apps sync | +| §4 Bootstrap AppProjects | verbatim | +| §5 Infra ApplicationSet | verbatim, but `--repo ` (the documented command still names the upstream repo) | +| §6 Secrets (`scripts/gen_secrets.sh`) | wait for `federation-a` to exist. **The script reads its prompts from `/dev/tty`**, so it cannot be piped — run it interactively, or create `mip-secret` and `keycloak-credentials` directly against the schema in the script. The `mip-common-datacatalog` half *is* scriptable via `COMMON_KC_CLIENT_ID` / `COMMON_KC_CLIENT_SECRET` / `COMMON_DB_USER` / `COMMON_DB_PASSWORD` | + +### Verification (what a good run looks like) + +```sh +kubectl get applications -n argocd-mip-team # 21/23 Synced+Healthy — see below +kubectl -n ingress-nginx get svc haproxy-public-controller # EXTERNAL-IP = vip_haproxy +curl -skI https://argocd..sslip.io/ # 200 +curl -skI https://federation-a..sslip.io/ # 200 +curl -skI https://datacatalogue..sslip.io/ # 200 +``` + +**Two Applications do not reach Synced+Healthy on EKS, both understood:** + +| Application | State | Why | +|---|---|---| +| `eck-stack` | `OutOfSync` / **Healthy** | every child resource is Healthy and every pod Running. The ECK operator writes defaults back into its own `Elasticsearch`/`Kibana`/`Beat` CRs, which Argo CD sees as drift. Cosmetic, and not EKS-specific | +| `submariner-operator` | `Synced` / `Progressing` | its gateway Service never gets an address, for two compounding reasons — see below | + +**Why the Submariner gateway stays pending on EKS.** Its Service exposes **UDP** (IPsec 500/4500). The Submariner operator rewrites that Service and drops the `loadBalancerClass` the fork patch adds, so MetalLB — which is scoped to `metallb.io/metallb` here — never claims it. EKS's in-tree controller then claims the now-unclassed Service and fails outright: + +``` +SyncLoadBalancerFailed: failed to ensure load balancer: +Protocol UDP not supported by LoadBalancer +``` + +AWS Classic Load Balancers do not support UDP. This costs nothing in practice — +Submariner has no remote cluster to reach on a single dev cluster, which the +[reference stack](../upcloud-rke2/README.md) documents as a known-acceptable +deviation too — but on EKS the gateway does not even get its VIP, whereas there +it does. + +## Likely first-run fixes + +| Unknown | Where | How to check / fix | +|---|---|---| +| `k8s_version` not on offer, or already on extended support | `terraform.tfvars` | `aws eks describe-cluster-versions` — extended support costs 6× the control-plane fee | +| Instance type not offered in the chosen AZ | `terraform.tfvars` | `aws ec2 describe-instance-type-offerings --location-type availability-zone` | +| Traefik chart values drifted | `addons.tf` | values keys are pinned against chart `41.1.1`; on a bump re-check `deployment.kind`, `service.spec.type`, `ports.*.nodePort` — Helm ignores unknown keys **silently** | +| EFS access-point provisioning permissions | `storage.tf` | `kubectl -n kube-system logs deploy/efs-csi-controller -c csi-provisioner`; the Pod Identity association must be present (`aws eks list-pod-identity-associations --cluster-name mip-eks`) | +| NLB targets never become healthy | `edge.tf` | `aws elbv2 describe-target-health --target-group-arn ...`; check the cluster security group has the rule for that port and that `haproxy-edge`/Traefik is actually listening | +| Argo CD ingress 404s | — | Traefik's IngressClass must literally be `traefik`: `kubectl get ingressclass` | + +## Tearing down + +**Delete the PVCs before `tofu destroy`.** The EFS CSI driver creates one +**access point per PVC**, and Terraform does not manage them — it only knows +about the file system. Destroying the cluster first orphans them, and orphaned +access points can block the file system's own deletion: + +```sh +kubectl -n argocd-mip-team delete applications --all # or Argo CD recreates the PVCs +kubectl delete pvc --all -A +kubectl delete pv --all # the `-retain` class leaves these behind +tofu destroy +``` + +Then confirm nothing survived — Terraform reporting success is not the same as +the account being clean: + +```sh +aws eks list-clusters +aws elbv2 describe-load-balancers --query 'LoadBalancers[].LoadBalancerName' +aws elb describe-load-balancers --query 'LoadBalancerDescriptions[].LoadBalancerName' # classic, if anything made one +aws efs describe-file-systems --query 'FileSystems[].FileSystemId' +aws ec2 describe-addresses --query 'Addresses[].PublicIp' +``` + +The classic-LB check matters because `type: LoadBalancer` Services are +reconciled by EKS's in-tree controller, which Terraform knows nothing about. In +this stack MetalLB owns those Services via `loadBalancerClass`, so none are +created — but a fork that drops the class would leak one per Service. + +## State & credentials + +- State is **local** (`terraform.tfstate`, gitignored). +- Kubeconfig is written to `./kubeconfig` (gitignored) and uses the + `aws eks get-token` exec plugin — **no long-lived token in state or on disk**. +- Everything is version-pinned in [versions.tf](versions.tf) and the chart + versions in [addons.tf](addons.tf); EKS add-on versions are deliberately + unpinned so EKS picks the default for the cluster version (see + [docs/maintenance.md](../../docs/maintenance.md)). Commit + `.terraform.lock.hcl`. diff --git a/providers/aws-eks/addons.tf b/providers/aws-eks/addons.tf new file mode 100644 index 0000000..3a401d1 --- /dev/null +++ b/providers/aws-eks/addons.tf @@ -0,0 +1,161 @@ +# Managed-cluster add-on layer. EKS has no RKE2 manifests directory, so instead +# of modules/rke2-bootstrap the same mip-infra contract pieces are installed via +# the helm and kubernetes providers. EKS supplies the CNI (VPC CNI) and the +# CSI drivers itself, so relative to the self-managed layer this drops the Canal +# pin and Longhorn — but unlike upcloud-uks it does install Traefik (the +# contract's Argo CD ingress class) and the haproxy-edge forwarder. + +locals { + # EKS's CoreDNS Service always sits at the 10th address of the cluster service + # CIDR. haproxy-edge needs it to resolve the haproxy-public Service at runtime. + cluster_dns = cidrhost(aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr, 10) + + # issuer_mode = "auto": self-sign unless acme_email is a real address. The + # reserved domains below (RFC 2606 / RFC 6761) are exactly the ones Let's + # Encrypt refuses, so attempting ACME with one can only fail. + acme_email_is_placeholder = var.acme_email == "" || can(regex("@(example\\.(com|org|net)|test|invalid|localhost)$", lower(var.acme_email))) + issuer_mode = ( + var.issuer_mode != "auto" ? var.issuer_mode : + local.acme_email_is_placeholder ? "selfsigned" : "acme" + ) +} + +# --- cert-manager -------------------------------------------------------------- +resource "helm_release" "cert_manager" { + name = "cert-manager" + repository = "https://charts.jetstack.io" + chart = "cert-manager" + version = "v1.20.3" + namespace = "cert-manager" + create_namespace = true + + set { + name = "crds.enabled" + value = "true" + } + + depends_on = [aws_eks_node_group.workers, aws_eks_addon.core] +} + +# --- ECK operator (contract: operator + CRDs in kube-system) ------------------- +# mip-infra's eck-stack deploys with operator.enabled=false and skipCrds=true, so +# both must already exist. +resource "helm_release" "eck_operator" { + name = "eck-operator" + repository = "https://helm.elastic.co" + chart = "eck-operator" + version = "3.4.1" + namespace = "kube-system" + + depends_on = [aws_eks_node_group.workers, aws_eks_addon.core] +} + +# --- Traefik (contract: ingress class `traefik` for the Argo CD UI) ------------ +# The self-managed stacks get this from RKE2's bundled Traefik; upcloud-uks +# leaves the gap open. Deployed as a DaemonSet on pinned NodePorts so the argocd +# load balancer (edge.tf) has a stable target — NOT on host ports 80/443, which +# haproxy-edge needs for the MIP apps. +resource "helm_release" "traefik" { + count = var.install_traefik ? 1 : 0 + name = "traefik" + repository = "https://traefik.github.io/charts" + chart = "traefik" + version = "41.1.1" + namespace = "traefik" + create_namespace = true + + # Key paths verified against this chart version's values.yaml — Helm ignores + # unknown values silently, so a typo here would be a no-op rather than an error. + values = [yamlencode({ + deployment = { kind = "DaemonSet" } + service = { enabled = true, spec = { type = "NodePort" } } + ports = { + web = { nodePort = var.traefik_node_ports.web } + websecure = { nodePort = var.traefik_node_ports.websecure } + } + # Named explicitly rather than derived from the release name: `traefik` is + # the class argo-setup's Argo CD ingress references by name. + ingressClass = { + enabled = true + isDefaultClass = false + name = "traefik" + } + })] + + depends_on = [aws_eks_node_group.workers, aws_eks_addon.core] +} + +# --- MetalLB (contract: pool `pool-no-auto` with fixed VIPs) ------------------- +# Scoped to loadBalancerClass "metallb.io/metallb" on purpose: EKS's legacy +# in-tree Service Controller claims every UNCLASSED type=LoadBalancer Service +# and hands it a Classic LB, so without the class both controllers reconcile the +# same Service and flap its status. See the README's known divergences for the +# two-line fork patch that puts the class on mip-infra's Services. +# +# As on the self-managed stacks MetalLB only ASSIGNS the VIPs so the Services go +# Healthy — no L2 announcement is involved (a VPC would not honour it anyway), +# and the public traffic path does not depend on MetalLB at all. +resource "helm_release" "metallb" { + count = var.install_metallb ? 1 : 0 + name = "metallb" + repository = "https://metallb.github.io/metallb" + chart = "metallb" + version = "0.16.1" + namespace = "metallb-system" + create_namespace = true + + values = [yamlencode({ + controller = { logLevel = "info" } + speaker = { logLevel = "info" } + loadBalancerClass = "metallb.io/metallb" + })] + + depends_on = [aws_eks_node_group.workers, aws_eks_addon.core] +} + +# --- the remaining contract objects, as a local chart ------------------------- +# The ClusterIssuers, the MetalLB pool and the haproxy-edge forwarder are plain +# Kubernetes objects, and on the other managed stacks they are applied one by one +# with the `kubectl` provider. That does not work here: this stack CREATES the +# cluster, so the provider's host and CA are unknown at plan time, and the +# kubectl provider refuses to configure with unknown values (the `kubernetes` and +# `helm` providers tolerate it and defer to apply). Passing it a `config_path` +# fails the same way — it stats the file during plan, before anything wrote it. +# +# Delivering them through the helm provider instead sidesteps that entirely, and +# drops the only third-party provider this stack had. The chart lives beside the +# stack in charts/mip-contract. +resource "helm_release" "mip_contract" { + name = "mip-contract" + chart = "${path.module}/charts/mip-contract" + namespace = "kube-system" + + values = [yamlencode({ + acmeEmail = var.acme_email + issuerMode = local.issuer_mode + # Served by Traefik rather than haproxy-public, so it needs its own solver. + argocdHostname = var.install_traefik ? "argocd.${aws_eip.argocd.public_ip}.sslip.io" : "" + metallb = { + enabled = var.install_metallb + poolName = "pool-no-auto" + vipHaproxy = var.vip_haproxy + vipSubmariner = var.vip_submariner + } + haproxyEdge = { + enabled = true + clusterDns = local.cluster_dns + hostPorts = { + http = var.haproxy_edge_host_ports.http + https = var.haproxy_edge_host_ports.https + } + } + })] + + # cert-manager and MetalLB must have established their CRDs before the custom + # resources here can be validated by the API server. + depends_on = [ + helm_release.cert_manager, + helm_release.metallb, + helm_release.traefik, + ] +} diff --git a/providers/aws-eks/charts/mip-contract/Chart.yaml b/providers/aws-eks/charts/mip-contract/Chart.yaml new file mode 100644 index 0000000..ba98f48 --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: mip-contract +description: >- + The pieces of the mip-infra contract that are plain Kubernetes objects: + the cert-manager ClusterIssuers, the MetalLB pool, and the haproxy-edge + forwarder. Delivered as a local chart rather than individual manifests + because the helm provider tolerates a provider configuration that is not + known until apply time — which is the case for a cluster this same run + creates. See addons.tf. +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/providers/aws-eks/charts/mip-contract/templates/clusterissuers.yaml b/providers/aws-eks/charts/mip-contract/templates/clusterissuers.yaml new file mode 100644 index 0000000..f87d9a3 --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/templates/clusterissuers.yaml @@ -0,0 +1,43 @@ +{{- /* +The issuer named literally `letsencrypt` is part of the contract — argo-setup's +Argo CD Ingress references it by name — so both modes below keep the same two +names and differ only in how they are backed. +*/ -}} +{{- range $mode := list "letsencrypt" "letsencrypt-staging" }} +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ $mode }} +spec: +{{- if eq $.Values.issuerMode "selfsigned" }} + # ACME cannot be used here (no real acme_email, or the cluster is unreachable + # from Let's Encrypt). Same names, self-signed certs, browser warning expected. + selfSigned: {} +{{- else }} + acme: + server: {{ if eq $mode "letsencrypt" }}https://acme-v02.api.letsencrypt.org/directory{{ else }}https://acme-staging-v02.api.letsencrypt.org/directory{{ end }} + email: {{ $.Values.acmeEmail | quote }} + privateKeySecretRef: + name: {{ $mode }}-account-key + solvers: + {{- if $.Values.argocdHostname }} + # Served by Traefik, not haproxy-public, so it needs its own solver. + - selector: + dnsNames: + - {{ $.Values.argocdHostname | quote }} + http01: + ingress: + ingressClassName: traefik + {{- end }} + - http01: + ingress: + ingressClassName: haproxy-public + # mip-infra's haproxy sets ssl-redirect globally; ACME must stay on + # port 80 (443 is operator-only, and Let's Encrypt follows redirects). + ingressTemplate: + metadata: + annotations: + haproxy.org/ssl-redirect: "false" +{{- end }} +{{- end }} diff --git a/providers/aws-eks/charts/mip-contract/templates/haproxy-edge-configmap.yaml b/providers/aws-eks/charts/mip-contract/templates/haproxy-edge-configmap.yaml new file mode 100644 index 0000000..571295a --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/templates/haproxy-edge-configmap.yaml @@ -0,0 +1,46 @@ +{{- if .Values.haproxyEdge.enabled }} +{{- /* +Public entry for the MIP apps: the same role the DaemonSet in +modules/rke2-bootstrap plays on the self-managed stacks — a minimal hostNetwork +TCP forwarder relaying :80/:443 to mip-infra's haproxy-public. + +One deliberate difference from the self-managed version: the backend is the +haproxy-public Service's cluster DNS name, resolved at runtime, rather than the +MetalLB VIP. That keeps the app path working whichever controller ends up owning +that Service on EKS. `init-addr none` lets haproxy start before mip-infra exists. +*/ -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: haproxy-edge + namespace: kube-system +data: + haproxy.cfg: | + global + log stdout format raw local0 + maxconn 4096 + defaults + mode tcp + log global + option tcplog + timeout connect 5s + timeout client 300s + timeout server 300s + resolvers kube + nameserver dns {{ .Values.haproxyEdge.clusterDns }}:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + hold nx 5s + frontend http-in + bind :{{ .Values.haproxyEdge.hostPorts.http }} + default_backend http-pub + backend http-pub + server pub {{ .Values.haproxyEdge.backend }}:80 resolvers kube init-addr none + frontend https-in + bind :{{ .Values.haproxyEdge.hostPorts.https }} + default_backend https-pub + backend https-pub + server pub {{ .Values.haproxyEdge.backend }}:443 resolvers kube init-addr none +{{- end }} diff --git a/providers/aws-eks/charts/mip-contract/templates/haproxy-edge.yaml b/providers/aws-eks/charts/mip-contract/templates/haproxy-edge.yaml new file mode 100644 index 0000000..b752cb0 --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/templates/haproxy-edge.yaml @@ -0,0 +1,58 @@ +{{- if .Values.haproxyEdge.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: haproxy-edge + namespace: kube-system + labels: + app: haproxy-edge +spec: + selector: + matchLabels: + app: haproxy-edge + template: + metadata: + labels: + app: haproxy-edge + annotations: + # A mounted ConfigMap updates on disk but haproxy only reads it at + # start — hashing the rendered ConfigMap into the pod template rolls the + # DaemonSet whenever the config above changes. + checksum/config: {{ include (print $.Template.BasePath "/haproxy-edge-configmap.yaml") . | sha256sum }} + spec: + nodeSelector: + {{- toYaml .Values.haproxyEdge.nodeSelector | nindent 8 }} + hostNetwork: true + # hostNetwork pods use the host resolver by default; the backend above is + # a cluster DNS name. + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + containers: + - name: haproxy + image: {{ .Values.haproxyEdge.image }} + # The official image drops to a non-root user; binding :80/:443 in the + # host network namespace needs root (dev-cluster trade-off). + securityContext: + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: cfg + mountPath: /usr/local/etc/haproxy/haproxy.cfg + subPath: haproxy.cfg + readOnly: true + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + livenessProbe: + tcpSocket: + port: {{ .Values.haproxyEdge.hostPorts.http }} + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: cfg + configMap: + name: haproxy-edge +{{- end }} diff --git a/providers/aws-eks/charts/mip-contract/templates/metallb.yaml b/providers/aws-eks/charts/mip-contract/templates/metallb.yaml new file mode 100644 index 0000000..f998c83 --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/templates/metallb.yaml @@ -0,0 +1,30 @@ +{{- if .Values.metallb.enabled }} +{{- /* +The pool name is part of the contract: mip-infra's Services carry +`metallb.io/address-pool: pool-no-auto` as a literal annotation. + +MetalLB only ASSIGNS these VIPs so the Services go Healthy. The L2Advertisement +is paired with the pool because MetalLB's docs are ambiguous about whether a +pool with no advertisement is allocated from at all — announcement itself is +inert in a VPC, and nothing in this stack depends on it. +*/ -}} +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: {{ .Values.metallb.poolName }} + namespace: metallb-system +spec: + autoAssign: false + addresses: + - {{ .Values.metallb.vipHaproxy }}/32 + - {{ .Values.metallb.vipSubmariner }}/32 +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: {{ .Values.metallb.poolName }} + namespace: metallb-system +spec: + ipAddressPools: + - {{ .Values.metallb.poolName }} +{{- end }} diff --git a/providers/aws-eks/charts/mip-contract/values.yaml b/providers/aws-eks/charts/mip-contract/values.yaml new file mode 100644 index 0000000..2f11d26 --- /dev/null +++ b/providers/aws-eks/charts/mip-contract/values.yaml @@ -0,0 +1,33 @@ +# All values are supplied by Terraform (see addons.tf); these are placeholders +# that make `helm template` / `helm lint` work standalone. +acmeEmail: email@example.com + +# "acme" (real Let's Encrypt) or "selfsigned" (same issuer names, self-signed). +issuerMode: acme + +# Hostname served by Traefik rather than haproxy-public — gets its own ACME +# solver so the Argo CD certificate validates against the right ingress class. +argocdHostname: "" + +metallb: + enabled: true + poolName: pool-no-auto + vipHaproxy: "" + vipSubmariner: "" + +haproxyEdge: + enabled: true + # NOT 80/443: eks-pod-identity-agent already binds 169.254.170.23:80 on every + # EKS node, and a wildcard bind collides with it. The load balancer listens on + # 80/443 publicly and forwards here. See variables.tf. + hostPorts: + http: 8080 + https: 8443 + # The in-cluster Service haproxy-edge forwards to. Resolved at runtime, so the + # DaemonSet can start before mip-infra is deployed. + backend: haproxy-public-controller.ingress-nginx.svc.cluster.local + clusterDns: "" + nodeSelector: + mip.dev/edge: haproxy + # renovate: datasource=docker depName=haproxy + image: haproxy:3.2-alpine diff --git a/providers/aws-eks/edge.tf b/providers/aws-eks/edge.tf new file mode 100644 index 0000000..2984987 --- /dev/null +++ b/providers/aws-eks/edge.tf @@ -0,0 +1,183 @@ +# The two public entry points, and the reason this stack needs load balancers at +# all: sslip.io hostnames must be anchored on a STABLE IPv4 address, and a +# managed node group's instances (and their auto-assigned public IPs) come and go +# whenever the ASG replaces one. A Network Load Balancer with an Elastic IP is +# the AWS way to get a fixed address; the self-managed stack uses the nodes' own +# EIPs instead. +# +# Two load balancers, mirroring the reference stack's two-anchor split: +# argocd NLB (EIP) :80/:443 -> Traefik NodePorts -> Argo CD UI +# apps NLB (EIP) :80/:443 -> node :80/:443 haproxy-edge -> haproxy-public -> MIP apps +# +# Both are internet-facing but NLBs honour the security groups attached to them, +# so 443 stays scoped to app_access_cidr and only 80 is world-open (for ACME). + +resource "aws_security_group" "lb" { + name = "${var.cluster_name}-lb" + description = "${var.cluster_name}: public load balancers" + vpc_id = aws_vpc.this.id + tags = { Name = "${var.cluster_name}-lb" } +} + +resource "aws_vpc_security_group_ingress_rule" "lb_http" { + security_group_id = aws_security_group.lb.id + # World-open on purpose: Let's Encrypt HTTP-01 validates from arbitrary, + # undisclosed IPs. Serves only redirects and ACME challenge responses. + description = "HTTP (ACME HTTP-01 + redirect only)" + ip_protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "lb_https" { + security_group_id = aws_security_group.lb.id + description = "HTTPS (app_access_cidr only)" + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_ipv4 = local.app_cidr +} + +resource "aws_vpc_security_group_egress_rule" "lb_all" { + security_group_id = aws_security_group.lb.id + description = "All egress (to the node target groups)" + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +# The nodes must accept the load balancers' traffic on the target ports. EKS's +# own cluster security group is attached to every node, so the rules go there. +resource "aws_vpc_security_group_ingress_rule" "nodes_from_lb" { + for_each = toset([ + tostring(var.haproxy_edge_host_ports.http), # haproxy-edge, MIP apps + tostring(var.haproxy_edge_host_ports.https), # haproxy-edge, MIP apps + tostring(var.traefik_node_ports.web), + tostring(var.traefik_node_ports.websecure), + ]) + security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id + description = "Load balancer to node port ${each.value}" + ip_protocol = "tcp" + from_port = tonumber(each.value) + to_port = tonumber(each.value) + referenced_security_group_id = aws_security_group.lb.id +} + +# --- Elastic IPs: the stable sslip.io anchors -------------------------------- +resource "aws_eip" "argocd" { + domain = "vpc" + tags = { Name = "${var.cluster_name}-argocd" } +} + +resource "aws_eip" "apps" { + domain = "vpc" + tags = { Name = "${var.cluster_name}-apps" } +} + +# --- load balancers --------------------------------------------------------- +# One subnet each: an NLB gets one address per subnet it is attached to, and one +# fixed IP is exactly what sslip.io needs. Cross-zone is on so a single-subnet +# NLB can still reach nodes in other AZs when node_az_count > 1. +resource "aws_lb" "argocd" { + name = "${var.cluster_name}-argocd" + load_balancer_type = "network" + security_groups = [aws_security_group.lb.id] + enable_cross_zone_load_balancing = true + + subnet_mapping { + subnet_id = aws_subnet.this[0].id + allocation_id = aws_eip.argocd.id + } + + tags = { Name = "${var.cluster_name}-argocd" } +} + +resource "aws_lb" "apps" { + name = "${var.cluster_name}-apps" + load_balancer_type = "network" + security_groups = [aws_security_group.lb.id] + enable_cross_zone_load_balancing = true + + subnet_mapping { + subnet_id = aws_subnet.this[0].id + allocation_id = aws_eip.apps.id + } + + tags = { Name = "${var.cluster_name}-apps" } +} + +# --- target groups ---------------------------------------------------------- +# target_type = instance: the targets are the worker nodes, reached on the +# haproxy-edge host ports and Traefik's pinned NodePorts. `instance` (rather than +# the `ip` type AWS generally recommends) is what lets a Terraform-managed NLB +# attach to the node group's ASG without an in-cluster controller. +locals { + target_ports = { + argocd_http = { lb = "argocd", listen = 80, target = var.traefik_node_ports.web } + argocd_https = { lb = "argocd", listen = 443, target = var.traefik_node_ports.websecure } + apps_http = { lb = "apps", listen = 80, target = var.haproxy_edge_host_ports.http } + apps_https = { lb = "apps", listen = 443, target = var.haproxy_edge_host_ports.https } + } +} + +resource "aws_lb_target_group" "this" { + for_each = local.target_ports + + # The port is part of the name, and the group is created before the old one is + # destroyed. Both are needed to change a target port at all: the port forces + # replacement, but a listener still referencing the old group blocks its + # deletion (ResourceInUse), and create_before_destroy alone would collide on + # the name. With the port in the name the new group is distinct, so the order + # becomes create -> repoint the listener -> delete. + name = "${var.cluster_name}-${replace(each.key, "_", "-")}-${each.value.target}" + vpc_id = aws_vpc.this.id + port = each.value.target + protocol = "TCP" + target_type = "instance" + + # TCP health check on the same port: a node only becomes a target once + # haproxy-edge / Traefik is actually listening there. + health_check { + protocol = "TCP" + port = "traffic-port" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + } + + # Client IPs are not preserved through haproxy-edge anyway (documented + # trade-off, same as the self-managed stacks), and disabling it avoids the + # asymmetric-routing pitfalls of NLB source-IP preservation to host ports. + preserve_client_ip = false + + deregistration_delay = 30 + + tags = { Name = "${var.cluster_name}-${replace(each.key, "_", "-")}" } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_listener" "this" { + for_each = local.target_ports + + load_balancer_arn = each.value.lb == "argocd" ? aws_lb.argocd.arn : aws_lb.apps.arn + port = each.value.listen + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.this[each.key].arn + } +} + +# Attach every node in the managed node group to every target group. The node +# group creates an Auto Scaling group; attaching to it means replaced nodes +# register automatically, which is what makes the NLB+EIP anchor stable. +resource "aws_autoscaling_attachment" "this" { + for_each = local.target_ports + + autoscaling_group_name = aws_eks_node_group.workers.resources[0].autoscaling_groups[0].name + lb_target_group_arn = aws_lb_target_group.this[each.key].arn +} diff --git a/providers/aws-eks/main.tf b/providers/aws-eks/main.tf new file mode 100644 index 0000000..b97c770 --- /dev/null +++ b/providers/aws-eks/main.tf @@ -0,0 +1,285 @@ +# Managed Kubernetes on AWS (EKS). The counterpart of providers/upcloud-uks: +# a managed control plane plus one worker node group, with the mip-infra +# contract add-ons applied through the helm and kubernetes providers +# instead of modules/rke2-bootstrap (there is no RKE2 manifests directory). +# +# EKS provides more of the contract natively than UKS does — the VPC CNI, the +# EBS and EFS CSI drivers — but the EFS layer (storage.tf) is what makes RWX +# work here at all, which is the one thing the UKS stack cannot do. + +locals { + app_cidr = coalesce(var.app_access_cidr, var.operator_cidr) + api_cidrs = distinct(concat([var.operator_cidr], var.extra_control_plane_cidrs)) + + # Subnets follow availability_zones one-for-one; the node group gets the first + # node_az_count of them. + node_subnet_ids = [for i in range(var.node_az_count) : aws_subnet.this[i].id] + single_az = var.node_az_count == 1 +} + +# --- network ---------------------------------------------------------------- +# Public subnets: the nodes get public IPs and egress straight through the +# internet gateway, so there is no NAT gateway to pay for (~$33/month). The +# security groups, not a private subnet, are what lock the nodes down. +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = var.cluster_name } +} + +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = { Name = var.cluster_name } +} + +resource "aws_subnet" "this" { + count = length(var.availability_zones) + vpc_id = aws_vpc.this.id + cidr_block = var.subnet_cidrs[count.index] + availability_zone = var.availability_zones[count.index] + map_public_ip_on_launch = true + + tags = { + Name = "${var.cluster_name}-${var.availability_zones[count.index]}" + # Lets the in-cluster AWS controllers discover the subnets they may use. + "kubernetes.io/role/elb" = "1" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.this.id + } + + tags = { Name = "${var.cluster_name}-public" } +} + +resource "aws_route_table_association" "this" { + count = length(aws_subnet.this) + subnet_id = aws_subnet.this[count.index].id + route_table_id = aws_route_table.public.id +} + +# --- IAM -------------------------------------------------------------------- +data "aws_iam_policy_document" "eks_assume" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cluster" { + name = "${var.cluster_name}-cluster" + assume_role_policy = data.aws_iam_policy_document.eks_assume.json +} + +resource "aws_iam_role_policy_attachment" "cluster" { + role = aws_iam_role.cluster.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" +} + +data "aws_iam_policy_document" "ec2_assume" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "node" { + name = "${var.cluster_name}-node" + assume_role_policy = data.aws_iam_policy_document.ec2_assume.json +} + +resource "aws_iam_role_policy_attachment" "node" { + for_each = toset([ + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + ]) + role = aws_iam_role.node.name + policy_arn = each.value +} + +# Trust policy for the CSI drivers' EKS Pod Identity roles (storage.tf). Pod +# Identity replaces IRSA: no OIDC provider, no per-cluster trust-policy editing. +data "aws_iam_policy_document" "pods_assume" { + statement { + actions = ["sts:AssumeRole", "sts:TagSession"] + principals { + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] + } + } +} + +# --- cluster ---------------------------------------------------------------- +resource "aws_eks_cluster" "this" { + name = var.cluster_name + version = var.k8s_version + role_arn = aws_iam_role.cluster.arn + + vpc_config { + subnet_ids = aws_subnet.this[*].id + endpoint_private_access = true + endpoint_public_access = true + # The AWS analogue of UKS's control_plane_ip_filter. Must include the IP + # `tofu apply` runs from, or the in-cluster providers cannot connect. + public_access_cidrs = local.api_cidrs + } + + # API-only auth: access entries, not the legacy aws-auth ConfigMap. The + # creating principal is bootstrapped as cluster admin so the helm/kubernetes + # providers can act in the same run. + access_config { + authentication_mode = "API" + bootstrap_cluster_creator_admin_permissions = true + } + + depends_on = [aws_iam_role_policy_attachment.cluster] +} + +# --- worker node group ------------------------------------------------------ +# Only worker nodes are reachable; the EKS control plane has no host/etcd/ +# apiserver access at all (see docs/why-self-managed.md). +# +# SSH to the nodes takes TWO pieces, and the trap is that the obvious one does +# nothing on its own. `remote_access.source_security_group_ids` names security +# groups whose MEMBERS may SSH in — it is not a CIDR allowlist. Pointing it at +# an empty group (as below) therefore grants nobody access; its only real job is +# to stop EKS applying its default, which is to open port 22 to 0.0.0.0/0 +# whenever an SSH key is set and no source group is given. +# +# What actually lets the operator in is the rule further down on the cluster +# security group, which EKS attaches to every node. +# NOTE: do not edit this description. A security group's description is +# ForceNew, the node group references this group's id through +# remote_access.source_security_group_ids, and that field is ForceNew too — so a +# one-word change here destroys and recreates every worker node. +resource "aws_security_group" "node_ssh" { + name = "${var.cluster_name}-node-ssh" + description = "${var.cluster_name}: SSH to worker nodes from the operator" + vpc_id = aws_vpc.this.id + tags = { Name = "${var.cluster_name}-node-ssh" } +} + +resource "aws_vpc_security_group_ingress_rule" "node_ssh" { + security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id + description = "SSH from operator" + ip_protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_ipv4 = var.operator_cidr +} + +resource "aws_key_pair" "operator" { + key_name = "${var.cluster_name}-operator" + public_key = var.ssh_public_key +} + +resource "aws_eks_node_group" "workers" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "workers" + node_role_arn = aws_iam_role.node.arn + subnet_ids = local.node_subnet_ids + + # AL2023 x86-64: the MIP container images are amd64-only. + ami_type = "AL2023_x86_64_STANDARD" + instance_types = [var.node_instance_type] + capacity_type = "ON_DEMAND" + disk_size = var.node_disk_size + + scaling_config { + desired_size = var.node_count + min_size = var.node_count + max_size = var.node_count + } + + remote_access { + ec2_ssh_key = aws_key_pair.operator.key_name + source_security_group_ids = [aws_security_group.node_ssh.id] + } + + labels = { + project = "mip-dev-cluster" + # haproxy-edge selects on this, mirroring the self-managed stacks' node + # label — here every node carries it so any node can serve public traffic. + "mip.dev/edge" = "haproxy" + } + + # The node group would otherwise race the CNI policy attachment and come up + # with nodes that cannot get pod addresses. + depends_on = [aws_iam_role_policy_attachment.node] + + lifecycle { + # desired_size drifts if anything ever scales the group; do not fight it. + ignore_changes = [scaling_config[0].desired_size] + } +} + +# --- managed add-ons -------------------------------------------------------- +# addon_version is deliberately omitted throughout: EKS then installs the +# default version for the cluster's Kubernetes version, so there is no pin to +# drift and nothing for Renovate to chase (see docs/maintenance.md). +resource "aws_eks_addon" "core" { + for_each = toset([ + "vpc-cni", + "coredns", + "kube-proxy", + "eks-pod-identity-agent", + "metrics-server", + ]) + cluster_name = aws_eks_cluster.this.name + addon_name = each.value + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + + depends_on = [aws_eks_node_group.workers] +} + +# --- kubeconfig ------------------------------------------------------------- +# Written next to the stack (gitignored) for kubectl/argocd. Uses the same exec +# plugin as the providers, so no long-lived token is stored. +resource "local_sensitive_file" "kubeconfig" { + filename = "${path.module}/kubeconfig" + file_permission = "0600" + content = yamlencode({ + apiVersion = "v1" + kind = "Config" + clusters = [{ + name = aws_eks_cluster.this.arn + cluster = { + server = aws_eks_cluster.this.endpoint + certificate-authority-data = aws_eks_cluster.this.certificate_authority[0].data + } + }] + users = [{ + name = aws_eks_cluster.this.arn + user = { + exec = { + apiVersion = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = local.kube_exec_args + } + } + }] + contexts = [{ + name = aws_eks_cluster.this.arn + context = { + cluster = aws_eks_cluster.this.arn + user = aws_eks_cluster.this.arn + } + }] + current-context = aws_eks_cluster.this.arn + }) +} diff --git a/providers/aws-eks/outputs.tf b/providers/aws-eks/outputs.tf new file mode 100644 index 0000000..ea1785e --- /dev/null +++ b/providers/aws-eks/outputs.tf @@ -0,0 +1,83 @@ +# Interface-contract outputs — same names as the self-managed stacks so shared +# tooling (../../scripts/prepare-mip-infra-fork.sh via STACK_DIR) works +# unchanged. Some values differ in nature on a managed cluster; see README. + +output "cp_public_ip" { + description = "Elastic IP of the Argo CD load balancer. On a managed cluster this is NOT a node IP — the kube-API lives at its own endpoint (see kube_api)." + value = aws_eip.argocd.public_ip +} + +output "edge_public_ip" { + description = "Elastic IP of the apps load balancer: public entry for all MIP apps (haproxy-edge -> haproxy-public)." + value = aws_eip.apps.public_ip +} + +output "argocd_hostname" { + description = "Hostname to use in the argo-setup bootstrap sed (replaces argocd.example.com)." + value = "argocd.${aws_eip.argocd.public_ip}.sslip.io" +} + +output "app_hostname_suffix" { + description = "Suffix for MIP app hostnames, e.g. federation-a. and datacatalogue.." + value = "${aws_eip.apps.public_ip}.sslip.io" +} + +output "kube_api" { + description = "Kubernetes API endpoint (allowlisted to operator_cidr + extra_control_plane_cidrs). Note it serves :443, not :6443." + value = aws_eks_cluster.this.endpoint +} + +output "kubeconfig_path" { + description = "Path to the written kubeconfig (gitignored). Or use: aws eks update-kubeconfig --name --region " + value = local_sensitive_file.kubeconfig.filename +} + +output "cluster_name" { + description = "EKS cluster name." + value = aws_eks_cluster.this.name +} + +output "region" { + description = "Region the cluster runs in." + value = var.region +} + +output "vip_haproxy" { + description = "Contract VIP for mip-infra's haproxy-public Service (only meaningful when install_metallb = true AND the Service carries loadBalancerClass metallb.io/metallb — see README)." + value = var.install_metallb ? var.vip_haproxy : "not installed (see README)" +} + +output "vip_submariner" { + description = "Contract VIP for the Submariner gateway Service (same conditions as vip_haproxy)." + value = var.install_metallb ? var.vip_submariner : "not installed (see README)" +} + +output "metallb_pool" { + description = "MetalLB pool name (only if install_metallb = true)." + value = var.install_metallb ? "pool-no-auto" : "not installed (managed LB path on EKS — see README)" +} + +output "storage_classes" { + description = "EFS-backed StorageClass aliases created for the mip-infra contract — genuinely RWX, unlike the block-only managed stacks." + value = ["ceph-corbo-cephfs (default, Delete)", "ceph-corbo-cephfs-retain (Retain)"] +} + +output "cluster_issuers" { + description = "cert-manager ClusterIssuers created." + value = ["letsencrypt", "letsencrypt-staging"] +} + +output "ingress_classes" { + description = "Ingress classes this stack provides. `haproxy-public` comes from mip-infra itself." + value = var.install_traefik ? ["traefik"] : [] +} + +output "cni" { + description = "EKS ships the AWS VPC CNI (not Canal), which gives pods VPC addresses — a known divergence from mip-infra's pod-CIDR assumption. See README." + value = "aws-vpc-cni" +} + +output "efs_id" { + description = "EFS file system backing the RWX StorageClasses." + value = aws_efs_file_system.this.id +} diff --git a/providers/aws-eks/storage.tf b/providers/aws-eks/storage.tf new file mode 100644 index 0000000..f3fb83b --- /dev/null +++ b/providers/aws-eks/storage.tf @@ -0,0 +1,194 @@ +# Storage for the mip-infra contract: RWX StorageClasses named +# `ceph-corbo-cephfs` (default) and `ceph-corbo-cephfs-retain`. +# +# This is where EKS beats the other managed stacks. UKS and SKS only offer +# block/RWO CSI drivers, so exareme2's ReadWriteMany PVC cannot bind there at +# all (see providers/upcloud-uks/README.md). EFS is a real shared filesystem, +# so the aliases below are genuinely RWX — the same property Longhorn's +# share-manager provides on the self-managed stacks, and the closest analogue +# to the CephFS the upstream names come from. +# +# The EBS CSI driver is installed too (block/RWO) so anything that asks for the +# gp3 class still works. + +# --- EFS file system -------------------------------------------------------- +resource "aws_efs_file_system" "this" { + creation_token = var.cluster_name + encrypted = true + + # One Zone ($0.16/GB-month) whenever the nodes are in a single AZ; Standard + # ($0.30) once they are spread, because a One Zone filesystem can only be + # mounted from its own AZ. + availability_zone_name = local.single_az ? var.availability_zones[0] : null + + throughput_mode = var.efs_throughput_mode + performance_mode = "generalPurpose" + + tags = { Name = var.cluster_name } +} + +resource "aws_security_group" "efs" { + name = "${var.cluster_name}-efs" + description = "${var.cluster_name}: NFS from the cluster nodes" + vpc_id = aws_vpc.this.id + tags = { Name = "${var.cluster_name}-efs" } +} + +# Only the cluster's own nodes may mount it. EKS creates and manages this +# security group and attaches it to every node. +resource "aws_vpc_security_group_ingress_rule" "efs_nfs" { + security_group_id = aws_security_group.efs.id + description = "NFS from cluster nodes" + ip_protocol = "tcp" + from_port = 2049 + to_port = 2049 + referenced_security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id +} + +# One mount target per AZ the nodes run in — a pod can only reach the target in +# its own AZ. +resource "aws_efs_mount_target" "this" { + count = var.node_az_count + file_system_id = aws_efs_file_system.this.id + subnet_id = aws_subnet.this[count.index].id + security_groups = [aws_security_group.efs.id] +} + +# --- CSI driver IAM (EKS Pod Identity) -------------------------------------- +resource "aws_iam_role" "efs_csi" { + name = "${var.cluster_name}-efs-csi" + assume_role_policy = data.aws_iam_policy_document.pods_assume.json +} + +resource "aws_iam_role_policy_attachment" "efs_csi" { + role = aws_iam_role.efs_csi.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" +} + +resource "aws_iam_role" "ebs_csi" { + name = "${var.cluster_name}-ebs-csi" + assume_role_policy = data.aws_iam_policy_document.pods_assume.json +} + +resource "aws_iam_role_policy_attachment" "ebs_csi" { + role = aws_iam_role.ebs_csi.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" +} + +# --- CSI driver add-ons ----------------------------------------------------- +# pod_identity_association wires the driver's service account to its role in +# one step; it needs the eks-pod-identity-agent add-on, hence the depends_on. +resource "aws_eks_addon" "efs_csi" { + cluster_name = aws_eks_cluster.this.name + addon_name = "aws-efs-csi-driver" + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + + pod_identity_association { + role_arn = aws_iam_role.efs_csi.arn + service_account = "efs-csi-controller-sa" + } + + depends_on = [ + aws_eks_addon.core, + aws_iam_role_policy_attachment.efs_csi, + aws_efs_mount_target.this, + ] +} + +resource "aws_eks_addon" "ebs_csi" { + cluster_name = aws_eks_cluster.this.name + addon_name = "aws-ebs-csi-driver" + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + + pod_identity_association { + role_arn = aws_iam_role.ebs_csi.arn + service_account = "ebs-csi-controller-sa" + } + + depends_on = [ + aws_eks_addon.core, + aws_iam_role_policy_attachment.ebs_csi, + ] +} + +# --- the contract StorageClasses -------------------------------------------- +# Named after upstream's CephFS classes so the mip-infra fork needs zero storage +# changes. Dynamic provisioning in `efs-ap` mode gives every PVC its own EFS +# access point, so claims are isolated but all RWX-capable. +# +# Unlike the self-managed stacks there is no ext4/xfs split to make: EFS is NFS, +# so neither the ext4 `lost+found` that breaks exareme2's CSV scanner nor +# Longhorn's 300Mi XFS floor applies. Both classes use identical parameters and +# differ only in reclaim policy. +resource "kubernetes_storage_class_v1" "ceph_corbo_cephfs" { + metadata { + name = "ceph-corbo-cephfs" + annotations = { "storageclass.kubernetes.io/is-default-class" = "true" } + } + storage_provisioner = "efs.csi.aws.com" + reclaim_policy = "Delete" + volume_binding_mode = "Immediate" + allow_volume_expansion = false # EFS is elastic; there is no size to expand + + parameters = { + provisioningMode = "efs-ap" + fileSystemId = aws_efs_file_system.this.id + directoryPerms = "700" + basePath = "/dynamic" + # The access point enforces this POSIX identity for every operation on the + # volume. Root is deliberate: EFS otherwise squashes root, and any image + # that chowns its data directory at startup — every stock Postgres image + # does — dies with "chown: Operation not permitted" and CrashLoopBackOffs + # forever. Longhorn on the self-managed stacks is block storage and has no + # such restriction, so this has no analogue there. + uid = "0" + gid = "0" + ensureUniqueDirectory = "true" + } + + depends_on = [aws_eks_addon.efs_csi] +} + +resource "kubernetes_storage_class_v1" "ceph_corbo_cephfs_retain" { + metadata { + name = "ceph-corbo-cephfs-retain" + } + storage_provisioner = "efs.csi.aws.com" + reclaim_policy = "Retain" + volume_binding_mode = "Immediate" + allow_volume_expansion = false + + parameters = { + provisioningMode = "efs-ap" + fileSystemId = aws_efs_file_system.this.id + directoryPerms = "700" + basePath = "/dynamic-retain" + # See the default class above: root identity so chown-on-startup works. + uid = "0" + gid = "0" + ensureUniqueDirectory = "true" + } + + depends_on = [aws_eks_addon.efs_csi] +} + +# The EBS CSI add-on ships a `gp2` StorageClass marked default. Two defaults +# make Kubernetes warn and pick arbitrarily, so drop the annotation from that +# one and leave ceph-corbo-cephfs as the only default. +resource "kubernetes_annotations" "undefault_gp2" { + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + force = true + + metadata { + name = "gp2" + } + + annotations = { + "storageclass.kubernetes.io/is-default-class" = "false" + } + + depends_on = [aws_eks_addon.ebs_csi] +} diff --git a/providers/aws-eks/terraform.tfvars.example b/providers/aws-eks/terraform.tfvars.example new file mode 100644 index 0000000..6f4c956 --- /dev/null +++ b/providers/aws-eks/terraform.tfvars.example @@ -0,0 +1,52 @@ +# Copy to terraform.tfvars and adjust. terraform.tfvars is gitignored. + +# --- required ----------------------------------------------------------------- + +# Who may reach the EKS API endpoint and HTTPS on the load balancers. MUST +# include the IP `tofu apply` runs from — the kubernetes/helm/kubectl providers +# connect to the control plane during the same apply. +# Your workstation/VPN egress IPv4: curl -4 -s ifconfig.me +operator_cidr = "192.0.2.10/32" + +# Installed on worker nodes for debugging (content, not a path). +ssh_public_key = "ssh-ed25519 AAAA... you@workstation" + +# Let's Encrypt account email for the cluster ClusterIssuers. +acme_email = "email@example.com" + +# --- optional (sensible defaults) ---------------------------------------------- + +# Default region is the lowest-carbon European one (~35 gCO2e/kWh) and also the +# cheapest of the three candidates. Only a default — switch freely, but set the +# AZs to match. +# region = "eu-north-1" +# availability_zones = ["eu-north-1a", "eu-north-1b", "eu-north-1c"] +# +# node_az_count = 1 # 1 keeps nodes in one AZ: no cross-AZ transfer charge, +# # and EFS can use One Zone storage ($0.16 vs $0.30/GB-mo) +# +# cluster_name = "mip-eks" +# k8s_version = "1.35" # keep on standard support ($0.10/h, not $0.60/h) +# # check: aws eks describe-cluster-versions +# +# x86-64 only: MIP container images are amd64-only, so Graviton is out. +# node_instance_type = "m7i.xlarge" # 4 vCPU / 16 GiB +# node_count = 3 # >= 3 for full mip-infra (Argo CD redis-ha) +# node_disk_size = 50 +# +# install_traefik = true # provides the contract's `traefik` ingress class +# install_metallb = true # provides pool-no-auto; see README for the fork patch +# # the Services need before MetalLB can own them +# +# efs_throughput_mode = "elastic" # or "bursting" (no per-GB transfer charge) +# +# vpc_cidr = "10.101.0.0/16" +# subnet_cidrs = ["10.101.0.0/20", "10.101.16.0/20", "10.101.32.0/20"] +# +# VIPs sit in a /24 of the VPC that has no subnet, so AWS can never hand these +# addresses to an ENI. +# vip_haproxy = "10.101.255.200" +# vip_submariner = "10.101.255.201" +# +# extra_control_plane_cidrs = [] # e.g. a CI runner that also runs tofu +# app_access_cidr = null # null = inherit operator_cidr; "0.0.0.0/0" = public diff --git a/providers/aws-eks/variables.tf b/providers/aws-eks/variables.tf new file mode 100644 index 0000000..fd33072 --- /dev/null +++ b/providers/aws-eks/variables.tf @@ -0,0 +1,268 @@ +variable "region" { + description = <<-EOT + AWS region. The default is a preference, not a constraint: eu-north-1 + (Stockholm) has the lowest-carbon grid of the European regions + (~35 vs ~256 in Ireland and ~330 gCO2e/kWh in Germany — see + docs/cost-comparison.md for the source) and is also the cheapest of the + three for the instance families this stack uses. EKS, EFS and 3 AZs are all + available there. Switch freely — set availability_zones to match. + EOT + type = string + default = "eu-north-1" +} + +variable "availability_zones" { + description = <<-EOT + AZs to create subnets in. EKS requires subnets in at least two AZs for the + control plane, even when the nodes all sit in one (see node_az_count). + Must all be inside var.region. + EOT + type = list(string) + default = ["eu-north-1a", "eu-north-1b", "eu-north-1c"] + + validation { + condition = length(var.availability_zones) >= 2 + error_message = "EKS needs subnets in at least two availability zones." + } +} + +variable "node_az_count" { + description = <<-EOT + How many of availability_zones the node group may place nodes in. 1 + (default) keeps every node in one AZ, which matches the self-managed + stacks and avoids AWS's $0.01/GB-each-way cross-AZ charge; it also lets the + EFS file system use the cheaper One Zone storage class ($0.16 vs + $0.30/GB-month). Raise it for AZ redundancy at that cost — but note that + crossing the 1 <-> >1 boundary later flips the EFS file system between One + Zone and Standard, which REPLACES it: every existing EFS volume (all PVC + data) is destroyed. Decide before the cluster holds data you care about. + EOT + type = number + default = 1 + + validation { + condition = var.node_az_count >= 1 && var.node_az_count <= length(var.availability_zones) + error_message = "node_az_count must be between 1 and length(availability_zones)." + } +} + +variable "cluster_name" { + description = "EKS cluster name. Also prefixes the load balancers and target groups, whose names AWS caps at 32 characters — hence the length limit." + type = string + default = "mip-eks" + + validation { + # The longest derived name is "-argocd-https-" — 19 + # characters of suffix — and AWS caps target-group names at 32. + condition = can(regex("^[a-zA-Z][a-zA-Z0-9-]{0,12}$", var.cluster_name)) + error_message = "cluster_name must start with a letter, contain only letters/digits/hyphens, and be at most 13 characters (AWS caps target-group names at 32, and this prefixes them along with a port suffix)." + } +} + +variable "k8s_version" { + description = <<-EOT + Kubernetes version for the EKS control plane. Keep it on standard support: + extended support costs $0.60/hour per cluster instead of $0.10. + List what is on offer with: aws eks describe-cluster-versions + EOT + type = string + default = "1.35" +} + +variable "node_instance_type" { + description = <<-EOT + Instance type for the managed node group (4 vCPU / 16 GiB, the tested + mip-infra worker shape). Must be x86-64: the MIP container images are + amd64-only, so the cheaper Graviton families cannot be used. + EOT + type = string + default = "m7i.xlarge" +} + +variable "node_count" { + description = "Worker node count. Use >= 3 to run full mip-infra (Argo CD redis-ha anti-affinity); 2 is fine to smoke-test the stack." + type = number + default = 3 +} + +variable "node_disk_size" { + description = "Root EBS volume size (GB) per node. Larger than the self-managed stacks' 35 GB because there is no separate data disk: EFS carries the PVCs, but images and ephemeral storage share the root volume." + type = number + default = 50 +} + +variable "vpc_cidr" { + description = "VPC CIDR. Must contain every subnet plus both MetalLB VIPs." + type = string + default = "10.101.0.0/16" +} + +variable "subnet_cidrs" { + description = <<-EOT + One subnet CIDR per entry in availability_zones. Sized /20 rather than /24 + because the VPC CNI gives every pod a VPC address out of the node's subnet — + a /24 runs out after a handful of nodes. + EOT + type = list(string) + default = ["10.101.0.0/20", "10.101.16.0/20", "10.101.32.0/20"] + + validation { + condition = length(var.subnet_cidrs) >= length(var.availability_zones) + error_message = "subnet_cidrs needs one entry per availability zone." + } +} + +variable "operator_cidr" { + description = <<-EOT + Primary IPv4 CIDR allowed to reach the EKS public API endpoint (also where + `tofu apply` runs from, so the kubernetes and helm providers can + connect) and to reach HTTPS on the public load balancers. + Get yours: curl -4 -s ifconfig.me + EOT + type = string + + validation { + condition = can(cidrnetmask(var.operator_cidr)) + error_message = "operator_cidr must be a valid IPv4 CIDR, e.g. 192.0.2.10/32." + } +} + +variable "extra_control_plane_cidrs" { + description = "Additional IPv4 CIDRs allowed to reach the EKS API endpoint (e.g. a CI runner or a jump host running tofu). Merged with operator_cidr." + type = list(string) + default = [] +} + +variable "app_access_cidr" { + description = <<-EOT + IPv4 CIDR allowed to reach HTTPS on the two public load balancers (the + Argo CD UI and all MIP apps). Defaults to operator_cidr (locked down). Set + to "0.0.0.0/0" to make the apps publicly reachable. Port 80 always stays + world-reachable for Let's Encrypt HTTP-01. See docs/firewall.md. + EOT + type = string + default = null + + validation { + condition = var.app_access_cidr == null || can(cidrnetmask(var.app_access_cidr)) + error_message = "app_access_cidr must be a valid IPv4 CIDR (or null to inherit operator_cidr)." + } +} + +variable "ssh_public_key" { + description = "SSH public key (content, not path) installed on worker nodes for debugging. Reachable from operator_cidr only." + type = string +} + +variable "acme_email" { + description = "Let's Encrypt account email for the cluster ClusterIssuers." + type = string +} + +variable "vip_haproxy" { + description = <<-EOT + Contract VIP for mip-infra's haproxy-public Service. Inside vpc_cidr but in + a /24 with no subnet, so AWS can never assign it to an ENI. Only used when + install_metallb = true — and see the README: on EKS the Services also need + a loadBalancerClass for MetalLB to own them. + EOT + type = string + default = "10.101.255.200" +} + +variable "vip_submariner" { + description = "Contract VIP for the Submariner gateway Service (same reserved, subnet-less range as vip_haproxy)." + type = string + default = "10.101.255.201" +} + +variable "issuer_mode" { + description = <<-EOT + How the `letsencrypt` / `letsencrypt-staging` ClusterIssuers are backed. + + "auto" (default) decides from acme_email: a placeholder address gets + self-signed issuers, a real one gets real Let's Encrypt. That is the useful + default because Let's Encrypt rejects reserved domains outright + ("contact email has forbidden domain") — and since the Argo CD Ingress is + websecure-only, a failed registration means no certificate, no TLS secret, + and a 404 with nothing in the logs pointing at the email. + + Force it with "acme" or "selfsigned". Either way the issuer NAMES stay the + same, because argo-setup's Ingress references `letsencrypt` literally. + Mirrors the issuer_mode knob in modules/rke2-bootstrap. + EOT + type = string + default = "auto" + + validation { + condition = contains(["auto", "acme", "selfsigned"], var.issuer_mode) + error_message = "issuer_mode must be \"auto\", \"acme\" or \"selfsigned\"." + } +} + +variable "install_metallb" { + description = <<-EOT + Install MetalLB and the pool-no-auto pool so the contract's fixed VIPs are + assignable. On by default, but scoped to loadBalancerClass + "metallb.io/metallb": EKS's legacy in-tree Service Controller still claims + every unclassed type=LoadBalancer Service (and gives it a Classic LB), so + without the class both controllers fight over the same Service. See the + README's known-divergences section for the two-line fork patch. + EOT + type = bool + default = true +} + +variable "install_traefik" { + description = <<-EOT + Install Traefik as a NodePort DaemonSet so the contract's `traefik` ingress + class exists for the Argo CD UI (the argo-setup ingress references it by + name). The argocd load balancer targets these NodePorts. + EOT + type = bool + default = true +} + +variable "traefik_node_ports" { + description = "Pinned NodePorts for Traefik's web/websecure entrypoints — pinned so the load-balancer target groups can reference them." + type = object({ + web = number + websecure = number + }) + default = { + web = 30080 + websecure = 30443 + } +} + +variable "haproxy_edge_host_ports" { + description = <<-EOT + Host ports the haproxy-edge DaemonSet binds, and the ports the apps load + balancer forwards to. NOT 80/443, and that is not a style choice: EKS's + eks-pod-identity-agent already listens on 169.254.170.23:80 on every node, + and a wildcard bind (0.0.0.0:80) collides with an existing bind on a + specific address on the same port — so a hostNetwork pod simply cannot take + port 80 on an EKS node. The load balancer still listens publicly on 80/443 + and translates. (The self-managed stacks bind 80/443 directly, because + nothing on an RKE2 node occupies them.) + EOT + type = object({ + http = number + https = number + }) + default = { + http = 8080 + https = 8443 + } +} + +variable "efs_throughput_mode" { + description = "EFS throughput mode. `elastic` scales automatically and bills per GB transferred ($0.06 write / $0.03 read); `bursting` bills nothing extra but scales with stored size." + type = string + default = "elastic" + + validation { + condition = contains(["elastic", "bursting", "provisioned"], var.efs_throughput_mode) + error_message = "efs_throughput_mode must be elastic, bursting or provisioned." + } +} diff --git a/providers/aws-eks/versions.tf b/providers/aws-eks/versions.tf new file mode 100644 index 0000000..54ad4ad --- /dev/null +++ b/providers/aws-eks/versions.tf @@ -0,0 +1,69 @@ +terraform { + required_version = ">= 1.9.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.31" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + local = { + source = "hashicorp/local" + version = "~> 2.5" + } + } +} + +# Credentials from the environment (never in files): +# export AWS_PROFILE=... or AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY +provider "aws" { + region = var.region + + default_tags { + tags = { + project = "mip-dev-cluster" + stack = "aws-eks" + } + } +} + +# The kubernetes and helm providers authenticate with the `aws eks get-token` +# exec plugin rather than a data-source token: the credential is minted at apply +# time and never written to state. Requires the `aws` CLI v2 on PATH. +# var.operator_cidr must include the IP `tofu apply` runs from — the EKS public +# endpoint is allowlisted to it (see public_access_cidrs in main.tf). +locals { + kube_exec_args = ["eks", "get-token", "--cluster-name", var.cluster_name, "--region", var.region] +} + +provider "kubernetes" { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = local.kube_exec_args + } +} + +provider "helm" { + kubernetes { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = local.kube_exec_args + } + } +} + diff --git a/providers/aws-rke2/.terraform.lock.hcl b/providers/aws-rke2/.terraform.lock.hcl new file mode 100644 index 0000000..d205d92 --- /dev/null +++ b/providers/aws-rke2/.terraform.lock.hcl @@ -0,0 +1,76 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "6.57.1" + constraints = "~> 6.0" + hashes = [ + "h1:8wbrCS3R5lunN2HMVfQ+oMB3uFVAO7gfUbItdZa+tdA=", + "h1:C4XEcAWUKXExKCF6NhvMxkSyZ/JqNWjtwtwwEqQtl70=", + "h1:Fah+p3BD+/E7wdgazQFp4fsjuvTh5/Q7fQFyMsv9juM=", + "h1:S6eLURiAh6L00nr+9s2zwuhHQOU04rEG60Z9xO1ORRM=", + "h1:UdllVJ4482tLgz5q8se3SPzpPoXuFXwvxiykpgdipWs=", + "h1:ZAkIIdmQryKnldenUI2fqThlAciNz2UYKVTZ5F/57ME=", + "h1:a/i6TVZX4jSNSvHIWWTqLIlaugg7SYfQMZNVtzDWlKM=", + "h1:gGtYen1GfQvynKT8lHguY+cvmxNww+TOqCaapWoNW1w=", + "h1:iFr4+F/b94MEadHCiaNm6o6TapyvLT6TrwmbPyDQs9E=", + "h1:k0OzZI0aBgeEhCUcoK00/6C3hBK59ETo7g5Jq62wRqQ=", + "h1:nlvu+qO8eMFdnMptYSUK1c5dOXoK6r3HG87jzE7EYvQ=", + "h1:rVP8Z9tc37nwsW+NLSvEQyUBoDG/zsMtPSMeHhBrR3k=", + "h1:rxnmHWTUE2Aou/otNocOI4Rvcm69LQsoNYssj0s7hIo=", + "h1:va0Q0eDSt7xyvkSX7GbXmAO/QUqugZb/05gMdcQlR0I=", + "h1:yXF1EzUTMepZv4SeQnBNUvjOj2HKEKSl+jVndIKksGU=", + "zh:053910cce06bba506fb7a3dcecdcd719280d7df8b8a7cce06ae3e2dd20b1cc0d", + "zh:14feabeefef742b528c978a78f6896240a7ca3464a4714a5b1e1b2d448db4fd5", + "zh:22dd549e3c479b2ff2121e00a005aa3b6b43a166e727ef3208a529a59feb1608", + "zh:3ca5eab63ac77da335d42ebe0af8569c5c617dc5b239750b111f5ba9f2f889cd", + "zh:3d686f9fca9752850958a353971cd7eb1f5a48e55c2c56fcd96d18cc6e54f8ae", + "zh:512e46d35abfec0ba85fc0f2da7dc39fece9106f02165e95096b6549f920453a", + "zh:5fe83903b4bef5b7318de4b61e0ff9c88aa8ac42b7abeae83d1599cb054b0908", + "zh:61ec8a18a7270e28f43c0f810c82a147af7a5de622eb1f243c3a773e234b2172", + "zh:6428cd62e615ccaaa3a26a22f68a84f80a0d497a04f0405648119ef69e709ff9", + "zh:776e57bb2c94716f452eea5803c90b97d304c9f343e6746780a255d8217b95f9", + "zh:99bacf79ce92568d086ed62b287d9b77fa0c4e86a6c25ad398bec04ea0e44897", + "zh:b4361d4ad5074df3e83dc0e40452b63f231ae5959b4007763739d3a02caffe29", + "zh:d96c0f8603a1f8ab61c633a7a707922313f6bdfa8d17fa21d9ccf103fade6af7", + "zh:edfd955636489371e5ad1f40fc77a023d7eb7d09140342b151f0ea15451ef41b", + "zh:eeb03674754bd059fafa62b58cf4015c2c07bc25477def3c64eef487d583a63c", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.6" + hashes = [ + "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", + "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", + "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", + "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", + "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", + "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", + "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", + "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", + "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", + "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", + "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", + "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", + "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", + "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", + "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", + "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", + "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", + "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", + "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", + "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", + "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", + "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", + "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", + "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", + "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", + "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", + "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", + "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", + "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", + "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", + ] +} diff --git a/providers/aws-rke2/Makefile b/providers/aws-rke2/Makefile new file mode 100644 index 0000000..bf6dcc1 --- /dev/null +++ b/providers/aws-rke2/Makefile @@ -0,0 +1,36 @@ +TOFU ?= tofu + +.PHONY: init validate plan apply destroy kubeconfig contract up down status fork-help + +init: ## download providers + $(TOFU) init + +validate: + $(TOFU) validate + +plan: + $(TOFU) plan + +apply: + $(TOFU) apply + +destroy: ## tears the whole cluster down (Longhorn data volumes included!) + $(TOFU) destroy + +kubeconfig: ## fetch ./kubeconfig from the control plane + ./scripts/fetch-kubeconfig.sh + +contract: ## print the values the mip-infra fork needs + ./scripts/contract.sh + +up: ## power the cluster on (CP first, then workers) + ./scripts/cluster-power.sh start + +down: ## power the cluster off (EC2 stops billing compute; EBS + EIPs keep billing) + ./scripts/cluster-power.sh stop + +status: + ./scripts/cluster-power.sh status + +fork-help: + @echo "usage: STACK_DIR=providers/aws-rke2 ../../scripts/prepare-mip-infra-fork.sh [path-to-mip-infra]" diff --git a/providers/aws-rke2/README.md b/providers/aws-rke2/README.md new file mode 100644 index 0000000..e065ab1 --- /dev/null +++ b/providers/aws-rke2/README.md @@ -0,0 +1,242 @@ +# provider: aws-rke2 (self-managed RKE2 on EC2) — ⚠️ UNTESTED + +The AWS twin of the reference stack +[`upcloud-rke2`](../upcloud-rke2), written file-for-file against it so the two +stay diffable. Run all deploy commands from **this directory** +(`providers/aws-rke2/`). The shared add-on layer lives in +[`../../modules/rke2-bootstrap`](../../modules/rke2-bootstrap). + +> **Status: code complete, never `apply`-ed.** Written against the current +> `hashicorp/aws` v6 provider and AWS documentation. Treat it as a strong +> starting point, not a turnkey deploy — the [likely first-run +> fixes](#likely-first-run-fixes) are listed at the bottom. + +- **Nodes:** **Ubuntu 24.04 LTS** (Canonical's AMI, resolved from its public + SSM parameter — no region-specific AMI id to maintain). RKE2 is installed at + first boot, pinned via `rke2_version`. +- **Kubernetes:** **RKE2** (pinned), Canal CNI, default CIDRs + (`10.42.0.0/16` pods / `10.43.0.0/16` services — mip-infra's submariner + values assume exactly this). +- **Pre-provided for mip-infra** (it assumes these but installs none of them): + cert-manager (+ `letsencrypt` / `letsencrypt-staging` ClusterIssuers), + MetalLB with pool `pool-no-auto`, RWX-capable StorageClasses named + `ceph-corbo-cephfs` / `ceph-corbo-cephfs-retain` (Longhorn-backed — zero + storage diff in the fork), the ECK operator, Traefik ingress. +- **DNS/certs:** sslip.io wildcard hostnames anchored on **Elastic IPs** (they + persist across power cycles) — no DNS to manage. +- **Cost:** roughly **$655/month at 24/7**, **~$193/month at 8 h × 22 + workdays** with `make down` overnight (EBS + Elastic IPs bill while stopped). + Breakdown and comparison against every other provider: + [docs/cost-comparison.md](../../docs/cost-comparison.md). + +## Region: why `eu-north-1` is the default + +`eu-north-1` (Stockholm) has the **lowest-carbon grid of AWS's European +regions** — roughly 35 gCO₂e/kWh, against ~256 in Ireland (`eu-west-1`) and +~330 in Germany (`eu-central-1`); see +[docs/cost-comparison.md](../../docs/cost-comparison.md) for the sourced +figures. It is also the **cheapest of the three** for the instance families +this stack uses (~2 % under Ireland, ~11 % under Frankfurt), and everything the +stack needs is available there (EC2 7th-gen Intel, EBS gp3, 3 AZs). + +This is a **default, not a constraint**. Move the stack anywhere: + +```hcl +region = "eu-west-1" # or eu-central-1 +availability_zone = "eu-west-1a" +``` + +Two things to re-check after a region change: instance-type availability +(`eu-north-1` for instance has no AMD `*6a` families) and, if you care about the +carbon column, the local grid figure. + +## Architecture + +``` +Internet + ├── argocd..sslip.io ───► mip-cp :80/:443 RKE2 Traefik (hostPort DaemonSet, + │ pinned to the CP via node label) + └── *..sslip.io ────────► mip-w1 :80/:443 haproxy-edge (hostNetwork TCP proxy) + │ + ▼ + MetalLB VIP 10.100.255.200 = mip-infra's haproxy-public + MetalLB VIP 10.100.255.201 = submariner gateway + +Nodes (VPC 10.100.0.0/16, subnet 10.100.1.0/24, one AZ): + mip-cp .10 c7i.xlarge 4c/8GB control plane, schedulable, Traefik edge + mip-w1 .11 m7i.xlarge 4c/16GB worker, haproxy-edge, Longhorn volume + mip-w2 .12 m7i.xlarge 4c/16GB worker, Longhorn volume + mip-w3 .13 m7i.xlarge 4c/16GB worker, Longhorn volume +``` + +**Where to look & log in:** all UIs, dashboards, and credentials are listed in +**[docs/access.md](../../docs/access.md)**. + +**Access control:** everything is operator-allowlisted by default — SSH + +kube-API to `operator_cidr`, HTTPS (Argo CD UI and all MIP apps) to +`app_access_cidr` (defaults to `operator_cidr`). Only port 80 is +world-reachable, exclusively for Let's Encrypt HTTP-01. Full model: +**[docs/firewall.md](../../docs/firewall.md)**. + +### Design notes (and how they differ from the UpCloud reference) + +- **Elastic IPs are the stable anchor.** Auto-assigned public IPs change on + stop/start, but an EIP stays associated across an EC2 stop — so `make down` / + `make up` preserves the public IPs, the sslip.io hostnames, the issued + certificates and the kubeconfig. This is the property the whole + no-DNS-to-manage design rests on. Cost: $3.65/month per address, whether + attached or not, running or stopped. +- **Security groups are stateful.** The reference stack's most intricate part — + return-traffic accepts scoped to the ephemeral port range, because the + UpCloud firewall is stateless — has no analogue here. A single egress rule + covers it, and there is no source-port bypass to defend against. +- **Single AZ, on purpose.** AWS charges $0.01/GB in *each* direction for + cross-AZ traffic, which would apply to every flannel VXLAN packet and every + Longhorn replica write. The reference stack is single-zone too. +- **No NAT gateway.** Nodes egress directly through the internet gateway via + their EIPs, saving ~$33/month. The security groups, not a private subnet, are + what lock the nodes down. +- **VIPs live in `10.100.255.0/24`** — inside the VPC CIDR (so the VPC's local + route covers them) but with **no subnet defined**, so AWS can never hand + those addresses to an ENI. MetalLB only *assigns* them; public traffic + reaches them by kube-proxy DNAT from the edge nodes, so nothing depends on L2 + announcement, which a VPC would not permit anyway. +- **No AWS cloud provider / CCM.** Longhorn and MetalLB serve storage and + LoadBalancer exactly as in the reference. Enabling + `--cloud-provider-name=aws` would pull in EBS-CSI and ELB semantics the + mip-infra contract does not want. +- **x86-64 only.** Graviton (`m8g`/`c8g`) is ~11 % cheaper for the same shape, + but the MIP container images are amd64-only — see the four blocked apps in + [`lima-rke2`](../lima-rke2/README.md). +- **Node names are set explicitly.** EC2 derives hostnames from the private IP + (`ip-10-100-1-10..compute.internal`), so the module sets both the OS + hostname and RKE2's `node-name`, and the cluster comes up as + `mip-cp`/`mip-w1..3` like every other stack. +- **IMDSv2 is enforced** (`http_tokens = "required"`), so the first-boot + public-IP lookup does the token `PUT` before the metadata `GET`. +- **The default login user is `ubuntu`**, not `root` — Canonical's AMIs have no + root login. The scripts here use `ubuntu@` + `sudo` accordingly. + +## Prerequisites + +- `tofu` (>= 1.9), the `aws` CLI v2, `jq` — all available via Homebrew. +- AWS credentials in the environment (**never in files**): + `export AWS_PROFILE=...` (shared config / SSO, preferred) or + `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY`. + This stack needs EC2 + VPC write, SSM parameter read, and **no `iam:*` at + all** — the nodes carry no instance profile, because nothing on them calls + AWS. Exact permission list, the rationale for each, and a generator: + **[docs/aws-credentials.md](../../docs/aws-credentials.md)**: + ```sh + ./scripts/aws-iam/create-deploy-identity.sh --stack rke2 --region eu-north-1 + ``` + The generated credential is **locked to the public IP you run that from** — + use the same address for `operator_cidr` below, since it is what the security + groups allowlist too. If your address changes, both need updating. +- An SSH keypair. +- Once, per account/region: confirm the instance types exist there — + `aws ec2 describe-instance-type-offerings --filters Name=instance-type,Values=c7i.xlarge,m7i.xlarge --region `. + +## Quickstart + +```sh +# 1. configure +cp terraform.tfvars.example terraform.tfvars # fill in operator_cidr, ssh_public_key, acme_email + +# 2. provision (~3 min for the instances + a few min for RKE2/add-ons to converge) +tofu init +tofu apply + +# 3. cluster access +./scripts/fetch-kubeconfig.sh +export KUBECONFIG=$PWD/kubeconfig + +# 4. smoke checks +kubectl get nodes # 4 Ready: mip-cp, mip-w1..3 +kubectl get sc # ceph-corbo-cephfs (default), ceph-corbo-cephfs-retain +kubectl get clusterissuer # letsencrypt, letsencrypt-staging Ready +kubectl -n metallb-system get ipaddresspool # pool-no-auto +kubectl -n kube-system get ds haproxy-edge # 1 pod, on mip-w1 + +# 5. print the values the mip-infra fork needs +./scripts/contract.sh +``` + +Add-ons are installed by RKE2's built-in helm-controller from +`/var/lib/rancher/rke2/server/manifests/` on the CP. Convergence takes a few +minutes after first boot; watch with +`kubectl -n kube-system get helmcharts,jobs`. + +## Deploying mip-infra + +Identical to the reference stack, and it follows mip-infra's **own** docs. Only +the fork prep is dev-specific: + +```sh +./scripts/setup-fork-git-server.sh # optional: bare repo on the CP if you have no git hosting +STACK_DIR=providers/aws-rke2 ../../scripts/prepare-mip-infra-fork.sh /path/to/mip-infra +``` + +Then follow the step table in the [reference stack's +README](../upcloud-rke2/README.md#2-follow-docsgetting-startedmd-with-these-parameters) +— it applies verbatim here, with hostnames from `./scripts/contract.sh`. + +## Power management + +```sh +make down # workers then CP; an EC2 stop is an ACPI shutdown -> clean RKE2/Longhorn stop +make up # CP first, then workers +make status +``` + +Compute billing stops while stopped; **EBS volumes and Elastic IPs keep +billing** (~$46/month for the default shape). Public IPs, sslip.io hostnames, +issued certificates and the kubeconfig all survive power cycles. Never power +off via `tofu` — state does not track power, so `tofu plan` stays clean. + +## Upgrades (deliberately manual) + +- **RKE2 / AMI bump:** change `rke2_version` (or `ami_id`) in + `terraform.tfvars`, then roll nodes one at a time: + `kubectl drain mip-w1 --ignore-daemonsets --delete-emptydir-data && tofu apply -replace='aws_instance.worker[0]'`. + Longhorn rebuilds replicas automatically (data volumes are separate resources + and survive). Workers first, CP last. Note `ami` and `user_data` are both in + `ignore_changes`, so a bump needs the explicit `-replace`. +- **Add-on chart bumps:** edit the version in + `../../modules/rke2-bootstrap/cloudinit/manifests/*.tftpl`. For an existing + cluster, also apply the rendered change directly + (`kubectl edit helmchart -n kube-system `) — cloud-init content is + first-boot only, so prefer on-cluster edits until the next rebuild. + +## Likely first-run fixes + +The specific unknowns for this stack, in rough order of likelihood: + +| Unknown | Where | How to check / fix | +|---|---|---| +| Primary NIC name (`ens5` vs `enX0`) for the Canal flannel bind | `locals.tf` → `private_iface` | `ssh ubuntu@ ip -br link`; then `kubectl get nodes -o json \| jq -r '.items[].metadata.annotations["flannel.alpha.coreos.com/public-ip"]'` must show `10.100.1.x` | +| NVMe index of the Longhorn volume | `locals.tf` → `data_disk_device` | `ssh ubuntu@ lsblk`; if the data volume is not `/dev/nvme1n1`, set the right name (instance-store types add devices) | +| IMDS hop limit / token lookup | `locals.tf` → `public_ipv4_lookup` | `/var/log/cloud-init-output.log` on the CP; `grep tls-san /etc/rancher/rke2/config.yaml` must show the EIP, not `__PUBLIC_IP__` | +| Instance types not offered in the chosen AZ | `terraform.tfvars` | `aws ec2 describe-instance-type-offerings --location-type availability-zone` | +| Jumbo frames (VPC MTU 9001) vs flannel VXLAN | — | flannel auto-detects; if large packets hang, pin the CNI MTU the way `hetzner-rke2` does for 1450 | + +## Troubleshooting + +| Symptom | Look at | +|---|---| +| Node never joins | `ssh ubuntu@`; `cloud-init status --long`, `/var/log/cloud-init-output.log`, `journalctl -u rke2-server -u rke2-agent` | +| Add-on missing | `kubectl -n kube-system get helmcharts`; `kubectl -n kube-system logs job/helm-install-` | +| Pods can't resolve DNS / cross-node traffic dead | check flannel picked the private NIC (see the table above); if the canal HelmChartConfig didn't land, delete `job/helm-install-rke2-canal` and `kubectl -n kube-system rollout restart ds/rke2-canal` | +| VIP not assigned | `kubectl -n metallb-system get ipaddresspool,l2advertisement,pods` | +| Cert not issued | `kubectl describe certificate/order/challenge -A`; check port 80 reachability of the right edge node; try `letsencrypt-staging` | +| No public app access | `kubectl -n kube-system get ds haproxy-edge` (runs on mip-w1); from mip-w1: `curl -H 'Host: x' http://10.100.255.200/` | +| Longhorn volumes won't attach | `ssh ubuntu@ systemctl status iscsid` — must be active; `lsblk` must show `/var/lib/longhorn` mounted | + +## State & credentials + +- State is **local** (`terraform.tfstate`, gitignored) — fine for a single + operator. It contains the RKE2 join token. If a second operator appears, move + to an S3 backend (`backend "s3"` with `use_lockfile = true`). +- Everything is version-pinned: provider (`~> 6.0`), RKE2 (`rke2_version`), + charts (`../../modules/rke2-bootstrap/cloudinit/manifests/*.tftpl`). Commit + `.terraform.lock.hcl`. diff --git a/providers/aws-rke2/locals.tf b/providers/aws-rke2/locals.tf new file mode 100644 index 0000000..4f9c580 --- /dev/null +++ b/providers/aws-rke2/locals.tf @@ -0,0 +1,42 @@ +locals { + worker_names = [for i in range(var.worker_count) : "mip-w${i + 1}"] + + # IMDSv2: unlike the other clouds' plain metadata GET, EC2 requires a PUT to + # obtain a session token first (and the instances below enforce v2). Kept to a + # single shell line because the module substitutes it into cloud-init verbatim. + public_ipv4_lookup = join(" ", [ + "TOKEN=$(curl -sf -X PUT --max-time 5 http://169.254.169.254/latest/api/token", + "-H 'X-aws-ec2-metadata-token-ttl-seconds: 300') &&", + "curl -sf --max-time 5 -H \"X-aws-ec2-metadata-token: $TOKEN\"", + "http://169.254.169.254/latest/meta-data/public-ipv4", + ]) +} + +# The cloud-init user_data (RKE2 config + add-on manifests) is rendered by the +# shared, provider-agnostic module. AWS specifics: +# - EC2 instances have a single ENI, so flannel binds the primary interface; +# on Ubuntu 24.04 Nitro guests that is ens5 (verify on first boot). +# - the public IP is NAT'd by AWS and never appears on the interface, so it +# has to come from IMDS. +# - Nitro renames EBS volumes to NVMe names, whatever the block-device +# mapping says. +# - no qemu-guest-agent exists on Nitro. +# - EC2 derives hostnames from the private IP, so node names are set here. +module "bootstrap" { + source = "../../modules/rke2-bootstrap" + rke2_token = random_password.rke2_token.result + rke2_version = var.rke2_version + cp_node_ip = var.cp_private_ip + worker_node_ips = slice(var.worker_private_ips, 0, var.worker_count) + edge_worker_index = 0 + ssh_public_key = var.ssh_public_key + vip_haproxy = var.vip_haproxy + vip_submariner = var.vip_submariner + acme_email = var.acme_email + private_iface = "ens5" + public_ipv4_lookup = local.public_ipv4_lookup + data_disk_device = "/dev/nvme1n1" + guest_agent_package = "" + cp_node_name = "mip-cp" + worker_node_names = local.worker_names +} diff --git a/providers/aws-rke2/network.tf b/providers/aws-rke2/network.tf new file mode 100644 index 0000000..d065b63 --- /dev/null +++ b/providers/aws-rke2/network.tf @@ -0,0 +1,52 @@ +# VPC with a single public subnet holding all four nodes. Nodes reach the +# internet directly through the internet gateway using their Elastic IPs, so +# there is no NAT gateway to pay for (~$33/month saved) — the security groups, +# not a private subnet, are what lock the nodes down. +# +# One subnet in one AZ on purpose: the reference stack is single-zone, and AWS +# charges $0.01/GB in EACH direction for cross-AZ traffic — which would apply +# to every flannel VXLAN packet and every Longhorn replica write. +resource "aws_vpc" "cluster" { + cidr_block = var.vpc_cidr + + # Both needed so nodes resolve the AMI's default apt/EC2 endpoints and each + # other's internal DNS names. + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = "mip-dev-cluster" } +} + +resource "aws_internet_gateway" "cluster" { + vpc_id = aws_vpc.cluster.id + tags = { Name = "mip-dev-cluster" } +} + +resource "aws_subnet" "nodes" { + vpc_id = aws_vpc.cluster.id + cidr_block = var.node_subnet_cidr + availability_zone = var.availability_zone + + # Elastic IPs are attached explicitly instead: auto-assigned public IPs + # change on stop/start, which would break the sslip.io hostnames baked into + # the mip-infra fork. + map_public_ip_on_launch = false + + tags = { Name = "mip-dev-cluster-nodes" } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.cluster.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.cluster.id + } + + tags = { Name = "mip-dev-cluster-public" } +} + +resource "aws_route_table_association" "nodes" { + subnet_id = aws_subnet.nodes.id + route_table_id = aws_route_table.public.id +} diff --git a/providers/aws-rke2/outputs.tf b/providers/aws-rke2/outputs.tf new file mode 100644 index 0000000..cf3b28d --- /dev/null +++ b/providers/aws-rke2/outputs.tf @@ -0,0 +1,84 @@ +# --------------------------------------------------------------------------- +# Interface contract for the mip-infra fork — same output names as every other +# stack, so the shared tooling is provider-agnostic. +# scripts/contract.sh prints a friendly summary of these, and +# ../../scripts/prepare-mip-infra-fork.sh consumes them (via STACK_DIR) to +# produce the fork diff. +# --------------------------------------------------------------------------- + +output "cp_public_ip" { + description = "Control-plane Elastic IP: kube-API endpoint + Argo CD entry (Traefik)." + value = aws_eip.cp.public_ip +} + +output "edge_public_ip" { + description = "Worker-0 Elastic IP: public entry for all MIP apps (haproxy-edge -> haproxy-public VIP)." + value = aws_eip.worker[0].public_ip +} + +output "worker_public_ips" { + description = "All worker Elastic IPs, in worker order (worker-0 first — it is the apps entry point)." + value = [for e in aws_eip.worker : e.public_ip] +} + +output "argocd_hostname" { + description = "Hostname to use in the argo-setup bootstrap sed (replaces argocd.example.com)." + value = "argocd.${aws_eip.cp.public_ip}.sslip.io" +} + +output "app_hostname_suffix" { + description = "Suffix for MIP app hostnames, e.g. federation-a. and datacatalogue.." + value = "${aws_eip.worker[0].public_ip}.sslip.io" +} + +output "vip_haproxy" { + description = "MetalLB VIP for mip-infra's haproxy-public Service (the fork replaces the hardcoded production IP with this)." + value = var.vip_haproxy +} + +output "vip_submariner" { + description = "MetalLB VIP for the Submariner gateway Service." + value = var.vip_submariner +} + +output "metallb_pool" { + description = "MetalLB address pool name, matches the annotation mip-infra already carries." + value = "pool-no-auto" +} + +output "storage_classes" { + description = "Longhorn-backed StorageClasses named after upstream's CephFS classes (RWX-capable) — zero storage diff in the fork." + value = ["ceph-corbo-cephfs (default, Delete)", "ceph-corbo-cephfs-retain (Retain)"] +} + +output "cluster_issuers" { + description = "cert-manager ClusterIssuers provided by the cluster (argo-setup's ingress references `letsencrypt`)." + value = ["letsencrypt", "letsencrypt-staging"] +} + +output "kube_api" { + description = "Kubernetes API endpoint (allowlisted to operator_cidr). Fetch credentials with scripts/fetch-kubeconfig.sh." + value = "https://${aws_eip.cp.public_ip}:6443" +} + +output "cp_private_ip" { + description = "RKE2 registration address for agents, and the in-VPC API endpoint the Submariner broker uses." + value = var.cp_private_ip +} + +# --- AWS-specific extras ---------------------------------------------------- + +output "region" { + description = "Region the cluster runs in." + value = var.region +} + +output "vpc_id" { + description = "VPC id (handy for `aws ec2 describe-*` filters)." + value = aws_vpc.cluster.id +} + +output "node_instance_ids" { + description = "Instance ids by node name — what scripts/cluster-power.sh acts on." + value = merge({ "mip-cp" = aws_instance.cp.id }, { for i, s in aws_instance.worker : local.worker_names[i] => s.id }) +} diff --git a/providers/aws-rke2/scripts/cluster-power.sh b/providers/aws-rke2/scripts/cluster-power.sh new file mode 100755 index 0000000..d36244f --- /dev/null +++ b/providers/aws-rke2/scripts/cluster-power.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Power the whole dev cluster on/off without touching OpenTofu state. +# EC2 bills compute only while an instance is running; EBS volumes and Elastic +# IPs keep billing (and the EIPs stay associated across a stop, so public IPs, +# sslip.io hostnames, certificates and the kubeconfig all survive a power cycle). +# +# Usage: cluster-power.sh start|stop|status +set -euo pipefail +cd "$(dirname "$0")/.." + +cmd="${1:-status}" + +REGION="$(tofu output -raw region 2>/dev/null || echo "${AWS_REGION:-}")" +[ -n "$REGION" ] && export AWS_REGION="$REGION" + +# Filter on the tags tofu applies (project=mip-dev-cluster, stack=aws-rke2) so a +# second stack in the same account is never touched. +list_nodes() { # -> lines of " " + # shellcheck disable=SC2016 # the backticks are JMESPath, not shell + aws ec2 describe-instances \ + --filters "Name=tag:project,Values=mip-dev-cluster" \ + "Name=tag:stack,Values=aws-rke2" \ + "Name=instance-state-name,Values=pending,running,stopping,stopped" \ + --query 'Reservations[].Instances[].[InstanceId,Tags[?Key==`Name`]|[0].Value,State.Name]' \ + --output text | sort -k2 +} + +NODES="$(list_nodes)" +if [ -z "$NODES" ]; then + echo "No mip-cp / mip-w* instances found (is the cluster applied?)" >&2 + exit 1 +fi + +cp_id=$(awk '$2 == "mip-cp" {print $1}' <<<"$NODES") +worker_ids=$(awk '$2 ~ /^mip-w[0-9]+$/ {print $1}' <<<"$NODES") + +case "$cmd" in + status) + printf '%s\n' "$NODES" | awk '{printf "%-10s %-10s %s\n", $2, $3, $1}' + ;; + stop) + # Workers first (drains I/O before the control plane goes down). An EC2 + # stop is an ACPI shutdown, so RKE2 and Longhorn shut down cleanly. + echo "Stopping workers..." + # shellcheck disable=SC2086 # word splitting is the intent: one id per arg + aws ec2 stop-instances --instance-ids $worker_ids >/dev/null + # shellcheck disable=SC2086 + aws ec2 wait instance-stopped --instance-ids $worker_ids + echo "Stopping control plane..." + if [ -n "$cp_id" ]; then + aws ec2 stop-instances --instance-ids "$cp_id" >/dev/null + aws ec2 wait instance-stopped --instance-ids "$cp_id" + fi + echo "Cluster stopped. EBS volumes and Elastic IPs keep billing." + ;; + start) + echo "Starting control plane..." + if [ -n "$cp_id" ]; then + aws ec2 start-instances --instance-ids "$cp_id" >/dev/null + aws ec2 wait instance-running --instance-ids "$cp_id" + fi + echo "Starting workers..." + # shellcheck disable=SC2086 + aws ec2 start-instances --instance-ids $worker_ids >/dev/null + # shellcheck disable=SC2086 + aws ec2 wait instance-running --instance-ids $worker_ids + echo "Cluster starting. Give RKE2 a few minutes, then: kubectl get nodes" + ;; + *) + echo "usage: $0 start|stop|status" >&2 + exit 2 + ;; +esac diff --git a/providers/aws-rke2/scripts/contract.sh b/providers/aws-rke2/scripts/contract.sh new file mode 100755 index 0000000..77f9566 --- /dev/null +++ b/providers/aws-rke2/scripts/contract.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Pretty-print the interface contract this cluster hands to the mip-infra fork. +set -euo pipefail +cd "$(dirname "$0")/.." + +out() { tofu output -raw "$1" 2>/dev/null || echo "?"; } + +CP_IP=$(out cp_public_ip) +W0_IP=$(out edge_public_ip) + +cat < mip-infra interface contract +============================================================================ + Region : $(out region) + Kubernetes API : https://${CP_IP}:6443 (kubeconfig: scripts/fetch-kubeconfig.sh) + Argo CD hostname : argocd.${CP_IP}.sslip.io (use in the argo-setup bootstrap sed) + MIP app hostnames : .${W0_IP}.sslip.io + federation-a : federation-a.${W0_IP}.sslip.io + datacatalogue : datacatalogue.${W0_IP}.sslip.io + + Values consumed by ../../scripts/prepare-mip-infra-fork.sh: + haproxy-public VIP : $(out vip_haproxy) (replaces the upstream production LB IP) + submariner gw VIP : $(out vip_submariner) (replaces the upstream gateway LB IP) + broker API endpoint : $(out cp_private_ip):6443 (in-VPC; replaces the upstream production endpoint) + + Provided by the cluster (no fork change needed): + MetalLB pool : pool-no-auto (autoAssign: false) + StorageClasses : ceph-corbo-cephfs (default, RWX), ceph-corbo-cephfs-retain + ClusterIssuers : letsencrypt, letsencrypt-staging + Ingress (Argo CD) : class "traefik" on the control-plane node + ECK operator : kube-system (CRDs installed) + Pod / Service CIDR : 10.42.0.0/16 / 10.43.0.0/16 (RKE2 defaults) + + Run the fork prep with: + STACK_DIR=providers/aws-rke2 ../../scripts/prepare-mip-infra-fork.sh +============================================================================ +EOF diff --git a/providers/aws-rke2/scripts/fetch-kubeconfig.sh b/providers/aws-rke2/scripts/fetch-kubeconfig.sh new file mode 100755 index 0000000..7d5d130 --- /dev/null +++ b/providers/aws-rke2/scripts/fetch-kubeconfig.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Fetch the RKE2 kubeconfig from the control plane and point it at the Elastic +# IP (valid TLS thanks to tls-san). Written to ./kubeconfig (gitignored). +# +# Canonical's Ubuntu AMIs log you in as `ubuntu` with passwordless sudo — unlike +# the UpCloud template, where the default user is root. +set -euo pipefail +cd "$(dirname "$0")/.." + +CP_IP=$(tofu output -raw cp_public_ip) +SSH_USER="${SSH_USER:-ubuntu}" +# SSH_KEY: optional path to the private key matching var.ssh_public_key +ssh ${SSH_KEY:+-i "$SSH_KEY"} -o StrictHostKeyChecking=accept-new "${SSH_USER}@${CP_IP}" \ + 'sudo cat /etc/rancher/rke2/rke2.yaml' \ + | sed "s/127\.0\.0\.1/${CP_IP}/g" > kubeconfig +chmod 600 kubeconfig + +echo "kubeconfig written to $(pwd)/kubeconfig" +echo " export KUBECONFIG=$(pwd)/kubeconfig" diff --git a/providers/aws-rke2/scripts/setup-fork-git-server.sh b/providers/aws-rke2/scripts/setup-fork-git-server.sh new file mode 100755 index 0000000..3d29039 --- /dev/null +++ b/providers/aws-rke2/scripts/setup-fork-git-server.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# Interim fork hosting: a bare git repo on the control-plane node, served over +# SSH. Argo CD pulls it over the VPC subnet (unfiltered between nodes), you push +# via the Elastic IP (port 22 is operator-allowlisted). Replace with real git +# hosting whenever available — then re-run prepare-mip-infra-fork.sh with the +# new URL on a fresh upstream clone and re-register the repo in Argo CD. +# +# Usage: setup-fork-git-server.sh [argocd-deploy-pubkey-file] +# SSH_KEY: private key for the node (default: ssh's own default) +# SSH_USER: node login user (default ubuntu — Canonical AMIs have no root login) +# The argocd deploy pubkey (default ~/.ssh/argocd-deploy-key.pub, generated +# if missing) is authorized for the git user; register its PRIVATE half in +# Argo CD with `argocd repo add ... --ssh-private-key-path`. +# --------------------------------------------------------------------------- +set -euo pipefail +cd "$(dirname "$0")/.." + +DEPLOY_PUBKEY_FILE="${1:-$HOME/.ssh/argocd-deploy-key.pub}" +if [ ! -f "$DEPLOY_PUBKEY_FILE" ]; then + ssh-keygen -t ed25519 -N "" -C "argocd deploy key" -f "${DEPLOY_PUBKEY_FILE%.pub}" >/dev/null + echo "generated deploy keypair ${DEPLOY_PUBKEY_FILE%.pub}" +fi + +CP_PUBLIC_IP=$(tofu output -raw cp_public_ip) +CP_PRIVATE_IP=$(tofu output -raw cp_private_ip) +SSH_USER="${SSH_USER:-ubuntu}" + +ssh ${SSH_KEY:+-i "$SSH_KEY"} -o StrictHostKeyChecking=accept-new "${SSH_USER}@${CP_PUBLIC_IP}" " +set -e +export DEBIAN_FRONTEND=noninteractive +command -v git >/dev/null || sudo apt-get install -qqy git >/dev/null +id git >/dev/null 2>&1 || sudo useradd -m -s /bin/bash git +sudo mkdir -p /home/git/.ssh +sudo touch /home/git/.ssh/authorized_keys +sudo grep -qxF '$(cat "$DEPLOY_PUBKEY_FILE")' /home/git/.ssh/authorized_keys || echo '$(cat "$DEPLOY_PUBKEY_FILE")' | sudo tee -a /home/git/.ssh/authorized_keys >/dev/null +$(if [ -n "${SSH_KEY:-}" ] && [ -f "${SSH_KEY}.pub" ]; then + echo "sudo grep -qxF '$(cat "${SSH_KEY}.pub")' /home/git/.ssh/authorized_keys || echo '$(cat "${SSH_KEY}.pub")' | sudo tee -a /home/git/.ssh/authorized_keys >/dev/null" +fi) +sudo chmod 700 /home/git/.ssh; sudo chmod 600 /home/git/.ssh/authorized_keys +[ -d /home/git/mip-infra.git ] || sudo -u git git init --bare -b main /home/git/mip-infra.git >/dev/null +sudo chown -R git:git /home/git +echo git-server-ready +" + +cat < every node: all intra-cluster traffic + SSH from the operator +# api -> control plane only: kube-apiserver +# web -> control plane + worker-0: HTTP (ACME) and HTTPS (ingress) +# +# Note what is NOT reachable as a result: kubelet (10250), the RKE2 supervisor +# (9345), etcd, and every NodePort (30000-32767) bind 0.0.0.0 on the nodes but +# have no ingress rule beyond the intra-cluster one, so they are private to the +# VPC subnet. + +locals { + # HTTPS entry points: locked to the operator unless app_access_cidr says otherwise. + app_cidr = coalesce(var.app_access_cidr, var.operator_cidr) +} + +# --- nodes: intra-cluster everything + operator SSH ------------------------- +resource "aws_security_group" "nodes" { + name = "mip-dev-cluster-nodes" + description = "mip-dev-cluster: intra-cluster traffic and operator SSH" + vpc_id = aws_vpc.cluster.id + tags = { Name = "mip-dev-cluster-nodes" } +} + +# etcd, kubelet, RKE2 supervisor, flannel VXLAN, MetalLB memberlist, Longhorn +# replication — the reference leaves the private SDN unfiltered, and this +# self-reference is the same thing: any node may talk to any node, nothing else can. +resource "aws_vpc_security_group_ingress_rule" "nodes_self" { + security_group_id = aws_security_group.nodes.id + description = "All traffic between cluster nodes" + ip_protocol = "-1" + referenced_security_group_id = aws_security_group.nodes.id +} + +resource "aws_vpc_security_group_ingress_rule" "nodes_ssh" { + security_group_id = aws_security_group.nodes.id + description = "SSH from operator" + ip_protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_ipv4 = var.operator_cidr +} + +# Outbound is unrestricted (container registries, Helm repos, ACME, apt, NTP). +# Because the group is stateful, this single rule is also what lets the replies +# back in. +resource "aws_vpc_security_group_egress_rule" "nodes_all" { + security_group_id = aws_security_group.nodes.id + description = "All egress" + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +# --- api: kube-apiserver, control plane only ------------------------------- +resource "aws_security_group" "api" { + name = "mip-dev-cluster-api" + description = "mip-dev-cluster: kube-apiserver from the operator" + vpc_id = aws_vpc.cluster.id + tags = { Name = "mip-dev-cluster-api" } +} + +resource "aws_vpc_security_group_ingress_rule" "api_6443" { + security_group_id = aws_security_group.api.id + description = "kube-apiserver from operator" + ip_protocol = "tcp" + from_port = 6443 + to_port = 6443 + cidr_ipv4 = var.operator_cidr +} + +# --- web: the two public entry points -------------------------------------- +resource "aws_security_group" "web" { + name = "mip-dev-cluster-web" + description = "mip-dev-cluster: public HTTP/HTTPS entry points" + vpc_id = aws_vpc.cluster.id + tags = { Name = "mip-dev-cluster-web" } +} + +resource "aws_vpc_security_group_ingress_rule" "web_http" { + security_group_id = aws_security_group.web.id + # World-open on purpose: Let's Encrypt HTTP-01 validates from arbitrary, + # undisclosed IPs, so it cannot be allowlisted. Serves only HTTPS redirects + # and ACME challenge responses — no application content. + description = "HTTP (ACME HTTP-01 + redirect only)" + ip_protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "web_https" { + security_group_id = aws_security_group.web.id + description = "HTTPS (ingress, app_access_cidr only)" + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_ipv4 = local.app_cidr +} + +# ICMP from the operator, for the same reason the reference allows it: ping and +# path-MTU discovery while debugging. +resource "aws_vpc_security_group_ingress_rule" "nodes_icmp" { + security_group_id = aws_security_group.nodes.id + description = "ICMP from operator" + ip_protocol = "icmp" + from_port = -1 + to_port = -1 + cidr_ipv4 = var.operator_cidr +} diff --git a/providers/aws-rke2/servers.tf b/providers/aws-rke2/servers.tf new file mode 100644 index 0000000..d1dc94b --- /dev/null +++ b/providers/aws-rke2/servers.tf @@ -0,0 +1,158 @@ +# Shared join token for RKE2 (server <-> agents). Lives in the state file — +# acceptable for a single-operator dev setup (state is gitignored). +resource "random_password" "rke2_token" { + length = 48 + special = false +} + +# Canonical publishes the current Ubuntu 24.04 LTS AMI id per region as a public +# SSM parameter, so there is no region-specific AMI id to maintain in tfvars. +data "aws_ssm_parameter" "ubuntu" { + count = var.ami_id == null ? 1 : 0 + name = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" +} + +resource "aws_key_pair" "operator" { + key_name = "mip-dev-cluster" + public_key = var.ssh_public_key +} + +locals { + ami = coalesce(var.ami_id, one(data.aws_ssm_parameter.ubuntu[*].value)) +} + +resource "aws_instance" "cp" { + ami = local.ami + instance_type = var.cp_instance_type + subnet_id = aws_subnet.nodes.id + private_ip = var.cp_private_ip + key_name = aws_key_pair.operator.key_name + vpc_security_group_ids = [aws_security_group.nodes.id, aws_security_group.api.id, aws_security_group.web.id] + + root_block_device { + volume_size = var.os_disk_size + volume_type = "gp3" + encrypted = true + tags = { Name = "mip-cp-root" } + } + + # IMDSv2 only. The first-boot public-IP lookup in locals.tf does the token + # dance accordingly. + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + } + + user_data = module.bootstrap.server_user_data + + # user_data is first-boot bootstrap only — refining cloud-init/add-on + # templates must not replace running nodes. New content applies to NEW nodes + # (or explicit -replace); for live clusters, apply add-on changes on-cluster + # (see README "Upgrades"). user_data_replace_on_change is the AWS-specific + # half of this: left false so a diff would at most reboot, never recreate. + user_data_replace_on_change = false + + lifecycle { + ignore_changes = [user_data, ami] + } + + tags = { + Name = "mip-cp" + role = "control-plane" + } +} + +# Dedicated Longhorn data volumes — separate resources, so they survive node +# rebuilds (tofu apply -replace of a worker keeps its data volume). +resource "aws_ebs_volume" "worker_data" { + count = var.worker_count + availability_zone = var.availability_zone + size = var.data_disk_size + type = var.data_disk_type + encrypted = true + + tags = { Name = "${local.worker_names[count.index]}-longhorn" } +} + +resource "aws_volume_attachment" "worker_data" { + count = var.worker_count + # Nitro ignores this name and presents the volume as /dev/nvme1n1 — which is + # what the cloud-init in modules/rke2-bootstrap formats (data_disk_device). + device_name = "/dev/sdf" + volume_id = aws_ebs_volume.worker_data[count.index].id + instance_id = aws_instance.worker[count.index].id +} + +resource "aws_instance" "worker" { + count = var.worker_count + ami = local.ami + instance_type = var.worker_instance_type + subnet_id = aws_subnet.nodes.id + private_ip = var.worker_private_ips[count.index] + key_name = aws_key_pair.operator.key_name + vpc_security_group_ids = concat( + [aws_security_group.nodes.id], + # worker-0 runs haproxy-edge, the public entry for all MIP apps. + count.index == 0 ? [aws_security_group.web.id] : [], + ) + + root_block_device { + volume_size = var.os_disk_size + volume_type = "gp3" + encrypted = true + tags = { Name = "${local.worker_names[count.index]}-root" } + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + } + + user_data = module.bootstrap.agent_user_data[count.index] + user_data_replace_on_change = false + + # See the control-plane note: user_data is first-boot bootstrap only. + lifecycle { + ignore_changes = [user_data, ami] + } + + # Not strictly required (rke2-agent retries joining), but avoids minutes of + # join backoff by letting the server come up first. + depends_on = [aws_instance.cp] + + tags = { + Name = local.worker_names[count.index] + role = "worker" + } +} + +# --- Elastic IPs ------------------------------------------------------------ +# The stable anchors. Auto-assigned public IPs change on stop/start, but an EIP +# stays associated across an EC2 stop — so `make down` / `make up` preserves the +# public IPs, the sslip.io hostnames, the issued certificates and the +# kubeconfig, exactly as the reference stack relies on. +resource "aws_eip" "cp" { + domain = "vpc" + tags = { Name = "mip-cp" } +} + +resource "aws_eip_association" "cp" { + instance_id = aws_instance.cp.id + allocation_id = aws_eip.cp.id +} + +# Every worker gets one, and not only for SSH: there is no NAT gateway in this +# VPC, so a public IP is a node's ONLY egress path — apt, get.rke2.io, image +# pulls. A worker without an EIP could never even bootstrap. Worker-0's EIP +# doubles as the public entry for all MIP apps. +resource "aws_eip" "worker" { + count = var.worker_count + domain = "vpc" + tags = { Name = local.worker_names[count.index] } +} + +resource "aws_eip_association" "worker" { + count = var.worker_count + instance_id = aws_instance.worker[count.index].id + allocation_id = aws_eip.worker[count.index].id +} diff --git a/providers/aws-rke2/terraform.tfvars.example b/providers/aws-rke2/terraform.tfvars.example new file mode 100644 index 0000000..0671282 --- /dev/null +++ b/providers/aws-rke2/terraform.tfvars.example @@ -0,0 +1,42 @@ +# Copy to terraform.tfvars and adjust. terraform.tfvars is gitignored. + +# --- required ----------------------------------------------------------------- + +# Who may reach SSH (all nodes) and the Kubernetes API (control plane). +# Your workstation/VPN egress IPv4: curl -4 -s ifconfig.me +operator_cidr = "192.0.2.10/32" + +# Installed via cloud-init for the AMI's default user (`ubuntu`) on every node +# (content, not a path). +ssh_public_key = "ssh-ed25519 AAAA... you@workstation" + +# Let's Encrypt account email for the cluster ClusterIssuers. +acme_email = "email@example.com" + +# --- optional (sensible defaults) ---------------------------------------------- + +# Default region is the lowest-carbon European one (~35 gCO2e/kWh) and also the +# cheapest of the three candidates. Only a default — switch freely, but set the +# AZ to match and re-check instance-type availability. +# region = "eu-north-1" +# availability_zone = "eu-north-1a" +# +# x86-64 only: MIP container images are amd64-only, so Graviton (m8g/c8g) is out. +# cp_instance_type = "c7i.xlarge" # 4 vCPU / 8 GiB (m7i.large = 2/8, cheaper) +# worker_instance_type = "m7i.xlarge" # 4 vCPU / 16 GiB (t3.xlarge cheaper, burstable) +# +# worker_count = 3 # >= 3 required (Argo CD redis-ha anti-affinity) +# os_disk_size = 35 +# data_disk_size = 80 # per-worker Longhorn volume +# data_disk_type = "gp3" +# +# ami_id = null # null = current Ubuntu 24.04 LTS via Canonical's SSM parameter +# rke2_version = "v1.36.2+rke2r1" +# +# vpc_cidr = "10.100.0.0/16" +# node_subnet_cidr = "10.100.1.0/24" +# +# VIPs sit in a /24 of the VPC that has no subnet, so AWS can never hand these +# addresses to an ENI. +# vip_haproxy = "10.100.255.200" # -> fork's haproxy-public loadBalancerIP +# vip_submariner = "10.100.255.201" # -> fork's submariner gateway loadBalancerIP diff --git a/providers/aws-rke2/variables.tf b/providers/aws-rke2/variables.tf new file mode 100644 index 0000000..cc2ba3e --- /dev/null +++ b/providers/aws-rke2/variables.tf @@ -0,0 +1,182 @@ +variable "region" { + description = <<-EOT + AWS region. The default is a preference, not a constraint: eu-north-1 + (Stockholm) has the lowest-carbon grid of the European regions + (~35 vs ~256 in Ireland and ~330 gCO2e/kWh in Germany — see + docs/cost-comparison.md for the source) and is also the cheapest of the + three for the instance families this stack uses. Everything the stack needs + is available there. Switch freely — set availability_zone to match, and + re-check instance-type availability (eu-north-1 has no *6a/AMD families). + List regions with: aws ec2 describe-regions --query 'Regions[].RegionName' + EOT + type = string + default = "eu-north-1" +} + +variable "availability_zone" { + description = <<-EOT + Single AZ for every node. Deliberately one zone: it matches the reference + stack's shape and avoids AWS's $0.01/GB-each-way cross-AZ charge, which + would otherwise apply to every flannel VXLAN packet and every Longhorn + replica write. Must be inside var.region. + List with: aws ec2 describe-availability-zones --region + EOT + type = string + default = "eu-north-1a" +} + +variable "ami_id" { + description = <<-EOT + AMI for the node OS. null (default) resolves Canonical's current Ubuntu + 24.04 LTS x86-64 image from its public SSM parameter, so the region is + handled automatically. Pin an AMI id here for a reproducible rebuild. + EOT + type = string + default = null +} + +variable "rke2_version" { + description = "Pinned RKE2 version, installed at first boot via get.rke2.io." + type = string + default = "v1.36.2+rke2r1" +} + +variable "cp_instance_type" { + description = <<-EOT + Instance type for the control-plane node (4 vCPU / 8 GiB, matching the + tested reference shape). EC2 bills by the second while running only, which + is what makes the power-off workflow save money. Cheaper option: + m7i.large (2 vCPU / 8 GiB) — the CP carries only Traefik and light add-ons. + Must be x86-64: the MIP container images are amd64-only, so the cheaper + Graviton families (m8g/c8g) cannot be used. + EOT + type = string + default = "c7i.xlarge" +} + +variable "worker_instance_type" { + description = <<-EOT + Instance type for worker nodes (4 vCPU / 16 GiB, the tested mip-infra + shape). Cheaper alternatives, both x86-64: m7i-flex.xlarge (same shape, + ~5% less, 40% sustained-CPU baseline) or t3.xlarge (~19% less, burstable — + Elasticsearch and exareme2 may throttle). + EOT + type = string + default = "m7i.xlarge" +} + +variable "worker_count" { + description = "Number of worker nodes." + type = number + default = 3 + + validation { + condition = var.worker_count >= 3 + error_message = "mip-infra requires at least 3 schedulable workers (Argo CD redis-ha uses hard pod anti-affinity across 3 replicas)." + } +} + +variable "vpc_cidr" { + description = "VPC CIDR. Must contain node_subnet_cidr and both MetalLB VIPs." + type = string + default = "10.100.0.0/16" +} + +variable "node_subnet_cidr" { + description = "Subnet the nodes' ENIs live in (node-to-node RKE2 traffic: supervisor, etcd, CNI, MetalLB memberlist)." + type = string + default = "10.100.1.0/24" +} + +variable "cp_private_ip" { + description = "Fixed private IP of the control-plane node (RKE2 registration address for agents). Must be inside node_subnet_cidr and outside its first four reserved addresses." + type = string + default = "10.100.1.10" +} + +variable "worker_private_ips" { + description = "Fixed private IPs of the worker nodes, in order." + type = list(string) + default = ["10.100.1.11", "10.100.1.12", "10.100.1.13"] + + validation { + condition = length(var.worker_private_ips) >= var.worker_count + error_message = "worker_private_ips must contain at least worker_count entries." + } +} + +variable "vip_haproxy" { + description = <<-EOT + MetalLB VIP handed to mip-infra's haproxy-public ingress Service. + scripts/prepare-mip-infra-fork.sh substitutes this for the hardcoded + production LoadBalancer IP. Deliberately inside vpc_cidr but in a /24 that + has NO subnet, so AWS can never assign the address to an ENI while the VPC + local route still covers it. + EOT + type = string + default = "10.100.255.200" +} + +variable "vip_submariner" { + description = "MetalLB VIP for the Submariner gateway Service (same reserved, subnet-less range as vip_haproxy)." + type = string + default = "10.100.255.201" +} + +variable "operator_cidr" { + description = "CIDR allowed to reach SSH (22, all nodes) and the Kubernetes API (6443, control plane), e.g. \"192.0.2.10/32\"." + type = string + + validation { + # Kept IPv4-only to match the reference stack: the security-group rules and + # the sslip.io hostnames are IPv4, and the nodes request no public IPv6. + condition = can(cidrnetmask(var.operator_cidr)) + error_message = "operator_cidr must be a valid IPv4 CIDR, e.g. 192.0.2.10/32 (IPv6 unsupported: node security-group rules are IPv4-only). Get yours with: curl -4 -s ifconfig.me" + } +} + +variable "app_access_cidr" { + description = <<-EOT + IPv4 CIDR allowed to reach the web entry points over HTTPS — port 443 on + the control plane (Argo CD UI/CLI) and on worker-0 (all MIP apps). + Defaults to operator_cidr (locked down). Set to "0.0.0.0/0" to make the + apps publicly reachable. Port 80 always stays world-reachable: Let's + Encrypt HTTP-01 validation comes from arbitrary IPs, and 80 only serves + HTTPS redirects and ACME challenge responses. See docs/firewall.md. + EOT + type = string + default = null + + validation { + condition = var.app_access_cidr == null || can(cidrnetmask(var.app_access_cidr)) + error_message = "app_access_cidr must be a valid IPv4 CIDR, e.g. 192.0.2.0/24 (or null to inherit operator_cidr)." + } +} + +variable "ssh_public_key" { + description = "SSH public key (content, not path) installed via cloud-init for the AMI's default user (`ubuntu` on Canonical images)." + type = string +} + +variable "acme_email" { + description = "Email for the Let's Encrypt account of the cluster-provided ClusterIssuers (letsencrypt / letsencrypt-staging)." + type = string +} + +variable "os_disk_size" { + description = "Root EBS volume size (GB) per node (holds RKE2 + container images under /var)." + type = number + default = 35 +} + +variable "data_disk_size" { + description = "Longhorn data volume size (GB) per worker." + type = number + default = 80 +} + +variable "data_disk_type" { + description = "EBS volume type for the Longhorn data disks. gp3 includes 3000 IOPS and 125 MB/s at no extra charge; gp2 costs ~25% more per GB." + type = string + default = "gp3" +} diff --git a/providers/aws-rke2/versions.tf b/providers/aws-rke2/versions.tf new file mode 100644 index 0000000..cfbf388 --- /dev/null +++ b/providers/aws-rke2/versions.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.9.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +# Credentials are taken from the environment — never put them in files: +# export AWS_PROFILE=... (shared config / SSO, preferred) +# or +# export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... +# An IAM principal with EC2 + VPC + SSM-parameter-read permissions is enough; +# no IAM instance profile is attached to the nodes (nothing on them calls AWS). +provider "aws" { + region = var.region + + # Applied to every taggable resource, so `cluster-power.sh` and the console + # can find the cluster by tag without per-resource repetition. + default_tags { + tags = { + project = "mip-dev-cluster" + stack = "aws-rke2" + } + } +} diff --git a/providers/upcloud-uks/addons.tf b/providers/upcloud-uks/addons.tf index d2c32c7..dc21c5a 100644 --- a/providers/upcloud-uks/addons.tf +++ b/providers/upcloud-uks/addons.tf @@ -144,3 +144,26 @@ resource "kubectl_manifest" "metallb_pool" { YAML depends_on = [helm_release.metallb] } + +# Paired with the pool the way every other stack pairs them: MetalLB's docs are +# ambiguous about whether a pool with no advertisement referencing it is still +# allocated from, and a Service that never gets an address is a Service Argo CD +# reports Unhealthy. Whether the L2 announcement itself reaches anything on the +# UKS node network is unverified — MetalLB is off by default here because UKS +# has its own managed LB path (see the README), so this path is unexercised — +# but allocation is all the `pool-no-auto` contract needs, and the +# advertisement is what guarantees it. +resource "kubectl_manifest" "metallb_advertisement" { + count = var.install_metallb ? 1 : 0 + yaml_body = <<-YAML + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: pool-no-auto + namespace: metallb-system + spec: + ipAddressPools: + - pool-no-auto + YAML + depends_on = [kubectl_manifest.metallb_pool] +} diff --git a/renovate.json b/renovate.json index 070d812..115a016 100644 --- a/renovate.json +++ b/renovate.json @@ -30,8 +30,8 @@ }, { "customType": "regex", - "description": "Container image tags flagged with a renovate hint comment", - "managerFilePatterns": ["/\\.tftpl$/", "/\\.ya?ml$/"], + "description": "Container image tags flagged with a renovate hint comment. Includes .tf: the managed stacks embed manifests in kubectl_manifest heredocs.", + "managerFilePatterns": ["/\\.tftpl$/", "/\\.ya?ml$/", "/\\.tf$/"], "matchStrings": [ "#\\s*renovate:\\s*datasource=(?\\S+)\\s+depName=(?\\S+)\\s*\\n\\s*image:\\s*\\S+:(?[\\w][\\w.-]*)" ] diff --git a/scripts/aws-iam/create-deploy-identity.sh b/scripts/aws-iam/create-deploy-identity.sh new file mode 100755 index 0000000..c2f4d8a --- /dev/null +++ b/scripts/aws-iam/create-deploy-identity.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# Create the least-privilege AWS identity that deploys the mip-dev-cluster AWS +# stacks, from the policy documents next to this script. +# +# Run it once, with an admin session, against the account you want the cluster +# in. It is idempotent: re-running updates the policies in place (new default +# version) and leaves the identity alone. +# +# ./create-deploy-identity.sh --stack rke2 # EC2 stack only +# ./create-deploy-identity.sh --stack eks --region eu-west-1 +# ./create-deploy-identity.sh --stack both --mode role \ +# --trust-arn arn:aws:iam::111122223333:user/you # assume-role, no static key +# ./create-deploy-identity.sh --source-ip 203.0.113.7/32 # lock to a specific IP +# ./create-deploy-identity.sh --source-ip 198.51.100.0/24 --source-ip 203.0.113.7/32 +# ./create-deploy-identity.sh --no-ip-lock # usable from anywhere +# ./create-deploy-identity.sh --print-only # render, create nothing +# +# SOURCE-IP LOCK (on by default): the credential is pinned to the public IP this +# script is run from, so a leaked key is unusable from anywhere else. This is the +# same IP the stacks want in operator_cidr, so one lookup feeds both. If your +# address changes, every AWS call starts returning AccessDenied except +# `sts get-caller-identity` (IAM policies cannot deny it) — re-run with +# --source-ip . Use --no-ip-lock if the deploy host has no stable +# egress address (a roaming laptop, or CI on shared runners). +# +# WHY A SEPARATE IDENTITY: the stacks need a specific, boring set of rights +# (see docs/aws-credentials.md). Deploying with an admin key works but means a +# leaked credential is an account takeover rather than a dev cluster rebuild. +# +# CREDENTIAL SHAPE, in order of preference: +# 1. --mode role : an IAM role you assume. Temporary credentials, nothing +# long-lived to leak. Best if you already have a user or +# SSO identity in the account. +# 2. IAM Identity Center permission set: attach the same policies to a +# permission set instead — this script does not automate +# that (it is an org-level operation), see the doc. +# 3. --mode user --with-access-key : a long-lived access key. Simplest, and +# what "an API key" usually means, but it never expires +# on its own. Rotate it; delete it when done testing. +# --------------------------------------------------------------------------- +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" + +STACK=both +REGION=eu-north-1 +ROLE_PREFIX=mip-eks +IDENTITY=mip-dev-cluster-deploy +MODE=user +TRUST_ARN="" +WITH_KEY=false +PRINT_ONLY=false +IP_LOCK=true +SOURCE_IPS=() + +die() { echo "error: $*" >&2; exit 1; } + +while [ $# -gt 0 ]; do + case "$1" in + --stack) STACK="${2:?}"; shift 2 ;; + --region) REGION="${2:?}"; shift 2 ;; + --role-prefix) ROLE_PREFIX="${2:?}"; shift 2 ;; + --identity-name) IDENTITY="${2:?}"; shift 2 ;; + --mode) MODE="${2:?}"; shift 2 ;; + --trust-arn) TRUST_ARN="${2:?}"; shift 2 ;; + --source-ip) SOURCE_IPS+=("${2:?}"); shift 2 ;; + --no-ip-lock) IP_LOCK=false; shift ;; + --with-access-key) WITH_KEY=true; shift ;; + --print-only) PRINT_ONLY=true; shift ;; + -h|--help) awk 'NR==1{next} /^#/{print; next} {exit}' "$0"; exit 0 ;; + *) die "unknown option: $1 (try --help)" ;; + esac +done + +case "$STACK" in rke2|eks|both) ;; *) die "--stack must be rke2, eks or both" ;; esac +case "$MODE" in user|role) ;; *) die "--mode must be user or role" ;; esac +[ "$MODE" = role ] && [ -z "$TRUST_ARN" ] && die "--mode role needs --trust-arn " +[ "$MODE" = role ] && [ "$WITH_KEY" = true ] && die "--with-access-key is meaningless for a role (it issues temporary credentials)" + +command -v aws >/dev/null || die "the aws CLI is required" +command -v jq >/dev/null || die "jq is required" + +[ ${#SOURCE_IPS[@]} -gt 0 ] && [ "$IP_LOCK" = false ] \ + && die "--source-ip and --no-ip-lock contradict each other" + +# --- source-IP lock --------------------------------------------------------- +# Default: discover this host's public IPv4 and pin the credential to it. +if [ "$IP_LOCK" = true ] && [ ${#SOURCE_IPS[@]} -eq 0 ]; then + MYIP=$(curl -4 -sf --max-time 10 https://checkip.amazonaws.com 2>/dev/null | tr -d '[:space:]') \ + || MYIP=$(curl -4 -sf --max-time 10 https://ifconfig.me 2>/dev/null | tr -d '[:space:]') || true + [ -n "${MYIP:-}" ] || die "could not determine this host's public IPv4. Pass it explicitly with --source-ip , or --no-ip-lock to skip the restriction." + case "$MYIP" in + *.*.*.*) SOURCE_IPS=("${MYIP}/32") ;; + *) die "unexpected value from the IP lookup: '$MYIP'. Pass --source-ip ." ;; + esac +fi + +# Normalise: a bare address means /32. +for i in "${!SOURCE_IPS[@]}"; do + case "${SOURCE_IPS[i]}" in + */*) ;; + *) SOURCE_IPS[i]="${SOURCE_IPS[i]}/32" ;; + esac +done + +if [ "$IP_LOCK" = true ]; then + SOURCE_IP_JSON=$(printf '%s\n' "${SOURCE_IPS[@]}" | jq -R . | jq -sc .) +else + # No lock: 0.0.0.0/0 makes NotIpAddress never match, so the Deny never fires. + SOURCE_IP_JSON='["0.0.0.0/0"]' +fi + +# Which policy documents this stack selection needs. +DOCS="mip-ec2-base" +case "$STACK" in + rke2) DOCS="$DOCS mip-rke2" ;; + eks) DOCS="$DOCS mip-eks" ;; + both) DOCS="$DOCS mip-rke2 mip-eks" ;; +esac + +render() { # render -> stdout (placeholders substituted, Comment stripped) + # The source-IP substitution is scoped to the "aws:SourceIp" line: it splices + # in a JSON array (quotes included), which must never land inside the Comment + # string — where the placeholder is also mentioned by name. + sed -e "s/__REGION__/${REGION}/g" \ + -e "s/__ROLE_PREFIX__/${ROLE_PREFIX}/g" \ + -e "/\"aws:SourceIp\"/s|__SOURCE_IP_JSON__|${SOURCE_IP_JSON}|" \ + "$HERE/$1.json" | jq 'del(.Comment)' +} + +echo "==> mip-dev-cluster AWS deploy identity" +echo " stack : $STACK" +echo " region : $REGION" +echo " role prefix : ${ROLE_PREFIX}-* (must equal var.cluster_name in providers/aws-eks)" +echo " identity : $MODE/$IDENTITY" +echo " policies : $DOCS" +if [ "$IP_LOCK" = true ]; then + echo " source IP : ${SOURCE_IPS[*]} <- the credential works ONLY from here" + echo " use the same value for operator_cidr in terraform.tfvars" +else + echo " source IP : UNRESTRICTED (--no-ip-lock) — usable from any address" +fi +echo + +if [ "$PRINT_ONLY" = true ]; then + for d in $DOCS; do + echo "----- ${IDENTITY}-${d} -----" + render "$d" + done + echo + echo "Nothing was created (--print-only). Sizes (IAM caps a managed policy at 6144 chars):" + for d in $DOCS; do printf ' %-32s %s chars\n' "$d" "$(render "$d" | tr -d '[:space:]' | wc -c | tr -d ' ')"; done + exit 0 +fi + +ACCOUNT=$(aws sts get-caller-identity --query Account --output text) \ + || die "no usable AWS credentials — this script needs an admin session for the account you are setting up" +echo " account : $ACCOUNT" +echo " running as : $(aws sts get-caller-identity --query Arn --output text)" +echo + +# --- policies --------------------------------------------------------------- +POLICY_ARNS=() +for d in $DOCS; do + NAME="${IDENTITY}-${d}" + ARN="arn:aws:iam::${ACCOUNT}:policy/${NAME}" + BODY=$(render "$d") + if aws iam get-policy --policy-arn "$ARN" >/dev/null 2>&1; then + echo "==> updating policy $NAME" + # Keep at most 5 versions; prune the oldest non-default before adding one. + COUNT=$(aws iam list-policy-versions --policy-arn "$ARN" --query 'length(Versions)' --output text) + if [ "$COUNT" -ge 5 ]; then + # shellcheck disable=SC2016 # the backticks are JMESPath, not shell + OLD=$(aws iam list-policy-versions --policy-arn "$ARN" \ + --query 'sort_by(Versions[?IsDefaultVersion==`false`], &CreateDate)[0].VersionId' --output text) + [ "$OLD" != "None" ] && aws iam delete-policy-version --policy-arn "$ARN" --version-id "$OLD" + fi + aws iam create-policy-version --policy-arn "$ARN" \ + --policy-document "$BODY" --set-as-default >/dev/null + else + echo "==> creating policy $NAME" + aws iam create-policy --policy-name "$NAME" \ + --description "mip-dev-cluster deploy rights ($d, region $REGION)" \ + --policy-document "$BODY" >/dev/null + fi + POLICY_ARNS+=("$ARN") +done + +# --- identity --------------------------------------------------------------- +if [ "$MODE" = user ]; then + if aws iam get-user --user-name "$IDENTITY" >/dev/null 2>&1; then + echo "==> user $IDENTITY already exists" + else + echo "==> creating user $IDENTITY" + aws iam create-user --user-name "$IDENTITY" \ + --tags Key=project,Value=mip-dev-cluster >/dev/null + fi + for arn in "${POLICY_ARNS[@]}"; do + aws iam attach-user-policy --user-name "$IDENTITY" --policy-arn "$arn" + done + echo " attached ${#POLICY_ARNS[@]} policies" +else + TRUST=$(jq -n --arg arn "$TRUST_ARN" '{ + Version: "2012-10-17", + Statement: [{Effect:"Allow", Principal:{AWS:$arn}, Action:"sts:AssumeRole"}] + }') + if aws iam get-role --role-name "$IDENTITY" >/dev/null 2>&1; then + echo "==> updating trust policy on role $IDENTITY" + aws iam update-assume-role-policy --role-name "$IDENTITY" --policy-document "$TRUST" >/dev/null + else + echo "==> creating role $IDENTITY" + aws iam create-role --role-name "$IDENTITY" --assume-role-policy-document "$TRUST" \ + --max-session-duration 3600 --tags Key=project,Value=mip-dev-cluster >/dev/null + fi + for arn in "${POLICY_ARNS[@]}"; do + aws iam attach-role-policy --role-name "$IDENTITY" --policy-arn "$arn" + done + echo " attached ${#POLICY_ARNS[@]} policies" +fi + +# --- how to use it ---------------------------------------------------------- +echo +echo "============================================================================" +if [ "$MODE" = role ]; then + cat < + region = ${REGION} + + Verify: AWS_PROFILE=mip-dev-cluster aws sts get-caller-identity +EOF +elif [ "$WITH_KEY" = true ]; then + echo " Creating an access key. It is shown ONCE, is not written to disk by this" + echo " script, and does not expire on its own — rotate or delete it when done." + echo + KEY=$(aws iam create-access-key --user-name "$IDENTITY") + cat <