From 16da006b6cdecda357e4277d7d3afc180f0ab203 Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 09:38:36 +0800 Subject: [PATCH 1/5] feat: add multi-node kubeadm quickstart --- Makefile | 9 +- README.md | 38 ++- charts/site/templates/01-storage.yaml | 7 + charts/site/values.schema.json | 20 +- charts/site/values.yaml | 3 + dev/kubeadm/lima.yaml | 13 +- docs/DEPLOYMENT.md | 22 +- docs/STANDALONE.md | 21 +- scripts/quickstart-kubeadm.sh | 409 +++++++++++++++++++++++--- src/sites/exposure.py | 8 +- tests/test_exposure.py | 4 +- tests/test_helm_package.py | 36 ++- website/index.html | 4 +- 13 files changed, 513 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index 1d31c00..f3f113f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL := /usr/bin/env bash CONTROL_IMAGE ?= site-control:local -.PHONY: help install test test-db test-db-down test-supabase test-oss console homepage-check image chart-lint chart-render benchmark quickstart-doctor quickstart quickstart-status quickstart-access quickstart-token quickstart-clean standalone-install standalone-smoke standalone-uninstall +.PHONY: help install test test-db test-db-down test-supabase test-oss console homepage-check image chart-lint chart-render benchmark quickstart-doctor quickstart quickstart-scale quickstart-status quickstart-access quickstart-token quickstart-clean standalone-install standalone-smoke standalone-uninstall # Generated from the targets themselves rather than kept as a second list. The # hand-written listing this replaces had already lost `help`, and nothing could @@ -82,9 +82,12 @@ benchmark: ## — run the fail-closed deterministic benchmark profile quickstart-doctor: ## — check every host prerequisite and local port before the trial scripts/quickstart-kubeadm.sh doctor -quickstart: ## — build and prove Site on a disposable kubeadm VM +quickstart: ## — build and prove Site on a disposable multi-node kubeadm cluster scripts/quickstart-kubeadm.sh up +quickstart-scale: ## — reconcile the trial to SITES_QUICKSTART_WORKERS=1-4 + scripts/quickstart-kubeadm.sh scale + quickstart-status: ## — inspect the disposable kubeadm trial scripts/quickstart-kubeadm.sh status @@ -94,7 +97,7 @@ quickstart-access: ## — open and serve the console from the disposable kubeadm quickstart-token: ## — print the disposable trial's local admin token scripts/quickstart-kubeadm.sh token -quickstart-clean: ## — delete only the disposable Site kubeadm VM +quickstart-clean: ## — delete only the disposable Site kubeadm VMs and network scripts/quickstart-kubeadm.sh clean standalone-install: ## — bootstrap local Secrets and install the Helm Chart diff --git a/README.md b/README.md index e17db5e..75cd22f 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ environments, not a public hosting platform — [Known limitations](#known-limit ### Host prerequisites The supported source-checkout trial runs on macOS or Linux with hardware virtualization, -outbound HTTPS, and enough capacity for a Lima VM configured with **6 CPUs, 8 GiB RAM, -and a 40 GiB sparse disk**. Install Git, Lima, Docker, `kubectl`, Helm, `curl`, `uv`, +outbound HTTPS, and enough capacity for three Lima VMs configured with **8 CPUs, 10 GiB +RAM, and 70 GiB of sparse disk in total**. Install Git, Lima, Docker, `kubectl`, Helm, `curl`, `uv`, `lsof`, and Python 3.12+. Docker must be running, not merely installed. On macOS with Homebrew, the command-line dependencies are: @@ -42,11 +42,11 @@ Linux users should install [Docker Engine](https://docs.docker.com/engine/instal [Helm](https://helm.sh/docs/intro/install/) using their distribution's supported path. Run `make quickstart-doctor` at any time for an explicit dependency, Docker-daemon, Python, and local-port check. The first uncached run downloads a VM and several container images; -depending on the network it can take 5–25 minutes and needs ports 18090–18098 and 18447. +depending on the network it can take 8–30 minutes and needs ports 18090–18098 and 18447. The fastest path is deliberately end to end. With the prerequisites above, one command -creates a disposable single-node Kubernetes -cluster with `kubeadm`, builds the image from this checkout, installs Site and Prometheus, +creates a disposable Kubernetes cluster with one control-plane and two worker nodes using +`kubeadm`, builds the image from this checkout, installs Site and Prometheus, deploys the included HTML example, and refuses to pass until both server-side HTTP verification and the administrative observability APIs return real evidence: @@ -61,6 +61,9 @@ A successful run ends with facts rather than only a Helm exit code: ```text Site kubeadm quickstart passed elapsed: + topology: 1 control-plane + 2 workers (3 Ready) + example node: site-quickstart-w<1-or-2> + control-plane scheduling: protected by NoSchedule taint phase: Running HTTP verification: 200 body SHA-256: @@ -93,12 +96,27 @@ application from an existing container image; choose **Public** to receive a bro or **Internal** when no public URL should exist. The token remains in a Kubernetes Secret and is never written into this checkout or browser storage. Inspect the running objects with `make quickstart-status`. Pressing Ctrl-C in terminal 1 only -closes console access; the application and cluster keep running. `make quickstart-clean` -permanently deletes the disposable `site-quickstart` VM, all applications inside it, and -the repository-local kubeconfig; it does not touch another cluster or repository. +closes console access; the application and cluster keep running. -This trial harness is entirely owned by this repository: it uses Lima's default network, -bootstraps Kubernetes directly with kubeadm, and builds the control image locally. No +Worker count can be changed in place from one through four. Expansion creates and joins +new kubeadm worker VMs; contraction drains the highest-numbered workers before deleting +them. Quickstart binds local persistent volumes to `site-quickstart-w1`, which is always +retained, and refuses to delete a worker if a local persistent volume is unexpectedly +pinned there: + +```bash +SITES_QUICKSTART_WORKERS=3 make quickstart-scale # add w3 +SITES_QUICKSTART_WORKERS=1 make quickstart-scale # drain/delete w3 and w2 +``` + +This changes worker capacity only. The single control-plane makes the trial a real +multi-node functional environment, not a highly available production topology. +`make quickstart-clean` permanently deletes the three default `site-quickstart` VMs, all +applications inside them, the repository-owned Lima network, and the repository-local +kubeconfig; it does not touch another cluster, network, or repository. + +This trial harness is entirely owned by this repository: it creates its own Lima user-v2 +network, bootstraps Kubernetes directly with kubeadm, and builds the control image locally. No other checkout, shared cluster configuration, or pre-published Site image is required. ```text diff --git a/charts/site/templates/01-storage.yaml b/charts/site/templates/01-storage.yaml index e456c71..768c063 100644 --- a/charts/site/templates/01-storage.yaml +++ b/charts/site/templates/01-storage.yaml @@ -134,6 +134,13 @@ metadata: provisioner: rancher.io/local-path volumeBindingMode: WaitForFirstConsumer reclaimPolicy: Delete +{{- with .Values.localPathProvisioner.allowedNodeNames }} +allowedTopologies: + - matchLabelExpressions: + - key: kubernetes.io/hostname + values: +{{ toYaml . | indent 10 }} +{{- end }} --- kind: ConfigMap apiVersion: v1 diff --git a/charts/site/values.schema.json b/charts/site/values.schema.json index 4b52e7b..b6a06f8 100644 --- a/charts/site/values.schema.json +++ b/charts/site/values.schema.json @@ -90,7 +90,25 @@ } }, "localPathProvisioner": { - "$ref": "#/$defs/feature" + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "allowedNodeNames" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "allowedNodeNames": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + } + } }, "postgresql": { "type": "object", diff --git a/charts/site/values.yaml b/charts/site/values.yaml index dbf7a5a..8ecd435 100644 --- a/charts/site/values.yaml +++ b/charts/site/values.yaml @@ -58,6 +58,9 @@ storage: localPathProvisioner: enabled: false + # Optional safety boundary for disposable multi-node clusters. All local + # volumes bind to one retained worker so other workers can be drained safely. + allowedNodeNames: [] postgresql: embedded: diff --git a/dev/kubeadm/lima.yaml b/dev/kubeadm/lima.yaml index cf0cbf7..fc15b57 100644 --- a/dev/kubeadm/lima.yaml +++ b/dev/kubeadm/lima.yaml @@ -4,13 +4,14 @@ images: - location: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img arch: aarch64 -cpus: 6 -memory: 8GiB -disk: 40GiB +# Control-plane defaults. The quickstart overrides these for worker VMs. +cpus: 4 +memory: 4GiB +disk: 30GiB -# The host talks to the API only through this repository-owned forward. The VM -# uses Lima's default network, so it does not require a pre-created network or -# configuration from another project. +# The host talks to the API only through this repository-owned forward. The +# quickstart attaches every VM to a user-v2 network that it creates and owns; +# no pre-created network or configuration from another project is required. portForwards: - guestPort: 6443 hostPort: 18447 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index c6e10bb..082f3c9 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -66,13 +66,15 @@ workflow, plus a separate `site-values-VERSION.yaml` digest-pinned override and Prerequisites for the complete local trial: Lima, a running Docker daemon, kubectl, Helm, curl, uv, lsof, and Python 3.12+. Run `make quickstart-doctor` for the executable preflight. -The VM allocates 6 CPUs, 8 GiB RAM, and a 40 GiB sparse disk; outbound HTTPS and host -ports 18090–18098 plus 18447 are required. The repository creates one disposable VM and -bootstraps Kubernetes with kubeadm; it does not use a shared cluster or another checkout. +The default three VMs allocate 8 CPUs, 10 GiB RAM, and 70 GiB of sparse disk in total; +outbound HTTPS and host ports 18090–18098 plus 18447 are required. The repository creates +one control-plane and two worker VMs on its own Lima network and bootstraps Kubernetes with +kubeadm; it does not use a shared cluster or another checkout. ```bash make quickstart-doctor make quickstart +SITES_QUICKSTART_WORKERS=3 make quickstart-scale # supported range: 1-4 workers make quickstart-status make quickstart-access make quickstart-token # run in a second terminal, then log in @@ -80,12 +82,14 @@ make quickstart-clean ``` `make quickstart` exits after verification. `make quickstart-access` must remain running -while the console is in use; Ctrl-C stops only that tunnel. `make quickstart-clean` -permanently removes the disposable VM and all applications stored inside it. - -The quickstart uses a single node to minimize first-run time. Production cluster -provisioning, upgrades, backup, and disaster recovery are deliberately not -implemented in this repository. +while the console is in use; Ctrl-C stops only that tunnel. Worker scale-down drains nodes +and retains `w1`, where quickstart constrains local persistent volumes. It refuses to +remove a worker carrying an unexpected local volume. `make quickstart-clean` permanently +removes the disposable VMs, their repository-owned network, and all applications inside. + +The quickstart is a multi-node functional topology with one control-plane, not a highly +available production cluster. Production provisioning, control-plane HA, upgrades, backup, +and disaster recovery are deliberately not implemented in this repository. The chart defaults deploy the core API, operator, PostgreSQL, registry, build plane, and console on the NodePort backend. Gateway routing and scale-to-zero diff --git a/docs/STANDALONE.md b/docs/STANDALONE.md index 6d5f530..cc9d14f 100644 --- a/docs/STANDALONE.md +++ b/docs/STANDALONE.md @@ -6,20 +6,22 @@ root: ```bash make quickstart-doctor # checks commands, Docker, Python, and fixed host ports make quickstart +SITES_QUICKSTART_WORKERS=3 make quickstart-scale # optional: resize to 1-4 workers make quickstart-access # terminal 1: opens and serves the console make quickstart-token # terminal 2: paste into 管理员 token / Admin token -make quickstart-clean # deletes only the site-quickstart Lima VM and local state +make quickstart-clean # deletes only the Site trial VMs, network, and local state ``` -The trial VM is configured for 6 CPUs, 8 GiB RAM, and a 40 GiB sparse disk. It needs +The default three VMs allocate 8 CPUs, 10 GiB RAM, and 70 GiB of sparse disk in total. They need outbound HTTPS while the Ubuntu, Kubernetes, Cilium, and workload images download, plus -host ports 18090–18098 and 18447. A first uncached install may take 5–25 minutes. +host ports 18090–18098 and 18447. A first uncached install may take 8–30 minutes. `make quickstart` exits after printing the proof. Keep `make quickstart-access` running; pressing Ctrl-C there closes only the console tunnel, not the cluster. In the default Chinese UI choose **进入控制台**; in English choose **Enter the console**. -That path creates a single-node Lima VM, installs Kubernetes 1.36 with kubeadm, installs -a pinned Cilium release, and builds the control image locally. The Helm release supplies +That path creates one control-plane and two worker Lima VMs, installs Kubernetes 1.36 with +kubeadm, joins the workers, installs a pinned Cilium release, and builds the control image +locally. The Helm release supplies its own local storage provisioner, enables Prometheus, deploys `examples/hello-site`, and gates success on the deployment's HTTP status/body digest, a host-openable public URL with the same body digest, plus the admin metrics and deployment-snapshot APIs. It needs no @@ -30,8 +32,15 @@ for its resource samples. **Deploy application** creates or updates from an exis container image: public applications receive a browser URL, while internal applications are explicitly labeled as having no public URL. +`SITES_QUICKSTART_WORKERS=N make quickstart-scale` reconciles the live trial to one through +four workers. Scale-down drains the highest-numbered workers first. All local persistent +volumes are constrained to worker `w1`, which is retained; the command refuses removal if +it detects a local volume on a candidate node. This only scales workers: one control-plane +is not an HA topology. + `make quickstart-clean` is intentionally destructive only to the disposable trial: it -deletes that VM, every application inside it, and this checkout's local kubeconfig. +deletes its VMs, repository-owned Lima network, every application inside it, and this +checkout's local kubeconfig. The lower-level path installs Site into the Kubernetes context selected by `kubectl`. Review the current context before continuing: diff --git a/scripts/quickstart-kubeadm.sh b/scripts/quickstart-kubeadm.sh index 406b270..e518070 100755 --- a/scripts/quickstart-kubeadm.sh +++ b/scripts/quickstart-kubeadm.sh @@ -4,6 +4,12 @@ set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) action=${1:-up} vm=${SITES_QUICKSTART_VM:-site-quickstart} +worker_count=${SITES_QUICKSTART_WORKERS:-2} +network=${SITES_QUICKSTART_NETWORK:-${vm}-net} +network_gateway=${SITES_QUICKSTART_NETWORK_GATEWAY:-192.168.107.1/24} +worker_cpus=${SITES_QUICKSTART_WORKER_CPUS:-2} +worker_memory=${SITES_QUICKSTART_WORKER_MEMORY_GIB:-3} +worker_disk=${SITES_QUICKSTART_WORKER_DISK_GIB:-20} api_port=${SITES_QUICKSTART_KUBE_API_PORT:-18447} console_port=${SITES_QUICKSTART_CONSOLE_PORT:-18091} pod_cidr=${SITES_QUICKSTART_POD_CIDR:-10.201.0.0/16} @@ -12,29 +18,60 @@ cilium_version=${SITES_QUICKSTART_CILIUM_VERSION:-1.19.6} namespace=sites-local state_dir="$root/.site-kubeadm" kubeconfig="$state_dir/kubeconfig" +network_marker="$state_dir/network" +instances_dir="$state_dir/instances" image=site-control:quickstart context=site-quickstart started_at=$SECONDS -[[ "$vm" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || { - echo "SITES_QUICKSTART_VM must be a lowercase DNS label" >&2 +[[ "$vm" =~ ^[a-z0-9][a-z0-9-]{0,58}$ ]] || { + echo "SITES_QUICKSTART_VM must be a lowercase DNS label of at most 59 characters" >&2 exit 2 } +[[ "$worker_count" =~ ^[1-4]$ ]] || { + echo "SITES_QUICKSTART_WORKERS must be an integer from 1 through 4" >&2 + exit 2 +} +[[ "$network" =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || { + echo "SITES_QUICKSTART_NETWORK must be a lowercase DNS label" >&2 + exit 2 +} +[[ "$network_gateway" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.1/24$ ]] || { + echo "SITES_QUICKSTART_NETWORK_GATEWAY must be an IPv4 /24 gateway ending in .1/24" >&2 + exit 2 +} +for resource_value in "$worker_cpus" "$worker_memory" "$worker_disk"; do + [[ "$resource_value" =~ ^[1-9][0-9]*$ ]] || { + echo "worker CPU, memory, and disk settings must be positive integers" >&2 + exit 2 + } +done + +worker_vms=() +for index in $(seq 1 "$worker_count"); do + worker_vms+=("${vm}-w${index}") +done +all_worker_vms=() +for index in $(seq 1 4); do + all_worker_vms+=("${vm}-w${index}") +done usage() { cat <<'EOF' -Usage: scripts/quickstart-kubeadm.sh +Usage: scripts/quickstart-kubeadm.sh doctor Check every host dependency and fixed local port before installation. -up Create one disposable Lima VM, bootstrap Kubernetes with kubeadm, +up Create one control-plane and two worker Lima VMs, bootstrap Kubernetes with kubeadm, build Site from this checkout, deploy the example, and prove it. +scale Reconcile the running cluster to SITES_QUICKSTART_WORKERS (1-4, default 2). status Show the node, Site workloads, example, verification, and monitoring. access Forward and open the console at http://127.0.0.1:18091/console/. token Print the disposable trial's local admin token for console login. -clean Delete only the site-quickstart VM and this checkout's .site-kubeadm state. +clean Delete only the site-quickstart VMs, network, and this checkout's local state. Prerequisites: Lima, a running Docker daemon, kubectl, Helm, curl, uv, lsof, -and Python 3.12+; the VM is configured for 6 CPUs, 8 GiB RAM, and a 40 GiB disk. +and Python 3.12+. The default three-node topology allocates 8 CPUs, 10 GiB RAM, +and 70 GiB of sparse disk across its VMs. No other repository, pre-created Lima network, or published Site image is used. EOF } @@ -88,13 +125,16 @@ PY fi fi - cat <<'EOF' + cat </dev/null | grep -Fxq Running } +instance_exists() { + limactl list --quiet | grep -Fxq "$1" +} + +instance_owned() { + [[ -f "$instances_dir/$1" ]] +} + +mark_instance_owned() { + mkdir -p "$instances_dir" + : >"$instances_dir/$1" +} + +network_exists() { + limactl network list --json 2>/dev/null | NETWORK_NAME="$network" python3 -c ' +import json, os, sys +name = os.environ["NETWORK_NAME"] +raise SystemExit(0 if any(json.loads(line).get("name") == name for line in sys.stdin if line.strip()) else 1) +' +} + +guest_ip() { + limactl shell "$1" -- sh -lc \ + "ip route get 1.1.1.1 | sed -n 's/.* src \\([^ ]*\\).*/\\1/p' | head -1" +} + +ensure_guest_hostname() { + local instance=$1 + if ! limactl shell "$instance" -- getent hosts "$instance" >/dev/null 2>&1; then + printf '127.0.1.1 %s\n' "$instance" \ + | limactl shell "$instance" -- sudo tee -a /etc/hosts >/dev/null + fi +} + +create_worker_instance() { + local worker=$1 + if ! instance_exists "$worker"; then + echo " Creating worker VM $worker" + limactl create --name="$worker" --tty=false \ + --cpus="$worker_cpus" --memory="$worker_memory" --disk="$worker_disk" \ + --network="lima:$network" --set='.portForwards = []' \ + "$root/dev/kubeadm/lima.yaml" + mark_instance_owned "$worker" + else + instance_owned "$worker" || { + echo "Lima VM $worker exists but is not owned by this checkout; choose SITES_QUICKSTART_VM" >&2 + exit 2 + } + echo " Reusing worker VM $worker" + fi + limactl start "$worker" --timeout=15m + ensure_guest_hostname "$worker" +} + +join_worker_nodes() { + local join_command worker + local join_args=() + join_command=$(limactl shell "$vm" -- sudo kubeadm token create --ttl=2h --print-join-command) + read -r -a join_args <<<"$join_command" + for worker in "$@"; do + if limactl shell "$worker" -- test -f /etc/kubernetes/kubelet.conf \ + && kube get node "$worker" >/dev/null 2>&1; then + echo " $worker is already joined" + else + if limactl shell "$worker" -- test -f /etc/kubernetes/kubelet.conf; then + limactl shell "$worker" -- sudo kubeadm reset --force >/dev/null + fi + limactl shell "$worker" -- sudo ${join_args[@]+"${join_args[@]}"} --node-name="$worker" + fi + for _ in $(seq 1 60); do + kube get node "$worker" >/dev/null 2>&1 && break + sleep 1 + done + kube label node "$worker" node-role.kubernetes.io/worker=worker --overwrite >/dev/null + kube uncordon "$worker" >/dev/null 2>&1 || true + done +} + +verify_node_topology() { + kube get nodes -o json | CONTROL_NODE="$vm" EXPECTED_WORKERS="${worker_vms[*]}" python3 -c ' +import json, os, sys +items = {item["metadata"]["name"]: item for item in json.load(sys.stdin)["items"]} +control = os.environ["CONTROL_NODE"] +workers = os.environ["EXPECTED_WORKERS"].split() +expected = {control, *workers} +if set(items) != expected: + raise SystemExit(f"expected nodes {sorted(expected)}, found {sorted(items)}") +for name in expected: + ready = next((c["status"] for c in items[name]["status"]["conditions"] if c["type"] == "Ready"), None) + if ready != "True": + raise SystemExit(f"node {name} is not Ready") +taints = items[control].get("spec", {}).get("taints", []) +if not any(t.get("key") == "node-role.kubernetes.io/control-plane" and t.get("effect") == "NoSchedule" for t in taints): + raise SystemExit("control-plane node must retain its NoSchedule taint") +' +} + kube() { kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" } @@ -156,80 +293,232 @@ wait_for_api_forward() { } create_cluster() { + local control_ip old_context worker mkdir -p "$state_dir" chmod 0700 "$state_dir" + + if vm_exists && ! instance_owned "$vm"; then + echo "Lima VM $vm exists but is not owned by this checkout; choose SITES_QUICKSTART_VM" >&2 + exit 2 + fi + for worker in ${worker_vms[@]+"${worker_vms[@]}"}; do + if instance_exists "$worker" && ! instance_owned "$worker"; then + echo "Lima VM $worker exists but is not owned by this checkout; choose SITES_QUICKSTART_VM" >&2 + exit 2 + fi + done + + if network_exists; then + if [[ ! -f "$network_marker" ]] || [[ "$(cat "$network_marker")" != "$network" ]]; then + printf 'Lima network %s already exists but is not owned by this checkout; choose SITES_QUICKSTART_NETWORK\n' "$network" >&2 + exit 2 + fi + echo "[1/8] Reusing repository-owned Lima network $network" + else + echo "[1/8] Creating repository-owned Lima user-v2 network $network" + limactl network create "$network" --mode=user-v2 --gateway="$network_gateway" + printf '%s\n' "$network" >"$network_marker" + fi + if ! vm_exists; then if lsof -nP -iTCP:"$api_port" -sTCP:LISTEN >/dev/null 2>&1; then printf 'host port %s is already in use; set SITES_QUICKSTART_KUBE_API_PORT and update the Lima template forward\n' "$api_port" >&2 exit 2 fi - echo "[1/7] Creating the disposable Lima VM $vm" + echo " Creating control-plane VM $vm" limactl create --name="$vm" --tty=false \ + --cpus=4 --memory=4 --disk=30 \ + --network="lima:$network" \ --set=".portForwards[0].hostPort = $api_port" \ "$root/dev/kubeadm/lima.yaml" + mark_instance_owned "$vm" else - echo "[1/7] Reusing the repository-owned Lima VM $vm" + echo " Reusing control-plane VM $vm" fi limactl start "$vm" --timeout=15m - local guest_ip - guest_ip=$(limactl shell "$vm" -- sh -lc \ - "ip route get 1.1.1.1 | sed -n 's/.* src \\([^ ]*\\).*/\\1/p' | head -1") - [[ "$guest_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + for worker in ${worker_vms[@]+"${worker_vms[@]}"}; do + create_worker_instance "$worker" + done + + control_ip=$(guest_ip "$vm") + [[ "$control_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "could not discover the kubeadm advertise address" >&2 exit 1 } # Lima assigns its final instance hostname after system provisioning. Add it # immediately before kubeadm so the preflight check is clean and deterministic. - if ! limactl shell "$vm" -- getent hosts "$vm" >/dev/null 2>&1; then - printf '127.0.1.1 %s\n' "$vm" \ - | limactl shell "$vm" -- sudo tee -a /etc/hosts >/dev/null - fi + ensure_guest_hostname "$vm" if ! limactl shell "$vm" -- test -f /etc/kubernetes/admin.conf; then - echo "[2/7] Bootstrapping Kubernetes with kubeadm" + echo "[2/8] Bootstrapping the Kubernetes control plane with kubeadm" limactl shell "$vm" -- sudo kubeadm init \ - --apiserver-advertise-address="$guest_ip" \ + --apiserver-advertise-address="$control_ip" \ --pod-network-cidr="$pod_cidr" \ --service-cidr="$service_cidr" \ --node-name="$vm" else - echo "[2/7] kubeadm is already initialized" + echo "[2/8] kubeadm control plane is already initialized" fi limactl shell "$vm" -- sudo cat /etc/kubernetes/admin.conf >"$kubeconfig" chmod 0600 "$kubeconfig" kubectl config --kubeconfig "$kubeconfig" set-cluster kubernetes \ - --server="https://127.0.0.1:$api_port" --tls-server-name="$guest_ip" >/dev/null - local old_context + --server="https://127.0.0.1:$api_port" --tls-server-name="$control_ip" >/dev/null old_context=$(kubectl config --kubeconfig "$kubeconfig" current-context) if [[ "$old_context" != "$context" ]]; then kubectl config --kubeconfig "$kubeconfig" rename-context "$old_context" "$context" >/dev/null fi - echo "[3/7] Installing the pinned Cilium pod network" + echo "[3/8] Joining $worker_count worker nodes" + join_worker_nodes ${worker_vms[@]+"${worker_vms[@]}"} + + echo "[4/8] Installing the pinned Cilium pod network on every node" helm upgrade --install cilium oci://quay.io/cilium/charts/cilium \ --version "$cilium_version" --namespace kube-system \ --kubeconfig "$kubeconfig" --kube-context "$context" \ --set routingMode=tunnel --set tunnelProtocol=vxlan \ --set ipam.mode=kubernetes --set kubeProxyReplacement=false \ --set operator.replicas=1 --wait --timeout=10m - kube taint nodes "$vm" node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true - kube wait --for=condition=Ready "node/$vm" --timeout=5m + kube wait --for=condition=Ready nodes --all --timeout=10m + + verify_node_topology +} + +scale_workers() { + local desired worker index existing_node result phase verification public_url public_sha proof_sha app_node + desired=$worker_count + [[ -f "$kubeconfig" ]] || { echo "quickstart is not installed; run make quickstart first" >&2; exit 1; } + vm_running || { echo "control-plane VM $vm is not running; run make quickstart first" >&2; exit 1; } + [[ -f "$network_marker" ]] && [[ "$(cat "$network_marker")" == "$network" ]] && network_exists || { + echo "repository-owned Lima network $network is unavailable; refusing to change nodes" >&2 + exit 1 + } + + echo "Scaling Site kubeadm workers to $desired" + for worker in ${worker_vms[@]+"${worker_vms[@]}"}; do + create_worker_instance "$worker" + done + join_worker_nodes ${worker_vms[@]+"${worker_vms[@]}"} + + # Every node may receive a control-plane or tenant Pod after a drain. Load the + # checkout image before changing placement so a scale operation cannot strand it. + docker build -t "$image" "$root" + for worker in ${worker_vms[@]+"${worker_vms[@]}"}; do + docker save "$image" | limactl shell "$worker" -- sudo ctr -n k8s.io images import - >/dev/null + done + + for ((index=4; index>desired; index--)); do + worker="${vm}-w${index}" + if kube get node "$worker" >/dev/null 2>&1; then + echo " Draining $worker before removal" + # Quickstart local volumes are constrained to w1. Refuse removal if that + # invariant has been bypassed instead of orphaning a local persistent volume. + kube get pv -o json | REMOVE_NODE="$worker" python3 -c ' +import json, os, sys +node = os.environ["REMOVE_NODE"] +for pv in json.load(sys.stdin).get("items", []): + terms = pv.get("spec", {}).get("nodeAffinity", {}).get("required", {}).get("nodeSelectorTerms", []) + for term in terms: + for expr in term.get("matchExpressions", []): + if expr.get("key") == "kubernetes.io/hostname" and node in expr.get("values", []): + name = pv.get("metadata", {}).get("name", "") + raise SystemExit(f"refusing to remove {node}: persistent volume {name} is pinned there") +' + kube drain "$worker" --ignore-daemonsets --delete-emptydir-data --timeout=10m + limactl shell "$worker" -- sudo kubeadm reset --force >/dev/null + kube delete node "$worker" --wait=true --timeout=2m + fi + if instance_exists "$worker"; then + instance_owned "$worker" || { + echo "refusing to delete unowned Lima VM $worker" >&2 + exit 1 + } + limactl delete --force "$worker" + rm -f "$instances_dir/$worker" + fi + done + + kube wait --for=condition=Ready nodes --all --timeout=10m + verify_node_topology + + for deployment in sites-api sites-operator sites-activator sites-registry sites-prometheus; do + kube -n "$namespace" rollout status "deployment/$deployment" --timeout=10m + done + kube -n "$namespace" rollout status statefulset/sites-postgres --timeout=10m + for _ in $(seq 1 120); do + result=$(kube get sitedeployments.sites.local -A -o json) + phase=$(printf '%s' "$result" | python3 -c ' +import json, sys +items = [item for item in json.load(sys.stdin)["items"] if item["spec"].get("serviceName") == "hello-site"] +print(items[0].get("status", {}).get("phase", "") if len(items) == 1 else "") +') + verification=$(printf '%s' "$result" | python3 -c ' +import json, sys +items = [item for item in json.load(sys.stdin)["items"] if item["spec"].get("serviceName") == "hello-site"] +print(str(items[0].get("status", {}).get("verification", {}).get("ok", False)).lower() if len(items) == 1 else "false") +') + [[ "$phase" == Running && "$verification" == true ]] && break + sleep 1 + done + [[ "$phase" == Running && "$verification" == true ]] || { + echo "hello-site did not recover after worker scaling" >&2 + exit 1 + } + public_url=$(printf '%s' "$result" | python3 -c ' +import json, sys +item = next(item for item in json.load(sys.stdin)["items"] if item["spec"].get("serviceName") == "hello-site") +print(item["status"]["url"]) +') + proof_sha=$(printf '%s' "$result" | python3 -c ' +import json, sys +item = next(item for item in json.load(sys.stdin)["items"] if item["spec"].get("serviceName") == "hello-site") +print(item["status"]["verification"]["bodySha256"]) +') + public_sha=$(curl --fail --silent --show-error "$public_url" | python3 -c \ + 'import hashlib,sys; print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())') + [[ "$public_sha" == "$proof_sha" ]] || { + echo "hello-site public URL digest changed after worker scaling" >&2 + exit 1 + } + app_node=$(kube get pods -A -o json | python3 -c ' +import json, sys +pods = [item for item in json.load(sys.stdin)["items"] if item["metadata"]["name"].startswith("hello-site-") and item.get("status", {}).get("phase") == "Running"] +if len(pods) != 1: + raise SystemExit(f"expected one running hello-site pod, found {len(pods)}") +print(pods[0]["spec"]["nodeName"]) +') + case " ${worker_vms[*]} " in + *" $app_node "*) ;; + *) echo "hello-site recovered on unexpected node $app_node" >&2; exit 1 ;; + esac + existing_node=$(kube get nodes --no-headers | awk '$2 == "Ready" {count++} END {print count+0}') + cat </dev/null + for node in "$vm" ${worker_vms[@]+"${worker_vms[@]}"}; do + docker save "$image" | limactl shell "$node" -- sudo ctr -n k8s.io images import - >/dev/null + done - echo "[5/7] Installing Site and local observability" + echo "[6/8] Installing Site and local observability" KUBECONFIG="$kubeconfig" \ SITES_KUBE_CONTEXT="$context" \ "$root/scripts/standalone.sh" install \ --set images.control.repository=site-control \ --set images.control.tag=quickstart \ + --set "localPathProvisioner.allowedNodeNames[0]=${vm}-w1" \ --set-value monitoring.enabled=true # A truly new node may need several minutes to pull the pinned database, # registry, proxy, and Prometheus images. Do not spend the API rollout budget @@ -251,7 +540,7 @@ prove_site() { KUBECONFIG="$kubeconfig" SITES_KUBE_CONTEXT="$context" \ "$root/scripts/standalone.sh" smoke - local forward_log forward_pid forward_port api_url token result phase verification sha status_code source public_url public_sha + local forward_log forward_pid forward_port api_url token result phase verification sha status_code source public_url public_sha app_node ready_nodes forward_log=$(mktemp "${TMPDIR:-/tmp}/site-api-forward.XXXXXX") kube -n "$namespace" port-forward service/sites-api :8080 >"$forward_log" 2>&1 & forward_pid=$! @@ -278,7 +567,7 @@ prove_site() { token=$(admin_token) ensure_demo_tenant "$token" "$api_url" - echo "[6/7] Deploying the included local/local application and waiting for measured HTTP proof" + echo "[7/8] Deploying the included local/local application and waiting for measured HTTP proof" SITES_URL="$api_url" SITES_TOKEN="$token" \ uv run --locked sites deploy-static --name hello-site \ --directory "$root/examples/hello-site" --exposure public >/dev/null @@ -313,7 +602,7 @@ prove_site() { exit 1 } - echo "[7/7] Verifying persisted admin evidence and Prometheus availability" + echo "[8/8] Verifying multi-node placement, persisted evidence, and Prometheus" for _ in $(seq 1 60); do source=$(curl --fail --silent -H "X-Sites-Service-Token: $token" \ "$api_url/v1/admin/metrics/cluster" | python3 -c \ @@ -332,12 +621,44 @@ if not row or not row.get("verification", {}).get("ok") or not row.get("artifact raise SystemExit("admin deployment snapshot is missing persisted proof") ' + app_node=$(kube get pods -A -o json | python3 -c ' +import json, sys +pods = [ + item for item in json.load(sys.stdin)["items"] + if item["metadata"]["name"].startswith("hello-site-") + and item.get("status", {}).get("phase") == "Running" +] +if len(pods) != 1: + raise SystemExit(f"expected one running hello-site pod, found {len(pods)}") +print(pods[0]["spec"]["nodeName"]) +') + case " ${worker_vms[*]} " in + *" $app_node "*) ;; + *) printf 'hello-site was scheduled on %s instead of a worker node\n' "$app_node" >&2; exit 1 ;; + esac + kube -n "$namespace" get pods -o json | CONTROL_NODE="$vm" python3 -c ' +import json, os, sys +control = os.environ["CONTROL_NODE"] +pods = json.load(sys.stdin)["items"] +bad = [item["metadata"]["name"] for item in pods if item.get("spec", {}).get("nodeName") == control] +if bad: + raise SystemExit(f"application control workloads escaped onto the control-plane node: {bad}") +' + ready_nodes=$(kube get nodes --no-headers | awk '$2 == "Ready" {count++} END {print count+0}') + [[ "$ready_nodes" -eq $((worker_count + 1)) ]] || { + printf 'expected %s Ready nodes, found %s\n' "$((worker_count + 1))" "$ready_nodes" >&2 + exit 1 + } + cleanup_forward trap - EXIT cat <&2; exit 1; } kube get nodes -o wide - kube -n "$namespace" get deployment,statefulset,pod,service + kube -n "$namespace" get deployment,statefulset,service + kube -n "$namespace" get pods -o wide + kube get pods -A -o wide | awk 'NR == 1 || $2 ~ /^hello-site-/' kube -n "$namespace" get sitedeployments.sites.local \ -o custom-columns='NAME:.metadata.name,PHASE:.status.phase,HTTP:.status.verification.httpStatus,SHA256:.status.verification.bodySha256' } @@ -411,14 +735,23 @@ show_token() { } clean() { + local worker require_command limactl - if vm_exists; then + for worker in ${all_worker_vms[@]+"${all_worker_vms[@]}"}; do + if instance_exists "$worker" && instance_owned "$worker"; then + limactl delete --force "$worker" + fi + done + if vm_exists && instance_owned "$vm"; then limactl delete --force "$vm" fi + if [[ -f "$network_marker" ]] && [[ "$(cat "$network_marker")" == "$network" ]] && network_exists; then + limactl network delete --force "$network" + fi if [[ -d "$state_dir" ]]; then rm -rf -- "$state_dir" fi - echo "removed only the $vm Lima VM and $state_dir" + echo "removed only the $vm cluster VMs, repository-owned $network network, and $state_dir" } case "$action" in @@ -428,6 +761,10 @@ case "$action" in create_cluster prove_site ;; + scale) + check_prerequisites + scale_workers + ;; status) require_command kubectl; show_status ;; access) require_command kubectl; access_console ;; token) require_command kubectl; show_token ;; diff --git a/src/sites/exposure.py b/src/sites/exposure.py index 6c697d3..7412e01 100644 --- a/src/sites/exposure.py +++ b/src/sites/exposure.py @@ -188,11 +188,13 @@ class NodePortExposure(ExposureBackend): allocates_ports = True def service_overrides(self, spec: dict[str, Any]) -> dict[str, Any]: - # externalTrafficPolicy: Local - the workload only falls on w1, which hosts NodePort; - # Changing to Cluster will cause the CP side to respond but there will be no local endpoint, and the host access will time out. + # The public host port enters through the control-plane VM in the + # repository-owned multi-node quickstart, while tenant workloads stay + # on workers. Cluster is therefore a correctness requirement: Local + # would drop traffic whenever the entry node has no local endpoint. return { "type": "NodePort", - "externalTrafficPolicy": "Local", + "externalTrafficPolicy": "Cluster", "nodePort": int(spec["nodePort"]), } diff --git a/tests/test_exposure.py b/tests/test_exposure.py index 235fa94..0b396a3 100644 --- a/tests/test_exposure.py +++ b/tests/test_exposure.py @@ -128,11 +128,11 @@ def test_gateway_does_not_allocate_ports(self) -> None: class ServiceShapeTest(unittest.TestCase): - def test_nodeport_backend_keeps_existing_shape(self) -> None: + def test_nodeport_backend_routes_from_entry_node_to_worker(self) -> None: with using_backend("nodeport"): service = service_resource(_spec(), "ulocal-local") self.assertEqual(service["spec"]["type"], "NodePort") - self.assertEqual(service["spec"]["externalTrafficPolicy"], "Local") + self.assertEqual(service["spec"]["externalTrafficPolicy"], "Cluster") self.assertEqual(service["spec"]["ports"][0]["nodePort"], 30080) def test_gateway_backend_uses_clusterip_and_drops_the_port(self) -> None: diff --git a/tests/test_helm_package.py b/tests/test_helm_package.py index c255810..79009bf 100644 --- a/tests/test_helm_package.py +++ b/tests/test_helm_package.py @@ -25,7 +25,13 @@ class HelmPackageContractTests(unittest.TestCase): def test_quickstart_is_a_repository_owned_kubeadm_workflow(self) -> None: script = (ROOT / "scripts" / "quickstart-kubeadm.sh").read_text(encoding="utf-8") self.assertIn("sudo kubeadm init", script) - self.assertIn('getent hosts "$vm"', script) + self.assertIn("sudo kubeadm token create", script) + self.assertIn('getent hosts "$instance"', script) + self.assertIn('limactl network create "$network" --mode=user-v2', script) + self.assertIn('--network="lima:$network"', script) + self.assertIn("node-role.kubernetes.io/worker=worker", script) + self.assertIn("control-plane node must retain its NoSchedule taint", script) + self.assertNotIn('node-role.kubernetes.io/control-plane-', script) self.assertIn("dev/kubeadm/lima.yaml", script) def test_quickstart_lima_template_needs_no_precreated_network(self) -> None: @@ -33,6 +39,9 @@ def test_quickstart_lima_template_needs_no_precreated_network(self) -> None: (ROOT / "dev" / "kubeadm" / "lima.yaml").read_text(encoding="utf-8") ) self.assertNotIn("networks", template) + self.assertEqual(template["cpus"], 4) + self.assertEqual(template["memory"], "4GiB") + self.assertEqual(template["disk"], "30GiB") self.assertEqual(template["portForwards"][0]["guestPort"], 6443) forwards = { item["guestPort"]: item["hostPort"] @@ -42,9 +51,22 @@ def test_quickstart_lima_template_needs_no_precreated_network(self) -> None: self.assertEqual(forwards[30088], 18098) self.assertNotIn(30081, forwards) + rendered = subprocess.check_output([ + "helm", "template", "site", str(CHART), *POD_CIDR, + "--set", "localPathProvisioner.enabled=true", + "--set-string", "localPathProvisioner.allowedNodeNames[0]=site-quickstart-w1", + ], text=True) + storage_class = next( + item for item in yaml.safe_load_all(rendered) + if isinstance(item, dict) and item.get("kind") == "StorageClass" + ) + expression = storage_class["allowedTopologies"][0]["matchLabelExpressions"][0] + self.assertEqual(expression["key"], "kubernetes.io/hostname") + self.assertEqual(expression["values"], ["site-quickstart-w1"]) + def test_makefile_exposes_the_complete_kubeadm_trial_lifecycle(self) -> None: makefile = (ROOT / "Makefile").read_text(encoding="utf-8") - for target in ("quickstart-doctor", "quickstart", "quickstart-status", "quickstart-access", "quickstart-token", "quickstart-clean"): + for target in ("quickstart-doctor", "quickstart", "quickstart-scale", "quickstart-status", "quickstart-access", "quickstart-token", "quickstart-clean"): self.assertRegex(makefile, rf"(?m)^{target}:") script = (ROOT / "scripts" / "quickstart-kubeadm.sh").read_text(encoding="utf-8") self.assertIn('open "$url"', script) @@ -53,6 +75,10 @@ def test_makefile_exposes_the_complete_kubeadm_trial_lifecycle(self) -> None: self.assertIn('"merchantId":"local","userId":"local"', script) self.assertIn('--exposure public', script) self.assertIn('public URL body digest', script) + self.assertIn('kube drain "$worker"', script) + self.assertIn('localPathProvisioner.allowedNodeNames[0]', script) + self.assertIn('refusing to remove {node}: persistent volume', script) + self.assertIn('instance_owned "$worker"', script) self.assertIn('rollout restart', script) self.assertIn('deployment/sites-api deployment/sites-operator deployment/sites-activator', script) dependency_wait = script.index('statefulset/sites-postgres') @@ -74,8 +100,10 @@ def test_quickstart_explains_every_newcomer_handoff(self) -> None: readme = (ROOT / "README.md").read_text(encoding="utf-8") for fact in ( "make quickstart-doctor", - "6 CPUs, 8 GiB RAM", - "5–25 minutes", + "8 CPUs, 10 GiB", + "8–30 minutes", + "SITES_QUICKSTART_WORKERS=3 make quickstart-scale", + "not a highly available production topology", "permanently deletes", ): self.assertIn(fact, readme) diff --git a/website/index.html b/website/index.html index b4d536d..9cef6e7 100644 --- a/website/index.html +++ b/website/index.html @@ -144,7 +144,7 @@

Small surface. Hard boundaries.
Ridiculous leverage
-
Kubeadm-first quickstart

One command.
A real cluster.
A live URL.

The repository-owned trial creates a disposable Lima VM, bootstraps Kubernetes with kubeadm, installs Cilium and Site, deploys the included app, and refuses to pass without measured HTTP proof.

Git · Lima · running Docker · kubectl · Helm · uv · Python 3.12+
6 CPU · 8 GiB RAM · 40 GiB sparse disk · outbound HTTPS
+
Kubeadm-first quickstart

One command.
Three real nodes.
A live URL.

The repository-owned trial creates one control-plane and two worker Lima VMs, bootstraps Kubernetes with kubeadm, installs Cilium and Site, deploys the included app on a worker, and refuses to pass without measured HTTP proof.

Git · Lima · running Docker · kubectl · Helm · uv · Python 3.12+
8 CPU · 10 GiB RAM · 70 GiB sparse disk · outbound HTTPS
site — zsh
$ git clone https://github.com/hullwork/site.git
@@ -152,6 +152,8 @@ 
Small surface. Hard boundaries.
Ridiculous leverage
 $ make quickstart
 
 Site kubeadm quickstart passed
+  topology: 1 control-plane + 2 workers (3 Ready)
+  example node: site-quickstart-w2
   phase: Running
   HTTP verification: 200
   public URL: http://127.0.0.1:18090

From 2560141933354fa27c43b95f510ca26002d5a150 Mon Sep 17 00:00:00 2001
From: convee 
Date: Sat, 5 Sep 2026 09:49:39 +0800
Subject: [PATCH 2/5] fix: keep quickstart workloads on workers

---
 README.md                                |  3 ++-
 charts/site/templates/11-monitoring.yaml |  6 ------
 docs/STANDALONE.md                       |  3 +++
 tests/test_helm_package.py               | 17 +++++++++++++++++
 website/index.html                       |  5 +++--
 5 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 75cd22f..7397a10 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,8 @@ depending on the network it can take 8–30 minutes and needs ports 18090–1809
 
 The fastest path is deliberately end to end. With the prerequisites above, one command
 creates a disposable Kubernetes cluster with one control-plane and two worker nodes using
-`kubeadm`, builds the image from this checkout, installs Site and Prometheus,
+`kubeadm`, keeps all Site components off the tainted control-plane, builds the image from
+this checkout, installs Site and Prometheus,
 deploys the included HTML example, and refuses to pass until both server-side HTTP
 verification and the administrative observability APIs return real evidence:
 
diff --git a/charts/site/templates/11-monitoring.yaml b/charts/site/templates/11-monitoring.yaml
index b4911e1..7baccb8 100644
--- a/charts/site/templates/11-monitoring.yaml
+++ b/charts/site/templates/11-monitoring.yaml
@@ -260,12 +260,6 @@ spec:
     spec:
       serviceAccountName: sites-prometheus
       enableServiceLinks: false
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: ""
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
       securityContext:
         runAsNonRoot: true
         runAsUser: 65534
diff --git a/docs/STANDALONE.md b/docs/STANDALONE.md
index cc9d14f..cc0f377 100644
--- a/docs/STANDALONE.md
+++ b/docs/STANDALONE.md
@@ -38,6 +38,9 @@ volumes are constrained to worker `w1`, which is retained; the command refuses r
 it detects a local volume on a candidate node. This only scales workers: one control-plane
 is not an HA topology.
 
+The control-plane keeps its kubeadm `NoSchedule` taint. Site, Prometheus, and tenant Pods
+run on workers; the quickstart fails if a Site workload crosses that boundary.
+
 `make quickstart-clean` is intentionally destructive only to the disposable trial: it
 deletes its VMs, repository-owned Lima network, every application inside it, and this
 checkout's local kubeconfig.
diff --git a/tests/test_helm_package.py b/tests/test_helm_package.py
index 79009bf..38c580a 100644
--- a/tests/test_helm_package.py
+++ b/tests/test_helm_package.py
@@ -63,6 +63,23 @@ def test_quickstart_lima_template_needs_no_precreated_network(self) -> None:
         expression = storage_class["allowedTopologies"][0]["matchLabelExpressions"][0]
         self.assertEqual(expression["key"], "kubernetes.io/hostname")
         self.assertEqual(expression["values"], ["site-quickstart-w1"])
+        for workload in (
+            item for item in yaml.safe_load_all(rendered)
+            if isinstance(item, dict) and item.get("kind") in {"Deployment", "StatefulSet"}
+        ):
+            pod_spec = workload["spec"]["template"]["spec"]
+            self.assertNotIn(
+                "node-role.kubernetes.io/control-plane",
+                pod_spec.get("nodeSelector", {}),
+                workload["metadata"]["name"],
+            )
+            self.assertFalse(
+                any(
+                    item.get("key") == "node-role.kubernetes.io/control-plane"
+                    for item in pod_spec.get("tolerations", [])
+                ),
+                workload["metadata"]["name"],
+            )
 
     def test_makefile_exposes_the_complete_kubeadm_trial_lifecycle(self) -> None:
         makefile = (ROOT / "Makefile").read_text(encoding="utf-8")
diff --git a/website/index.html b/website/index.html
index 9cef6e7..bf977ad 100644
--- a/website/index.html
+++ b/website/index.html
@@ -144,7 +144,7 @@ 
Small surface. Hard boundaries.
Ridiculous leverage
-
Kubeadm-first quickstart

One command.
Three real nodes.
A live URL.

The repository-owned trial creates one control-plane and two worker Lima VMs, bootstraps Kubernetes with kubeadm, installs Cilium and Site, deploys the included app on a worker, and refuses to pass without measured HTTP proof.

Git · Lima · running Docker · kubectl · Helm · uv · Python 3.12+
8 CPU · 10 GiB RAM · 70 GiB sparse disk · outbound HTTPS
+
Kubeadm-first quickstart

One command.
Three real nodes.
A live URL.

The repository-owned trial creates one tainted control-plane and two worker Lima VMs, bootstraps Kubernetes with kubeadm, installs Cilium and Site, deploys the included app on a worker, and refuses to pass without measured HTTP proof. Resize to 1–4 workers in place; scale-down drains before removal and every change rechecks the live URL.

Git · Lima · running Docker · kubectl · Helm · uv · Python 3.12+
8 CPU · 10 GiB RAM · 70 GiB sparse disk · outbound HTTPS
site — zsh
$ git clone https://github.com/hullwork/site.git
@@ -160,7 +160,8 @@ 
Small surface. Hard boundaries.
Ridiculous leverage
   public URL body: matches verification digest
 
 $ make quickstart-access  # keep terminal 1 open
-$ make quickstart-token   # run in terminal 2
+$ make quickstart-token # run in terminal 2 +$ SITES_QUICKSTART_WORKERS=3 make quickstart-scale
kubeadm cluster readyproof, not promises
From 70eab14816a3e1905b05d6332897ec5480b293e9 Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 09:56:21 +0800 Subject: [PATCH 3/5] fix: route host ports across quickstart nodes --- README.md | 7 ++-- dev/kubeadm/lima.yaml | 10 +++--- docs/DEPLOYMENT.md | 4 +++ docs/STANDALONE.md | 4 +++ scripts/quickstart-kubeadm.sh | 61 +++++++++++++++++++++++++++++++++++ tests/test_helm_package.py | 3 ++ 6 files changed, 83 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7397a10..a9c3a98 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,11 @@ applications inside them, the repository-owned Lima network, and the repository- kubeconfig; it does not touch another cluster, network, or repository. This trial harness is entirely owned by this repository: it creates its own Lima user-v2 -network, bootstraps Kubernetes directly with kubeadm, and builds the control image locally. No -other checkout, shared cluster configuration, or pre-published Site image is required. +network, bootstraps Kubernetes directly with kubeadm, and builds the control image locally. +Fixed host ports enter through the control-plane VM; a repository-managed, reboot-persistent +relay sends only those ports to retained worker `w1`, and Kubernetes routes them to whichever +worker owns the application Pod. No other checkout, shared cluster configuration, or +pre-published Site image is required. ```text ┌─ an agent asks for a deployment ────────────────────────────────────────┐ diff --git a/dev/kubeadm/lima.yaml b/dev/kubeadm/lima.yaml index fc15b57..bea4f1d 100644 --- a/dev/kubeadm/lima.yaml +++ b/dev/kubeadm/lima.yaml @@ -16,10 +16,12 @@ portForwards: - guestPort: 6443 hostPort: 18447 static: true - # NodePort exposure is the quickstart's public application path. Keep this - # mapping aligned with src/sites/exposure.py: 30080, 30082-30088 become - # 18090, 18092-18098. 30081 is deliberately excluded and 18091 serves the - # admin console through kubectl port-forward instead. + # NodePort exposure is the quickstart's public application path. Lima sends + # these fixed forwards to guest loopback; quickstart-kubeadm.sh owns a narrow + # systemd/iptables relay from loopback to retained worker w1 so the target Pod + # may live on any worker. Keep this mapping aligned with src/sites/exposure.py: + # 30080, 30082-30088 become 18090, 18092-18098. 30081 is deliberately excluded + # and 18091 serves the admin console through kubectl port-forward instead. - guestPort: 30080 hostPort: 18090 static: true diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 082f3c9..aaeeb58 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -90,6 +90,10 @@ removes the disposable VMs, their repository-owned network, and all applications The quickstart is a multi-node functional topology with one control-plane, not a highly available production cluster. Production provisioning, control-plane HA, upgrades, backup, and disaster recovery are deliberately not implemented in this repository. +The host-visible application ports terminate on control-plane loopback and pass through a +repository-managed systemd/iptables relay to retained worker `w1`; kube-proxy and Cilium +route onward to the selected workload Pod. The relay covers only the documented NodePort +pool, survives VM reboot, and disappears when the disposable control-plane VM is deleted. The chart defaults deploy the core API, operator, PostgreSQL, registry, build plane, and console on the NodePort backend. Gateway routing and scale-to-zero diff --git a/docs/STANDALONE.md b/docs/STANDALONE.md index cc0f377..be588a9 100644 --- a/docs/STANDALONE.md +++ b/docs/STANDALONE.md @@ -40,6 +40,10 @@ is not an HA topology. The control-plane keeps its kubeadm `NoSchedule` taint. Site, Prometheus, and tenant Pods run on workers; the quickstart fails if a Site workload crosses that boundary. +Lima's fixed host ports terminate on control-plane loopback, so the repository installs a +narrow systemd/iptables relay for ports 30080 and 30082–30088 to retained worker `w1`. +Kube-proxy and Cilium then route to the actual application worker. The relay is recreated +on control-plane reboot and removed with the disposable VM. `make quickstart-clean` is intentionally destructive only to the disposable trial: it deletes its VMs, repository-owned Lima network, every application inside it, and this diff --git a/scripts/quickstart-kubeadm.sh b/scripts/quickstart-kubeadm.sh index e518070..b316490 100755 --- a/scripts/quickstart-kubeadm.sh +++ b/scripts/quickstart-kubeadm.sh @@ -244,6 +244,64 @@ if not any(t.get("key") == "node-role.kubernetes.io/control-plane" and t.get("ef ' } +configure_nodeport_relay() { + local control_ip worker_ip + control_ip=$(guest_ip "$vm") + worker_ip=$(guest_ip "${vm}-w1") + [[ "$control_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "could not discover the control-plane relay address" >&2 + exit 1 + } + [[ "$worker_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "could not discover the retained worker relay address" >&2 + exit 1 + } + + # Lima's static host forwards terminate on guest loopback. A loopback source + # cannot traverse to a remote NodePort endpoint, so translate the fixed trial + # ports to retained worker w1 and SNAT them to the control-plane node address. + # The systemd unit reapplies the narrow rules after a VM reboot. + limactl shell "$vm" -- sudo tee /usr/local/sbin/site-nodeport-relay >/dev/null </dev/null || true +iptables -t nat -N SITE-NODEPORT-SNAT 2>/dev/null || true +iptables -t nat -F SITE-NODEPORT-OUTPUT +iptables -t nat -F SITE-NODEPORT-SNAT +iptables -t nat -C OUTPUT -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT 2>/dev/null \ + || iptables -t nat -I OUTPUT 1 -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT +iptables -t nat -C POSTROUTING -j SITE-NODEPORT-SNAT 2>/dev/null \ + || iptables -t nat -I POSTROUTING 1 -j SITE-NODEPORT-SNAT +for port in \$ports; do + iptables -t nat -A SITE-NODEPORT-OUTPUT -p tcp --dport "\$port" \ + -j DNAT --to-destination "\$worker_ip:\$port" + iptables -t nat -A SITE-NODEPORT-SNAT -d "\$worker_ip/32" -p tcp --dport "\$port" \ + -j SNAT --to-source "\$control_ip" +done +EOF + limactl shell "$vm" -- sudo chmod 0755 /usr/local/sbin/site-nodeport-relay + limactl shell "$vm" -- sudo tee /etc/systemd/system/site-nodeport-relay.service >/dev/null <<'EOF' +[Unit] +Description=Site quickstart loopback to worker NodePort relay +After=network-online.target kubelet.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/site-nodeport-relay +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +EOF + limactl shell "$vm" -- sudo systemctl daemon-reload + limactl shell "$vm" -- sudo systemctl enable site-nodeport-relay.service >/dev/null + limactl shell "$vm" -- sudo systemctl restart site-nodeport-relay.service +} + kube() { kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" } @@ -380,10 +438,12 @@ create_cluster() { --kubeconfig "$kubeconfig" --kube-context "$context" \ --set routingMode=tunnel --set tunnelProtocol=vxlan \ --set ipam.mode=kubernetes --set kubeProxyReplacement=false \ + --set policyCIDRMatchMode=nodes \ --set operator.replicas=1 --wait --timeout=10m kube wait --for=condition=Ready nodes --all --timeout=10m verify_node_topology + configure_nodeport_relay } scale_workers() { @@ -442,6 +502,7 @@ for pv in json.load(sys.stdin).get("items", []): kube wait --for=condition=Ready nodes --all --timeout=10m verify_node_topology + configure_nodeport_relay for deployment in sites-api sites-operator sites-activator sites-registry sites-prometheus; do kube -n "$namespace" rollout status "deployment/$deployment" --timeout=10m diff --git a/tests/test_helm_package.py b/tests/test_helm_package.py index 38c580a..eb60b25 100644 --- a/tests/test_helm_package.py +++ b/tests/test_helm_package.py @@ -31,6 +31,9 @@ def test_quickstart_is_a_repository_owned_kubeadm_workflow(self) -> None: self.assertIn('--network="lima:$network"', script) self.assertIn("node-role.kubernetes.io/worker=worker", script) self.assertIn("control-plane node must retain its NoSchedule taint", script) + self.assertIn("policyCIDRMatchMode=nodes", script) + self.assertIn("site-nodeport-relay.service", script) + self.assertIn("SITE-NODEPORT-SNAT", script) self.assertNotIn('node-role.kubernetes.io/control-plane-', script) self.assertIn("dev/kubeadm/lima.yaml", script) From f186bd34c490abc14b26f09203e378fa25245cd9 Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 09:59:26 +0800 Subject: [PATCH 4/5] fix: detect running clusters under pipefail --- scripts/quickstart-kubeadm.sh | 12 +++++++----- tests/test_helm_package.py | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/quickstart-kubeadm.sh b/scripts/quickstart-kubeadm.sh index b316490..132c94a 100755 --- a/scripts/quickstart-kubeadm.sh +++ b/scripts/quickstart-kubeadm.sh @@ -55,6 +55,8 @@ all_worker_vms=() for index in $(seq 1 4); do all_worker_vms+=("${vm}-w${index}") done +worker_label=workers +[[ "$worker_count" -eq 1 ]] && worker_label=worker usage() { cat <<'EOF' @@ -131,7 +133,7 @@ Quickstart doctor passed Python: 3.12 or newer Docker daemon: reachable local ports: available or owned by this trial - topology: 1 control-plane + $worker_count workers ($((worker_count + 1)) kubeadm nodes) + topology: 1 control-plane + $worker_count $worker_label ($((worker_count + 1)) kubeadm nodes) control-plane allocation: 4 CPUs, 4 GiB RAM, 30 GiB sparse disk worker allocation: ${worker_cpus} CPUs, ${worker_memory} GiB RAM, ${worker_disk} GiB sparse disk each total allocation: $((4 + worker_count * worker_cpus)) CPUs, $((4 + worker_count * worker_memory)) GiB RAM, $((30 + worker_count * worker_disk)) GiB sparse disk @@ -140,7 +142,7 @@ EOF } vm_exists() { - limactl list --quiet | grep -Fxq "$vm" + limactl list --quiet | grep -Fx "$vm" >/dev/null } vm_running() { @@ -148,7 +150,7 @@ vm_running() { } instance_exists() { - limactl list --quiet | grep -Fxq "$1" + limactl list --quiet | grep -Fx "$1" >/dev/null } instance_owned() { @@ -557,7 +559,7 @@ print(pods[0]["spec"]["nodeName"]) existing_node=$(kube get nodes --no-headers | awk '$2 == "Ready" {count++} END {print count+0}') cat < None: self.assertIn("policyCIDRMatchMode=nodes", script) self.assertIn("site-nodeport-relay.service", script) self.assertIn("SITE-NODEPORT-SNAT", script) + self.assertIn('grep -Fx "$vm" >/dev/null', script) + self.assertNotIn('grep -Fxq "$vm"', script) self.assertNotIn('node-role.kubernetes.io/control-plane-', script) self.assertIn("dev/kubeadm/lima.yaml", script) From 3d811ed06feeb7048333df7e1ba8476c7128e2b6 Mon Sep 17 00:00:00 2001 From: convee Date: Sat, 5 Sep 2026 10:05:16 +0800 Subject: [PATCH 5/5] fix: restore nodeport relay after agent restarts --- scripts/quickstart-kubeadm.sh | 47 +++++++++++++++++++++++++++-------- tests/test_helm_package.py | 4 +++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/scripts/quickstart-kubeadm.sh b/scripts/quickstart-kubeadm.sh index 132c94a..93fa24d 100755 --- a/scripts/quickstart-kubeadm.sh +++ b/scripts/quickstart-kubeadm.sh @@ -273,10 +273,14 @@ iptables -t nat -N SITE-NODEPORT-OUTPUT 2>/dev/null || true iptables -t nat -N SITE-NODEPORT-SNAT 2>/dev/null || true iptables -t nat -F SITE-NODEPORT-OUTPUT iptables -t nat -F SITE-NODEPORT-SNAT -iptables -t nat -C OUTPUT -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT 2>/dev/null \ - || iptables -t nat -I OUTPUT 1 -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT -iptables -t nat -C POSTROUTING -j SITE-NODEPORT-SNAT 2>/dev/null \ - || iptables -t nat -I POSTROUTING 1 -j SITE-NODEPORT-SNAT +while iptables -t nat -C OUTPUT -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT 2>/dev/null; do + iptables -t nat -D OUTPUT -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT +done +while iptables -t nat -C POSTROUTING -j SITE-NODEPORT-SNAT 2>/dev/null; do + iptables -t nat -D POSTROUTING -j SITE-NODEPORT-SNAT +done +iptables -t nat -I OUTPUT 1 -d 127.0.0.1/32 -j SITE-NODEPORT-OUTPUT +iptables -t nat -I POSTROUTING 1 -j SITE-NODEPORT-SNAT for port in \$ports; do iptables -t nat -A SITE-NODEPORT-OUTPUT -p tcp --dport "\$port" \ -j DNAT --to-destination "\$worker_ip:\$port" @@ -294,14 +298,23 @@ Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/site-nodeport-relay -RemainAfterExit=yes +EOF + limactl shell "$vm" -- sudo tee /etc/systemd/system/site-nodeport-relay.timer >/dev/null <<'EOF' +[Unit] +Description=Reassert Site quickstart NodePort relay after node-agent rule changes + +[Timer] +OnBootSec=15s +OnUnitActiveSec=15s +AccuracySec=1s +Unit=site-nodeport-relay.service [Install] -WantedBy=multi-user.target +WantedBy=timers.target EOF limactl shell "$vm" -- sudo systemctl daemon-reload - limactl shell "$vm" -- sudo systemctl enable site-nodeport-relay.service >/dev/null limactl shell "$vm" -- sudo systemctl restart site-nodeport-relay.service + limactl shell "$vm" -- sudo systemctl enable --now site-nodeport-relay.timer >/dev/null } kube() { @@ -352,6 +365,20 @@ wait_for_api_forward() { return 1 } +public_body_sha() { + local url=$1 digest + for _ in $(seq 1 30); do + if digest=$(curl --fail --silent --show-error --connect-timeout 2 --max-time 5 "$url" \ + | python3 -c 'import hashlib,sys; print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())'); then + printf '%s\n' "$digest" + return 0 + fi + sleep 1 + done + echo "public URL did not return a successful response within the bounded retry window: $url" >&2 + return 1 +} + create_cluster() { local control_ip old_context worker mkdir -p "$state_dir" @@ -539,8 +566,7 @@ import json, sys item = next(item for item in json.load(sys.stdin)["items"] if item["spec"].get("serviceName") == "hello-site") print(item["status"]["verification"]["bodySha256"]) ') - public_sha=$(curl --fail --silent --show-error "$public_url" | python3 -c \ - 'import hashlib,sys; print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())') + public_sha=$(public_body_sha "$public_url") [[ "$public_sha" == "$proof_sha" ]] || { echo "hello-site public URL digest changed after worker scaling" >&2 exit 1 @@ -658,8 +684,7 @@ prove_site() { printf 'quickstart did not return a local public URL: %s\n' "$public_url" >&2 exit 1 } - public_sha=$(curl --fail --silent --show-error "$public_url" | python3 -c \ - 'import hashlib,sys; print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())') + public_sha=$(public_body_sha "$public_url") [[ "$public_sha" == "$sha" ]] || { printf 'public URL body digest %s does not match control-plane proof %s\n' "$public_sha" "$sha" >&2 exit 1 diff --git a/tests/test_helm_package.py b/tests/test_helm_package.py index 9ad84f3..19f71e7 100644 --- a/tests/test_helm_package.py +++ b/tests/test_helm_package.py @@ -33,6 +33,8 @@ def test_quickstart_is_a_repository_owned_kubeadm_workflow(self) -> None: self.assertIn("control-plane node must retain its NoSchedule taint", script) self.assertIn("policyCIDRMatchMode=nodes", script) self.assertIn("site-nodeport-relay.service", script) + self.assertIn("site-nodeport-relay.timer", script) + self.assertIn("OnUnitActiveSec=15s", script) self.assertIn("SITE-NODEPORT-SNAT", script) self.assertIn('grep -Fx "$vm" >/dev/null', script) self.assertNotIn('grep -Fxq "$vm"', script) @@ -97,6 +99,8 @@ def test_makefile_exposes_the_complete_kubeadm_trial_lifecycle(self) -> None: self.assertIn('"merchantId":"local","userId":"local"', script) self.assertIn('--exposure public', script) self.assertIn('public URL body digest', script) + self.assertIn('--connect-timeout 2 --max-time 5', script) + self.assertIn('public URL did not return a successful response within the bounded retry window', script) self.assertIn('kube drain "$worker"', script) self.assertIn('localPathProvisioner.allowedNodeNames[0]', script) self.assertIn('refusing to remove {node}: persistent volume', script)