From 1947b70277a23cbec8e51091faec954d18c34536 Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Fri, 18 Sep 2026 12:41:36 +0200 Subject: [PATCH] feat(platform): the site's images name a version, and Flagger promotes the pair The api and the frontend ran :latest and Keel rolled each on its own two-minute poll, so two services built from one commit reached production minutes apart and nothing in git said what was running. The overlay now names one version for both, in one place, and the keel.sh annotations come off those two Deployments. On top of that, each service becomes a Flagger blue/green canary whose confirm-promotion gate reads the sibling's phase, so a release promotes both or neither and a failed half parks the other. Flagger installs from its own apps-delivery Kustomization: everything depends on apps-core, so a controller only apps-stateless needs must not be able to stall cert-manager, Traefik and Vault. The tag is bumped by hand until the release pipeline writes a digest there, so cutting a release publishes images without deploying them. Applying this prunes the hand-written Services and Flagger recreates them only once its -primary Deployments are Ready, which costs about one api cold start; the runbook carries the suspend-merge-resume procedure. --- .github/workflows/build.yml | 40 +++++------- README.md | 12 +++- .../apps/delivery/flagger/kustomization.yaml | 8 +++ .../apps/delivery/flagger/loadtester.yaml | 58 +++++++++++++++++ .../flux/apps/delivery/flagger/namespace.yaml | 4 ++ .../flux/apps/delivery/flagger/release.yaml | 36 ++++++++++ .../flux/apps/delivery/flagger/source.yaml | 8 +++ .../flux/apps/delivery/kustomization.yaml | 8 +++ .../flux/apps/stateless/api/canary.yaml | 50 ++++++++++++++ .../flux/apps/stateless/api/deployment.yaml | 46 ++++--------- .../apps/stateless/api/kustomization.yaml | 1 + .../flux/apps/stateless/frontend/canary.yaml | 42 ++++++++++++ .../apps/stateless/frontend/deployment.yaml | 54 ++++++++------- .../stateless/frontend/kustomization.yaml | 1 + .../flux/apps/stateless/kustomization.yaml | 10 +++ .../clusters/production/kustomizations.yaml | 40 ++++++++++++ platform/docs/bringup-v2.md | 11 ++-- platform/docs/flux-bootstrap.md | 9 ++- platform/docs/runbook.md | 65 ++++++++++++++++++- 19 files changed, 412 insertions(+), 91 deletions(-) create mode 100644 platform/cluster/flux/apps/delivery/flagger/kustomization.yaml create mode 100644 platform/cluster/flux/apps/delivery/flagger/loadtester.yaml create mode 100644 platform/cluster/flux/apps/delivery/flagger/namespace.yaml create mode 100644 platform/cluster/flux/apps/delivery/flagger/release.yaml create mode 100644 platform/cluster/flux/apps/delivery/flagger/source.yaml create mode 100644 platform/cluster/flux/apps/delivery/kustomization.yaml create mode 100644 platform/cluster/flux/apps/stateless/api/canary.yaml create mode 100644 platform/cluster/flux/apps/stateless/frontend/canary.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8e9ce59b..80fc39a90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,11 +7,9 @@ name: Build # its own GHA cache scope. # # main tags :sha- only. release.yml calls this with release-tag set, -# which additionally publishes : and then, in a single promote job, moves -# :latest — the tag Keel polls — for every service at once, so only releases -# reach production and no service reaches it alone. It is called rather than -# triggered by `release: published` because a release cut with GITHUB_TOKEN -# raises no event. +# which additionally publishes :, the tag Flux pins api and frontend to. +# It is called rather than triggered by `release: published` because a release +# cut with GITHUB_TOKEN raises no event. on: push: branches: [main] @@ -28,9 +26,9 @@ on: inputs: release-tag: description: >- - Tag to publish and point :latest at, e.g. v1.2.0. Empty builds and pushes - images without moving :latest. Naming an older tag moves :latest backwards and - Keel will roll it out, so that is a rollback rather than a retry. + Tag to republish, e.g. v1.2.0, when a release half-finished. Empty builds and + pushes images without moving any :latest. Rolling api or frontend back is a git + change to the pinned tag, not a run of this workflow. type: string default: '' @@ -322,20 +320,9 @@ jobs: fi promote: - # Moves :latest, the tag Keel polls, for every service in this release — - # in one job, one after the other, and only once every image exists under - # its release tag. - # - # A matrix would publish each service's :latest in its own job, and Keel - # polling in the gap between two of them would roll one Deployment against - # the previous version of another. The gap is seconds and the poll is every - # two minutes, which makes it the kind of race that shows up in production - # rather than in a pipeline. - # - # Skew is narrowed rather than eliminated: Keel still rolls each Deployment - # separately and they come up at different speeds, so an api and a frontend - # from adjacent releases can still meet briefly. One release of backward - # compatibility remains the contract. + # Moves :latest for the Keel-driven images only. api and frontend deploy + # from the tag pinned in git, so a :latest for them would be a second, + # racing trigger on the same Deployments. name: Promote release to :latest needs: [detect-changes, merge] if: needs.detect-changes.outputs.has-matrix == 'true' && inputs.release-tag != '' @@ -361,6 +348,8 @@ jobs: env: SERVICES: ${{ needs.detect-changes.outputs.services }} RELEASE_TAG: ${{ inputs.release-tag }} + # Images whose Deployment carries keel.sh annotations. + KEEL_SERVICES: '["stalwart-tools"]' run: | set -euo pipefail @@ -379,7 +368,12 @@ jobs: # earlier can still read as missing; retrying was not enough and the # release failed for a tag that had in fact moved. `imagetools create` # reports its own failure, which is the signal that means something. - mapfile -t services < <(jq -r '.[]' <<< "$SERVICES") + mapfile -t services < <(jq -r --argjson keel "$KEEL_SERVICES" \ + '. - (. - $keel) | .[]' <<< "$SERVICES") + if [[ ${#services[@]} -eq 0 ]]; then + echo "No Keel-driven image in this release — nothing to promote." + exit 0 + fi for service in "${services[@]}"; do image="${REGISTRY}/${IMAGE_PREFIX}/${service}" echo "::group::${service}" diff --git a/README.md b/README.md index d106ec746..a668c5db5 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,15 @@ IntelliJ: **Remote JVM Debug → host: localhost, port: 5005**. ## Production deployment Production runs on a single-node NixOS + k3s + FluxCD stack. Flux reconciles -manifests from `platform/cluster/flux/` against `main`; Keel polls -`ghcr.io/esa-blueshell/*` for new `:latest` tags and rolls the matching -Deployments. There is no CI deploy step — pushing to `main` is the deploy. +manifests from `platform/cluster/flux/` against `main`. The api and the +frontend run one release tag, pinned in +`platform/cluster/flux/apps/stateless/kustomization.yaml`. Flagger runs each as +a blue/green canary and the two `confirm-promotion` gates wait for one another, +so a release promotes both or neither. Keel still polls `:latest` for the +remaining images. + +That tag is bumped by hand for now. Until the release pipeline writes it +(#1293), cutting a release publishes images but does not deploy them. Runbook: [`platform/docs/runbook.md`](platform/docs/runbook.md). diff --git a/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml b/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml new file mode 100644 index 000000000..3cd09aa18 --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - source.yaml + - release.yaml + - loadtester.yaml diff --git a/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml b/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml new file mode 100644 index 000000000..9b1164088 --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml @@ -0,0 +1,58 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: flagger-loadtester + namespace: flagger-system +spec: + interval: 30m + chart: + spec: + chart: loadtester + sourceRef: + kind: HelmRepository + name: flagger + namespace: flagger-system + values: + # Runs the Canary webhooks: the acceptance tests against the -canary + # Services, and the confirm-promotion gate that reads the other Canary's + # phase with the kubectl the image ships. + logLevel: info + cmd: + timeout: 1h + rbac: + create: true + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi +--- +# The confirm-promotion gate reads the sibling Canary. Namespaced Role rather +# than a ClusterRole: the only Canaries it may see are these two. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: canary-phase-reader + namespace: default +rules: + - apiGroups: + - flagger.app + resources: + - canaries + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: canary-phase-reader + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: canary-phase-reader +subjects: + - kind: ServiceAccount + name: flagger-loadtester + namespace: flagger-system diff --git a/platform/cluster/flux/apps/delivery/flagger/namespace.yaml b/platform/cluster/flux/apps/delivery/flagger/namespace.yaml new file mode 100644 index 000000000..1f7400c3b --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flagger-system diff --git a/platform/cluster/flux/apps/delivery/flagger/release.yaml b/platform/cluster/flux/apps/delivery/flagger/release.yaml new file mode 100644 index 000000000..80898405d --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/release.yaml @@ -0,0 +1,36 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: flagger + namespace: flagger-system +spec: + interval: 30m + chart: + spec: + chart: flagger + sourceRef: + kind: HelmRepository + name: flagger + namespace: flagger-system + values: + # L4 only. There is no service mesh here, so Flagger drives blue/green + # by swapping Deployments behind a plain Service rather than by shifting + # traffic weights. + meshProvider: kubernetes + # No Prometheus here, so metricsServer keeps its unreachable chart default + # and Flagger logs one warning at boot. The Canaries define no metric + # checks, only webhooks, so nothing ever queries it. + prometheus: + install: false + podMonitor: + enabled: false + rbac: + create: true + serviceAccount: + create: true + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 256Mi diff --git a/platform/cluster/flux/apps/delivery/flagger/source.yaml b/platform/cluster/flux/apps/delivery/flagger/source.yaml new file mode 100644 index 000000000..bb9afad8b --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/source.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: flagger + namespace: flagger-system +spec: + interval: 1h + url: https://flagger.app diff --git a/platform/cluster/flux/apps/delivery/kustomization.yaml b/platform/cluster/flux/apps/delivery/kustomization.yaml new file mode 100644 index 000000000..f3c485aaa --- /dev/null +++ b/platform/cluster/flux/apps/delivery/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Flagger sits here rather than in apps-core because everything else depends on +# apps-core: a failed install there stops cert-manager, Traefik and Vault +# reconciling to deploy a canary controller. Only apps-stateless waits on this. +resources: + - flagger diff --git a/platform/cluster/flux/apps/stateless/api/canary.yaml b/platform/cluster/flux/apps/stateless/api/canary.yaml new file mode 100644 index 000000000..614cd18e2 --- /dev/null +++ b/platform/cluster/flux/apps/stateless/api/canary.yaml @@ -0,0 +1,50 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: api + namespace: default +spec: + provider: kubernetes + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: api + # Flagger owns the api, api-primary and api-canary Services from this block. + # The Traefik IngressRoute keeps pointing at `api`, which Flagger holds on + # the primary pods except during a promotion. + service: + port: 8080 + portName: http + targetPort: http + # Matches the Deployment: a cold Spring Boot start on a fresh image pull. + progressDeadlineSeconds: 900 + analysis: + # Blue/green, not weighted traffic shifting: iterations without stepWeight. + # Four 30s rounds of acceptance tests against api-canary before promotion. + interval: 30s + iterations: 4 + threshold: 3 + webhooks: + - name: canary-answers-http + type: rollout + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: curl -sf http://api-canary.default:8080/health + + # The rendezvous. Both Canaries hold here until the other one is out of + # Progressing and not Failed, so the release promotes as a pair or not at + # all. A gate that keeps failing holds the phase at WaitingPromotion + # without burning the failure threshold, so a broken frontend parks the + # api instead of shipping half a release. + - name: wait-for-frontend + type: confirm-promotion + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: >- + phase=$(kubectl -n default get canary frontend -o jsonpath='{.status.phase}'); + echo "frontend=$phase"; + [ "$phase" != Progressing ] && [ "$phase" != Failed ] diff --git a/platform/cluster/flux/apps/stateless/api/deployment.yaml b/platform/cluster/flux/apps/stateless/api/deployment.yaml index dcb810847..fe382d0fb 100644 --- a/platform/cluster/flux/apps/stateless/api/deployment.yaml +++ b/platform/cluster/flux/apps/stateless/api/deployment.yaml @@ -9,30 +9,22 @@ kind: Deployment metadata: name: api namespace: default - annotations: - # Keel polls GHCR every 2 min for the :latest digest and rolls when it - # changes. match-tag scopes Keel to the tag already on the Deployment - # rather than scanning every tag in the repo and picking the "highest" — - # without it Keel would pin to an old : tag that sorts above latest. - keel.sh/policy: force - keel.sh/match-tag: 'true' - keel.sh/trigger: poll - keel.sh/pollSchedule: '@every 2m' - # Keel's rollout message can only name digests on :latest, so it - # appends this link. - keel.sh/releaseNotes: https://github.com/ESA-Blueshell/website/releases + # Keel annotations removed: Keel rolls each Deployment on its own, which is + # the skew this pinning exists to prevent. spec: - replicas: 1 + # No `replicas`: Flagger scales this Deployment between 0 and 1 around a + # canary run, and a value here would make Flux reset it every 2 minutes. + # # Zero-downtime rollout: bring a second pod up first (maxSurge=1) and only # tear the old one down once the new pod's readiness probe reports # ACCEPTING_TRAFFIC (maxUnavailable=0). A broken image — failed Flyway # migration, Vault token miss, anything that keeps /actuator/health/readiness - # red — never displaces the healthy old pod, so the rollout silently - # ProgressDeadline-stalls instead of taking the api offline. + # red — never displaces the healthy old pod, so the rollout stalls instead of + # taking the api offline. # - # progressDeadlineSeconds is generous because Spring Boot's cold start with - # Flyway + Hibernate + Vault agent handshake can take several minutes on a - # fresh image pull. + # progressDeadlineSeconds is generous: Spring Boot cold start takes minutes, + # and the release commit reaches Flux before the release image is built, so + # the new pod sits in ImagePullBackOff first. strategy: type: RollingUpdate rollingUpdate: @@ -105,8 +97,9 @@ spec: - name: ghcr-pull-secret containers: - name: api - image: ghcr.io/esa-blueshell/api:latest - imagePullPolicy: Always + # Tag comes from the images block in ../kustomization.yaml. + image: ghcr.io/esa-blueshell/api + imagePullPolicy: IfNotPresent command: - /bin/sh - -ec @@ -319,16 +312,3 @@ spec: - name: storage persistentVolumeClaim: claimName: api-storage ---- -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: default -spec: - selector: - app.kubernetes.io/name: api - ports: - - name: http - port: 8080 - targetPort: http diff --git a/platform/cluster/flux/apps/stateless/api/kustomization.yaml b/platform/cluster/flux/apps/stateless/api/kustomization.yaml index c96545467..886873892 100644 --- a/platform/cluster/flux/apps/stateless/api/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/api/kustomization.yaml @@ -4,3 +4,4 @@ kind: Kustomization resources: - pvc.yaml - deployment.yaml + - canary.yaml diff --git a/platform/cluster/flux/apps/stateless/frontend/canary.yaml b/platform/cluster/flux/apps/stateless/frontend/canary.yaml new file mode 100644 index 000000000..dc5f05ca4 --- /dev/null +++ b/platform/cluster/flux/apps/stateless/frontend/canary.yaml @@ -0,0 +1,42 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: frontend + namespace: default +spec: + provider: kubernetes + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + # Flagger owns the frontend, frontend-primary and frontend-canary Services + # from this block. + service: + port: 3000 + portName: http + targetPort: http + progressDeadlineSeconds: 900 + analysis: + interval: 30s + iterations: 4 + threshold: 3 + webhooks: + - name: canary-answers-http + type: rollout + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: curl -sf http://frontend-canary.default:3000/healthz + + # Mirror of the api's gate; see the comment there. + - name: wait-for-api + type: confirm-promotion + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: >- + phase=$(kubectl -n default get canary api -o jsonpath='{.status.phase}'); + echo "api=$phase"; + [ "$phase" != Progressing ] && [ "$phase" != Failed ] diff --git a/platform/cluster/flux/apps/stateless/frontend/deployment.yaml b/platform/cluster/flux/apps/stateless/frontend/deployment.yaml index 8a1ed77b9..027370398 100644 --- a/platform/cluster/flux/apps/stateless/frontend/deployment.yaml +++ b/platform/cluster/flux/apps/stateless/frontend/deployment.yaml @@ -3,21 +3,23 @@ kind: Deployment metadata: name: frontend namespace: default - annotations: - keel.sh/policy: force - keel.sh/match-tag: 'true' - keel.sh/trigger: poll - keel.sh/pollSchedule: '@every 2m' - keel.sh/releaseNotes: https://github.com/ESA-Blueshell/website/releases + # Keel annotations removed: Keel rolls each Deployment on its own, which is + # the skew this pinning exists to prevent. spec: - replicas: 1 - # Single replica, but replaced with zero downtime: surge a second pod and - # only retire the old one once the new pod is Ready (maxUnavailable=0). + # No `replicas`: Flagger owns it; see the api Deployment. + # + # Surge first, retire the old pod only once the new one is Ready, so a broken + # image stalls the rollout instead of taking the site down. + # + # progressDeadlineSeconds matches the api's: the release commit reaches Flux + # before the release image is built, so the new pod sits in ImagePullBackOff + # first. strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 maxUnavailable: 0 + progressDeadlineSeconds: 900 selector: matchLabels: app.kubernetes.io/name: frontend @@ -33,19 +35,36 @@ spec: - name: ghcr-pull-secret containers: - name: frontend - image: ghcr.io/esa-blueshell/frontend:latest - imagePullPolicy: Always + # Tag comes from the images block in ../kustomization.yaml. + image: ghcr.io/esa-blueshell/frontend + imagePullPolicy: IfNotPresent ports: - containerPort: 3000 name: http + # /healthz is nginx's own `return 200` (nginx.conf) and does not + # probe the api upstream: an api outage must not empty the frontend's + # endpoints too. + startupProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 2 + failureThreshold: 30 + timeoutSeconds: 2 readinessProbe: httpGet: path: /healthz port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 livenessProbe: httpGet: path: /healthz port: http + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 6 resources: requests: cpu: 50m @@ -53,16 +72,3 @@ spec: limits: cpu: 200m memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend - namespace: default -spec: - selector: - app.kubernetes.io/name: frontend - ports: - - name: http - port: 3000 - targetPort: http diff --git a/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml b/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml index 88a04b541..762b3aca3 100644 --- a/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - deployment.yaml + - canary.yaml diff --git a/platform/cluster/flux/apps/stateless/kustomization.yaml b/platform/cluster/flux/apps/stateless/kustomization.yaml index 289d02678..177b38379 100644 --- a/platform/cluster/flux/apps/stateless/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/kustomization.yaml @@ -4,3 +4,13 @@ kind: Kustomization resources: - api - frontend + +# Only place the deployed version is written, and it is written once for both, +# so api and frontend cannot diverge. Bumped by hand until the release pipeline +# writes a digest here (#1293); until then a release does not reach production +# on its own. +images: + - name: ghcr.io/esa-blueshell/api + newTag: v1.7.1 + - name: ghcr.io/esa-blueshell/frontend + newTag: v1.7.1 diff --git a/platform/cluster/flux/clusters/production/kustomizations.yaml b/platform/cluster/flux/clusters/production/kustomizations.yaml index 3eac5d37a..85b868ce0 100644 --- a/platform/cluster/flux/clusters/production/kustomizations.yaml +++ b/platform/cluster/flux/clusters/production/kustomizations.yaml @@ -44,6 +44,27 @@ spec: kind: GitRepository name: flux-system --- +# Flagger and its loadtester, kept out of apps-core on purpose: everything else +# depends on apps-core, so a failed install there would stop cert-manager, +# Traefik and Vault reconciling. wait, because apps-stateless applies Canary CRs +# that need Flagger's CRDs present. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps-delivery + namespace: flux-system +spec: + interval: 2m0s + path: ./platform/cluster/flux/apps/delivery + prune: true + wait: true + timeout: 10m + dependsOn: + - name: apps-core + sourceRef: + kind: GitRepository + name: flux-system +--- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -105,6 +126,13 @@ spec: kind: GitRepository name: flux-system --- +# api and frontend are one release: one apply carries both pinned tags, and +# Flagger's paired confirm-promotion gates hold each canary until the other is +# ready, so the pair promotes together or neither does. +# +# The health checks name the Flagger-managed -primary Deployments, which are +# what the Services point at; the source Deployments sit at 0 replicas between +# canary runs. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -114,9 +142,21 @@ spec: interval: 2m0s path: ./platform/cluster/flux/apps/stateless prune: true + timeout: 20m + retryInterval: 2m0s + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: api-primary + namespace: default + - apiVersion: apps/v1 + kind: Deployment + name: frontend-primary + namespace: default dependsOn: - name: apps-core - name: apps-data + - name: apps-delivery - name: apps-vso-secrets - name: apps-edge sourceRef: diff --git a/platform/docs/bringup-v2.md b/platform/docs/bringup-v2.md index 277978c6d..24bcf3dad 100644 --- a/platform/docs/bringup-v2.md +++ b/platform/docs/bringup-v2.md @@ -130,8 +130,9 @@ flux get kustomizations --watch Reconciliation order: `flux-system` → `apps-core` (2–5 min) → `apps-data` (stalls pending Vault unseal — expected, proceed to -step 6 in parallel) → `apps-edge` / `apps-vso-secrets` / -`apps-mail` / `apps-stateless` / `apps-utility-system`. +step 6 in parallel) / `apps-delivery` → `apps-edge` / +`apps-vso-secrets` / `apps-mail` / `apps-stateless` / +`apps-utility-system`. ## 6. Unseal Vault and seed secrets @@ -298,5 +299,7 @@ mv ~/.ssh/blueshell-admin.pub ~/.ssh/blueshell-admin.pub.retired post-activation SSH health check fails. - **Apps** (anything under `platform/cluster/flux/`): Flux reconciles - `main` every minute. Keel polls GHCR every 2 min and rolls api + - frontend when their `:latest` digest changes. No manual step. + `main` every minute. api + frontend roll together when a release + bumps their pinned tag in `apps/stateless/kustomization.yaml`, via + paired Flagger canaries; Keel polls GHCR every 2 min for the + remaining `:latest` images. No manual step. diff --git a/platform/docs/flux-bootstrap.md b/platform/docs/flux-bootstrap.md index 5194462ac..59acf636c 100644 --- a/platform/docs/flux-bootstrap.md +++ b/platform/docs/flux-bootstrap.md @@ -35,10 +35,15 @@ Expected order (each step waits for the previous): 1. `flux-system` — Kustomization + GitRepository seeded. 2. `apps-core` — HelmReleases for cert-manager, external-dns, Traefik, Vault Secrets Operator install and report Ready. Takes 2–5 minutes. -3. `apps-edge` — ClusterIssuer, wildcard Certificate + TLSStore, and + Everything else depends on this one, which is why Flagger is not in + it: a controller that only `apps-stateless` needs must not be able + to stall cert-manager and Traefik. +3. `apps-delivery` — Flagger and its loadtester. Waits for Ready so the + Canary CRDs exist before `apps-stateless` applies Canaries. +4. `apps-edge` — ClusterIssuer, wildcard Certificate + TLSStore, and forward-auth Middleware land. The wildcard cert takes the longest (DNS-01 propagation + ACME order). -4. `apps-utility-system` — Headlamp + Keel deploy. Headlamp login +5. `apps-utility-system` — Headlamp + Keel deploy. Headlamp login will return errors until the website api OIDC issuer lands in its own PR; the pod is otherwise healthy. diff --git a/platform/docs/runbook.md b/platform/docs/runbook.md index 53f46d55f..e5116a296 100644 --- a/platform/docs/runbook.md +++ b/platform/docs/runbook.md @@ -5,8 +5,9 @@ The `platform/` tree manages the production stack: Production runs on a single Contabo VPS (`frankfurt-contabo-1`) under `esa-blueshell.nl`. Flux reconciles every manifest from this -repository against `main`; Keel rolls Deployments when new -`ghcr.io/esa-blueshell/*:latest` images appear. +repository against `main`. The api and the frontend are pinned to one +release tag and promoted as a pair by Flagger (see below); Keel still +rolls the remaining Deployments when a new `:latest` image appears. Detailed setup guides: @@ -19,6 +20,66 @@ Detailed setup guides: - [`nix-flake.md`](nix-flake.md) — NixOS flake structure + host definitions. +## Releasing api + frontend + +Both images are pinned to one tag in +[`apps/stateless/kustomization.yaml`](../cluster/flux/apps/stateless/kustomization.yaml), +bumped by hand until the release pipeline writes a digest there (#1293). +Cutting a release therefore publishes images without deploying them; +edit that one file and push to roll both services. + +Flux applies the pair; Flagger turns each one +into a blue/green rollout and the two `confirm-promotion` gates hold +each canary until the other is out of `Progressing` and not `Failed`. +One release of backward compatibility is still the contract: promotion +copies the canary spec onto the `-primary` Deployment, which then rolls +normally, so the two apex Services finish flipping seconds apart. + +```bash +kubectl -n default get canaries +kubectl -n default describe canary api | tail -30 +flux -n flux-system get kustomization apps-stateless +``` + +Phases: `Initializing` → `Initialized` on first install, then +`Progressing` → `WaitingPromotion` → `Promoting` → `Succeeded` per +release. A canary parked in `WaitingPromotion` is waiting for its +sibling — check the other one's phase before touching anything. + +`Failed` means the analysis or the acceptance webhook failed; Flagger +scales the canary down and leaves the primary serving the previous +release. Recover by reverting the tag in git, not by deleting pods. + +Manual rollback: set both `newTag` values back to the previous release +tag and push. Never move one without the other. + +### Merging a change you do not want applied immediately + +Flux reconciles `main` every minute and there is no other gate, so a +merge is a deploy. Suspend the Kustomization that owns the change +first, merge, read what it would apply, then resume while watching: + +```bash +flux -n flux-system suspend kustomization apps-stateless +# merge, then: +flux -n flux-system build kustomization apps-stateless --path ./platform/cluster/flux/apps/stateless +flux -n flux-system resume kustomization apps-stateless +flux -n flux-system get kustomization apps-stateless --watch +``` + +Suspending stops drift correction for that subtree as well, so resume +the same day. `flux diff kustomization` shows the change against the +live cluster if you want it before resuming. + +### First rollout onto Flagger + +Expect an outage of roughly one api cold start. Flux prunes the +hand-written `api` and `frontend` Services in the same apply that +creates the Canaries, and Flagger only recreates them once its +`-primary` Deployments are Ready. Do it in a quiet window, with +`apps-stateless` suspended per the procedure above, and watch +`kubectl -n default get canaries,svc`. + ## User uploads The api persists uploads to `/srv/blueshell/storage`, backed by a