diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8e9ce59b..80fc39a90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,11 +7,9 @@ name: Build # its own GHA cache scope. # # main tags :sha- only. release.yml calls this with release-tag set, -# which additionally publishes : and then, in a single promote job, moves -# :latest — the tag Keel polls — for every service at once, so only releases -# reach production and no service reaches it alone. It is called rather than -# triggered by `release: published` because a release cut with GITHUB_TOKEN -# raises no event. +# which additionally publishes :, the tag Flux pins api and frontend to. +# It is called rather than triggered by `release: published` because a release +# cut with GITHUB_TOKEN raises no event. on: push: branches: [main] @@ -28,9 +26,9 @@ on: inputs: release-tag: description: >- - Tag to publish and point :latest at, e.g. v1.2.0. Empty builds and pushes - images without moving :latest. Naming an older tag moves :latest backwards and - Keel will roll it out, so that is a rollback rather than a retry. + Tag to republish, e.g. v1.2.0, when a release half-finished. Empty builds and + pushes images without moving any :latest. Rolling api or frontend back is a git + change to the pinned tag, not a run of this workflow. type: string default: '' @@ -322,20 +320,9 @@ jobs: fi promote: - # Moves :latest, the tag Keel polls, for every service in this release — - # in one job, one after the other, and only once every image exists under - # its release tag. - # - # A matrix would publish each service's :latest in its own job, and Keel - # polling in the gap between two of them would roll one Deployment against - # the previous version of another. The gap is seconds and the poll is every - # two minutes, which makes it the kind of race that shows up in production - # rather than in a pipeline. - # - # Skew is narrowed rather than eliminated: Keel still rolls each Deployment - # separately and they come up at different speeds, so an api and a frontend - # from adjacent releases can still meet briefly. One release of backward - # compatibility remains the contract. + # Moves :latest for the Keel-driven images only. api and frontend deploy + # from the tag pinned in git, so a :latest for them would be a second, + # racing trigger on the same Deployments. name: Promote release to :latest needs: [detect-changes, merge] if: needs.detect-changes.outputs.has-matrix == 'true' && inputs.release-tag != '' @@ -361,6 +348,8 @@ jobs: env: SERVICES: ${{ needs.detect-changes.outputs.services }} RELEASE_TAG: ${{ inputs.release-tag }} + # Images whose Deployment carries keel.sh annotations. + KEEL_SERVICES: '["stalwart-tools"]' run: | set -euo pipefail @@ -379,7 +368,12 @@ jobs: # earlier can still read as missing; retrying was not enough and the # release failed for a tag that had in fact moved. `imagetools create` # reports its own failure, which is the signal that means something. - mapfile -t services < <(jq -r '.[]' <<< "$SERVICES") + mapfile -t services < <(jq -r --argjson keel "$KEEL_SERVICES" \ + '. - (. - $keel) | .[]' <<< "$SERVICES") + if [[ ${#services[@]} -eq 0 ]]; then + echo "No Keel-driven image in this release — nothing to promote." + exit 0 + fi for service in "${services[@]}"; do image="${REGISTRY}/${IMAGE_PREFIX}/${service}" echo "::group::${service}" diff --git a/README.md b/README.md index d106ec746..a668c5db5 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,15 @@ IntelliJ: **Remote JVM Debug → host: localhost, port: 5005**. ## Production deployment Production runs on a single-node NixOS + k3s + FluxCD stack. Flux reconciles -manifests from `platform/cluster/flux/` against `main`; Keel polls -`ghcr.io/esa-blueshell/*` for new `:latest` tags and rolls the matching -Deployments. There is no CI deploy step — pushing to `main` is the deploy. +manifests from `platform/cluster/flux/` against `main`. The api and the +frontend run one release tag, pinned in +`platform/cluster/flux/apps/stateless/kustomization.yaml`. Flagger runs each as +a blue/green canary and the two `confirm-promotion` gates wait for one another, +so a release promotes both or neither. Keel still polls `:latest` for the +remaining images. + +That tag is bumped by hand for now. Until the release pipeline writes it +(#1293), cutting a release publishes images but does not deploy them. Runbook: [`platform/docs/runbook.md`](platform/docs/runbook.md). diff --git a/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml b/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml new file mode 100644 index 000000000..3cd09aa18 --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - source.yaml + - release.yaml + - loadtester.yaml diff --git a/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml b/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml new file mode 100644 index 000000000..9b1164088 --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/loadtester.yaml @@ -0,0 +1,58 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: flagger-loadtester + namespace: flagger-system +spec: + interval: 30m + chart: + spec: + chart: loadtester + sourceRef: + kind: HelmRepository + name: flagger + namespace: flagger-system + values: + # Runs the Canary webhooks: the acceptance tests against the -canary + # Services, and the confirm-promotion gate that reads the other Canary's + # phase with the kubectl the image ships. + logLevel: info + cmd: + timeout: 1h + rbac: + create: true + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi +--- +# The confirm-promotion gate reads the sibling Canary. Namespaced Role rather +# than a ClusterRole: the only Canaries it may see are these two. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: canary-phase-reader + namespace: default +rules: + - apiGroups: + - flagger.app + resources: + - canaries + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: canary-phase-reader + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: canary-phase-reader +subjects: + - kind: ServiceAccount + name: flagger-loadtester + namespace: flagger-system diff --git a/platform/cluster/flux/apps/delivery/flagger/namespace.yaml b/platform/cluster/flux/apps/delivery/flagger/namespace.yaml new file mode 100644 index 000000000..1f7400c3b --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flagger-system diff --git a/platform/cluster/flux/apps/delivery/flagger/release.yaml b/platform/cluster/flux/apps/delivery/flagger/release.yaml new file mode 100644 index 000000000..80898405d --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/release.yaml @@ -0,0 +1,36 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: flagger + namespace: flagger-system +spec: + interval: 30m + chart: + spec: + chart: flagger + sourceRef: + kind: HelmRepository + name: flagger + namespace: flagger-system + values: + # L4 only. There is no service mesh here, so Flagger drives blue/green + # by swapping Deployments behind a plain Service rather than by shifting + # traffic weights. + meshProvider: kubernetes + # No Prometheus here, so metricsServer keeps its unreachable chart default + # and Flagger logs one warning at boot. The Canaries define no metric + # checks, only webhooks, so nothing ever queries it. + prometheus: + install: false + podMonitor: + enabled: false + rbac: + create: true + serviceAccount: + create: true + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 256Mi diff --git a/platform/cluster/flux/apps/delivery/flagger/source.yaml b/platform/cluster/flux/apps/delivery/flagger/source.yaml new file mode 100644 index 000000000..bb9afad8b --- /dev/null +++ b/platform/cluster/flux/apps/delivery/flagger/source.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: flagger + namespace: flagger-system +spec: + interval: 1h + url: https://flagger.app diff --git a/platform/cluster/flux/apps/delivery/kustomization.yaml b/platform/cluster/flux/apps/delivery/kustomization.yaml new file mode 100644 index 000000000..f3c485aaa --- /dev/null +++ b/platform/cluster/flux/apps/delivery/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Flagger sits here rather than in apps-core because everything else depends on +# apps-core: a failed install there stops cert-manager, Traefik and Vault +# reconciling to deploy a canary controller. Only apps-stateless waits on this. +resources: + - flagger diff --git a/platform/cluster/flux/apps/stateless/api/canary.yaml b/platform/cluster/flux/apps/stateless/api/canary.yaml new file mode 100644 index 000000000..614cd18e2 --- /dev/null +++ b/platform/cluster/flux/apps/stateless/api/canary.yaml @@ -0,0 +1,50 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: api + namespace: default +spec: + provider: kubernetes + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: api + # Flagger owns the api, api-primary and api-canary Services from this block. + # The Traefik IngressRoute keeps pointing at `api`, which Flagger holds on + # the primary pods except during a promotion. + service: + port: 8080 + portName: http + targetPort: http + # Matches the Deployment: a cold Spring Boot start on a fresh image pull. + progressDeadlineSeconds: 900 + analysis: + # Blue/green, not weighted traffic shifting: iterations without stepWeight. + # Four 30s rounds of acceptance tests against api-canary before promotion. + interval: 30s + iterations: 4 + threshold: 3 + webhooks: + - name: canary-answers-http + type: rollout + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: curl -sf http://api-canary.default:8080/health + + # The rendezvous. Both Canaries hold here until the other one is out of + # Progressing and not Failed, so the release promotes as a pair or not at + # all. A gate that keeps failing holds the phase at WaitingPromotion + # without burning the failure threshold, so a broken frontend parks the + # api instead of shipping half a release. + - name: wait-for-frontend + type: confirm-promotion + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: >- + phase=$(kubectl -n default get canary frontend -o jsonpath='{.status.phase}'); + echo "frontend=$phase"; + [ "$phase" != Progressing ] && [ "$phase" != Failed ] diff --git a/platform/cluster/flux/apps/stateless/api/deployment.yaml b/platform/cluster/flux/apps/stateless/api/deployment.yaml index dcb810847..fe382d0fb 100644 --- a/platform/cluster/flux/apps/stateless/api/deployment.yaml +++ b/platform/cluster/flux/apps/stateless/api/deployment.yaml @@ -9,30 +9,22 @@ kind: Deployment metadata: name: api namespace: default - annotations: - # Keel polls GHCR every 2 min for the :latest digest and rolls when it - # changes. match-tag scopes Keel to the tag already on the Deployment - # rather than scanning every tag in the repo and picking the "highest" — - # without it Keel would pin to an old : tag that sorts above latest. - keel.sh/policy: force - keel.sh/match-tag: 'true' - keel.sh/trigger: poll - keel.sh/pollSchedule: '@every 2m' - # Keel's rollout message can only name digests on :latest, so it - # appends this link. - keel.sh/releaseNotes: https://github.com/ESA-Blueshell/website/releases + # Keel annotations removed: Keel rolls each Deployment on its own, which is + # the skew this pinning exists to prevent. spec: - replicas: 1 + # No `replicas`: Flagger scales this Deployment between 0 and 1 around a + # canary run, and a value here would make Flux reset it every 2 minutes. + # # Zero-downtime rollout: bring a second pod up first (maxSurge=1) and only # tear the old one down once the new pod's readiness probe reports # ACCEPTING_TRAFFIC (maxUnavailable=0). A broken image — failed Flyway # migration, Vault token miss, anything that keeps /actuator/health/readiness - # red — never displaces the healthy old pod, so the rollout silently - # ProgressDeadline-stalls instead of taking the api offline. + # red — never displaces the healthy old pod, so the rollout stalls instead of + # taking the api offline. # - # progressDeadlineSeconds is generous because Spring Boot's cold start with - # Flyway + Hibernate + Vault agent handshake can take several minutes on a - # fresh image pull. + # progressDeadlineSeconds is generous: Spring Boot cold start takes minutes, + # and the release commit reaches Flux before the release image is built, so + # the new pod sits in ImagePullBackOff first. strategy: type: RollingUpdate rollingUpdate: @@ -105,8 +97,9 @@ spec: - name: ghcr-pull-secret containers: - name: api - image: ghcr.io/esa-blueshell/api:latest - imagePullPolicy: Always + # Tag comes from the images block in ../kustomization.yaml. + image: ghcr.io/esa-blueshell/api + imagePullPolicy: IfNotPresent command: - /bin/sh - -ec @@ -319,16 +312,3 @@ spec: - name: storage persistentVolumeClaim: claimName: api-storage ---- -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: default -spec: - selector: - app.kubernetes.io/name: api - ports: - - name: http - port: 8080 - targetPort: http diff --git a/platform/cluster/flux/apps/stateless/api/kustomization.yaml b/platform/cluster/flux/apps/stateless/api/kustomization.yaml index c96545467..886873892 100644 --- a/platform/cluster/flux/apps/stateless/api/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/api/kustomization.yaml @@ -4,3 +4,4 @@ kind: Kustomization resources: - pvc.yaml - deployment.yaml + - canary.yaml diff --git a/platform/cluster/flux/apps/stateless/frontend/canary.yaml b/platform/cluster/flux/apps/stateless/frontend/canary.yaml new file mode 100644 index 000000000..dc5f05ca4 --- /dev/null +++ b/platform/cluster/flux/apps/stateless/frontend/canary.yaml @@ -0,0 +1,42 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: frontend + namespace: default +spec: + provider: kubernetes + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + # Flagger owns the frontend, frontend-primary and frontend-canary Services + # from this block. + service: + port: 3000 + portName: http + targetPort: http + progressDeadlineSeconds: 900 + analysis: + interval: 30s + iterations: 4 + threshold: 3 + webhooks: + - name: canary-answers-http + type: rollout + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: curl -sf http://frontend-canary.default:3000/healthz + + # Mirror of the api's gate; see the comment there. + - name: wait-for-api + type: confirm-promotion + url: http://flagger-loadtester.flagger-system/ + timeout: 15s + metadata: + type: bash + cmd: >- + phase=$(kubectl -n default get canary api -o jsonpath='{.status.phase}'); + echo "api=$phase"; + [ "$phase" != Progressing ] && [ "$phase" != Failed ] diff --git a/platform/cluster/flux/apps/stateless/frontend/deployment.yaml b/platform/cluster/flux/apps/stateless/frontend/deployment.yaml index 8a1ed77b9..027370398 100644 --- a/platform/cluster/flux/apps/stateless/frontend/deployment.yaml +++ b/platform/cluster/flux/apps/stateless/frontend/deployment.yaml @@ -3,21 +3,23 @@ kind: Deployment metadata: name: frontend namespace: default - annotations: - keel.sh/policy: force - keel.sh/match-tag: 'true' - keel.sh/trigger: poll - keel.sh/pollSchedule: '@every 2m' - keel.sh/releaseNotes: https://github.com/ESA-Blueshell/website/releases + # Keel annotations removed: Keel rolls each Deployment on its own, which is + # the skew this pinning exists to prevent. spec: - replicas: 1 - # Single replica, but replaced with zero downtime: surge a second pod and - # only retire the old one once the new pod is Ready (maxUnavailable=0). + # No `replicas`: Flagger owns it; see the api Deployment. + # + # Surge first, retire the old pod only once the new one is Ready, so a broken + # image stalls the rollout instead of taking the site down. + # + # progressDeadlineSeconds matches the api's: the release commit reaches Flux + # before the release image is built, so the new pod sits in ImagePullBackOff + # first. strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 maxUnavailable: 0 + progressDeadlineSeconds: 900 selector: matchLabels: app.kubernetes.io/name: frontend @@ -33,19 +35,36 @@ spec: - name: ghcr-pull-secret containers: - name: frontend - image: ghcr.io/esa-blueshell/frontend:latest - imagePullPolicy: Always + # Tag comes from the images block in ../kustomization.yaml. + image: ghcr.io/esa-blueshell/frontend + imagePullPolicy: IfNotPresent ports: - containerPort: 3000 name: http + # /healthz is nginx's own `return 200` (nginx.conf) and does not + # probe the api upstream: an api outage must not empty the frontend's + # endpoints too. + startupProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 2 + failureThreshold: 30 + timeoutSeconds: 2 readinessProbe: httpGet: path: /healthz port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 livenessProbe: httpGet: path: /healthz port: http + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 6 resources: requests: cpu: 50m @@ -53,16 +72,3 @@ spec: limits: cpu: 200m memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend - namespace: default -spec: - selector: - app.kubernetes.io/name: frontend - ports: - - name: http - port: 3000 - targetPort: http diff --git a/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml b/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml index 88a04b541..762b3aca3 100644 --- a/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/frontend/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - deployment.yaml + - canary.yaml diff --git a/platform/cluster/flux/apps/stateless/kustomization.yaml b/platform/cluster/flux/apps/stateless/kustomization.yaml index 289d02678..177b38379 100644 --- a/platform/cluster/flux/apps/stateless/kustomization.yaml +++ b/platform/cluster/flux/apps/stateless/kustomization.yaml @@ -4,3 +4,13 @@ kind: Kustomization resources: - api - frontend + +# Only place the deployed version is written, and it is written once for both, +# so api and frontend cannot diverge. Bumped by hand until the release pipeline +# writes a digest here (#1293); until then a release does not reach production +# on its own. +images: + - name: ghcr.io/esa-blueshell/api + newTag: v1.7.1 + - name: ghcr.io/esa-blueshell/frontend + newTag: v1.7.1 diff --git a/platform/cluster/flux/clusters/production/kustomizations.yaml b/platform/cluster/flux/clusters/production/kustomizations.yaml index 3eac5d37a..85b868ce0 100644 --- a/platform/cluster/flux/clusters/production/kustomizations.yaml +++ b/platform/cluster/flux/clusters/production/kustomizations.yaml @@ -44,6 +44,27 @@ spec: kind: GitRepository name: flux-system --- +# Flagger and its loadtester, kept out of apps-core on purpose: everything else +# depends on apps-core, so a failed install there would stop cert-manager, +# Traefik and Vault reconciling. wait, because apps-stateless applies Canary CRs +# that need Flagger's CRDs present. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps-delivery + namespace: flux-system +spec: + interval: 2m0s + path: ./platform/cluster/flux/apps/delivery + prune: true + wait: true + timeout: 10m + dependsOn: + - name: apps-core + sourceRef: + kind: GitRepository + name: flux-system +--- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -105,6 +126,13 @@ spec: kind: GitRepository name: flux-system --- +# api and frontend are one release: one apply carries both pinned tags, and +# Flagger's paired confirm-promotion gates hold each canary until the other is +# ready, so the pair promotes together or neither does. +# +# The health checks name the Flagger-managed -primary Deployments, which are +# what the Services point at; the source Deployments sit at 0 replicas between +# canary runs. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -114,9 +142,21 @@ spec: interval: 2m0s path: ./platform/cluster/flux/apps/stateless prune: true + timeout: 20m + retryInterval: 2m0s + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: api-primary + namespace: default + - apiVersion: apps/v1 + kind: Deployment + name: frontend-primary + namespace: default dependsOn: - name: apps-core - name: apps-data + - name: apps-delivery - name: apps-vso-secrets - name: apps-edge sourceRef: diff --git a/platform/docs/bringup-v2.md b/platform/docs/bringup-v2.md index 277978c6d..24bcf3dad 100644 --- a/platform/docs/bringup-v2.md +++ b/platform/docs/bringup-v2.md @@ -130,8 +130,9 @@ flux get kustomizations --watch Reconciliation order: `flux-system` → `apps-core` (2–5 min) → `apps-data` (stalls pending Vault unseal — expected, proceed to -step 6 in parallel) → `apps-edge` / `apps-vso-secrets` / -`apps-mail` / `apps-stateless` / `apps-utility-system`. +step 6 in parallel) / `apps-delivery` → `apps-edge` / +`apps-vso-secrets` / `apps-mail` / `apps-stateless` / +`apps-utility-system`. ## 6. Unseal Vault and seed secrets @@ -298,5 +299,7 @@ mv ~/.ssh/blueshell-admin.pub ~/.ssh/blueshell-admin.pub.retired post-activation SSH health check fails. - **Apps** (anything under `platform/cluster/flux/`): Flux reconciles - `main` every minute. Keel polls GHCR every 2 min and rolls api + - frontend when their `:latest` digest changes. No manual step. + `main` every minute. api + frontend roll together when a release + bumps their pinned tag in `apps/stateless/kustomization.yaml`, via + paired Flagger canaries; Keel polls GHCR every 2 min for the + remaining `:latest` images. No manual step. diff --git a/platform/docs/flux-bootstrap.md b/platform/docs/flux-bootstrap.md index 5194462ac..59acf636c 100644 --- a/platform/docs/flux-bootstrap.md +++ b/platform/docs/flux-bootstrap.md @@ -35,10 +35,15 @@ Expected order (each step waits for the previous): 1. `flux-system` — Kustomization + GitRepository seeded. 2. `apps-core` — HelmReleases for cert-manager, external-dns, Traefik, Vault Secrets Operator install and report Ready. Takes 2–5 minutes. -3. `apps-edge` — ClusterIssuer, wildcard Certificate + TLSStore, and + Everything else depends on this one, which is why Flagger is not in + it: a controller that only `apps-stateless` needs must not be able + to stall cert-manager and Traefik. +3. `apps-delivery` — Flagger and its loadtester. Waits for Ready so the + Canary CRDs exist before `apps-stateless` applies Canaries. +4. `apps-edge` — ClusterIssuer, wildcard Certificate + TLSStore, and forward-auth Middleware land. The wildcard cert takes the longest (DNS-01 propagation + ACME order). -4. `apps-utility-system` — Headlamp + Keel deploy. Headlamp login +5. `apps-utility-system` — Headlamp + Keel deploy. Headlamp login will return errors until the website api OIDC issuer lands in its own PR; the pod is otherwise healthy. diff --git a/platform/docs/runbook.md b/platform/docs/runbook.md index 53f46d55f..e5116a296 100644 --- a/platform/docs/runbook.md +++ b/platform/docs/runbook.md @@ -5,8 +5,9 @@ The `platform/` tree manages the production stack: Production runs on a single Contabo VPS (`frankfurt-contabo-1`) under `esa-blueshell.nl`. Flux reconciles every manifest from this -repository against `main`; Keel rolls Deployments when new -`ghcr.io/esa-blueshell/*:latest` images appear. +repository against `main`. The api and the frontend are pinned to one +release tag and promoted as a pair by Flagger (see below); Keel still +rolls the remaining Deployments when a new `:latest` image appears. Detailed setup guides: @@ -19,6 +20,66 @@ Detailed setup guides: - [`nix-flake.md`](nix-flake.md) — NixOS flake structure + host definitions. +## Releasing api + frontend + +Both images are pinned to one tag in +[`apps/stateless/kustomization.yaml`](../cluster/flux/apps/stateless/kustomization.yaml), +bumped by hand until the release pipeline writes a digest there (#1293). +Cutting a release therefore publishes images without deploying them; +edit that one file and push to roll both services. + +Flux applies the pair; Flagger turns each one +into a blue/green rollout and the two `confirm-promotion` gates hold +each canary until the other is out of `Progressing` and not `Failed`. +One release of backward compatibility is still the contract: promotion +copies the canary spec onto the `-primary` Deployment, which then rolls +normally, so the two apex Services finish flipping seconds apart. + +```bash +kubectl -n default get canaries +kubectl -n default describe canary api | tail -30 +flux -n flux-system get kustomization apps-stateless +``` + +Phases: `Initializing` → `Initialized` on first install, then +`Progressing` → `WaitingPromotion` → `Promoting` → `Succeeded` per +release. A canary parked in `WaitingPromotion` is waiting for its +sibling — check the other one's phase before touching anything. + +`Failed` means the analysis or the acceptance webhook failed; Flagger +scales the canary down and leaves the primary serving the previous +release. Recover by reverting the tag in git, not by deleting pods. + +Manual rollback: set both `newTag` values back to the previous release +tag and push. Never move one without the other. + +### Merging a change you do not want applied immediately + +Flux reconciles `main` every minute and there is no other gate, so a +merge is a deploy. Suspend the Kustomization that owns the change +first, merge, read what it would apply, then resume while watching: + +```bash +flux -n flux-system suspend kustomization apps-stateless +# merge, then: +flux -n flux-system build kustomization apps-stateless --path ./platform/cluster/flux/apps/stateless +flux -n flux-system resume kustomization apps-stateless +flux -n flux-system get kustomization apps-stateless --watch +``` + +Suspending stops drift correction for that subtree as well, so resume +the same day. `flux diff kustomization` shows the change against the +live cluster if you want it before resuming. + +### First rollout onto Flagger + +Expect an outage of roughly one api cold start. Flux prunes the +hand-written `api` and `frontend` Services in the same apply that +creates the Canaries, and Flagger only recreates them once its +`-primary` Deployments are Ready. Do it in a quiet window, with +`apps-stateless` suspended per the procedure above, and watch +`kubectl -n default get canaries,svc`. + ## User uploads The api persists uploads to `/srv/blueshell/storage`, backed by a