feat: add deployment topology and shared state

.github/workflows/cd.yml

@@ -40,3 +40,48 @@ jobs:
           path: ${{ runner.temp }}/local-llm-router.tar
           retention-days: 14
 
+
+  deployment-plan:
+    # Renders the deployment topology and the canary plan that names its own
+    # rollback target. Applying to a cluster stays disabled until a deployment
+    # destination is configured; nothing here contacts a live environment.
+    needs: release-artifact
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.event.workflow_run.head_sha }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.RELEASE_REF }}
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -e ".[dev]"
+      - name: Verify the serving configuration matches the catalog
+        run: python -m llm_router.serving > /tmp/ray-serve.yaml && diff -u config/ray-serve.yaml /tmp/ray-serve.yaml
+      - name: Render the canary and rollback plan
+        run: |
+          python - <<'PY' > canary-plan.json
+          import json
+          from llm_router.registry import load_registry
+          from llm_router.serving import canary_config
+          registry = load_registry("config/registry.yaml")
+          production = [item for item in registry.deployments if item.stage.value == "production"]
+          plan = canary_config(registry, production[0].id)
+          plan["release_ref"] = "${{ env.RELEASE_REF }}"
+          print(json.dumps(plan, indent=2))
+          PY
+      - name: Validate the Kubernetes manifests
+        run: |
+          curl -sSLo kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.7.0/kubeconform-linux-amd64.tar.gz
+          tar xf kubeconform.tar.gz kubeconform
+          ./kubeconform -strict -ignore-missing-schemas -summary deploy/kubernetes
+      - uses: actions/upload-artifact@v7
+        with:
+          name: deployment-plan-${{ env.RELEASE_REF }}
+          path: |
+            canary-plan.json
+            config/ray-serve.yaml
+            deploy/kubernetes
+          retention-days: 14

README.md

@@ -83,6 +83,31 @@ python -m llm_router.serving > config/ray-serve.yaml
 It carries per-tier autoscaling (latency-sensitive tiers keep a warm replica), GPU pool
 placement, tensor parallelism, prefix caching, quantization, and Multi-LoRA settings.
 
+## Deployment topology
+
+[`deploy/kubernetes`](deploy/kubernetes) holds the namespaced manifests: gateway
+Deployment and Service, GPU serving pool, Redis, KEDA autoscaling on queue depth and p95
+latency, a Prometheus `ServiceMonitor`, network policy, and credentials sourced from the
+cluster secret manager. No secret material is committed. Unit tests enforce the contract:
+unprivileged workloads, digest-pinned images, bounded resources, real probes, GPU pool
+pinning, and `/metrics` reachable only from monitoring.
+
+```bash
+kubectl apply -k deploy/kubernetes
+```
+
+Stateless ingress scales separately from GPU replicas. Set `ROUTER_REDIS_URL` so cache and
+quota state are shared once the gateway runs more than one replica; without it both are
+in-process and correct for a single replica only. Install the client with the extra:
+
+```bash
+python -m pip install -e ".[redis]"
+```
+
+CD renders the canary plan (with its rollback target and triggers), verifies
+`config/ray-serve.yaml` against the catalog, and validates the manifests with kubeconform.
+Applying to a cluster stays disabled until a deployment destination is configured.
+
 ## Model registry
 
 [`config/registry.yaml`](config/registry.yaml) is the governed source of truth for what may
@@ -149,6 +174,7 @@ All settings use the `ROUTER_` prefix.
 | `ROUTER_ADMISSION_TIMEOUT_SECONDS` | `0.25` | Time allowed to wait for capacity. |
 | `ROUTER_QUOTA_REQUESTS_PER_MINUTE` | `120` | Per-token sliding-window quota. |
 | `ROUTER_EXTERNAL_FALLBACK_ENABLED` | `false` | Operator gate for external fallback. |
+| `ROUTER_REDIS_URL` | _(empty)_ | Shared cache and quota state; in-process when empty. |
 | `ROUTER_BACKEND` | `mock` | `mock` or `vllm`. |
 | `ROUTER_VLLM_BASE_URL` | `http://127.0.0.1:8001` | vLLM OpenAI-compatible endpoint. |
 | `ROUTER_BACKEND_TIMEOUT_SECONDS` | `60` | Per-request engine timeout. |

deploy/kubernetes/autoscaling.yaml

@@ -0,0 +1,25 @@
+# Stateless ingress scales on queue depth, independent of GPU replicas.
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: llm-gateway
+  namespace: llm-routing
+spec:
+  scaleTargetRef:
+    name: llm-gateway
+  minReplicaCount: 2
+  maxReplicaCount: 20
+  cooldownPeriod: 120
+  triggers:
+    - type: prometheus
+      metadata:
+        serverAddress: http://prometheus.monitoring.svc.cluster.local:9090
+        metricName: router_queued_requests
+        query: sum(router_queued_requests{namespace="llm-routing"})
+        threshold: "4"
+    - type: prometheus
+      metadata:
+        serverAddress: http://prometheus.monitoring.svc.cluster.local:9090
+        metricName: router_request_latency_p95
+        query: histogram_quantile(0.95, sum(rate(router_request_latency_seconds_bucket[5m])) by (le))
+        threshold: "2"

deploy/kubernetes/gateway.yaml

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llm-gateway
+  namespace: llm-routing
+  labels:
+    app.kubernetes.io/name: llm-gateway
+    app.kubernetes.io/part-of: local-llm-router
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: llm-gateway
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: llm-gateway
+        app.kubernetes.io/part-of: local-llm-router
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: gateway
+          # Replaced at deploy time with the digest recorded in the DeploymentRevision.
+          image: ghcr.io/REPLACE_ME/local-llm-router@sha256:REPLACE_ME
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8000
+          env:
+            - name: ROUTER_ENVIRONMENT
+              value: production
+            - name: ROUTER_BACKEND
+              value: vllm
+            - name: ROUTER_VLLM_BASE_URL
+              value: http://vllm-serve.llm-routing.svc.cluster.local:8000
+            - name: ROUTER_API_KEYS
+              valueFrom:
+                secretKeyRef:
+                  name: llm-gateway-credentials
+                  key: api-keys
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          lifecycle:
+            preStop:
+              exec:
+                command: ["sleep", "10"]
+      terminationGracePeriodSeconds: 60
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llm-gateway
+  namespace: llm-routing
+  labels:
+    app.kubernetes.io/name: llm-gateway
+spec:
+  selector:
+    app.kubernetes.io/name: llm-gateway
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

deploy/kubernetes/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: llm-routing
+resources:
+  - namespace.yaml
+  - gateway.yaml
+  - autoscaling.yaml
+  - vllm-serve.yaml
+  - state.yaml
+  - network-policy.yaml
+  - observability.yaml

deploy/kubernetes/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: llm-routing
+  labels:
+    app.kubernetes.io/part-of: local-llm-router
+    pod-security.kubernetes.io/enforce: restricted

deploy/kubernetes/network-policy.yaml

@@ -0,0 +1,59 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: llm-gateway
+  namespace: llm-routing
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: llm-gateway
+  policyTypes: [Ingress, Egress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: applications
+      ports:
+        - protocol: TCP
+          port: 8000
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+      ports:
+        - protocol: TCP
+          port: 8000
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: vllm-serve
+      ports:
+        - protocol: TCP
+          port: 8000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: redis
+      ports:
+        - protocol: TCP
+          port: 6379
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: vllm-serve
+  namespace: llm-routing
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: vllm-serve
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: llm-gateway
+      ports:
+        - protocol: TCP
+          port: 8000

deploy/kubernetes/observability.yaml

@@ -0,0 +1,16 @@
+# Metrics are unauthenticated by design and reachable only from monitoring.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: llm-gateway
+  namespace: llm-routing
+  labels:
+    release: prometheus
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: llm-gateway
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 15s

deploy/kubernetes/state.yaml

@@ -0,0 +1,88 @@
+# Cache and quota state. Credentials come from the cluster secret manager;
+# no secret material is committed to this repository.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis
+  namespace: llm-routing
+  labels:
+    app.kubernetes.io/name: redis
+spec:
+  serviceName: redis
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: redis
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: redis
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: redis
+          image: docker.io/library/redis@sha256:REPLACE_ME
+          ports:
+            - name: redis
+              containerPort: 6379
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          livenessProbe:
+            tcpSocket:
+              port: redis
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            tcpSocket:
+              port: redis
+            initialDelaySeconds: 5
+            periodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  namespace: llm-routing
+spec:
+  selector:
+    app.kubernetes.io/name: redis
+  ports:
+    - name: redis
+      port: 6379
+      targetPort: redis
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: llm-gateway-credentials
+  namespace: llm-routing
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: platform-secret-store
+    kind: ClusterSecretStore
+  target:
+    name: llm-gateway-credentials
+  data:
+    - secretKey: api-keys
+      remoteRef:
+        key: llm-routing/gateway
+        property: api_keys
+    - secretKey: external-provider-key
+      remoteRef:
+        key: llm-routing/gateway
+        property: external_provider_key