Umbrella chart charts/ai-rm, rm-controller image, simulated-cluster e2e in CI

.github/workflows/rm-controller-image.yml

@@ -0,0 +1,65 @@
+name: rm-controller-image
+
+# ghcr.io/ai-computing/rm-controller (packages/rm-controller/Dockerfile), used by charts/ai-rm.
+# Pull requests build and smoke-test it; master pushes :master, tags rm-controller-vX.Y.Z push :X.Y.Z.
+on:
+  push:
+    branches: [master]
+    tags: ["rm-controller-v*"]
+    paths: ["packages/rm-controller/**", ".github/workflows/rm-controller-image.yml"]
+  pull_request:
+    paths: ["packages/rm-controller/**", ".github/workflows/rm-controller-image.yml"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE: ghcr.io/ai-computing/rm-controller
+
+jobs:
+  image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+      - uses: docker/setup-buildx-action@v3
+      - name: Tags
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=match,pattern=rm-controller-v(\d+\.\d+\.\d+),group=1
+            type=ref,event=branch
+            type=sha,format=short
+      - name: Build
+        uses: docker/build-push-action@v6
+        with:
+          context: packages/rm-controller
+          load: true
+          tags: ${{ env.IMAGE }}:test
+          cache-from: type=gha,scope=rm-controller
+          cache-to: type=gha,mode=max,scope=rm-controller
+      - name: Smoke test
+        run: |
+          docker run --rm ${{ env.IMAGE }}:test --help
+          test "$(docker run --rm --entrypoint id ${{ env.IMAGE }}:test -u)" = 65534
+      - name: Log in to ghcr.io
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Push
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          context: packages/rm-controller
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=rm-controller

.github/workflows/sim-e2e.yml

@@ -0,0 +1,69 @@
+name: sim-e2e
+
+# Umbrella chart lint/template, and the simulated cluster end to end on kind:
+# L1 sim provider -> AcceleratorProfiles -> rm-controller -> airm-placement, rm-bench pd-smoke against
+# a fake OpenAI-compatible PD pool (tools/fake_openai.py). Same steps as make up-sim / make e2e-sim.
+on:
+  push:
+    branches: [master]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  chart:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+      - name: Fetch the rm-accel submodule (L1 subchart and node image)
+        run: git submodule update --init components/l1-accel/rm-accel
+      - uses: azure/setup-helm@v4
+        with:
+          version: v3.16.4
+      - name: helm lint / template (default, example site, sim)
+        run: |
+          make chart
+          helm lint charts/ai-rm -f deploy/clusters/example/ai-rm.values.yaml
+          helm template airm charts/ai-rm -n airm-system -f deploy/clusters/example/ai-rm.values.yaml > /dev/null
+
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+      - name: Fetch the rm-accel submodule (L1 subchart and node image)
+        run: git submodule update --init components/l1-accel/rm-accel
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install rm-controller, rm-bench
+        run: python -m pip install pyyaml pytest -e packages/rm-controller -e packages/rm-bench
+      - name: Unit tests (fake PD pool, checks, e2e-sim --no-cluster)
+        run: python -m pytest -q tools/tests
+      - uses: azure/setup-helm@v4
+        with:
+          version: v3.16.4
+      - uses: helm/kind-action@v1
+        with:
+          install_only: true
+      - name: make up-sim (kind, images, charts/ai-rm with sim values)
+        run: make up-sim PY=python
+      - name: make e2e-sim (rm-controller once, airm-placement, rm-bench pd-smoke)
+        run: make e2e-sim PY=python
+      - name: Cluster state on failure
+        if: failure()
+        run: |
+          kubectl --context kind-airm-sim get nodes,pods -A -o wide || true
+          kubectl --context kind-airm-sim get acceleratorprofiles || true
+          kubectl --context kind-airm-sim -n airm describe pods || true
+          kubectl --context kind-airm-sim -n airm logs -l app.kubernetes.io/name=rm-controller --tail=100 || true
+          kubectl --context kind-airm-sim -n airm logs -l app.kubernetes.io/component=node-daemon --tail=50 --prefix || true
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: sim-e2e-results
+          path: results/
+          if-no-files-found: ignore

.gitignore

@@ -10,3 +10,8 @@ __pycache__/
 .venv/
 *.egg-info/
 .pytest_cache/
+
+# Umbrella chart dependencies (helm dependency update charts/ai-rm)
+charts/ai-rm/charts/
+charts/ai-rm/Chart.lock
+results/

Makefile

@@ -6,7 +6,7 @@ CLUSTER ?= local
 LAYER_ARG = $(if $(LAYER),--layer $(LAYER),)
 DRY = $(if $(DRY_RUN),--dry-run,)
 
-.PHONY: init list doctor setup test deploy check bump
+.PHONY: init list doctor setup test deploy check bump chart up-sim e2e-sim down-sim
 
 init:            ## fetch all component submodules (needs ai-computing org access)
 	git submodule update --init
@@ -31,3 +31,18 @@ check:           ## scan ai-rm and fetched components for IPs, tokens, keys
 
 bump:            ## move a component to its latest commit: make bump COMPONENT=lmd-top
 	$(AIRM) bump $(COMPONENT)
+
+chart:           ## helm lint + template of the umbrella chart (needs the rm-accel submodule)
+	helm dependency update charts/ai-rm
+	helm lint charts/ai-rm
+	helm lint charts/ai-rm -f charts/ai-rm/ci/sim-values.yaml
+	helm template airm charts/ai-rm -f charts/ai-rm/ci/sim-values.yaml > /dev/null
+
+up-sim:          ## kind cluster airm-sim: sim accelerators, rm-controller, charts/ai-rm (docker, kind, kubectl, helm)
+	$(AIRM) up-sim
+
+e2e-sim:         ## placement + rm-bench pd-smoke on the sim cluster (NO_CLUSTER=1: in-process, no kind)
+	$(AIRM) e2e-sim $(if $(NO_CLUSTER),--no-cluster,)
+
+down-sim:        ## delete the kind cluster
+	$(AIRM) down-sim

charts/ai-rm/.helmignore

@@ -0,0 +1,4 @@
+.git/
+*.tgz.prov
+*.swp
+*~

charts/ai-rm/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: ai-rm
+description: ai-rm umbrella - L1 node daemon and AcceleratorProfile CRD, L4 rm-controller and the airm-placement ConfigMap
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+home: https://github.com/ai-computing/ai-rm
+dependencies:
+  # L1: the rm-accel chart from the pinned submodule (git submodule update --init components/l1-accel/rm-accel;
+  # helm dependency update charts/ai-rm)
+  - name: rm-abstract-node
+    alias: node
+    version: ">=0.5.0"
+    repository: file://../../components/l1-accel/rm-accel/deploy/helm/rm-abstract-node
+    condition: node.enabled

charts/ai-rm/ci/sim-values.yaml

@@ -0,0 +1,28 @@
+# Simulated cluster (kind; make up-sim, CI sim-e2e): rm-core's sim provider on every node (one RNGD and
+# one ATOM each, placeholder figures), images built locally and loaded into kind, no host access.
+node:
+  provider: sim
+  image: {repository: airm/rm-accel, tag: sim, pullPolicy: Never}
+  privileged: false
+  hostDev: false
+  hostBin: ""
+  intervalSeconds: 10
+
+controller:
+  image: {repository: airm/rm-controller, tag: sim, pullPolicy: Never}
+  intervalSeconds: 15
+  # Two RNGD-prefill / ATOM-decode pools over host-local KV paths: each needs one node to itself, so on
+  # the two-node sim cluster they land on different nodes.
+  pools:
+    - pool: sim-pd-a
+      model: sim-model
+      kv_transport: shm
+      slo: {tpot_p90_ms: 500}
+      prefill: {accelerator: furiosa.RNGD, count: 1}
+      decode: {accelerator: rebellions.ATOM, count: 1}
+    - pool: sim-pd-b
+      model: sim-model
+      kv_transport: p2p
+      slo: {tpot_p90_ms: 500}
+      prefill: {accelerator: furiosa.RNGD, count: 1}
+      decode: {accelerator: rebellions.ATOM, count: 1}

charts/ai-rm/crds/acceleratorprofile.crd.yaml

@@ -0,0 +1,50 @@
+# AcceleratorProfile CRD, copied verbatim from rm-core v0.1.1 k8s/acceleratorprofile.crd.yaml; update it
+# with the rm-core pin. Helm installs crds/ before the templates and never upgrades or deletes them.
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: acceleratorprofiles.airm.ai-computing.io
+spec:
+  group: airm.ai-computing.io
+  scope: Cluster
+  names:
+    kind: AcceleratorProfile
+    plural: acceleratorprofiles
+    singular: acceleratorprofile
+    shortNames: [accp]
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - {name: Node, type: string, jsonPath: .spec.node}
+        - {name: Vendor, type: string, jsonPath: .spec.vendor}
+        - {name: Model, type: string, jsonPath: .spec.model}
+        - {name: Memory, type: integer, jsonPath: .spec.memoryBytes}
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              required: [id, node, vendor, model, memoryBytes, memoryKind]
+              properties:
+                id: {type: string, description: "<node>/<vendor>:<index>"}
+                node: {type: string}
+                vendor: {type: string, enum: [nvidia, furiosa, rebellions, cpu, amd, intel, other]}
+                model: {type: string}
+                memoryBytes: {type: integer, minimum: 1}
+                memoryKind: {type: string, enum: [hbm, gddr, dram, unified, lpddr]}
+                peakTflops:
+                  type: object
+                  description: precision -> dense TFLOPS (as strings to keep exact values)
+                  additionalProperties: {type: string}
+                peakSource: {type: string}
+                powerLimitW: {type: string, nullable: true}
+                interconnect:
+                  type: array
+                  items: {type: string}
+                extra:
+                  type: object
+                  description: vendor details (PCI address, firmware, product name); no addresses or secrets
+                  additionalProperties: {type: string}

charts/ai-rm/templates/NOTES.txt

@@ -0,0 +1,8 @@
+ai-rm {{ .Chart.Version }} installed in {{ .Release.Namespace }}.
+
+  kubectl get acceleratorprofiles          # L1: one per device{{ if .Values.node.enabled }} (provider {{ .Values.node.provider }}){{ end }}
+  kubectl -n {{ include "ai-rm.placementNamespace" . }} get configmap airm-placement -o jsonpath='{.data.decisions\.json}'
+{{- if .Values.controller.enabled }}
+  kubectl -n {{ .Release.Namespace }} logs deploy/{{ include "ai-rm.controller" . }}
+  kubectl -n {{ .Release.Namespace }} exec deploy/{{ include "ai-rm.controller" . }} -- rm-controller --spec /etc/rm-controller/spec.yaml --once
+{{- end }}

charts/ai-rm/templates/_helpers.tpl

@@ -0,0 +1,27 @@
+{{- define "ai-rm.controller" -}}
+{{- printf "%s-rm-controller" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "ai-rm.labels" -}}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/part-of: ai-rm
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }}
+{{- end -}}
+
+{{- define "ai-rm.controllerSelector" -}}
+app.kubernetes.io/name: rm-controller
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "ai-rm.placementNamespace" -}}
+{{- default .Release.Namespace .Values.placementNamespace -}}
+{{- end -}}
+
+{{/* rm-controller spec (YAML): the pools, and the namespace of airm-placement. */}}
+{{- define "ai-rm.controllerSpec" -}}
+{{- if not .Values.controller.pools }}
+{{- fail "controller.pools must list at least one pool" }}
+{{- end }}
+{{- toYaml (dict "namespace" (include "ai-rm.placementNamespace" .) "pools" .Values.controller.pools) }}
+{{- end -}}

charts/ai-rm/templates/controller-configmap.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.controller.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "ai-rm.controller" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "ai-rm.labels" . | nindent 4 }}
+    app.kubernetes.io/name: rm-controller
+data:
+  spec.yaml: |
+    {{- include "ai-rm.controllerSpec" . | nindent 4 }}
+{{- end }}

charts/ai-rm/templates/controller-deployment.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.controller.enabled }}
+{{- $c := .Values.controller }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "ai-rm.controller" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "ai-rm.labels" . | nindent 4 }}
+    app.kubernetes.io/name: rm-controller
+    app.kubernetes.io/component: placement-controller
+spec:
+  replicas: 1                    # the only writer of airm-placement
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "ai-rm.controllerSelector" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "ai-rm.labels" . | nindent 8 }}
+        app.kubernetes.io/name: rm-controller
+        app.kubernetes.io/component: placement-controller
+      annotations:
+        checksum/spec: {{ include "ai-rm.controllerSpec" . | sha256sum }}
+    spec:
+      serviceAccountName: {{ include "ai-rm.controller" . }}
+      {{- with $c.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $c.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        runAsGroup: 65534
+      containers:
+        - name: rm-controller
+          image: "{{ required "controller.image.repository is required" $c.image.repository }}:{{ $c.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ $c.image.pullPolicy }}
+          command: ["rm-controller"]
+          args:
+            - --spec=/etc/rm-controller/spec.yaml
+            - --interval={{ $c.intervalSeconds }}
+            - --kube=client
+            {{- with $c.extraArgs }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          {{- with $c.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: spec
+              mountPath: /etc/rm-controller
+              readOnly: true
+      volumes:
+        - name: spec
+          configMap:
+            name: {{ include "ai-rm.controller" . }}
+{{- end }}