diff --git a/.github/workflows/rm-controller-image.yml b/.github/workflows/rm-controller-image.yml new file mode 100644 index 0000000..b852172 --- /dev/null +++ b/.github/workflows/rm-controller-image.yml @@ -0,0 +1,65 @@ +name: rm-controller-image + +# ghcr.io/ai-computing/rm-controller (packages/rm-controller/Dockerfile), used by charts/ai-rm. +# Pull requests build and smoke-test it; master pushes :master, tags rm-controller-vX.Y.Z push :X.Y.Z. +on: + push: + branches: [master] + tags: ["rm-controller-v*"] + paths: ["packages/rm-controller/**", ".github/workflows/rm-controller-image.yml"] + pull_request: + paths: ["packages/rm-controller/**", ".github/workflows/rm-controller-image.yml"] + workflow_dispatch: + +permissions: + contents: read + packages: write + +env: + IMAGE: ghcr.io/ai-computing/rm-controller + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - uses: docker/setup-buildx-action@v3 + - name: Tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + tags: | + type=match,pattern=rm-controller-v(\d+\.\d+\.\d+),group=1 + type=ref,event=branch + type=sha,format=short + - name: Build + uses: docker/build-push-action@v6 + with: + context: packages/rm-controller + load: true + tags: ${{ env.IMAGE }}:test + cache-from: type=gha,scope=rm-controller + cache-to: type=gha,mode=max,scope=rm-controller + - name: Smoke test + run: | + docker run --rm ${{ env.IMAGE }}:test --help + test "$(docker run --rm --entrypoint id ${{ env.IMAGE }}:test -u)" = 65534 + - name: Log in to ghcr.io + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v6 + with: + context: packages/rm-controller + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=rm-controller diff --git a/.github/workflows/sim-e2e.yml b/.github/workflows/sim-e2e.yml new file mode 100644 index 0000000..7114e71 --- /dev/null +++ b/.github/workflows/sim-e2e.yml @@ -0,0 +1,69 @@ +name: sim-e2e + +# Umbrella chart lint/template, and the simulated cluster end to end on kind: +# L1 sim provider -> AcceleratorProfiles -> rm-controller -> airm-placement, rm-bench pd-smoke against +# a fake OpenAI-compatible PD pool (tools/fake_openai.py). Same steps as make up-sim / make e2e-sim. +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +jobs: + chart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - name: Fetch the rm-accel submodule (L1 subchart and node image) + run: git submodule update --init components/l1-accel/rm-accel + - uses: azure/setup-helm@v4 + with: + version: v3.16.4 + - name: helm lint / template (default, example site, sim) + run: | + make chart + helm lint charts/ai-rm -f deploy/clusters/example/ai-rm.values.yaml + helm template airm charts/ai-rm -n airm-system -f deploy/clusters/example/ai-rm.values.yaml > /dev/null + + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - name: Fetch the rm-accel submodule (L1 subchart and node image) + run: git submodule update --init components/l1-accel/rm-accel + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install rm-controller, rm-bench + run: python -m pip install pyyaml pytest -e packages/rm-controller -e packages/rm-bench + - name: Unit tests (fake PD pool, checks, e2e-sim --no-cluster) + run: python -m pytest -q tools/tests + - uses: azure/setup-helm@v4 + with: + version: v3.16.4 + - uses: helm/kind-action@v1 + with: + install_only: true + - name: make up-sim (kind, images, charts/ai-rm with sim values) + run: make up-sim PY=python + - name: make e2e-sim (rm-controller once, airm-placement, rm-bench pd-smoke) + run: make e2e-sim PY=python + - name: Cluster state on failure + if: failure() + run: | + kubectl --context kind-airm-sim get nodes,pods -A -o wide || true + kubectl --context kind-airm-sim get acceleratorprofiles || true + kubectl --context kind-airm-sim -n airm describe pods || true + kubectl --context kind-airm-sim -n airm logs -l app.kubernetes.io/name=rm-controller --tail=100 || true + kubectl --context kind-airm-sim -n airm logs -l app.kubernetes.io/component=node-daemon --tail=50 --prefix || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: sim-e2e-results + path: results/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index bb62af7..92ea80b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ __pycache__/ .venv/ *.egg-info/ .pytest_cache/ + +# Umbrella chart dependencies (helm dependency update charts/ai-rm) +charts/ai-rm/charts/ +charts/ai-rm/Chart.lock +results/ diff --git a/Makefile b/Makefile index 6d51048..fe73ae6 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CLUSTER ?= local LAYER_ARG = $(if $(LAYER),--layer $(LAYER),) DRY = $(if $(DRY_RUN),--dry-run,) -.PHONY: init list doctor setup test deploy check bump +.PHONY: init list doctor setup test deploy check bump chart up-sim e2e-sim down-sim init: ## fetch all component submodules (needs ai-computing org access) git submodule update --init @@ -31,3 +31,18 @@ check: ## scan ai-rm and fetched components for IPs, tokens, keys bump: ## move a component to its latest commit: make bump COMPONENT=lmd-top $(AIRM) bump $(COMPONENT) + +chart: ## helm lint + template of the umbrella chart (needs the rm-accel submodule) + helm dependency update charts/ai-rm + helm lint charts/ai-rm + helm lint charts/ai-rm -f charts/ai-rm/ci/sim-values.yaml + helm template airm charts/ai-rm -f charts/ai-rm/ci/sim-values.yaml > /dev/null + +up-sim: ## kind cluster airm-sim: sim accelerators, rm-controller, charts/ai-rm (docker, kind, kubectl, helm) + $(AIRM) up-sim + +e2e-sim: ## placement + rm-bench pd-smoke on the sim cluster (NO_CLUSTER=1: in-process, no kind) + $(AIRM) e2e-sim $(if $(NO_CLUSTER),--no-cluster,) + +down-sim: ## delete the kind cluster + $(AIRM) down-sim diff --git a/charts/ai-rm/.helmignore b/charts/ai-rm/.helmignore new file mode 100644 index 0000000..cda671f --- /dev/null +++ b/charts/ai-rm/.helmignore @@ -0,0 +1,4 @@ +.git/ +*.tgz.prov +*.swp +*~ diff --git a/charts/ai-rm/Chart.yaml b/charts/ai-rm/Chart.yaml new file mode 100644 index 0000000..ef77fe6 --- /dev/null +++ b/charts/ai-rm/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: ai-rm +description: ai-rm umbrella - L1 node daemon and AcceleratorProfile CRD, L4 rm-controller and the airm-placement ConfigMap +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/ai-computing/ai-rm +dependencies: + # L1: the rm-accel chart from the pinned submodule (git submodule update --init components/l1-accel/rm-accel; + # helm dependency update charts/ai-rm) + - name: rm-abstract-node + alias: node + version: ">=0.5.0" + repository: file://../../components/l1-accel/rm-accel/deploy/helm/rm-abstract-node + condition: node.enabled diff --git a/charts/ai-rm/ci/sim-values.yaml b/charts/ai-rm/ci/sim-values.yaml new file mode 100644 index 0000000..820da02 --- /dev/null +++ b/charts/ai-rm/ci/sim-values.yaml @@ -0,0 +1,28 @@ +# Simulated cluster (kind; make up-sim, CI sim-e2e): rm-core's sim provider on every node (one RNGD and +# one ATOM each, placeholder figures), images built locally and loaded into kind, no host access. +node: + provider: sim + image: {repository: airm/rm-accel, tag: sim, pullPolicy: Never} + privileged: false + hostDev: false + hostBin: "" + intervalSeconds: 10 + +controller: + image: {repository: airm/rm-controller, tag: sim, pullPolicy: Never} + intervalSeconds: 15 + # Two RNGD-prefill / ATOM-decode pools over host-local KV paths: each needs one node to itself, so on + # the two-node sim cluster they land on different nodes. + pools: + - pool: sim-pd-a + model: sim-model + kv_transport: shm + slo: {tpot_p90_ms: 500} + prefill: {accelerator: furiosa.RNGD, count: 1} + decode: {accelerator: rebellions.ATOM, count: 1} + - pool: sim-pd-b + model: sim-model + kv_transport: p2p + slo: {tpot_p90_ms: 500} + prefill: {accelerator: furiosa.RNGD, count: 1} + decode: {accelerator: rebellions.ATOM, count: 1} diff --git a/charts/ai-rm/crds/acceleratorprofile.crd.yaml b/charts/ai-rm/crds/acceleratorprofile.crd.yaml new file mode 100644 index 0000000..85943ca --- /dev/null +++ b/charts/ai-rm/crds/acceleratorprofile.crd.yaml @@ -0,0 +1,50 @@ +# AcceleratorProfile CRD, copied verbatim from rm-core v0.1.1 k8s/acceleratorprofile.crd.yaml; update it +# with the rm-core pin. Helm installs crds/ before the templates and never upgrades or deletes them. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: acceleratorprofiles.airm.ai-computing.io +spec: + group: airm.ai-computing.io + scope: Cluster + names: + kind: AcceleratorProfile + plural: acceleratorprofiles + singular: acceleratorprofile + shortNames: [accp] + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - {name: Node, type: string, jsonPath: .spec.node} + - {name: Vendor, type: string, jsonPath: .spec.vendor} + - {name: Model, type: string, jsonPath: .spec.model} + - {name: Memory, type: integer, jsonPath: .spec.memoryBytes} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: [id, node, vendor, model, memoryBytes, memoryKind] + properties: + id: {type: string, description: "/:"} + node: {type: string} + vendor: {type: string, enum: [nvidia, furiosa, rebellions, cpu, amd, intel, other]} + model: {type: string} + memoryBytes: {type: integer, minimum: 1} + memoryKind: {type: string, enum: [hbm, gddr, dram, unified, lpddr]} + peakTflops: + type: object + description: precision -> dense TFLOPS (as strings to keep exact values) + additionalProperties: {type: string} + peakSource: {type: string} + powerLimitW: {type: string, nullable: true} + interconnect: + type: array + items: {type: string} + extra: + type: object + description: vendor details (PCI address, firmware, product name); no addresses or secrets + additionalProperties: {type: string} diff --git a/charts/ai-rm/templates/NOTES.txt b/charts/ai-rm/templates/NOTES.txt new file mode 100644 index 0000000..01393ce --- /dev/null +++ b/charts/ai-rm/templates/NOTES.txt @@ -0,0 +1,8 @@ +ai-rm {{ .Chart.Version }} installed in {{ .Release.Namespace }}. + + kubectl get acceleratorprofiles # L1: one per device{{ if .Values.node.enabled }} (provider {{ .Values.node.provider }}){{ end }} + kubectl -n {{ include "ai-rm.placementNamespace" . }} get configmap airm-placement -o jsonpath='{.data.decisions\.json}' +{{- if .Values.controller.enabled }} + kubectl -n {{ .Release.Namespace }} logs deploy/{{ include "ai-rm.controller" . }} + kubectl -n {{ .Release.Namespace }} exec deploy/{{ include "ai-rm.controller" . }} -- rm-controller --spec /etc/rm-controller/spec.yaml --once +{{- end }} diff --git a/charts/ai-rm/templates/_helpers.tpl b/charts/ai-rm/templates/_helpers.tpl new file mode 100644 index 0000000..d61b80a --- /dev/null +++ b/charts/ai-rm/templates/_helpers.tpl @@ -0,0 +1,27 @@ +{{- define "ai-rm.controller" -}} +{{- printf "%s-rm-controller" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "ai-rm.labels" -}} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/part-of: ai-rm +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +{{- end -}} + +{{- define "ai-rm.controllerSelector" -}} +app.kubernetes.io/name: rm-controller +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "ai-rm.placementNamespace" -}} +{{- default .Release.Namespace .Values.placementNamespace -}} +{{- end -}} + +{{/* rm-controller spec (YAML): the pools, and the namespace of airm-placement. */}} +{{- define "ai-rm.controllerSpec" -}} +{{- if not .Values.controller.pools }} +{{- fail "controller.pools must list at least one pool" }} +{{- end }} +{{- toYaml (dict "namespace" (include "ai-rm.placementNamespace" .) "pools" .Values.controller.pools) }} +{{- end -}} diff --git a/charts/ai-rm/templates/controller-configmap.yaml b/charts/ai-rm/templates/controller-configmap.yaml new file mode 100644 index 0000000..a32ca19 --- /dev/null +++ b/charts/ai-rm/templates/controller-configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.controller.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ai-rm.controller" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} + app.kubernetes.io/name: rm-controller +data: + spec.yaml: | + {{- include "ai-rm.controllerSpec" . | nindent 4 }} +{{- end }} diff --git a/charts/ai-rm/templates/controller-deployment.yaml b/charts/ai-rm/templates/controller-deployment.yaml new file mode 100644 index 0000000..a4ae8a4 --- /dev/null +++ b/charts/ai-rm/templates/controller-deployment.yaml @@ -0,0 +1,70 @@ +{{- if .Values.controller.enabled }} +{{- $c := .Values.controller }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ai-rm.controller" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} + app.kubernetes.io/name: rm-controller + app.kubernetes.io/component: placement-controller +spec: + replicas: 1 # the only writer of airm-placement + strategy: + type: Recreate + selector: + matchLabels: + {{- include "ai-rm.controllerSelector" . | nindent 6 }} + template: + metadata: + labels: + {{- include "ai-rm.labels" . | nindent 8 }} + app.kubernetes.io/name: rm-controller + app.kubernetes.io/component: placement-controller + annotations: + checksum/spec: {{ include "ai-rm.controllerSpec" . | sha256sum }} + spec: + serviceAccountName: {{ include "ai-rm.controller" . }} + {{- with $c.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $c.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + containers: + - name: rm-controller + image: "{{ required "controller.image.repository is required" $c.image.repository }}:{{ $c.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ $c.image.pullPolicy }} + command: ["rm-controller"] + args: + - --spec=/etc/rm-controller/spec.yaml + - --interval={{ $c.intervalSeconds }} + - --kube=client + {{- with $c.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + {{- with $c.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: spec + mountPath: /etc/rm-controller + readOnly: true + volumes: + - name: spec + configMap: + name: {{ include "ai-rm.controller" . }} +{{- end }} diff --git a/charts/ai-rm/templates/controller-rbac.yaml b/charts/ai-rm/templates/controller-rbac.yaml new file mode 100644 index 0000000..e665df7 --- /dev/null +++ b/charts/ai-rm/templates/controller-rbac.yaml @@ -0,0 +1,72 @@ +{{- if .Values.controller.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ai-rm.controller" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} +--- +# Reads: AcceleratorProfiles (L1) and node state (cordoned / not Ready). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ai-rm.controller" . }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} +rules: + - apiGroups: ["airm.ai-computing.io"] + resources: ["acceleratorprofiles"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ai-rm.controller" . }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ai-rm.controller" . }} +subjects: + - kind: ServiceAccount + name: {{ include "ai-rm.controller" . }} + namespace: {{ .Release.Namespace }} +--- +# The one write: the airm-placement ConfigMap in the placement namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "ai-rm.controller" . }} + namespace: {{ include "ai-rm.placementNamespace" . }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["airm-placement"] + verbs: ["get", "patch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] # create cannot be limited by name +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "ai-rm.controller" . }} + namespace: {{ include "ai-rm.placementNamespace" . }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ai-rm.controller" . }} +subjects: + - kind: ServiceAccount + name: {{ include "ai-rm.controller" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/ai-rm/templates/placement-configmap.yaml b/charts/ai-rm/templates/placement-configmap.yaml new file mode 100644 index 0000000..4648c2b --- /dev/null +++ b/charts/ai-rm/templates/placement-configmap.yaml @@ -0,0 +1,22 @@ +{{- /* +airm-placement (hetero-llm-d deploy/airm/README.md, key decisions.json): an empty v0.1 document so +readers find a valid one before rm-controller's first placement. Rendered only while the ConfigMap +does not exist; after that rm-controller owns it, and the keep policy stops Helm deleting it. +*/ -}} +{{- if .Values.placement.create }} +{{- $ns := include "ai-rm.placementNamespace" . }} +{{- if not (lookup "v1" "ConfigMap" $ns "airm-placement") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: airm-placement + namespace: {{ $ns }} + labels: + {{- include "ai-rm.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +data: + decisions.json: | + {"protocol": "0.1", "generation": 0, "updatedAt": null, "decisions": []} +{{- end }} +{{- end }} diff --git a/charts/ai-rm/values.yaml b/charts/ai-rm/values.yaml new file mode 100644 index 0000000..da9a1db --- /dev/null +++ b/charts/ai-rm/values.yaml @@ -0,0 +1,48 @@ +# ai-rm umbrella chart: L1 node daemon + AcceleratorProfile CRD, L4 rm-controller, airm-placement. +# Defaults describe an example accelerator cluster. Site overrides go in an untracked +# deploy/clusters//ai-rm.values.yaml (start from deploy/clusters/example/ai-rm.values.yaml); +# the simulated kind cluster uses ci/sim-values.yaml (make up-sim). + +# Namespace of the airm-placement ConfigMap: the llm-d namespace, where the EPP scorer and lmd-top +# read it. "" = the release namespace. +placementNamespace: "" + +# L1: rm-accel node daemon (subchart rm-abstract-node from components/l1-accel/rm-accel, chart >= 0.5.0). +# Every value of that chart can be set here; see its values.yaml. +node: + enabled: true + provider: rm-abstract # "sim" for a simulated cluster + # image: ghcr.io/ai-computing/rm-accel: by default + # hostDev / hostBin / privileged: the host's vendor tools read the device nodes (chart defaults) + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {memory: 512Mi} + +# L4: rm-controller places the pools below and writes airm-placement (packages/rm-controller). +controller: + enabled: true + image: + repository: ghcr.io/ai-computing/rm-controller + tag: "0.1.0" # packages/rm-controller version; :master follows ai-rm master + pullPolicy: IfNotPresent + intervalSeconds: 60 # seconds between placements + # Pools to place, in order (rm-controller spec, packages/rm-controller/README.md). The spec's + # namespace is placementNamespace. accelerator is vendor.model as in airm.ai-computing.io/.. + pools: + - pool: llama8b-pd-pool # InferencePool name + model: meta-llama/Llama-3.1-8B-Instruct + kv_transport: shm + slo: {tpot_p90_ms: 100} + prefill: {accelerator: furiosa.RNGD, count: 1} + decode: {accelerator: furiosa.RNGD, count: 1} + extraArgs: [] + resources: + requests: {cpu: 20m, memory: 64Mi} + limits: {memory: 256Mi} + nodeSelector: {} + tolerations: [] + +# airm-placement ConfigMap: created at install, with an empty v0.1 document, only when it does not +# exist yet. rm-controller owns its contents afterwards; Helm keeps it on upgrade and uninstall. +placement: + create: true diff --git a/components/l1-accel/rm-accel b/components/l1-accel/rm-accel index c72f356..675a527 160000 --- a/components/l1-accel/rm-accel +++ b/components/l1-accel/rm-accel @@ -1 +1 @@ -Subproject commit c72f35609374bed5a54bb4c5ba6ee636a0f62af1 +Subproject commit 675a527500f878c0fac59e87bbccefcb2a365caa diff --git a/deploy/README.ko.md b/deploy/README.ko.md index 43aef4c..e832524 100644 --- a/deploy/README.ko.md +++ b/deploy/README.ko.md @@ -10,7 +10,9 @@ ai-rm의 기본 대상은 Kubernetes + llm-d 클러스터입니다. 클러스터 ``` deploy/clusters/ ├─ example/ 커밋됨: 키 목록과 자리표시자 -│ └─ cluster.example.env +│ ├─ cluster.example.env +│ ├─ ai-rm.values.yaml umbrella chart 사이트 값 +│ └─ .values.yaml chart별 예시 └─ / 추적 안 함: 실제 사이트 값 ├─ cluster.local.env 노드·게이트웨이 주소, 자격 증명 위치 └─ / (선택) 그 구성요소의 kustomize overlay @@ -28,6 +30,52 @@ make deploy CLUSTER=local LAYER=l3-serving `deploy/clusters///`에 overlay가 있으면 그것을 대신 적용합니다. `cluster.local.env`의 값은 환경 변수로 전달됩니다. +## Umbrella chart + +[`charts/ai-rm`](../charts/ai-rm)은 ai-rm 제어 경로를 `helm install` 한 번으로 설치합니다. + +| 구성 | 출처 | 리소스 | +|---|---|---| +| L1 노드 데몬 | rm-accel chart `rm-abstract-node` (subchart, alias `node`, 고정된 서브모듈에서) | DaemonSet, ServiceAccount, ClusterRole; 이미지 `ghcr.io/ai-computing/rm-accel` | +| `AcceleratorProfile` CRD | rm-core v0.1.1 `k8s/`를 `charts/ai-rm/crds/`로 복사 | CRD (먼저 설치, Helm이 업그레이드·삭제하지 않음) | +| L4 rm-controller | [`packages/rm-controller`](../packages/rm-controller) | Deployment, spec ConfigMap(`controller.pools`), ClusterRole(프로파일, 노드), Role(`airm-placement`만); 이미지 `ghcr.io/ai-computing/rm-controller` | +| `airm-placement` | hetero-llm-d `deploy/airm/README.md` 형식 | 빈 v0.1 문서를 담은 ConfigMap. 없을 때만 만들고 업그레이드·삭제 시 유지, 내용은 rm-controller가 관리 | + +```bash +git submodule update --init components/l1-accel/rm-accel +helm dependency update charts/ai-rm +mkdir -p deploy/clusters/local +cp deploy/clusters/example/ai-rm.values.yaml deploy/clusters/local/ai-rm.values.yaml # 풀, 네임스페이스 수정 +helm upgrade --install airm charts/ai-rm -n airm-system --create-namespace \ + -f deploy/clusters/local/ai-rm.values.yaml +make chart # helm lint + template (기본값, sim 값) +``` + +`placementNamespace`는 EPP scorer와 lmd-top이 `airm-placement`를 읽는 네임스페이스(llm-d 네임스페이스)입니다. +hetero-llm-d의 `airm-install.sh`가 CRD나 ConfigMap을 이미 만들었다면 chart는 그대로 둡니다. 노드 데몬에는 이제 +init container가 필요 없습니다. 이미지에 rm-abstract, rm-core, Kubernetes 클라이언트가 들어 있고 벤더 도구는 호스트 +`/usr/bin`에서 마운트합니다(`node.hostBin`). 이미지는 이 저장소의 `.github/workflows/rm-controller-image.yml`과 +rm-accel의 `Image` 워크플로가 배포합니다. + +## 시뮬레이션 클러스터 + +같은 chart를 [kind](https://kind.sigs.k8s.io/)에서 rm-core의 `sim` 가속기 provider(노드마다 RNGD 1개와 ATOM 1개, +수치는 자리표시자), 소스에서 빌드한 이미지, PD 풀 대신 [`tools/fake_openai.py`](../tools/fake_openai.py)로 실행합니다. +CI가 모든 pull request에서 실행합니다(`.github/workflows/sim-e2e.yml`). + +```bash +make up-sim # kind 클러스터 airm-sim(노드 2개), 이미지 빌드·로드, charts/ai-rm + ci/sim-values.yaml +make e2e-sim # rm-controller --once, airm-placement 확인, 가짜 풀에 rm-bench pd-smoke +make e2e-sim NO_CLUSTER=1 # 같은 확인을 프로세스 안에서, Docker·kind 없이 +make down-sim +``` + +`e2e-sim`은 프로파일이 있고(노드당 2개), `charts/ai-rm/ci/sim-values.yaml`의 각 풀에 요청한 종류의 카드로 한 +노드 안에 prefill·decode 결정이 있으며, 두 풀이 서로 다른 노드를 쓰고, `rm-bench`가 실패 요청 없이 +`slo_attainment_pct` 100을 보고하면(`results/sim-e2e.json`) 통과합니다. Docker, kind, kubectl, Helm과 +`pip install -e packages/rm-controller -e packages/rm-bench`가 필요합니다. 시뮬레이션은 제어 경로와 측정 경로를 +확인하며 서빙 성능을 뜻하지 않습니다. + ## 구성요소별 배포 상태 `make list`의 DEPLOY 열에서 확인합니다(✓ = 매니페스트 있음). 매니페스트가 없는 구성요소는 각 repo의 @@ -35,4 +83,4 @@ README `## ai-rm` 절에 무엇이 필요한지 적혀 있습니다. ## Optimus Prime 전환 경계 -[통합 설계](../docs/ko/engine-integration.md)와 [계획 설정](../config/engine-integration.yaml)은 향후 엔진·FASOP 연결을 설명하며 현재 CLI가 실행하지 않습니다. 기존 소스 배포는 유지하고, 계획된 엔진이나 존재하지 않는 브랜치를 배포하는 것으로 해석하지 마세요. 현재 CLI는 deploy.manifests만 kubectl로 적용하며 deploy.chart를 자동 Helm 배포하지 않습니다. cluster.local.env는 kubectl 환경 변수로 전달되며 임의 YAML을 자동 치환하지 않습니다. +[통합 설계](../docs/ko/engine-integration.md)와 [계획 설정](../config/engine-integration.yaml)은 향후 엔진·FASOP 연결을 설명하며 현재 CLI가 실행하지 않습니다. 기존 소스 배포는 유지하고, 계획된 엔진이나 존재하지 않는 브랜치를 배포하는 것으로 해석하지 마세요. 현재 CLI는 deploy.manifests만 kubectl로 적용하며 deploy.chart를 자동 Helm 배포하지 않습니다(Helm은 위 umbrella chart를 사용). cluster.local.env는 kubectl 환경 변수로 전달되며 임의 YAML을 자동 치환하지 않습니다. diff --git a/deploy/README.md b/deploy/README.md index 738a73d..bb9db75 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,7 +11,9 @@ AI-RM targets Kubernetes + llm-d. `hetero-llm-d` owns cluster setup, Gateway/EPP ```text deploy/clusters/ ├── example/ # tracked keys and placeholders -│ └── cluster.example.env +│ ├── cluster.example.env +│ ├── ai-rm.values.yaml # umbrella chart site values +│ └── .values.yaml # per-chart examples └── / # untracked actual site settings ├── cluster.local.env └── / # optional Kustomize overlay @@ -26,6 +28,53 @@ python3 tools/airm.py deploy --cluster local --dry-run --layer l3-serving Review actual manifests and cluster selection before omitting `--dry-run` to apply them. +## Umbrella chart + +[`charts/ai-rm`](../charts/ai-rm) installs the ai-rm control path with one `helm install`: + +| Piece | Source | Resources | +|---|---|---| +| L1 node daemon | rm-accel chart `rm-abstract-node` (subchart, alias `node`, from the pinned submodule) | DaemonSet, ServiceAccount, ClusterRole; image `ghcr.io/ai-computing/rm-accel` | +| `AcceleratorProfile` CRD | rm-core v0.1.1 `k8s/`, copied to `charts/ai-rm/crds/` | CRD (installed first, never upgraded or deleted by Helm) | +| L4 rm-controller | [`packages/rm-controller`](../packages/rm-controller) | Deployment, ConfigMap with its spec (`controller.pools`), ClusterRole (profiles, nodes), Role (`airm-placement` only); image `ghcr.io/ai-computing/rm-controller` | +| `airm-placement` | hetero-llm-d `deploy/airm/README.md` format | ConfigMap with an empty v0.1 document, created only if missing and kept on upgrade/uninstall; rm-controller owns its contents | + +```bash +git submodule update --init components/l1-accel/rm-accel +helm dependency update charts/ai-rm +mkdir -p deploy/clusters/local +cp deploy/clusters/example/ai-rm.values.yaml deploy/clusters/local/ai-rm.values.yaml # edit pools, namespace +helm upgrade --install airm charts/ai-rm -n airm-system --create-namespace \ + -f deploy/clusters/local/ai-rm.values.yaml +make chart # helm lint + template (default and sim values) +``` + +`placementNamespace` is where the EPP scorer and lmd-top read `airm-placement` (llm-d's namespace). If +hetero-llm-d's `airm-install.sh` already created the CRD or the ConfigMap, the chart leaves them as they are. The +node daemon needs no init container any more: the image carries rm-abstract, rm-core and the Kubernetes client, and +the host's vendor tools are mounted from `/usr/bin` (`node.hostBin`). Images are published by +`.github/workflows/rm-controller-image.yml` here and by rm-accel's `Image` workflow. + +## Simulated cluster + +The same chart runs on [kind](https://kind.sigs.k8s.io/) with rm-core's `sim` accelerator provider (one RNGD and +one ATOM per node, placeholder figures), images built from source, and +[`tools/fake_openai.py`](../tools/fake_openai.py) standing in for the PD pool. CI runs it on every pull request +(`.github/workflows/sim-e2e.yml`). + +```bash +make up-sim # kind cluster airm-sim (2 nodes), images built and loaded, charts/ai-rm + ci/sim-values.yaml +make e2e-sim # rm-controller --once, airm-placement checked, rm-bench pd-smoke against the fake pool +make e2e-sim NO_CLUSTER=1 # the same checks in-process, no Docker or kind +make down-sim +``` + +`e2e-sim` passes when the profiles exist (2 per node), each pool in `charts/ai-rm/ci/sim-values.yaml` has a +prefill and a decode decision with cards of the requested kind on one node, the two pools use different nodes, +and `rm-bench` reports `slo_attainment_pct` 100 with no failed request (`results/sim-e2e.json`). It needs +Docker, kind, kubectl, Helm, and `pip install -e packages/rm-controller -e packages/rm-bench`. The simulation +checks the control path and the measurement path, not serving performance. + ## Current dispatcher behavior - Reads each fetched component's `ai-rm.yaml` / `deploy.manifests`. @@ -33,6 +82,6 @@ Review actual manifests and cluster selection before omitting `--dry-run` to app - Prefers a site overlay at `deploy/clusters///` and applies it with Kustomize. - Passes `cluster.local.env` values in the command environment; does not automatically template arbitrary YAML. - Uses `--dry-run=client` when requested. A usable Kubernetes context/discovery setup may still be needed; dry run is not deployment validation. -- Skips components without manifests. `deploy.chart` alone does not invoke Helm. +- Skips components without manifests. `deploy.chart` alone does not invoke Helm; use the umbrella chart above. -Inspect `python3 tools/airm.py list` and the component README for the actual deployment assets at the pinned revision. A planned umbrella chart, simulation target or FASOP-driven controller is not provided by this documentation change. +Inspect `python3 tools/airm.py list` and the component README for the actual deployment assets at the pinned revision. A FASOP-driven controller is not provided. diff --git a/deploy/clusters/example/ai-rm.values.yaml b/deploy/clusters/example/ai-rm.values.yaml new file mode 100644 index 0000000..026e7a0 --- /dev/null +++ b/deploy/clusters/example/ai-rm.values.yaml @@ -0,0 +1,28 @@ +# Example site override for the umbrella chart charts/ai-rm; copy to deploy/clusters// (untracked). +# helm dependency update charts/ai-rm +# helm upgrade --install airm charts/ai-rm -n airm-system --create-namespace \ +# -f deploy/clusters//ai-rm.values.yaml +placementNamespace: llm-serving # the llm-d namespace (EPP scorer, lmd-top read airm-placement there) + +node: + provider: rm-abstract + nodeAffinity: {} # every node; restrict with your own node labels if needed + # hostBin: /usr/bin + +controller: + intervalSeconds: 60 + pools: + # RNGD prefill + RNGD decode over shared memory on one node (the end-to-end run's shape) + - pool: llama8b-pd-pool + model: meta-llama/Llama-3.1-8B-Instruct + kv_transport: shm + slo: {tpot_p90_ms: 100} + prefill: {accelerator: furiosa.RNGD, count: 1} + decode: {accelerator: furiosa.RNGD, count: 1} + # RNGD prefill + 2x ATOM decode over same-host P2P DMA + # - pool: llama8b-hetero-pool + # model: meta-llama/Llama-3.1-8B-Instruct + # kv_transport: p2p + # slo: {tpot_p90_ms: 100} + # prefill: {accelerator: furiosa.RNGD, count: 1} + # decode: {accelerator: rebellions.ATOM, count: 2} diff --git a/deploy/clusters/example/rm-abstract-node.values.yaml b/deploy/clusters/example/rm-abstract-node.values.yaml index c54d303..841cdc2 100644 --- a/deploy/clusters/example/rm-abstract-node.values.yaml +++ b/deploy/clusters/example/rm-abstract-node.values.yaml @@ -1,36 +1,8 @@ -# Example site override for rm-abstract-node (docs/e2e.md); copy to deploy/clusters//. -# No published image yet: stock Python image, -# rm-abstract + rm-core + kubernetes installed by an init container, host vendor tools on PATH. -image: - repository: python - tag: "3.11-slim" - -extraEnv: - - {name: PYTHONPATH, value: /opt/airm} - - {name: PYTHONDONTWRITEBYTECODE, value: "1"} - - {name: PATH, value: /usr/local/bin:/usr/bin:/bin:/host/usr/bin} - -initContainers: - - name: install - image: python:3.11-slim - command: ["sh", "-c"] - args: - - >- - pip install -q --no-cache-dir --target /opt/airm --no-deps - https://github.com/ai-computing/rm-accel/archive/refs/tags/v0.4.1.tar.gz - https://github.com/ai-computing/rm-core/archive/refs/tags/v0.1.1.tar.gz - && pip install -q --no-cache-dir --target /opt/airm "kubernetes>=24.2.0" "numpy>=1.20,<3" - volumeMounts: - - {name: airm-pkgs, mountPath: /opt/airm} - -extraVolumes: - - {name: airm-pkgs, emptyDir: {}} - - {name: host-bin, hostPath: {path: /usr/bin, type: Directory}} - -extraVolumeMounts: - - {name: airm-pkgs, mountPath: /opt/airm, readOnly: true} - - {name: host-bin, mountPath: /host/usr/bin, readOnly: true} - +# Example site override for rm-abstract-node (rm-accel chart >= 0.5.0); copy to deploy/clusters//. +# The chart's defaults run the published image ghcr.io/ai-computing/rm-accel with the host's vendor +# tools mounted from /usr/bin (hostBin); no init container is needed. Set only what differs per site. +# hostBin: /usr/bin # where nvidia-smi / furiosa-smi / rbln-stat live on the nodes +# image: {tag: master} # a newer image than the chart's appVersion resources: requests: {cpu: 50m, memory: 128Mi} limits: {memory: 512Mi} diff --git a/docs/getting-started.md b/docs/getting-started.md index 4f11fbf..dd84acd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -83,6 +83,22 @@ The current dispatcher applies `deploy.manifests` with `kubectl apply`; a site o After reviewing the actual resources and the selected cluster, omit `--dry-run` only when you intend to apply them. See [deployment](../deploy/README.md). +## 6. Try the stack on a simulated cluster + +No accelerator or site cluster needed: Docker, [kind](https://kind.sigs.k8s.io/), kubectl and Helm on Linux (or +WSL with Docker). The umbrella chart runs with rm-core's `sim` accelerators and a fake OpenAI-compatible PD pool. + +```bash +git submodule update --init components/l1-accel/rm-accel +python3 -m pip install PyYAML -e packages/rm-controller -e packages/rm-bench +make e2e-sim NO_CLUSTER=1 # placement + rm-bench in-process (no Docker): a quick first check +make up-sim # kind cluster airm-sim with charts/ai-rm +make e2e-sim # AcceleratorProfiles -> rm-controller -> airm-placement, rm-bench pd-smoke +make down-sim +``` + +On a real cluster the same chart installs with site values; see [deployment](../deploy/README.md#umbrella-chart). + ## Troubleshooting | Symptom | Check | @@ -94,5 +110,6 @@ After reviewing the actual resources and the selected cluster, omit `--dry-run` | Deployment says `no manifests` | The pinned component has no `deploy.manifests`; check its README and work card. | | P2P is unsupported | Verify same-host devices, ABI and declared support; select host staging explicitly if appropriate. | | Conformance skips hardware | Record SKIP and the reason; it is not a hardware PASS. | +| `make up-sim` says `missing tools` or rm-accel has no Dockerfile | Install Docker, kind, kubectl, Helm; fetch `components/l1-accel/rm-accel` at the pinned revision. | Next: [architecture](architecture.md), [integration guide](integration-guide.md), [status](status.md). diff --git a/docs/ko/getting-started.md b/docs/ko/getting-started.md index 896ed84..16d7039 100644 --- a/docs/ko/getting-started.md +++ b/docs/ko/getting-started.md @@ -71,6 +71,22 @@ python3 tools/airm.py deploy --cluster local --dry-run --layer l3-serving 실제 적용할 리소스와 클러스터를 검토한 뒤 배포를 의도할 때만 `--dry-run`을 제거하세요. [배포 문서](../../deploy/README.ko.md)를 참고하세요. +## 시뮬레이션 클러스터로 실행해 보기 + +가속기나 사이트 클러스터 없이 Linux(또는 Docker가 있는 WSL)에서 Docker, [kind](https://kind.sigs.k8s.io/), +kubectl, Helm만 있으면 됩니다. umbrella chart를 rm-core의 `sim` 가속기와 가짜 OpenAI 호환 PD 풀로 실행합니다. + +```bash +git submodule update --init components/l1-accel/rm-accel +python3 -m pip install PyYAML -e packages/rm-controller -e packages/rm-bench +make e2e-sim NO_CLUSTER=1 # 배치 + rm-bench를 프로세스 안에서(Docker 없이): 빠른 첫 확인 +make up-sim # charts/ai-rm을 설치한 kind 클러스터 airm-sim +make e2e-sim # AcceleratorProfile -> rm-controller -> airm-placement, rm-bench pd-smoke +make down-sim +``` + +실제 클러스터에는 같은 chart를 사이트 값으로 설치합니다. [배포 문서](../../deploy/README.ko.md#umbrella-chart)를 참고하세요. + ## 자주 만나는 상황 | 상황 | 확인할 것 | @@ -82,3 +98,4 @@ python3 tools/airm.py deploy --cluster local --dry-run --layer l3-serving | `no manifests` | 해당 고정 버전의 배포 자산과 작업 카드 확인 | | P2P 미지원 | 같은 호스트·장치·ABI 조건 확인, staging은 명시적으로 선택 | | 하드웨어 conformance SKIP | 이유와 함께 기록하며 하드웨어 PASS로 보고하지 않기 | +| `make up-sim`이 `missing tools`를 보고하거나 rm-accel에 Dockerfile이 없음 | Docker, kind, kubectl, Helm 설치; `components/l1-accel/rm-accel`을 고정 리비전으로 받기 | diff --git a/packages/rm-controller/.dockerignore b/packages/rm-controller/.dockerignore new file mode 100644 index 0000000..a455653 --- /dev/null +++ b/packages/rm-controller/.dockerignore @@ -0,0 +1,4 @@ +* +!pyproject.toml +!rm_controller/ +**/__pycache__ diff --git a/packages/rm-controller/Dockerfile b/packages/rm-controller/Dockerfile new file mode 100644 index 0000000..f266052 --- /dev/null +++ b/packages/rm-controller/Dockerfile @@ -0,0 +1,27 @@ +# rm-controller image (ghcr.io/ai-computing/rm-controller), used by the ai-rm umbrella chart. +# +# docker build -t ghcr.io/ai-computing/rm-controller:dev packages/rm-controller +# docker run --rm ghcr.io/ai-computing/rm-controller:dev --help +# +# rm-core is installed from its release tarball (no git in the slim image), then this package without +# re-resolving its git dependency. +FROM python:3.11-slim + +ARG RM_CORE_REF=v0.1.1 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +COPY pyproject.toml /src/ +COPY rm_controller /src/rm_controller +RUN pip install --no-cache-dir \ + "https://github.com/ai-computing/rm-core/archive/refs/tags/${RM_CORE_REF}.tar.gz" \ + pyyaml "kubernetes>=24.2.0" \ + && pip install --no-cache-dir --no-deps /src \ + && rm -rf /src \ + && rm-controller --help >/dev/null + +# Reads the cluster and writes one ConfigMap through its ServiceAccount; needs no root. +USER 65534:65534 +ENTRYPOINT ["rm-controller"] diff --git a/packages/rm-controller/README.ko.md b/packages/rm-controller/README.ko.md index eebb3c7..79cde91 100644 --- a/packages/rm-controller/README.ko.md +++ b/packages/rm-controller/README.ko.md @@ -106,7 +106,7 @@ pools: - [ ] 실시간 부하(`airm_*` 대기열, TPOT)와 워크로드 모양([트레이스](../../docs/ko/workload-traces.md))을 선택에 반영 - [ ] 결정대로 움직이는 서빙 파드 (지금은 사이트 값이 같은 카드를 고정) -- [ ] 클러스터에 상주할 이미지와 chart (umbrella chart의 일부) +- [x] 이미지(`ghcr.io/ai-computing/rm-controller`, [`Dockerfile`](Dockerfile))와 umbrella chart [`charts/ai-rm`](../../charts/ai-rm)의 Deployment - [ ] `airm-placement` 형식을 rm-core 프로토콜에 규범화 ## 📄 라이선스 diff --git a/packages/rm-controller/README.md b/packages/rm-controller/README.md index d94bb8b..cee808d 100644 --- a/packages/rm-controller/README.md +++ b/packages/rm-controller/README.md @@ -107,7 +107,7 @@ pools: - [ ] Use live load (`airm_*` queueing, TPOT) and workload shape ([traces](../../docs/workload-traces.md)) in the choice - [ ] Serving pods that follow the decisions (today site values still pin the same cards) -- [ ] An image and chart to run in the cluster, as part of the umbrella chart +- [x] An image (`ghcr.io/ai-computing/rm-controller`, [`Dockerfile`](Dockerfile)) and a Deployment in the umbrella chart [`charts/ai-rm`](../../charts/ai-rm) - [ ] The `airm-placement` format made normative in rm-core's protocol ## 📄 License diff --git a/packages/rm-controller/ai-rm.yaml b/packages/rm-controller/ai-rm.yaml index 2151da2..2f13e53 100644 --- a/packages/rm-controller/ai-rm.yaml +++ b/packages/rm-controller/ai-rm.yaml @@ -14,10 +14,10 @@ adapters: - group: airm.placements name: topology deploy: - target: none - chart: null + target: k8s + chart: charts/ai-rm # umbrella chart (repository root) manifests: null - image: null + image: ghcr.io/ai-computing/rm-controller config: env_example: null interfaces: diff --git a/tools/airm.py b/tools/airm.py index cb2b222..7a9017b 100644 --- a/tools/airm.py +++ b/tools/airm.py @@ -8,6 +8,7 @@ python3 tools/airm.py deploy --cluster NAME [--dry-run] [--layer L] [COMPONENT ...] python3 tools/airm.py check python3 tools/airm.py bump COMPONENT [--ref REF] + python3 tools/airm.py up-sim | e2e-sim [--no-cluster] | down-sim (tools/airm_sim.py) Components live under components// as git submodules, or in-tree under packages/ (rm-bench, rm-controller) with their layer in the manifest; each declares itself in /ai-rm.yaml @@ -117,10 +118,11 @@ def cmd_list(args): def cmd_doctor(_args): ok = True for tool, why in [("git", "submodules"), ("python3", "components"), ("kubectl", "deploy"), - ("helm", "llm-d charts"), ("cargo", "lmd-top build"), ("docker", "images")]: + ("helm", "charts"), ("cargo", "lmd-top build"), ("docker", "images"), + ("kind", "simulated cluster")]: found = shutil.which(tool) print(f"{'ok ' if found else '-- '} {tool:8} {found or 'not found'} ({why})") - ok &= bool(found) or tool in ("cargo", "docker", "helm") + ok &= bool(found) or tool in ("cargo", "docker", "helm", "kind") missing = [c["name"] for c in components() if not c["fetched"]] if missing: print("-- submodules not fetched: " + ", ".join(missing) + @@ -205,6 +207,11 @@ def main(): s = sub.add_parser("bump") s.add_argument("component") s.add_argument("--ref") + sub.add_parser("up-sim", help="kind cluster with sim accelerators and charts/ai-rm") + s = sub.add_parser("e2e-sim", help="placement + rm-bench checks on the simulated cluster") + s.add_argument("--no-cluster", action="store_true", help="in-process: sim profiles, rm-controller plan, fake pool") + s.add_argument("--out", help="rm-bench result path (default results/sim-e2e.json)") + sub.add_parser("down-sim", help="delete the kind cluster") args = ap.parse_args() if args.cmd == "list": return cmd_list(args) @@ -220,6 +227,9 @@ def main(): return cmd_check(args) if args.cmd == "bump": return cmd_bump(args) + if args.cmd in ("up-sim", "e2e-sim", "down-sim"): + import airm_sim + return {"up-sim": airm_sim.cmd_up, "e2e-sim": airm_sim.cmd_e2e, "down-sim": airm_sim.cmd_down}[args.cmd](args) if __name__ == "__main__": diff --git a/tools/airm_sim.py b/tools/airm_sim.py new file mode 100644 index 0000000..1f075c5 --- /dev/null +++ b/tools/airm_sim.py @@ -0,0 +1,229 @@ +"""Simulated ai-rm cluster on kind: rm-core's sim accelerators, rm-controller, a fake PD pool, rm-bench. + + python3 tools/airm.py up-sim # kind cluster airm-sim (2 nodes), images built and loaded, + # charts/ai-rm installed with ci/sim-values.yaml + python3 tools/airm.py e2e-sim # rm-controller once, airm-placement checked, + # rm-bench pd-smoke against tools/fake_openai.py + python3 tools/airm.py e2e-sim --no-cluster # the same checks in-process, without kind + python3 tools/airm.py down-sim + +up-sim needs docker, kind, kubectl, helm and the rm-accel submodule (git submodule update --init +components/l1-accel/rm-accel). e2e-sim needs rm-bench (pip install -e packages/rm-bench), and with +--no-cluster also rm-controller (pip install -e packages/rm-controller). +""" +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import time + +import fake_openai + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CLUSTER = "airm-sim" +NAMESPACE = "airm" +RELEASE = "airm" +CHART = os.path.join(ROOT, "charts", "ai-rm") +VALUES = os.path.join(CHART, "ci", "sim-values.yaml") +NODE_SRC = os.path.join(ROOT, "components", "l1-accel", "rm-accel") +IMAGES = {"airm/rm-accel:sim": NODE_SRC, + "airm/rm-controller:sim": os.path.join(ROOT, "packages", "rm-controller")} +SCENARIO = os.path.join(ROOT, "packages", "rm-bench", "scenarios", "pd-smoke.yaml") +SIM_MODEL = "sim-model" +DEVICES_PER_NODE = 2 # rm-core SimAcceleratorProvider default: one RNGD, one ATOM +HOST_LOCAL = ("shm", "p2p", "loopback") +KIND_CONFIG = """kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker +""" + + +def sh(cmd, check=True, capture=False, stdin=None, timeout=900): + print("$ " + " ".join(cmd), flush=True) + p = subprocess.run(cmd, input=stdin, capture_output=capture, text=True, timeout=timeout) + if check and p.returncode != 0: + detail = (p.stderr or p.stdout or "").strip() if capture else "" + raise SystemExit(f"failed ({p.returncode}): {' '.join(cmd)}" + (f"\n{detail}" if detail else "")) + return p + + +def kubectl_json(*args): + return json.loads(sh(["kubectl", "--context", f"kind-{CLUSTER}", *args, "-o", "json"], capture=True).stdout) + + +def require(*tools): + missing = [t for t in tools if not shutil.which(t)] + if missing: + raise SystemExit("missing tools: " + ", ".join(missing) + " (see docs/getting-started.md, simulated cluster)") + + +def sim_pools(): + import yaml + with open(VALUES, encoding="utf-8") as f: + return yaml.safe_load(f)["controller"]["pools"] + + +# -- checks (pure, unit-tested in tools/tests) --------------------------------------------------------- + +def check_placement(doc, pools, devices, min_nodes=1): + """Problems with an airm-placement document for `pools`; [] when it is as expected. + + devices: accelerator id -> (node, "vendor.model"), from the AcceleratorProfiles. + Expected: every pool has one prefill and one decode decision with `count` cards of the requested kind, + on one node for a host-local KV transport, no card used twice, and the pools spread over at least + `min_nodes` nodes. + """ + errs = [] + if doc.get("protocol") != "0.1" or int(doc.get("generation") or 0) < 1: + errs.append(f"document not written by rm-controller: protocol={doc.get('protocol')} " + f"generation={doc.get('generation')}") + decisions = doc.get("decisions") or [] + seen, nodes_used = {}, set() + for p in pools: + name = p["pool"] + mine = [d for d in decisions if d.get("pool") == name] + roles = sorted(d.get("workload", {}).get("kind", "?") for d in mine) + if roles != ["decode", "prefill"]: + errs.append(f"{name}: roles {roles}, expected ['decode', 'prefill']") + continue + pool_nodes = set() + for d in mine: + role = d["workload"]["kind"] + want = p[role] + ids = d.get("acceleratorIds") or [] + if len(ids) != int(want.get("count", 1)): + errs.append(f"{name}/{role}: {len(ids)} card(s) {ids}, expected {want.get('count', 1)} " + f"({d.get('reason', '')})") + if d.get("kvTransport") != p["kv_transport"]: + errs.append(f"{name}/{role}: kvTransport {d.get('kvTransport')}, expected {p['kv_transport']}") + for i in ids: + if i not in devices: + errs.append(f"{name}/{role}: {i} is not an AcceleratorProfile") + continue + node, kind = devices[i] + pool_nodes.add(node) + if want.get("accelerator") and kind != want["accelerator"]: + errs.append(f"{name}/{role}: {i} is {kind}, expected {want['accelerator']}") + if i in seen and seen[i] != (name, role): + errs.append(f"{name}/{role}: {i} already placed for {seen[i][0]}/{seen[i][1]}") + seen[i] = (name, role) + if p["kv_transport"] in HOST_LOCAL and len(pool_nodes) > 1: + errs.append(f"{name}: {p['kv_transport']} needs one node, got {sorted(pool_nodes)}") + nodes_used |= pool_nodes + if len(nodes_used) < min_nodes: + errs.append(f"pools use {len(nodes_used)} node(s) {sorted(nodes_used)}, expected at least {min_nodes}") + return errs + + +def check_bench(measurements, rc): + """Problems with an rm-bench result; [] when the run passed with no failed request.""" + errs = [] if rc == 0 else [f"rm-bench exited {rc}"] + by_name = {m["name"]: m["value"] for m in measurements} + if by_name.get("slo_attainment_pct") != 100.0: + errs.append(f"slo_attainment_pct {by_name.get('slo_attainment_pct')}, expected 100 (0 errors)") + if not by_name.get("tok_s"): + errs.append("no tokens received") + return errs + + +# -- steps ---------------------------------------------------------------------------------------------- + +def devices_from_profiles(items): + return {o["spec"]["id"]: (o["spec"]["node"], f"{o['spec']['vendor']}.{o['spec']['model']}") for o in items} + + +def wait_profiles(timeout_s=240): + nodes = [n["metadata"]["name"] for n in kubectl_json("get", "nodes")["items"]] + want = len(nodes) * DEVICES_PER_NODE + deadline = time.time() + timeout_s + while True: + items = kubectl_json("get", "acceleratorprofiles")["items"] + print(f"AcceleratorProfiles: {len(items)}/{want} on {len(nodes)} node(s)", flush=True) + if len(items) >= want: + return items, nodes + if time.time() > deadline: + raise SystemExit(f"timed out waiting for {want} AcceleratorProfiles") + time.sleep(5) + + +def cmd_up(args): + require("docker", "kind", "kubectl", "helm") + if not os.path.exists(os.path.join(NODE_SRC, "Dockerfile")): + raise SystemExit("rm-accel not fetched (or older than its Dockerfile): " + "git submodule update --init components/l1-accel/rm-accel") + clusters = sh(["kind", "get", "clusters"], capture=True).stdout.split() + if CLUSTER not in clusters: + sh(["kind", "create", "cluster", "--name", CLUSTER, "--config", "-", "--wait", "120s"], stdin=KIND_CONFIG) + for tag, ctx in IMAGES.items(): + sh(["docker", "build", "-t", tag, ctx]) + sh(["kind", "load", "docker-image", "--name", CLUSTER, tag]) + sh(["helm", "dependency", "update", CHART]) + sh(["helm", "upgrade", "--install", RELEASE, CHART, "--kube-context", f"kind-{CLUSTER}", + "-n", NAMESPACE, "--create-namespace", "-f", VALUES, "--wait", "--timeout", "5m"]) + items, nodes = wait_profiles() + for o in items: + s = o["spec"] + print(f" {s['id']:40} {s['vendor']}.{s['model']}") + print(f"up-sim: {len(items)} AcceleratorProfiles on {len(nodes)} nodes; next: python3 tools/airm.py e2e-sim") + return 0 + + +def run_bench(out): + if importlib.util.find_spec("rm_bench") is None: + raise SystemExit("rm-bench is not installed: python3 -m pip install -e packages/rm-bench") + srv = fake_openai.serve(model=SIM_MODEL) + gateway = "http://127.0.0.1:%d" % srv.server_address[1] + try: + rc = sh([sys.executable, "-m", "rm_bench.cli", "e2e", "--scenario", SCENARIO, "--gateway", gateway, + "--out", out, "--kv-transport", "shm"], check=False).returncode + finally: + srv.shutdown() + measurements = [] + if os.path.exists(out): + with open(out, encoding="utf-8") as f: + measurements = json.load(f) + return check_bench(measurements, rc) + + +def cmd_e2e(args): + pools = sim_pools() + if args.no_cluster: + from rm_core.reference import SimAcceleratorProvider + from rm_controller import controller, document + nodes = ["sim-node-a", "sim-node-b"] + accs = [a for n in nodes for a in SimAcceleratorProvider(n).discover()] + entries, unplaced = controller.plan({"namespace": NAMESPACE, "pools": pools}, accs, set()) + doc, _ = document.build(entries, document.parse(None), "topology") + devices = {a.id: (a.node, f"{a.vendor}.{a.model}") for a in accs} + print(f"rm-controller (in-process): {len(accs)} sim accelerators on {len(nodes)} nodes, " + f"{len(entries)} decisions" + (f"; not placed: {', '.join(unplaced)}" if unplaced else "")) + else: + require("kubectl") + items, nodes = wait_profiles(timeout_s=60) + devices = devices_from_profiles(items) + rc = sh(["kubectl", "--context", f"kind-{CLUSTER}", "-n", NAMESPACE, "exec", f"deploy/{RELEASE}-rm-controller", + "--", "rm-controller", "--spec", "/etc/rm-controller/spec.yaml", "--once"], check=False).returncode + if rc != 0: + raise SystemExit(f"rm-controller --once exited {rc}") + cm = kubectl_json("-n", NAMESPACE, "get", "configmap", "airm-placement") + doc = json.loads(cm["data"]["decisions.json"]) + print(json.dumps(doc, indent=2)) + errs = check_placement(doc, pools, devices, min_nodes=min(len(pools), len(nodes))) + out = args.out or os.path.join(ROOT, "results", "sim-e2e.json") + errs += run_bench(out) + if errs: + print("e2e-sim FAILED:\n " + "\n ".join(errs)) + return 1 + print(f"e2e-sim PASS: {len(pools)} pools placed (prefill + decode) on {len(nodes)} nodes, " + f"rm-bench pd-smoke 0 errors -> {os.path.relpath(out, ROOT)}") + return 0 + + +def cmd_down(_args): + require("kind") + sh(["kind", "delete", "cluster", "--name", CLUSTER]) + return 0 diff --git a/tools/fake_openai.py b/tools/fake_openai.py new file mode 100644 index 0000000..5079cab --- /dev/null +++ b/tools/fake_openai.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""A tiny OpenAI-compatible server that stands in for the PD pool in the simulated cluster. + + python3 tools/fake_openai.py --port 8000 [--model sim-model] [--token-delay-ms 2] + +Serves what rm-bench and the llm-d gateway clients use, with no model behind it: + +- GET /v1/models one model +- POST /v1/chat/completions streaming (SSE, chunked, one content chunk per token, a usage chunk, [DONE]) + or a plain JSON reply when "stream" is false +- POST /v1/completions non-streaming, with rngd-pd style server timing ("pd": {ttft_ms, wall_ms}) +- GET /health 200 + +Replies are "t0 t1 ..." with max_tokens tokens. Python 3.9+, stdlib only. +""" +import argparse +import json +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +DEFAULT_MODEL = "sim-model" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" # chunked streaming, as vLLM / llm-d send it + model = DEFAULT_MODEL + token_delay_s = 0.002 + requests_served = 0 + _lock = threading.Lock() + + def log_message(self, *args): + pass + + def _json(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _error(self, code, message): + self._json(code, {"error": {"message": message, "type": "invalid_request_error", "code": code}}) + + def do_GET(self): + path = self.path.split("?", 1)[0] + if path == "/v1/models": + self._json(200, {"object": "list", "data": [{"id": self.model, "object": "model", "owned_by": "airm-sim"}]}) + elif path == "/health": + self._json(200, {"status": "ok"}) + else: + self._error(404, f"no route {path}") + + def do_POST(self): + path = self.path.split("?", 1)[0] + try: + req = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + except ValueError: + self._error(400, "body is not JSON") + return + if not isinstance(req, dict): + self._error(400, "body is not a JSON object") + return + if req.get("model") not in (None, self.model): + self._error(404, f"model {req.get('model')!r} not served (have {self.model!r})") + return + n = max(1, int(req.get("max_tokens") or 16)) + with self._lock: + type(self).requests_served += 1 + if path == "/v1/chat/completions": + if req.get("stream"): + self._stream_chat(n) + else: + self._chat(n) + elif path == "/v1/completions": + self._completion(n) + else: + self._error(404, f"no route {path}") + + def _tokens(self, n): + for i in range(n): + time.sleep(self.token_delay_s) + yield f"t{i} " + + def _usage(self, n): + return {"prompt_tokens": 1, "completion_tokens": n, "total_tokens": n + 1} + + def _stream_chat(self, n): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + + def send(data): + event = f"data: {data}\n\n".encode() + self.wfile.write(b"%x\r\n%s\r\n" % (len(event), event)) + self.wfile.flush() + + rid = f"chatcmpl-sim-{time.monotonic_ns()}" + for tok in self._tokens(n): + send(json.dumps({"id": rid, "object": "chat.completion.chunk", "model": self.model, + "choices": [{"index": 0, "delta": {"content": tok}, "finish_reason": None}]})) + send(json.dumps({"id": rid, "object": "chat.completion.chunk", "model": self.model, "choices": [], + "usage": self._usage(n)})) + send("[DONE]") + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + def _chat(self, n): + text = "".join(self._tokens(n)) + self._json(200, {"id": "chatcmpl-sim", "object": "chat.completion", "model": self.model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, + "finish_reason": "length"}], + "usage": self._usage(n)}) + + def _completion(self, n): + t0 = time.perf_counter() + first = None + text = "" + for tok in self._tokens(n): + first = first if first is not None else time.perf_counter() + text += tok + wall_ms = (time.perf_counter() - t0) * 1000 + self._json(200, {"id": "cmpl-sim", "object": "text_completion", "model": self.model, + "choices": [{"index": 0, "text": text, "finish_reason": "length"}], + "usage": self._usage(n), + "pd": {"ttft_ms": round((first - t0) * 1000, 3), "wall_ms": round(wall_ms, 3)}}) + + +class _Server(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request, client_address): + if isinstance(sys.exc_info()[1], (ConnectionResetError, BrokenPipeError)): + return # a client dropped an idle keep-alive connection + super().handle_error(request, client_address) + + +def serve(host="127.0.0.1", port=0, model=DEFAULT_MODEL, token_delay_s=0.002): + """Start the server in a background thread; returns it (server.server_address, server.shutdown()).""" + handler = type("SimHandler", (Handler,), {"model": model, "token_delay_s": token_delay_s, + "requests_served": 0}) + srv = _Server((host, port), handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--host", default="127.0.0.1") + ap.add_argument("--port", type=int, default=8000) + ap.add_argument("--model", default=DEFAULT_MODEL) + ap.add_argument("--token-delay-ms", type=float, default=2.0) + args = ap.parse_args(argv) + srv = serve(args.host, args.port, args.model, args.token_delay_ms / 1000) + host, port = srv.server_address[:2] + print(f"fake OpenAI-compatible server on http://{host}:{port} (model {args.model})", flush=True) + try: + threading.Event().wait() + except KeyboardInterrupt: + srv.shutdown() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_sim.py b/tools/tests/test_sim.py new file mode 100644 index 0000000..53cfd9f --- /dev/null +++ b/tools/tests/test_sim.py @@ -0,0 +1,127 @@ +"""The simulated-cluster pieces that need no kind: the fake PD pool, the placement and bench checks, +and e2e-sim --no-cluster (when rm-controller and rm-bench are installed).""" +import copy +import json +import os +import sys +import urllib.request + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import airm_sim # noqa: E402 +import fake_openai # noqa: E402 + + +@pytest.fixture +def server(): + srv = fake_openai.serve(model="m", token_delay_s=0) + yield "http://127.0.0.1:%d" % srv.server_address[1] + srv.shutdown() + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + return urllib.request.urlopen(req, timeout=10) + + +def test_models(server): + data = json.load(urllib.request.urlopen(server + "/v1/models", timeout=10)) + assert [m["id"] for m in data["data"]] == ["m"] + + +def test_streaming_chat(server): + with post(server + "/v1/chat/completions", + {"model": "m", "stream": True, "max_tokens": 3, "messages": [{"role": "user", "content": "hi"}]}) as r: + assert r.headers["Content-Type"] == "text/event-stream" + events = [line[len("data: "):] for line in r.read().decode().splitlines() if line.startswith("data: ")] + assert events[-1] == "[DONE]" + chunks = [json.loads(e) for e in events[:-1]] + text = "".join(c["choices"][0]["delta"]["content"] for c in chunks if c["choices"]) + assert text == "t0 t1 t2 " + assert chunks[-1]["usage"]["completion_tokens"] == 3 + + +def test_plain_chat_and_completions(server): + with post(server + "/v1/chat/completions", {"model": "m", "max_tokens": 2, "messages": []}) as r: + assert json.load(r)["choices"][0]["message"]["content"] == "t0 t1 " + with post(server + "/v1/completions", {"model": "m", "max_tokens": 4, "prompt": "x"}) as r: + body = json.load(r) + assert body["usage"]["completion_tokens"] == 4 and body["pd"]["wall_ms"] >= body["pd"]["ttft_ms"] >= 0 + + +def test_unknown_model_and_route(server): + for path, body in (("/v1/chat/completions", {"model": "other"}), ("/v1/embeddings", {"model": "m"})): + with pytest.raises(urllib.error.HTTPError) as e: + post(server + path, body) + assert e.value.code == 404 + + +POOLS = [ + {"pool": "a", "kv_transport": "shm", "prefill": {"accelerator": "furiosa.RNGD", "count": 1}, + "decode": {"accelerator": "rebellions.ATOM", "count": 1}}, + {"pool": "b", "kv_transport": "p2p", "prefill": {"accelerator": "furiosa.RNGD", "count": 1}, + "decode": {"accelerator": "rebellions.ATOM", "count": 1}}, +] +DEVICES = {f"{n}/furiosa:0": (n, "furiosa.RNGD") for n in ("n1", "n2")} +DEVICES.update({f"{n}/rebellions:0": (n, "rebellions.ATOM") for n in ("n1", "n2")}) + + +def decision(pool, kind, ids, kv): + return {"pool": pool, "workload": {"kind": kind}, "acceleratorIds": ids, "kvTransport": kv, "reason": ""} + + +GOOD = {"protocol": "0.1", "generation": 1, "placement": "topology", "decisions": [ + decision("a", "prefill", ["n1/furiosa:0"], "shm"), decision("a", "decode", ["n1/rebellions:0"], "shm"), + decision("b", "prefill", ["n2/furiosa:0"], "p2p"), decision("b", "decode", ["n2/rebellions:0"], "p2p")]} + + +def test_check_placement_accepts_expected_document(): + assert airm_sim.check_placement(GOOD, POOLS, DEVICES, min_nodes=2) == [] + + +@pytest.mark.parametrize("mutate, expect", [ + (lambda d: d.update(generation=0), "not written by rm-controller"), + (lambda d: d["decisions"].pop(), "roles ['prefill']"), + (lambda d: d["decisions"][1].update(acceleratorIds=[]), "0 card(s)"), + (lambda d: d["decisions"][1].update(acceleratorIds=["n1/furiosa:0"]), "is furiosa.RNGD, expected rebellions.ATOM"), + (lambda d: d["decisions"][1].update(acceleratorIds=["n2/rebellions:0"]), "shm needs one node"), + (lambda d: d["decisions"][3].update(acceleratorIds=["n1/rebellions:0"]), "already placed"), + (lambda d: d["decisions"][0].update(kvTransport="loopback"), "kvTransport loopback"), + (lambda d: d["decisions"][0].update(acceleratorIds=["n9/furiosa:0"]), "not an AcceleratorProfile"), +]) +def test_check_placement_reports(mutate, expect): + doc = copy.deepcopy(GOOD) + mutate(doc) + errs = airm_sim.check_placement(doc, POOLS, DEVICES, min_nodes=1) + assert any(expect in e for e in errs), errs + + +def test_check_placement_spread(): + doc = copy.deepcopy(GOOD) + doc["decisions"] = doc["decisions"][:2] + assert any("expected at least 2" in e for e in airm_sim.check_placement(doc, POOLS[:1], DEVICES, min_nodes=2)) + + +def test_check_bench(): + ok = [{"name": "slo_attainment_pct", "value": 100.0}, {"name": "tok_s", "value": 50.0}] + assert airm_sim.check_bench(ok, 0) == [] + assert airm_sim.check_bench(ok, 1) == ["rm-bench exited 1"] + bad = [{"name": "slo_attainment_pct", "value": 96.875}, {"name": "tok_s", "value": 50.0}] + assert "expected 100" in airm_sim.check_bench(bad, 1)[1] + + +def test_sim_values_pools_match_the_checks(): + pools = airm_sim.sim_pools() + assert len(pools) == 2 and {p["kv_transport"] for p in pools} <= set(airm_sim.HOST_LOCAL) + + +@pytest.mark.skipif(not all(__import__("importlib").util.find_spec(m) for m in ("rm_controller", "rm_bench")), + reason="needs pip install -e packages/rm-controller packages/rm-bench") +def test_e2e_sim_no_cluster(tmp_path): + class Args: + no_cluster = True + out = str(tmp_path / "sim.json") + assert airm_sim.cmd_e2e(Args) == 0 + names = {m["name"] for m in json.load(open(Args.out))} + assert {"slo_attainment_pct", "tok_s", "ttft_p90_ms"} <= names