diff --git a/.claude/skills/nvca-chart-release b/.claude/skills/nvca-chart-release new file mode 120000 index 000000000..4faf728bc --- /dev/null +++ b/.claude/skills/nvca-chart-release @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-chart-release \ No newline at end of file diff --git a/.claude/skills/nvca-self-managed-install b/.claude/skills/nvca-self-managed-install new file mode 120000 index 000000000..46fc1f5a0 --- /dev/null +++ b/.claude/skills/nvca-self-managed-install @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-self-managed-install \ No newline at end of file diff --git a/.claude/skills/nvca-values-customization b/.claude/skills/nvca-values-customization new file mode 120000 index 000000000..c4f27f5d7 --- /dev/null +++ b/.claude/skills/nvca-values-customization @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-values-customization \ No newline at end of file diff --git a/.codex/skills/nvca-chart-release b/.codex/skills/nvca-chart-release new file mode 120000 index 000000000..4faf728bc --- /dev/null +++ b/.codex/skills/nvca-chart-release @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-chart-release \ No newline at end of file diff --git a/.codex/skills/nvca-self-managed-install b/.codex/skills/nvca-self-managed-install new file mode 120000 index 000000000..46fc1f5a0 --- /dev/null +++ b/.codex/skills/nvca-self-managed-install @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-self-managed-install \ No newline at end of file diff --git a/.codex/skills/nvca-values-customization b/.codex/skills/nvca-values-customization new file mode 120000 index 000000000..c4f27f5d7 --- /dev/null +++ b/.codex/skills/nvca-values-customization @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-values-customization \ No newline at end of file diff --git a/.cursor/skills/nvca-chart-release b/.cursor/skills/nvca-chart-release new file mode 120000 index 000000000..4faf728bc --- /dev/null +++ b/.cursor/skills/nvca-chart-release @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-chart-release \ No newline at end of file diff --git a/.cursor/skills/nvca-self-managed-install b/.cursor/skills/nvca-self-managed-install new file mode 120000 index 000000000..46fc1f5a0 --- /dev/null +++ b/.cursor/skills/nvca-self-managed-install @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-self-managed-install \ No newline at end of file diff --git a/.cursor/skills/nvca-values-customization b/.cursor/skills/nvca-values-customization new file mode 120000 index 000000000..c4f27f5d7 --- /dev/null +++ b/.cursor/skills/nvca-values-customization @@ -0,0 +1 @@ +../../ai-tooling/dev/skills/nvca-values-customization \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 7ae9c5a30..7925a2071 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,6 +216,9 @@ manually before finishing. | `documentation-style` | `ai-tooling/dev/skills/` | NVCF documentation conventions (no bold, no emojis, no em-dash) | | `nvcf-explore-stack` | `ai-tooling/dev/skills/` | Navigate the self-hosted stack topology and dependency graph | | `official-docs-style` | `ai-tooling/dev/skills/` | External-facing NVCF user documentation voice and structure | +| `nvca-chart-release` | `ai-tooling/dev/skills/` | Release NVCA Operator chart changes from monorepo source to the vendored Helm chart | +| `nvca-self-managed-install` | `ai-tooling/dev/skills/` | Install or validate the NVCA Operator chart against a self-managed control plane | +| `nvca-values-customization` | `ai-tooling/dev/skills/` | Customize NVCA Operator Helm chart values in the monorepo | | `nvcf-self-managed-cli` | `ai-tooling/user/skills/` | Install, operate, and manage self-managed NVCF through `nvcf-cli` | | `nvcf-self-managed-installation` | `ai-tooling/user/skills/` | Install and deploy the self-managed NVCF stack | | `nvcf-self-managed-prerequisite` | `ai-tooling/user/skills/` | Install cluster-level prerequisites such as KAI Scheduler and SMB CSI driver | diff --git a/ai-tooling/dev/skills/nvca-chart-release/SKILL.md b/ai-tooling/dev/skills/nvca-chart-release/SKILL.md new file mode 100644 index 000000000..0b69545ca --- /dev/null +++ b/ai-tooling/dev/skills/nvca-chart-release/SKILL.md @@ -0,0 +1,104 @@ +--- +name: nvca-chart-release +description: Release NVCA Operator chart changes from the native monorepo source to the vendored Helm chart. Use when updating the vendored NVCA Operator chart, changing NVCA image refs, publishing helm-nvca-operator, or validating the chart against a self-managed control plane. +license: Apache-2.0 +compatibility: Requires a local checkout of the NVCF monorepo with deploy/helm/nvca-operator/ and src/compute-plane-services/nvca/ present, plus helm and yq. +author: "nvcf-core-eng " +version: "1.0.0" +tags: [nvcf, nvca, helm, chart-release, self-managed] +tools: [Read, Grep, Glob, Shell] +metadata: + internal: false + author: "nvcf-core-eng " + version: "1.0" + tags: [nvcf, nvca, helm, chart-release] + languages: [bash] + frameworks: [helm] + domain: cloud-infrastructure +--- + +# NVCA Operator Chart Release Workflow + +Propagates Helm chart changes through the native monorepo paths: + +| Component | Path | Purpose | +|-----------|------|---------| +| NVCA source | `src/compute-plane-services/nvca` | Operator and agent source plus source chart at `deployments/nvca-operator/` | +| Vendored chart | `deploy/helm/nvca-operator` | Vendors the source chart, applies self-managed defaults, publishes `helm-nvca-operator` | +| Self-managed stack | `deploy/stacks/self-managed` | Control-plane Helmfile deployment and environment defaults | + +## Workflow + +1. Make source changes in `src/compute-plane-services/nvca` when operator, + agent, or source chart behavior changes. Test them there. +2. Vendor from the monorepo source chart at + `src/compute-plane-services/nvca/deployments/nvca-operator/`. +3. Set the version inputs, either in the environment or in + `deploy/helm/nvca-operator/.env`: + + ```bash + NVCA_OPERATOR_VERSION= + NVCA_VERSION= + NVCA_SHARED_STORAGE_IMAGE_TAG= + ``` + +4. Vendor and validate from `deploy/helm/nvca-operator`: + + ```bash + make vendor-chart + make lint + make template + make validate + ``` + +5. If the chart is tested against a local self-managed control plane, render + stack-aware values and install from this chart subtree: + + ```bash + make render-values-from-stack stack_repo=../../../deploy/stacks/self-managed stack_env=local + make install-from-stack stack_repo=../../../deploy/stacks/self-managed stack_env=local + ``` + + Use `additional_values=override.yml` for one-off validation. Do not edit the + stack just to test this chart. + +## CI and Release + +Umbrella CI is declared in `tools/ci/subproject-validations.yaml` with +subproject id `nvca-operator`. Do not add a chart-local `.gitlab-ci.yml`. + +Validate the same chart path CI uses: + +```bash +tools/ci/validate-helm-chart deploy/helm/nvca-operator/nvca-operator \ + -f tools/ci/helm-validate-values/nvca-operator.yaml +``` + +## Local Image Testing + +When testing local images in k3d, build them from `src/compute-plane-services/nvca` +and import them into the test cluster. Keep tags explicit and match them in the +chart values: + +```bash +export NVCA_OPERATOR_VERSION=dev-local +export NVCA_VERSION=dev-local + +# Run from src/compute-plane-services/nvca. +docker build -f docker/Dockerfile.nvca-operator \ + -t nvca-operator:$NVCA_OPERATOR_VERSION . +docker build -f docker/Dockerfile.nvca \ + -t nvca:$NVCA_VERSION . +``` + +Use the local-dev safety guidance before creating or deleting k3d clusters. + +## Gotchas + +- `make vendor-chart` overwrites the vendored `nvca-operator/` chart. +- Chart release generation depends on Conventional Commit semantics at the + umbrella level. Use `feat` or `fix` when a chart release is required. +- Keep `image.*`, `nvcaImage.*`, `ngcConfig.*`, and `selfManaged.*` values in + sync with the stack and source image tags. +- Never commit service keys, kubeconfigs, rendered secrets, or local registry + credentials. diff --git a/ai-tooling/dev/skills/nvca-self-managed-install/SKILL.md b/ai-tooling/dev/skills/nvca-self-managed-install/SKILL.md new file mode 100644 index 000000000..2abb06ec6 --- /dev/null +++ b/ai-tooling/dev/skills/nvca-self-managed-install/SKILL.md @@ -0,0 +1,71 @@ +--- +name: nvca-self-managed-install +description: Install or validate the NVCA Operator chart against a self-managed NVCF control plane from the native monorepo. Use when the control plane comes from deploy/stacks/self-managed and NVCA must be installed with stack-derived image repository settings. +license: Apache-2.0 +compatibility: Requires a local checkout of the NVCF monorepo with deploy/helm/nvca-operator/ and deploy/stacks/self-managed/ present, plus helm. +author: "nvcf-core-eng " +version: "1.0.0" +tags: [nvcf, nvca, helm, install, self-managed] +tools: [Read, Shell] +metadata: + internal: false + author: "nvcf-core-eng " + version: "1.0" + tags: [nvcf, nvca, helm, install] + languages: [bash] + frameworks: [helm] + domain: cloud-infrastructure +--- + +# NVCA Self-Managed Install + +Use `deploy/helm/nvca-operator` as the install surface for NVCA Operator chart +validation. Do not add NVCA back into the stack Helmfile just to test this +chart. + +## Source of Truth + +- Control-plane deployment: `deploy/stacks/self-managed` +- NVCA chart and install workflow: `deploy/helm/nvca-operator` +- NVCA source images and source chart: `src/compute-plane-services/nvca` +- Cluster prerequisites: use the public `nvcf-self-managed-prerequisite` skill +- Full stack install and teardown: use the public `nvcf-self-managed-installation` skill + +## Preferred Workflow + +Run from `deploy/helm/nvca-operator`: + +```bash +make render-values-from-stack \ + stack_env=local \ + stack_repo=../../../deploy/stacks/self-managed +``` + +Inspect the generated file under `bin/`, then install or upgrade: + +```bash +make install-from-stack \ + stack_env=local \ + stack_repo=../../../deploy/stacks/self-managed +``` + +## Optional Inputs + +- `additional_values=override.yml` for one-off overrides +- `NCA_ID=` to set the primary account ID explicitly +- `CLUSTER_NAME=` and `CLUSTER_ID=` to preserve or pin cluster identity +- `IMAGE_PULL_SECRET_NAME=` to attach a pre-created image pull secret + +## Existing Release Reuse + +If `nvca-operator` is already installed in the `nvca-operator` namespace, the +render helper reuses current release values first, then overlays repository +paths derived from the stack environment file. Use this to preserve cluster IDs +and operator settings during upgrades. + +## Safety + +Before creating, deleting, or reusing local k3d clusters, read the local-dev +safety guidance in `.cursor/skills/nvcf-self-hosted-local-dev/SKILL.md` from +the repo root. Do not delete clusters or Helm releases without explicit user +approval. diff --git a/ai-tooling/dev/skills/nvca-values-customization/SKILL.md b/ai-tooling/dev/skills/nvca-values-customization/SKILL.md new file mode 100644 index 000000000..a87ea609b --- /dev/null +++ b/ai-tooling/dev/skills/nvca-values-customization/SKILL.md @@ -0,0 +1,101 @@ +--- +name: nvca-values-customization +description: Customize NVCA Operator Helm chart values in the native monorepo. Use when modifying vendored defaults, changing stack-derived install values, adding deployment-time overrides, or updating scripts under deploy/helm/nvca-operator. +license: Apache-2.0 +compatibility: Requires a local checkout of the NVCF monorepo with deploy/helm/nvca-operator/ present, plus helm and yq. +author: "nvcf-core-eng " +version: "1.0.0" +tags: [nvcf, nvca, helm, values, customization] +tools: [Read, Grep] +metadata: + internal: false + author: "nvcf-core-eng " + version: "1.0" + tags: [nvcf, nvca, helm, values] + languages: [bash, yaml] + frameworks: [helm] + domain: cloud-infrastructure +--- + +# Customizing NVCA Operator Chart Values + +Use this skill from `deploy/helm/nvca-operator`. + +## Values Flow + +```text +src/compute-plane-services/nvca/deployments/nvca-operator/ source chart + -> scripts/ci_vendor_nvca_operator_chart applies self-managed defaults + -> nvca-operator/values.yaml vendored chart values + -> scripts/render_values_from_stack_env.sh stack-aware generated values + -> make install or make install-from-stack optional additional overrides +``` + +## Permanent Defaults + +For defaults that every self-managed deployment should receive, edit +`scripts/ci_vendor_nvca_operator_chart` and re-vendor: + +```bash +make vendor-chart +git diff nvca-operator/values.yaml +``` + +The vendoring script already applies defaults such as: + +- `ngcConfig.clusterSource = "self-managed"` +- `ngcConfig.serviceKey = "dummy-api-key"` +- `image.tag = "$NVCA_OPERATOR_VERSION"` +- `selfManaged.nvcaVersion = "$NVCA_VERSION"` +- `generateImagePullSecret = false` +- `selfManaged.sharedStorage.imageTag = "$NVCA_SHARED_STORAGE_IMAGE_TAG"` +- `nameOverride = "nvca-operator"` +- `fullnameOverride = "nvca-operator"` + +Do not edit `nvca-operator/values.yaml` directly for a permanent default. The +next vendor run will overwrite it. + +## Deploy-time Overrides + +Use `additional_values` for one-off validation or environment-specific values: + +```bash +make install-from-stack \ + stack_repo=../../../deploy/stacks/self-managed \ + stack_env=local \ + additional_values=override.yml +``` + +Use deploy-time overrides for secrets, credentials, cluster-specific IDs, and +temporary validation changes. + +## Adding .env Inputs + +For version-like values that the vendoring script needs, add a variable to +`.env`, require it in `scripts/ci_vendor_nvca_operator_chart`, and re-vendor: + +```bash +MY_NEW_CONFIG=some-value +``` + +```bash +update_yaml_key ".myConfig = \"${MY_NEW_CONFIG:?MY_NEW_CONFIG is not set}\"" "${TARGET_DIR}/values.yaml" +``` + +## Validation + +```bash +make lint +make template +make validate +tools/ci/validate-helm-chart deploy/helm/nvca-operator/nvca-operator \ + -f tools/ci/helm-validate-values/nvca-operator.yaml +``` + +## Gotchas + +- Install-time values are layered after generated stack-aware values. +- Use `yq` carefully for nested keys and quoted strings. +- Keep `Chart.yaml` name/version changes in the vendoring script when they are + part of the self-managed packaging contract. +- Never commit real service keys or rendered secret material. diff --git a/deploy/helm/admin-token-issuer-proxy/values.local.yaml b/deploy/helm/admin-token-issuer-proxy/values.local.yaml new file mode 100644 index 000000000..44eb05b80 --- /dev/null +++ b/deploy/helm/admin-token-issuer-proxy/values.local.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +adminIssuerProxy: + image: + registry: "" + repository: "" + tag: "1.0.2" diff --git a/deploy/helm/api-keys-colocated/Makefile b/deploy/helm/api-keys-colocated/Makefile new file mode 100644 index 000000000..874dc1b83 --- /dev/null +++ b/deploy/helm/api-keys-colocated/Makefile @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variables +release ?= api-keys +namespace ?= api-keys +helm_dir ?= ./api-keys +values := $(helm_dir)/values.yaml + +# OPTIONAL for deploy target: Path to an additional Helm values file. +# Example: make deploy values=my-values.yaml additional_values=override.yaml +additional_values ?= + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +.PHONY: deploy delete status lint template + +# Test targets +test-consul-template: + @echo "Testing consul-template..." + @bash api-keys/vault-agent-templates/test-template-render.sh + +install: +ifndef values + $(error "values" variable is not set. Please specify with 'make deploy values=') +endif + @echo "Deploying $(release) to namespace $(namespace) using values file '$(values)'..." + @echo "Additional values file: '$(if $(additional_values),$(additional_values),N/A)'" + helm install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +uninstall: + @echo "Deleting $(release) from namespace $(namespace)..." + helm uninstall $(release) --namespace $(namespace) + +status: + @echo "Checking status of $(release) in namespace $(namespace)..." + helm status $(release) --namespace $(namespace) + +lint: + @echo "Linting chart $(helm_dir)..." + helm lint $(helm_dir) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) + +template: + @echo "Templating chart $(helm_dir)..." + @mkdir -p bin + helm template $(release) $(helm_dir) \ + --namespace $(namespace) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + > bin/manifest.yaml + @echo "Rendered manifest to bin/manifest.yaml" + +validate: template + @echo "Validating manifest with kubeconform..." + @kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml + +# Publish Chart +# NOTE: this is manual until the CI pipeline is updated to push the chart to the NVCR OCI registry +clean: + rm -rf $(charts_dir)/*.tgz + rm -rf ./packaged-charts + +package: clean lint + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(helm_dir) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." diff --git a/deploy/helm/api-keys-colocated/keygen.sh b/deploy/helm/api-keys-colocated/keygen.sh new file mode 100755 index 000000000..0c61b0548 --- /dev/null +++ b/deploy/helm/api-keys-colocated/keygen.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Exit on any error +set -e + +# Function to generate a random kid (Key ID) +generate_kid() { + date_str=$(date +"%Y%m%d%H%M%S") + echo "kid-$date_str" +} + +# Function to check if OpenSSL is available +check_openssl() { + if ! command -v openssl &> /dev/null; then + echo "Error: OpenSSL is required but not installed." + exit 1 + fi +} + +# Function to generate data domain key +generate_domain_key() { + # Generate 2048 bits (256 bytes) random key and encode to base64url + local raw_key=$(openssl rand -base64 256) + echo -n "$raw_key" | tr '/+' '_-' | tr -d '=\n' +} + +# Main function to generate JWK +generate_jwk() { + # Generate 256-bit (32 byte) random key + local raw_key=$(openssl rand -base64 32) + + # Convert to base64url encoding (replace / with _, + with -, remove =) + local base64url_key=$(echo -n "$raw_key" | tr '/+' '_-' | tr -d '=\n') + + # Generate key ID + local kid=$(generate_kid) + + # Create JWK JSON + echo -n "{\"kty\":\"oct\",\"use\":\"enc\",\"kid\":\"$kid\",\"k\":\"$base64url_key\",\"alg\":\"A256GCM\"}" +} + +# Function to generate JWKS +generate_jwks() { + local jwk=$(generate_jwk) + echo -n "{\"keys\":[$jwk]}" +} + +# Check dependencies +check_openssl + +# Generate and output both keys +echo "=== JWKS for A256GCM ===" +generate_jwks + +echo -e "\n=== Data Domain Key ===" +echo "{\"value\":\"$(generate_domain_key)\"}" \ No newline at end of file diff --git a/deploy/helm/api-keys-colocated/values.local.yaml b/deploy/helm/api-keys-colocated/values.local.yaml new file mode 100644 index 000000000..37dcea99b --- /dev/null +++ b/deploy/helm/api-keys-colocated/values.local.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apikeys: + image: + registry: "" + repository: "" + tag: "1.5.0" diff --git a/deploy/helm/cassandra/Makefile b/deploy/helm/cassandra/Makefile new file mode 100644 index 000000000..16104069d --- /dev/null +++ b/deploy/helm/cassandra/Makefile @@ -0,0 +1,120 @@ +.PHONY: all deps build install upgrade uninstall clean ensure-helmignore package push-oci bump-version release prepare-local template + +# Directory variables +helm_dir := helm +charts_dir := $(helm_dir)/charts + +# Chart and namespace +release_name := cassandra +namespace := cassandra-system +values := $(helm_dir)/values.yaml + +# Optional additional values files +additional_values ?= + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +# Variable for the new version, to be passed as an argument to make +# Example: make bump-version NEW_VERSION=0.2.0 +NEW_VERSION ?= + +all: install + +deps: + @if ! ls $(charts_dir)/*.tgz 1>/dev/null 2>&1; then \ + helm dependency build $(helm_dir); \ + else \ + echo "[deps] helm dependencies already built, skipping"; \ + fi + +build: deps + @echo "[build] nothing to do" + +install: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + helm install $(release_name) $(helm_dir) -n $(namespace) --values $(values) $(if $(additional_values),--values $(additional_values),) --atomic --create-namespace --wait --wait-for-jobs --timeout 20m --set global.security.allowInsecureImages=true + +install-local: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + helm upgrade --install $(release_name) $(helm_dir) -n $(namespace) --values $(values) --values ./values.local.yml $(if $(atomic),--atomic,) --create-namespace --wait --wait-for-jobs --timeout 20m --set global.security.allowInsecureImages=true + +debug-local: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + helm install --debug --dry-run $(release_name) $(helm_dir) -n $(namespace) --values $(values) --values ./values.local.yml $(if $(atomic),--atomic,) --create-namespace --wait --wait-for-jobs --timeout 20m --set global.security.allowInsecureImages=true + +template: build + helm template $(release_name) $(helm_dir) -n $(namespace) --values $(values) $(if $(additional_values),--values $(additional_values),) + +upgrade: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + helm upgrade $(release_name) $(helm_dir) -n $(namespace) --values $(values) $(if $(additional_values),--values $(additional_values),) --atomic --wait --wait-for-jobs --timeout 20m --force + +uninstall: + helm uninstall $(release_name) -n $(namespace) || true + kubectl delete ns ${namespace} + +clean: + rm -rf $(charts_dir)/*.tgz + rm -rf ./packaged-charts + +# Ensure .helmignore is set up to exclude dependencies and secrets +ensure-helmignore: + # Do not ignore the charts/ directory itself, as helm package needs to access + # dependencies placed there by 'helm dependency build'. + @echo "secrets/" > $(helm_dir)/.helmignore + @echo ".DS_Store" >> $(helm_dir)/.helmignore + @echo "ci/" >> $(helm_dir)/.helmignore + @echo "[ensure-helmignore] .helmignore configured in $(helm_dir)" + +package: deps ensure-helmignore + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(helm_dir) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." + +bump-version: +ifndef NEW_VERSION + $(error NEW_VERSION is not set. Usage: make bump-version NEW_VERSION=x.y.z) +endif + @echo "[bump-version] Current chart version in $(helm_dir)/Chart.yaml (name: $(CHART_NAME)): $(CHART_VERSION)" + @echo "[bump-version] Bumping chart version to: $(NEW_VERSION)" + @yq e '.version = "$(NEW_VERSION)"' -i $(helm_dir)/Chart.yaml + $(eval CHART_VERSION := $(NEW_VERSION)) # Update CHART_VERSION for subsequent messages + @echo "[bump-version] Successfully bumped chart version in $(helm_dir)/Chart.yaml to $(CHART_VERSION)" + @echo "[bump-version] You might want to commit this change." + +release: bump-version package push-oci + @echo "[release] Successfully released chart $(CHART_NAME) version $(CHART_VERSION) to OCI registry." + @echo "[release] You might want to commit this change." + +DOCKER_CONFIG_JSON := ./helm/secrets/.dockerconfigjson +prepare-local: deps + @echo "[prepare-local] Preparing local environment" + @if [ -f ${DOCKER_CONFIG_JSON} ]; then \ + echo "[prepare-local] ${DOCKER_CONFIG_JSON} file exists, skipping creation"; \ + else \ + echo "[prepare-local] ${DOCKER_CONFIG_JSON} does not exist, creating it. please update the file with your auth token for nvcr.io"; \ + echo '{"auths":{"nvcr.io":{"auth":""}}}' > ${DOCKER_CONFIG_JSON}; \ + fi + @echo "[prepare-local] Successfully prepared local environment" diff --git a/deploy/helm/cassandra/README.md b/deploy/helm/cassandra/README.md index edd4d6ff5..590ba687a 100644 --- a/deploy/helm/cassandra/README.md +++ b/deploy/helm/cassandra/README.md @@ -15,14 +15,14 @@ cassandra: migrations: image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/nvcf-cassandra-migrations + repository: /nvcf-cassandra-migrations image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/bitnami-cassandra + repository: /bitnami-cassandra dynamicSeedDiscovery: image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/bitnami-cassandra + repository: /bitnami-cassandra ``` ## Prerequisites diff --git a/deploy/helm/cassandra/values.eks.yml b/deploy/helm/cassandra/values.eks.yml new file mode 100644 index 000000000..00f567568 --- /dev/null +++ b/deploy/helm/cassandra/values.eks.yml @@ -0,0 +1,6 @@ +cassandra: + global: + defaultStorageClass: gp3 + + nodeSelector: + node-type: cassandra diff --git a/deploy/helm/cassandra/values.local.yml b/deploy/helm/cassandra/values.local.yml new file mode 100644 index 000000000..4db8e5310 --- /dev/null +++ b/deploy/helm/cassandra/values.local.yml @@ -0,0 +1,74 @@ +# When running on local via colima it needs min of 6 CPU and 8GB of memory +# Use make install-local to install the chart locally + +cassandra_registry: &cassandra_registry "" + +cassandra_repo: &cassandra_repo "" +cassandra_tag: &cassandra_tag 5.0.6-nv-1 + +cassandra_migrations_tag: &cassandra_migrations_tag latest +cassandra_migrations_repo: &cassandra_migrations_repo "" + +# change this to Always if you want to pull the image every time +pullPolicy: &pullPolicy IfNotPresent + +cassandra: + global: + security: + allowInsecureImages: true + # accepts either a map or a list + # imagePullSecrets: + # - name: nvcr-secret + # - name: anotherSecret + # imagePullSecrets: + # - nvcr-secret + # - anotherSecret + imagePullSecrets: + - name: nvcr-secret + + migrations: + image: + registry: "" + repository: "" + tag: *cassandra_migrations_tag + pullPolicy: *pullPolicy + + # see: https://github.com/bitnami/charts/blob/main/bitnami/cassandra/values.yaml#L373 + # resourcesPreset: small + + # reduce compute usage + replicaCount: 1 + + # adjust probes to be more lenient + startupProbe: + initialDelaySeconds: 15 + + readinessProbe: + initialDelaySeconds: 15 + periodSeconds: 10 + + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 30 + + image: + registry: "" + repository: "" + # tag: *cassandra_tag + pullPolicy: *pullPolicy + + dynamicSeedDiscovery: + image: + registry: "" + repository: "" + # tag: *cassandra_tag + pullPolicy: *pullPolicy + + jvm: + # this should only be set for local development with only 1 replica is used. + # use of this with more than 1 replica will cause node startups to unexpectedly fail. + # cassandra-exporter-agent.jar is required to support prometheus metrics exposure + extraOpts: | + -javaagent:/opt/bitnami/cassandra/lib/cassandra-exporter-agent.jar + -Dcassandra.superuser_setup_delay_ms=100 + -Dcassandra.gossip_settle_min_wait_ms=1000 diff --git a/deploy/helm/container-cache/README.md b/deploy/helm/container-cache/README.md new file mode 100644 index 000000000..c3ca5460e --- /dev/null +++ b/deploy/helm/container-cache/README.md @@ -0,0 +1,539 @@ + +# NVCF Container Cache + +**Version:** 0.25.12 +**Application Version:** 1.2.1 +**Chart Name:** nvcf-container-cache + +## Table of Contents +- [Overview](#overview) +- [Architecture](#architecture) + - [Container Cache](#container-cache) + - [Proxy Cache](#proxy-cache) + - [Authentication Flow (Container Registries)](#authentication-flow-container-registries) + - [Authentication Flow (S3 Proxy Cache)](#authentication-flow-s3-proxy-cache) +- [Prerequisites](#prerequisites) +- [Configuration Reference](#configuration-reference) +- [Deployment Guide](#deployment-guide) +- [Container Runtime Support](#container-runtime-support) +- [Uninstallation and Cleanup](#uninstallation-and-cleanup) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +NVCF Container Cache is a high-performance caching system with two complementary modes: + +- **Container Cache** -- caches container image pulls (OCI/Docker registry protocol) for containerd and CRI-O +- **Proxy Cache** -- caches S3 objects, NGC assets, HuggingFace models, and Nucleus files via DNS-based MITM + +Both run in the same StatefulSet pod and share the OpenResty/Nginx engine with dynamic TLS certificate generation. + +### Key Features +- **Multi-runtime**: Supports both containerd and CRI-O for container image caching +- **Multi-endpoint**: S3, NGC Files, HuggingFace CDN, Nucleus/LFT for proxy caching +- **Multi-registry**: NGC, Docker Hub, GCR, ECR, and any OCI-compliant registry +- **Transparent fallback**: If the cache is unavailable, container pulls fall through to the upstream registry +- **DNS-based MITM**: nvcf-unbound DNS redirects S3/NGC/HF traffic to the proxy cache transparently +- **Dynamic TLS**: Generates leaf certificates on-the-fly signed by a bundled CA (SNI-based) +- **Auto-configuration**: DaemonSet configures container runtimes on every node automatically +- **Disk-pressure aware**: `min_free` watermark prevents the cache from filling the disk +- **Observable**: Prometheus metrics, structured JSON logs, optional OpenTelemetry traces + +--- + +## Architecture + +The system has two caching modes that serve different traffic types. + +### Container Cache + +Caches container image layers and manifests pulled by containerd or CRI-O. Routing is via explicit mirror configuration written by the DaemonSet -- no DNS trick involved. + +![Container Cache Architecture](docs/container-cache-architecture.png) + +``` + +---------------------------+ + | Upstream Registries | + | (nvcr.io, docker.io, etc) | + +------------+--------------+ + | + | HTTPS + v + +-------------------------------+ + | NVCF Container Cache Pod | + | (OpenResty/Nginx, StatefulSet) | + | | + | lua-ssl.lua Cache Engine | + | (dynamic TLS) (disk cache) | + | | + | Ports: 13128 (containerd) | + | 30346+ (CRI-O) | + +-------+--------------+--------+ + | | + cache HIT | | cache MISS + (from disk) | | (fetch upstream) + v v + +----------------------------------------------------+ + | Kubernetes Worker Nodes | + | | + | containerd CRI-O | + | hosts.toml: registries.conf: | + | 1. Try cache 1. Try cache | + | 2. Fallback 2. Fallback | + | | + | DaemonSet: writes mirror config, copies CA cert | + | | + | kubelet --> Pod: container image pull | + +----------------------------------------------------+ +``` + +**How a cached pull works:** + +1. **kubelet** asks the container runtime to pull `nvcr.io/nvidia/cuda:12.4.0` +2. **containerd/CRI-O** checks its mirror config (`hosts.toml` / `registries.conf`) +3. The request is routed to the **cache proxy** (via NodePort or pod IP) +4. The proxy checks its **on-disk cache**: + - **HIT**: Returns the cached layer immediately + - **MISS**: Fetches from the upstream registry, caches it, and returns it +5. If the cache proxy is **unreachable**, the runtime falls back to the **upstream registry** directly + +### Proxy Cache + +Caches S3 objects, NGC assets, and HuggingFace models. Uses DNS-based MITM via nvcf-unbound: DNS resolution for configured domains returns a CNAME pointing to the cache service, so pods connect to the cache thinking it is the real endpoint. The cache terminates TLS using SNI (lua-ssl.lua generates a matching leaf certificate on the fly) and proxies to the real upstream. + +![Proxy Cache Architecture](docs/proxy-cache-architecture.png) + +``` + +---------------------------------------+ + | Upstream Endpoints | + | AWS S3 | NGC Files | HuggingFace CDN | + +------------------+--------------------+ + ^ + | HTTPS (real upstream) + | + +---------------------------+ +----+------------------------------+ + | nvcf-unbound DNS | | NVCF Proxy Cache Pod | + | (dns-proxy namespace) | | (same StatefulSet, port 14128) | + | | | | + | Intercepts DNS: | | lua-ssl.lua S3/NGC/HF routing | + | s3.amazonaws.com | | (SNI certs) proxy_s3 cache | + | xfiles.ngc.nvidia.com +--->| proxy_ngc cache | + | huggingface.co | | | + | | | Proxy Cache PVC (disk) | + | Returns CNAME: | +----+------------------------------+ + | -> nvcf-proxy-cache.svc | | + +----------+----------------+ | cache HIT / MISS + ^ v + | 1. DNS query 3. HTTPS connect + | (thinks it's real S3) + +-------+------------------------------------+ + | Cluster Pods (Training/Inference Jobs) | + | | + | Transparent to the application: | + | no config changes needed in the pod. | + +---------------------------------------------+ +``` + +**How DNS-based MITM caching works:** + +1. A **training pod** calls `s3.us-east-1.amazonaws.com` to download a dataset +2. The DNS query goes to **nvcf-unbound**, which returns a CNAME pointing to `nvcf-proxy-cache.container-caching.svc.cluster.local` +3. The pod connects to the **cache pod** on port 14128 via TLS, presenting the original hostname as SNI +4. **lua-ssl.lua** dynamically generates a leaf certificate matching the SNI hostname +5. The cache checks its **on-disk cache** (`proxy_s3` or `proxy_ngc` zone): + - **HIT**: Returns the cached response immediately + - **MISS**: Proxies to the real upstream, caches the response, and returns it +6. The pod receives the response as if it came directly from S3 -- completely transparent + +### Authentication Flow (Container Registries) + +The cache validates credentials on **every request** by sending a HEAD to the upstream registry before serving cached content. This ensures revoked or rotated keys are rejected immediately. + +![Container Registry Auth Flow](docs/container-registry-auth-flow.png) + +Key behaviors: +- **Key rotation**: Old keys are rejected immediately. The cache never serves content with stale credentials. +- **Public registries**: HEAD with no auth header returns 200, so caching works for all users with no configuration needed. +- **Private registries**: Each request is validated against the upstream. Different users sharing the same cluster get the same cached blobs (blobs are content-addressed by SHA256, so identical content is safe to share). +- **Manifest by tag** (e.g., `:latest`): Never cached. Tags are mutable, so every request goes to the upstream registry to get the current digest. +- **Node-local cache**: Once an image is pulled to a node, it is available to all pods on that node without auth. Set `imagePullPolicy: Always` to force re-validation through the cache on every pod start. + +### Authentication Flow (S3 Proxy Cache) + +The S3 proxy cache validates access on every request using `lua-access.lua`. The logic differs by request type (HEAD, presigned URL GET, normal GET) and uses shared-memory caches to avoid redundant upstream calls during burst traffic. + +![S3 Proxy Cache Auth Flow](docs/s3-proxy-auth-flow.png) + +``` + +-- Cache disabled? (no-cache header) --YES--> Bypass, proxy direct to S3 + | + S3 request ----->+ + (port 14128) | + +-- Request method? + | + +-------------+-------------+ + | | | + HEAD GET+presigned Normal GET + | URL | + v | v + In HEAD auth v Public URL? + cache? ----YES--> Skip (no Authorization, + | auth no X-Amz-Signature) + NO | | + | In presigned YES: skip auth, + v URL cache? cache shared + Lock by URI, | by all users + HEAD to S3 YES: skip | + with headers auth NO: send full + | | request to S3 + v NO with Authorization + S3 200/404? | | + YES: cache Lock, GET v + auth result to S3 with S3 200? + (TTL-based) presigned YES: proceed + | params NO: return 403 + | | + | S3 200? + | YES: cache + | until expiry + | NO: return + | 403/404 + | | + +------+------+------+ + | + v + In disk cache + (proxy_s3)? + / \ + YES NO + | | + CACHE HIT CACHE MISS + serve from fetch from + disk S3, store, + return +``` + +Key behaviors: +- **Cache key**: `bucket_region | bucket_name | method | URI | versionId | If-Match | filtered_args | range`. This ensures different buckets, versions, and byte ranges are cached independently. +- **Presigned URLs**: Cached until their `X-Amz-Expires` time minus a 10-second safety margin. This avoids serving content after the presigned URL has expired. +- **HEAD auth caching**: HEAD results are cached in shared memory with a configurable TTL. This dramatically reduces upstream calls during burst traffic (e.g., hundreds of pods starting simultaneously and checking the same S3 object). +- **Public S3 objects**: If a request has no `Authorization` header and no `X-Amz-Signature` query parameter, it is treated as public. No upstream auth check is performed, and the cached object is shared across all users. +- **Locking**: Concurrent requests for the same URI are serialized using a lock (`resty.lock`) to avoid thundering herd -- only one request performs the upstream auth check, and subsequent requests use the cached result. +- **Stale cache resilience**: On upstream errors (500/502/503/504), stale cached content is served rather than returning an error to the client. +- **If-Match / ETag**: The `If-Match` header is preserved end-to-end (S3 normally expects it for content integrity) and is included in the cache key to avoid serving mismatched content. +- **Last-Modified validation**: On cache HIT, the proxy compares the cached `Last-Modified` timestamp against the upstream value. If upstream is newer, the stale entry is purged and a 429 is returned so the client retries and gets fresh content. + +--- + +## Prerequisites + +1. **Kubernetes** 1.19+ with containerd or CRI-O runtime +2. **Helm** 3.x +3. **NGC API Key** for pulling NVIDIA container images +4. **Storage**: Persistent volumes or sufficient node storage +5. **Network**: Outbound HTTPS to target registries + +--- + +## Configuration Reference + +### Core Settings + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `nodeSelector` | `{}` | Node labels for cache pod placement | +| `replicaCount` | `1` | Number of cache StatefulSet replicas | +| `targetHost` | `nvcr.io,stg.nvcr.io` | Comma-separated registries to cache | + +### Images + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `images.server` | `//nvcf-container-cache:v1.1.33` | Cache proxy image | +| `images.exporter` | `//nginx-prometheus-exporter:1.0` | Metrics exporter | +| `images.certificates` | `//nvcf-proxy-tls-certs:1.2.2` | TLS cert bundle | +| `images.secrets` | `[nvidia-ngcuser-pull-secret]` | Image pull secrets | + +### Cache Behavior + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `cache.keyStorageSize` | `50m` | Shared memory for cache keys | +| `cache.maxSize` | `500g` | Max on-disk cache size | +| `cache.inactive` | `30d` | Evict entries not accessed in this period | +| `cache.valid` | `12h` | Revalidate cached entries after this period | +| `cache.http2` | `off` | HTTP/2 for ingress listeners | +| `cache.workerConnection` | `1000` | Nginx worker connections | + +### Storage + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `persistentVolumeClaim.storageClassName` | `emptydir` | Storage class (`emptydir` for ephemeral) | +| `persistentVolumeClaim.sizeGB` | `100` | Container cache volume (Gi) | +| `persistentVolumeClaim.sizeProxyGB` | `200` | Proxy cache volume (Gi) | +| `persistentVolumeClaim.freeProxyPct` | `10` | Min free space percentage | + +### Service + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `service.type` | `NodePort` | `NodePort`, `ClusterIP`, or `LoadBalancer` | +| `service.port` | `30345` | Primary service port | + +### CRI-O + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `crio.registryPorts` | `[]` | Manual per-registry port mappings | +| `crio.basePort` | `service.port + 1` | Starting port for auto-derived CRI-O ports | +| `crio.mirrorHost` | `nvcf-container-cache..svc.cluster.local` | Mirror hostname for CRI-O | +| `crio.nodePortLocal` | `true` | Set `externalTrafficPolicy: Local` on NodePort | + +CRI-O requires a separate listening port per registry (unlike containerd which uses `?ns=` query params). Ports are auto-derived from `targetHost` starting at `basePort`: + +| targetHost | Auto port | +|------------|-----------| +| `nvcr.io` | 30346 | +| `stg.nvcr.io` | 30347 | + +### Security + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `vault.enabled` | `false` | Enable HashiCorp Vault for cert management | +| `vault.vaultAddress` | `https://vault.example.com` | Vault server URL | +| `vault.certLocation` | `""` | URL to download CA cert (vault mode) | + +When `vault.enabled=false`, the bundled TLS certificates from the `images.certificates` image are used. The proxy dynamically generates leaf certificates signed by the bundled intermediate CA. + +### Observability + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `monitoring.enabled` | `false` | Deploy the `nginx-prometheus-exporter` sidecar **and** create a PodMonitor for Prometheus. When `false`, neither is rendered and the `images.exporter` image does not need to be reachable. The lua-prometheus endpoint on `cache-metrics` (port 9145) is unaffected by this setting and remains available for in-cluster scraping. | +| `metrics.cacheMetricsStorageSize` | `300m` | Shared memory for Prometheus metrics | +| `traces.enabled` | `false` | Enable OpenTelemetry tracing | +| `nucleus.enabled` | `false` | Enable nucleus proxy integration | + +--- + +## Deployment Guide + +### 1. Create namespace and pull secret + +```bash +kubectl create namespace container-caching + +kubectl create secret docker-registry nvidia-ngcuser-pull-secret \ + --docker-server=nvcr.io \ + --docker-username='$oauthtoken' \ + --docker-password= \ + -n container-caching +``` + +### 2. Create a values file + +```yaml +# my-values.yaml +nodeSelector: + nodeGroup: monitoring + +replicaCount: 1 +targetHost: nvcr.io,stg.nvcr.io + +images: + server: //nvcf-container-cache:v1.1.33 + exporter: //nginx-prometheus-exporter:1.0 + certificates: //nvcf-proxy-tls-certs:1.2.2 + secrets: + - nvidia-ngcuser-pull-secret + +service: + type: NodePort + port: 30345 + +persistentVolumeClaim: + storageClassName: premium-rwo + sizeProxyGB: 200 + sizeGB: 100 + freeProxyPct: 10 + +vault: + enabled: false + +monitoring: + enabled: false +``` + +### 3. Install + +```bash +helm install nvcf-container-cache ./deploy \ + -n container-caching \ + -f my-values.yaml +``` + +### 4. Verify + +```bash +# Check all resources +kubectl -n container-caching get pods,svc,ds + +# Verify containerd config on a node +kubectl -n container-caching exec ds/nvcf-container-cache-cc -- \ + cat /host/etc/containerd/certs.d/nvcr.io/hosts.toml +``` + +Expected `hosts.toml`: +```toml +server = "https://nvcr.io" +[host."https://10.0.0.27:30345"] +capabilities = ["pull", "resolve"] +ca = "/etc/ssl/nginx_container_cache.crt" +skip_verify = true +[host."https://nvcr.io"] +capabilities = ["pull", "resolve"] +``` + +The `skip_verify = true` is needed because the proxy's dynamically-generated TLS certificate cannot match every node IP when traffic arrives via NodePort (kube-proxy NATs the original destination IP). The explicit `[host."https://nvcr.io"]` entry ensures containerd falls back to the upstream registry if the cache is unavailable. + +--- + +## Container Runtime Support + +### containerd + +The DaemonSet writes `hosts.toml` files to `/etc/containerd/certs.d//` on every node. containerd reads these dynamically on each pull without requiring a restart. The configuration: + +1. Routes pulls through the cache proxy (via NodePort) +2. Falls back to the upstream registry if the proxy is unreachable +3. Trusts the cache's CA certificate + +### CRI-O + +CRI-O uses a different configuration model (`registries.conf` instead of `hosts.toml`). Key differences: + +- CRI-O does not support `?ns=` query parameters, so each registry needs a **dedicated port** +- The DaemonSet writes to `/etc/containers/registries.conf.d/nvcf-container-cache.conf` +- CRI-O mirrors are configured using pod IPs (discovered via Kubernetes API) or the service hostname +- The DaemonSet refreshes CRI-O mirrors every 5 minutes and signals CRI-O to reload +- An RBAC Role/RoleBinding grants the DaemonSet permission to list pods (for IP discovery) + +CRI-O auto-detection: if `/etc/containers` exists on a node, the DaemonSet configures CRI-O. If `/etc/containerd` exists, it configures containerd. Both can coexist. + +--- + +## Uninstallation and Cleanup + +### 1. Uninstall the Helm release + +```bash +helm uninstall nvcf-container-cache -n container-caching +``` + +### 2. Clean up node configuration + +The DaemonSet modifies containerd/CRI-O configuration on every node. After uninstalling, run the cleanup tool to remove these modifications: + +```bash +# Deploy cleanup DaemonSet, wait, then remove it +./client/remove/remove-containerd-configuration.sh +``` + +Or manually: +```bash +kubectl create -f ./client/remove/remove-containerd-configuration.yaml +kubectl wait --for=condition=ready pod -l name=remove-containerd-configuration \ + -n kube-system --timeout=120s +kubectl delete -f ./client/remove/remove-containerd-configuration.yaml +``` + +The cleanup tool: +- Removes `hosts.toml` files from `/etc/containerd/certs.d/` for each target registry +- Restores `config.toml` from the backup created during installation +- Removes CRI-O registry mirror config (`nvcf-container-cache.conf`) +- Removes CRI-O drop-in config (`99-nvcf-registry.conf`) +- Removes per-registry CA certificates from `/etc/containers/certs.d/` +- Removes the shared CA cert (`/etc/ssl/nginx_container_cache.crt`) +- Restarts containerd and reloads CRI-O + +### 3. Delete persistent data + +```bash +kubectl -n container-caching delete pvc --all +kubectl delete namespace container-caching +``` + +--- + +## Troubleshooting + +### TLS errors: `certificate is valid for X, not Y` + +This means the proxy's dynamically-generated certificate SAN doesn't match the IP the client connected to. This is expected with NodePort traffic (kube-proxy NATs the destination IP). The `skip_verify = true` in `hosts.toml` resolves this for containerd. For CRI-O, `insecure = true` is set when using the service hostname. + +### Connection refused on NodePort + +The cache pod or service was deleted (e.g., by a GitOps controller pruning resources). With the fallback `[host."https://nvcr.io"]` entry in `hosts.toml`, containerd should fall through to the upstream registry. If not, check: + +```bash +# Verify cache pods are running +kubectl -n container-caching get pods + +# Check hosts.toml has fallback entry +kubectl -n container-caching exec ds/nvcf-container-cache-cc -- \ + cat /host/etc/containerd/certs.d/nvcr.io/hosts.toml +``` + +### Cache always shows MISS + +- Verify the DaemonSet has configured the runtime on the node +- Check that the cache service is accessible: `curl -k https://:30345/v2/` +- Confirm cache disk has space: `kubectl exec nvcf-container-cache-0 -- df -h /container_cache` + +### CRI-O mirrors not updating + +The DaemonSet refreshes CRI-O mirrors every 5 minutes. Check: + +```bash +# View DaemonSet logs +kubectl -n container-caching logs ds/nvcf-container-cache-cc + +# Check registries.conf on a node +kubectl -n container-caching exec ds/nvcf-container-cache-cc -- \ + cat /host/etc/containers/registries.conf.d/nvcf-container-cache.conf +``` + +--- + +### Debug commands + +```bash +# Nginx proxy logs +kubectl -n container-caching logs nvcf-container-cache-0 -c nginx-proxy --tail=50 + +# Cache hit/miss activity +kubectl -n container-caching logs nvcf-container-cache-0 -c nginx-proxy | \ + grep -oP '"upstream_cache_status":"\K[^"]+' | sort | uniq -c + +# Rendered nginx configuration +kubectl -n container-caching exec nvcf-container-cache-0 -c nginx-proxy -- nginx -T +``` diff --git a/deploy/helm/container-cache/helmfile.d/nvcf-container-cache.yaml b/deploy/helm/container-cache/helmfile.d/nvcf-container-cache.yaml new file mode 100644 index 000000000..9170fe0e8 --- /dev/null +++ b/deploy/helm/container-cache/helmfile.d/nvcf-container-cache.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Minimal helmfile so generate-sbom can discover this chart in-repo (workspace +# root is resolved via helmfile.d/). Chart has no external repo dependencies. +releases: + - name: nvcf-container-cache + namespace: default + chart: ../deploy diff --git a/deploy/helm/container-cache/tests/chart-render/verify-mirrors.sh b/deploy/helm/container-cache/tests/chart-render/verify-mirrors.sh new file mode 100644 index 000000000..2ac555dfc --- /dev/null +++ b/deploy/helm/container-cache/tests/chart-render/verify-mirrors.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUT_FILE="$(mktemp)" +trap 'rm -f "${OUT_FILE}"' EXIT + +cd "${ROOT_DIR}" + +# Render a multi-domain scenario covering both CRI-O and containerd paths. +helm template container-cache ./deploy \ + --set-string targetHost="nvcr.io\,stg.nvcr.io\,docker.io" \ + > "${OUT_FILE}" + +# Use `grep -F` (fixed-string) and POSIX-portable flags. Every assertion below +# is a literal substring match on a single rendered line, so we do not need +# regex semantics or ripgrep (which is not installed on the CI tools image). +assert_has() { + local needle="$1" + if ! grep -F -q -- "${needle}" "${OUT_FILE}"; then + echo "FAILED: expected pattern not found: ${needle}" >&2 + exit 1 + fi +} + +assert_not_has() { + local needle="$1" + if grep -F -q -- "${needle}" "${OUT_FILE}"; then + echo "FAILED: unexpected pattern found: ${needle}" >&2 + exit 1 + fi +} + +echo "Checking Service shape..." +assert_has 'kind: Service' +assert_has 'name: nvcf-container-cache' +# externalTrafficPolicy: Local was removed -- it caused connection-refused on +# nodes without a local cache pod (kube-proxy drops NodePort traffic). +assert_not_has 'externalTrafficPolicy: Local' +assert_not_has 'internalTrafficPolicy: Local' + +echo "Checking multi-domain NodePort listeners..." +assert_has 'nodePort: 30346' +assert_has 'nodePort: 30347' +assert_has 'nodePort: 30348' +assert_has 'name: crio-nvcr-io' +assert_has 'name: crio-stg-nvcr-i' +assert_has 'name: crio-docker-io' + +echo "Checking generated registry->port map used by both runtimes..." +assert_has 'CRIO_PORTS["nvcr.io"]="30346"' +assert_has 'CRIO_PORTS["stg.nvcr.io"]="30347"' +assert_has 'CRIO_PORTS["docker.io"]="30348"' + +echo "Checking containerd mirror behavior..." +assert_has '[host."https://${NODE_IP}:${port}"]' + +echo "Checking CRI-O static mirror behavior..." +# CRI-O mirror points at NODE_IP (NodePort), not cluster service DNS, because +# the CRI-O daemon runs in the host network namespace and typically cannot +# resolve *.svc.cluster.local. +assert_has 'location = "%s:%s"' +assert_not_has 'registry_mirror_host=' +# We do not rewrite /etc/containers/registries.conf -- only the drop-in. +assert_not_has 'crio_main=' +# No 5-minute refresh loop. +assert_not_has 'sleep 300' +# pull-from-mirror must live in the [[registry.mirror]] block, not on the +# parent [[registry]]; CRI-O / containers-image rejects the latter. +assert_has 'pull-from-mirror = "all"' +# We mirror the drop-in into $HOME/.config/containers/registries.conf.d/ for +# distros (e.g. Oracle Linux) that ship a user-level registries.conf for root. +assert_has '/host/root/.config/containers' +assert_has 'user_drop_in_dir=' + +echo "Mirror render checks passed." diff --git a/deploy/helm/container-cache/tests/chart-render/verify-monitoring.sh b/deploy/helm/container-cache/tests/chart-render/verify-monitoring.sh new file mode 100755 index 000000000..315b56aa4 --- /dev/null +++ b/deploy/helm/container-cache/tests/chart-render/verify-monitoring.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Verify that monitoring.enabled fully gates the Prometheus integration: +# - the nginx-prometheus-exporter sidecar container, +# - the metrics port (9113) on that sidecar, +# - and the PodMonitor resource. +# +# When monitoring is off, none of those should be rendered. The lua-prometheus +# endpoint that nginx itself exposes on cache-metrics (9145) is independent of +# this gate and stays whether or not the sidecar is in the pod. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OFF_FILE="$(mktemp)" +ON_FILE="$(mktemp)" +trap 'rm -f "${OFF_FILE}" "${ON_FILE}"' EXIT + +cd "${ROOT_DIR}" + +helm template c ./deploy --set monitoring.enabled=false > "${OFF_FILE}" +helm template c ./deploy --set monitoring.enabled=true > "${ON_FILE}" + +assert_has() { + local file="$1" needle="$2" + if ! grep -F -q -- "${needle}" "${file}"; then + echo "FAILED: expected pattern not found in $(basename "${file}"): ${needle}" >&2 + exit 1 + fi +} + +assert_not_has() { + local file="$1" needle="$2" + if grep -F -q -- "${needle}" "${file}"; then + echo "FAILED: unexpected pattern found in $(basename "${file}"): ${needle}" >&2 + exit 1 + fi +} + +# ----- Assertions: monitoring disabled -------------------------------------- + +echo "Checking monitoring.enabled=false render..." +assert_not_has "${OFF_FILE}" 'nginx-prometheus-exporter' +assert_not_has "${OFF_FILE}" 'containerPort: 9113' +assert_not_has "${OFF_FILE}" 'kind: PodMonitor' +# The lua-prometheus port on the cache pod stays regardless. +assert_has "${OFF_FILE}" 'containerPort: 9145' + +# ----- Assertions: monitoring enabled --------------------------------------- + +echo "Checking monitoring.enabled=true render..." +assert_has "${ON_FILE}" 'name: nginx-prometheus-exporter' +assert_has "${ON_FILE}" 'containerPort: 9113' +assert_has "${ON_FILE}" 'kind: PodMonitor' +assert_has "${ON_FILE}" 'containerPort: 9145' + +echo "Monitoring gating checks passed." diff --git a/deploy/helm/container-cache/tests/chart-render/verify-registry-auth.sh b/deploy/helm/container-cache/tests/chart-render/verify-registry-auth.sh new file mode 100755 index 000000000..59bff2841 --- /dev/null +++ b/deploy/helm/container-cache/tests/chart-render/verify-registry-auth.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Verify the container-cache.conf registry-auth probe shape: +# +# - The probe is per-resource: HEAD against the same URI the client is +# requesting, with the client's Authorization header. A weaker scheme +# (e.g. probing the registry root or /v2/) would let a caller with a +# bearer scoped to repoA read a cached blob that originally came from +# repoB -- which neutralises the registry's authorization model. +# - Anonymous requests (no Authorization header) DO NOT enable the cache. +# There are no credentials to validate, so we cannot prove the caller +# has rights to anything cached. The request falls through to upstream +# and the registry's own auth policy applies. +# - Probe responses 200-399 are accepted as "auth ok". 3xx covers Docker +# Hub redirecting blob fetches to S3 / Cloudflare -- the redirect +# itself proves the bearer was accepted. +# - Both blob-by-digest and manifest-by-digest locations have a +# content-type guard that refuses to write text/html responses to disk +# (defends against rate-limit HTML pages poisoning the cache). +# - `proxy_no_cache $disable_cache` is wired alongside `proxy_cache_bypass` +# so a probe failure or HTML response actually blocks cache writes (not +# just reads). +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUT_FILE="$(mktemp)" +trap 'rm -f "${OUT_FILE}"' EXIT + +cd "${ROOT_DIR}" + +helm template c ./deploy --set-string 'targetHost=docker.io\,nvcr.io' > "${OUT_FILE}" + +assert_count() { + local needle="$1" want="$2" + local got + got=$(grep -F -c -- "${needle}" "${OUT_FILE}" || true) + if [[ "${got}" -lt "${want}" ]]; then + echo "FAILED: expected at least ${want} occurrences of '${needle}', got ${got}" >&2 + exit 1 + fi +} + +assert_not_has() { + local needle="$1" + if grep -F -q -- "${needle}" "${OUT_FILE}"; then + echo "FAILED: unexpected pattern found: ${needle}" >&2 + exit 1 + fi +} + +echo "Checking probe is per-resource (uses ngx.var.request_uri)..." +# Old probe (broken): hits the registry root or /v2/ with the client's auth +# -- doesn't validate that the bearer covers THIS resource. +# New probe (correct): hits the same URI the client is asking for. +assert_count 'auth_host .. ngx.var.request_uri' 2 +assert_not_has 'request_uri("https://" .. auth_host, {' +assert_not_has 'request_uri("https://" .. auth_host .. "/v2/"' + +echo "Checking probe accepts 2xx and 3xx (registry redirects)..." +# Authenticated 200 = "you can read this". +# Authenticated 3xx = "you can read this; here's where it lives" (Docker +# Hub redirects blobs to S3 / Cloudflare). +assert_count 'res.status >= 200 and res.status < 400' 2 + +echo "Checking anonymous requests bypass cache (no false validation)..." +assert_count 'Anonymous request; cache bypassed' 2 +# Anonymous path must NOT set disable_cache=0; it returns early leaving +# the location-level default ("1") in place. +assert_not_has 'Anonymous pull; skipping registry auth probe.' + +echo "Checking proxy_no_cache is wired..." +# Both blob-by-digest and manifest-by-digest paths must gate cache *writes* +# on $disable_cache, not just reads. +assert_count 'proxy_no_cache $disable_cache' 2 + +echo "Checking text/html content-type guard..." +# Both digest paths defensively refuse to cache HTML responses (e.g. +# Docker Hub rate-limit pages). +assert_count 'string.find(string.lower(ct), "text/html"' 2 +assert_count 'Upstream returned text/html on a registry' 2 + +echo "Container-registry auth checks passed." diff --git a/deploy/helm/function-autoscaler/files/values-nvcf.yaml b/deploy/helm/function-autoscaler/files/values-nvcf.yaml new file mode 100644 index 000000000..7bc844db9 --- /dev/null +++ b/deploy/helm/function-autoscaler/files/values-nvcf.yaml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +rsfTemplate: + replicaCount: 1 + + nameOverride: nvcf-autoscaler-service + containerName: inference + + image: + repository: //nvcf-autoscaler-service + pullPolicy: Always + tag: latest + + entrypoint: + command: /bin/server + args: [] + + podLabels: + NCA_ID: "sf_metric" + + ingress: + enabled: false + + settings: + server: + metrics: + selectedMetricsBackend: "ott-metrics" + ottMetrics: + metrics: {} + # The OTEL_EXPORTER_OTLP_ENDPOINT (from ov-otel-collector) + # is honored implicitly by ott-metrics + + foundationsMetrics: + metrics_server: + addr: "0.0.0.0:8002" + + tracing: + selectedTracingBackend: "ott-tracing" + rsfTracing: + tracing: + endpoint_ip: ov-otel-collector + + env: + # These are honored by ott-metrics + OTEL_METRIC_EXPORT_INTERVAL: 5000 + +ov-otel-collector: + enabled: true + + lightstep: + accessToken: $LIGHTSTEP_ACCESS_TOKEN + enabled: true + endpoint: https://:8282 diff --git a/deploy/helm/function-autoscaler/files/values-nvks.yaml b/deploy/helm/function-autoscaler/files/values-nvks.yaml new file mode 100644 index 000000000..d80df10ca --- /dev/null +++ b/deploy/helm/function-autoscaler/files/values-nvks.yaml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +rsfTemplate: + replicaCount: 1 + + nameOverride: nvcf-autoscaler-service + + image: + repository: //nvcf-autoscaler-service + pullPolicy: IfNotPresent + tag: + + entrypoint: + command: /bin/server + args: [] + + podAnnotations: {} + + podLabels: + kratos_logging: "true" + kratos_metrics: "true" + + podMonitor: + enabled: true + + service: + host: "nvcf-autoscaler-service.service.odp.nvidia.com" + port: 50051 + tls_secret: "odp-common-dns" + annotations: + 'consul.hashicorp.com/service-name': 'nvcf-autoscaler-service' + 'consul.hashicorp.com/service-sync': 'true' + 'consul.hashicorp.com/service-tags': 'omni-backstage' + 'consul.hashicorp.com/service-port': 'grpc' + livenessProbe: + delaySeconds: 30 + + ingress: + enabled: true + + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + + settings: + server: + metrics: + selectedMetricsBackend: "ott-metrics" + ottMetrics: + metrics: {} + # see env section below + + foundationsMetrics: + metrics_server: + addr: 0.0.0.0:8080 + + tracing: + selectedTracingBackend: "ott-tracing" + rsfTracing: + tracing: + endpoint_ip: nvks-opentelemetry-agent-collector.otel-collector.svc.cluster.local + endpoint_port: 4317 + envfilter_directive: server=debug + + env: + # These are honored by ott-metrics + OTEL_METRIC_EXPORT_INTERVAL: 5000 + OTEL_EXPORTER_OTLP_ENDPOINT : http://nvks-opentelemetry-agent-collector.otel-collector.svc.cluster.local:4317 diff --git a/deploy/helm/gateway-routes/Makefile b/deploy/helm/gateway-routes/Makefile new file mode 100644 index 000000000..e3d0f06f7 --- /dev/null +++ b/deploy/helm/gateway-routes/Makefile @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +.PHONY: test test-render-routes test-vanity-gateway-route + +test: test-render-routes test-vanity-gateway-route + +test-render-routes: + @sh ./scripts/test-render-routes.sh + +test-vanity-gateway-route: + @sh ./scripts/test-vanity-gateway-route.sh diff --git a/deploy/helm/gateway-routes/scripts/test-render-routes.sh b/deploy/helm/gateway-routes/scripts/test-render-routes.sh new file mode 100755 index 000000000..86a064e88 --- /dev/null +++ b/deploy/helm/gateway-routes/scripts/test-render-routes.sh @@ -0,0 +1,236 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +default_render="$(mktemp)" +enabled_render="$(mktemp)" +annotated_render="$(mktemp)" +disabled_render="$(mktemp)" +trap 'rm -f "$default_render" "$enabled_render" "$annotated_render" "$disabled_render"' EXIT + +if ! command -v yq >/dev/null 2>&1; then + echo "yq is required for render tests" >&2 + exit 1 +fi + +assert_yq_eq() { + file="$1" + expr="$2" + expected="$3" + actual="$(yq ea -r "$expr" "$file")" + if [ "$actual" != "$expected" ]; then + echo "expected yq expression to equal '$expected', got '$actual': $expr" >&2 + exit 1 + fi +} + +assert_resource_count() { + file="$1" + kind="$2" + name="$3" + namespace="$4" + expected="$5" + assert_yq_eq "$file" "[select(.kind == \"$kind\" and .metadata.name == \"$name\" and .metadata.namespace == \"$namespace\")] | length" "$expected" +} + +assert_resource_field() { + file="$1" + kind="$2" + name="$3" + namespace="$4" + field="$5" + expected="$6" + assert_yq_eq "$file" "select(.kind == \"$kind\" and .metadata.name == \"$name\" and .metadata.namespace == \"$namespace\") | $field" "$expected" +} + +assert_yq_eq "$repo_root/chart/values.yaml" '.nvcfGatewayRoutes.routes.grpc | has("hostnames")' false +assert_yq_eq "$repo_root/chart/values.yaml" '.nvcfGatewayRoutes.routes.grpcWorker | has("hostnames")' false +assert_yq_eq "$repo_root/chart/values.yaml" '.nvcfGatewayRoutes.routes.nats | has("hostnames")' false + +helm template nvcf-gateway-routes "$repo_root/chart" > "$default_render" + +# Default-enabled HTTPRoutes. +assert_resource_count "$default_render" HTTPRoute nvcf-api gateway 1 +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.metadata.labels."app.kubernetes.io/component"' nvcf-api-route +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.parentRefs[0].name' gateway +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.parentRefs[0].sectionName' http +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.hostnames[0]' api.localhost +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.rules[0].backendRefs[0].name' api +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" HTTPRoute nvcf-api gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute nvct-api gateway 1 +assert_resource_field "$default_render" HTTPRoute nvct-api gateway '.spec.hostnames[0]' tasks.localhost +assert_resource_field "$default_render" HTTPRoute nvct-api gateway '.spec.rules[0].backendRefs[0].name' nvct-api +assert_resource_field "$default_render" HTTPRoute nvct-api gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" HTTPRoute nvct-api gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute api-keys gateway 1 +assert_resource_field "$default_render" HTTPRoute api-keys gateway '.metadata.labels."app.kubernetes.io/component"' api-keys-route +assert_resource_field "$default_render" HTTPRoute api-keys gateway '.spec.hostnames[0]' api-keys.localhost +assert_resource_field "$default_render" HTTPRoute api-keys gateway '.spec.rules[0].backendRefs[0].name' api-keys +assert_resource_field "$default_render" HTTPRoute api-keys gateway '.spec.rules[0].backendRefs[0].namespace' api-keys +assert_resource_field "$default_render" HTTPRoute api-keys gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute invocation-service gateway 1 +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.metadata.labels."app.kubernetes.io/component"' invocation-service-route +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.spec.hostnames[0]' '*.invocation.localhost' +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.spec.hostnames[1]' invocation.localhost +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.spec.rules[0].backendRefs[0].name' invocation +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" HTTPRoute invocation-service gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute llm-api-gateway gateway 1 +assert_resource_field "$default_render" HTTPRoute llm-api-gateway gateway '.metadata.labels."app.kubernetes.io/component"' llm-api-gateway-route +assert_resource_field "$default_render" HTTPRoute llm-api-gateway gateway '.spec.hostnames[0]' llm.localhost +assert_resource_field "$default_render" HTTPRoute llm-api-gateway gateway '.spec.rules[0].backendRefs[0].name' llm-api-gateway +assert_resource_field "$default_render" HTTPRoute llm-api-gateway gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" HTTPRoute llm-api-gateway gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute sis gateway 1 +assert_resource_field "$default_render" HTTPRoute sis gateway '.metadata.labels."app.kubernetes.io/component"' sis-route +assert_resource_field "$default_render" HTTPRoute sis gateway '.spec.hostnames[0]' sis.localhost +assert_resource_field "$default_render" HTTPRoute sis gateway '.spec.rules[0].backendRefs[0].name' api +assert_resource_field "$default_render" HTTPRoute sis gateway '.spec.rules[0].backendRefs[0].namespace' sis +assert_resource_field "$default_render" HTTPRoute sis gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$default_render" HTTPRoute reval gateway 1 +assert_resource_field "$default_render" HTTPRoute reval gateway '.metadata.labels."app.kubernetes.io/component"' reval-route +assert_resource_field "$default_render" HTTPRoute reval gateway '.spec.hostnames[0]' reval.localhost +assert_resource_field "$default_render" HTTPRoute reval gateway '.spec.rules[0].backendRefs[0].name' reval +assert_resource_field "$default_render" HTTPRoute reval gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" HTTPRoute reval gateway '.spec.rules[0].backendRefs[0].port' 8080 + +helm template nvcf-gateway-routes "$repo_root/chart" \ + --set nvcfGatewayRoutes.routes.reval.enabled=false \ + > "$disabled_render" + +assert_resource_count "$disabled_render" HTTPRoute reval gateway 0 + +# Default-enabled TCPRoute. +assert_resource_count "$default_render" TCPRoute grpc gateway 1 +assert_resource_field "$default_render" TCPRoute grpc gateway '.metadata.labels."app.kubernetes.io/component"' grpc-route +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.parentRefs[0].name' tcp-gateway +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.parentRefs[0].sectionName' tcp +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.hostnames' null +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.rules[0].backendRefs[0].name' grpc +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$default_render" TCPRoute grpc gateway '.spec.rules[0].backendRefs[0].port' 10081 + +# Cross-namespace grants for default routes. +assert_resource_count "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf 1 +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[0].kind' HTTPRoute +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[0].namespace' gateway +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[1].kind' TCPRoute +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[1].namespace' gateway +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[2].kind' GRPCRoute +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.from[2].namespace' gateway +assert_resource_field "$default_render" ReferenceGrant allow-routes-to-nvcf nvcf '.spec.to[0].kind' Service + +assert_resource_count "$default_render" ReferenceGrant allow-httproute-to-api-keys api-keys 1 +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-api-keys api-keys '.spec.from[0].kind' HTTPRoute +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-api-keys api-keys '.spec.from[0].namespace' gateway +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-api-keys api-keys '.spec.to[0].kind' Service + +assert_resource_count "$default_render" ReferenceGrant allow-httproute-to-sis sis 1 +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis sis '.spec.from[0].kind' HTTPRoute +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis sis '.spec.from[0].namespace' gateway +assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis sis '.spec.to[0].kind' Service + +# Routes disabled by default stay absent unless explicitly enabled. +assert_resource_count "$default_render" HTTPRoute llm-invocation gateway 0 +assert_resource_count "$default_render" GRPCRoute nvcf-api-grpc gateway 0 +assert_resource_count "$default_render" GRPCRoute nvct-api-grpc gateway 0 +assert_resource_count "$default_render" TCPRoute grpc-worker gateway 0 +assert_resource_count "$default_render" TCPRoute nats gateway 0 +assert_resource_count "$default_render" ReferenceGrant allow-tcproute-to-nats nats-system 0 + +helm template nvcf-gateway-routes "$repo_root/chart" \ + --set nvcfGatewayRoutes.routes.llmInvocation.enabled=true \ + --set nvcfGatewayRoutes.routes.nvcfApi.grpc.enabled=true \ + --set nvcfGatewayRoutes.routes.nvctApi.grpc.enabled=true \ + --set nvcfGatewayRoutes.routes.grpcWorker.enabled=true \ + --set nvcfGatewayRoutes.routes.nats.enabled=true \ + --set nvcfGatewayRoutes.gateways.nats.name=nats-gateway \ + --set nvcfGatewayRoutes.gateways.nats.namespace=gateway \ + --set nvcfGatewayRoutes.gateways.nats.listenerName=nats \ + > "$enabled_render" + +assert_resource_count "$enabled_render" HTTPRoute llm-invocation gateway 1 +assert_resource_field "$enabled_render" HTTPRoute llm-invocation gateway '.metadata.labels."app.kubernetes.io/component"' llm-invocation-route +assert_resource_field "$enabled_render" HTTPRoute llm-invocation gateway '.spec.hostnames[0]' llm.invocation.localhost +assert_resource_field "$enabled_render" HTTPRoute llm-invocation gateway '.spec.rules[0].backendRefs[0].name' llm-api-gateway +assert_resource_field "$enabled_render" HTTPRoute llm-invocation gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" HTTPRoute llm-invocation gateway '.spec.rules[0].backendRefs[0].port' 8080 + +assert_resource_count "$enabled_render" GRPCRoute nvcf-api-grpc gateway 1 +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.metadata.labels."app.kubernetes.io/component"' nvcf-api-grpc-route +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.parentRefs[0].name' gateway +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.parentRefs[0].sectionName' http +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.hostnames[0]' api.localhost +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.rules[0].backendRefs[0].name' api +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" GRPCRoute nvcf-api-grpc gateway '.spec.rules[0].backendRefs[0].port' 9090 + +assert_resource_count "$enabled_render" GRPCRoute nvct-api-grpc gateway 1 +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.metadata.labels."app.kubernetes.io/component"' nvct-api-grpc-route +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.parentRefs[0].name' gateway +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.parentRefs[0].sectionName' http +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.hostnames[0]' tasks.localhost +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.rules[0].backendRefs[0].name' nvct-api +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" GRPCRoute nvct-api-grpc gateway '.spec.rules[0].backendRefs[0].port' 9090 + +assert_resource_count "$enabled_render" TCPRoute grpc-worker gateway 1 +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.metadata.labels."app.kubernetes.io/component"' grpc-worker-route +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.parentRefs[0].name' tcp-gateway +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.parentRefs[0].sectionName' worker-tcp +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.rules[0].backendRefs[0].name' grpc +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.rules[0].backendRefs[0].port' 10086 +assert_resource_field "$enabled_render" TCPRoute grpc-worker gateway '.spec.hostnames' null + +assert_resource_count "$enabled_render" TCPRoute nats gateway 1 +assert_resource_field "$enabled_render" TCPRoute nats gateway '.metadata.labels."app.kubernetes.io/component"' nats-route +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.parentRefs[0].name' nats-gateway +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.parentRefs[0].sectionName' nats +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.rules[0].backendRefs[0].name' nats +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.rules[0].backendRefs[0].namespace' nats-system +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.rules[0].backendRefs[0].port' 4222 +assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.hostnames' null +assert_resource_field "$enabled_render" TCPRoute nats gateway '.metadata.annotations' null + +assert_resource_count "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system 1 +assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].kind' TCPRoute +assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].namespace' gateway +assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.to[0].kind' Service + +helm template nvcf-gateway-routes "$repo_root/chart" \ + --set nvcfGatewayRoutes.routes.nats.enabled=true \ + --set 'nvcfGatewayRoutes.routes.nats.routeAnnotations.example\.com/nats-route=true' \ + > "$annotated_render" + +assert_resource_field "$annotated_render" TCPRoute nats gateway '.metadata.annotations."example.com/nats-route"' true + +echo "Gateway route render checks passed." diff --git a/deploy/helm/gateway-routes/scripts/test-vanity-gateway-route.sh b/deploy/helm/gateway-routes/scripts/test-vanity-gateway-route.sh new file mode 100644 index 000000000..4f121175a --- /dev/null +++ b/deploy/helm/gateway-routes/scripts/test-vanity-gateway-route.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +umbrella_root="$(cd "$repo_root/../../.." && pwd)" +values_file="$umbrella_root/tools/ci/helm-validate-values/gateway-routes-vanity.yaml" +ci_render="$(mktemp)" +default_render="$(mktemp)" +trap 'rm -f "$ci_render" "$default_render"' EXIT + +helm lint "$repo_root/chart" -f "$values_file" +helm template nvcf-gateway-routes "$repo_root/chart" \ + -f "$values_file" \ + > "$ci_render" + +grep -q 'name: ci-vanity-gateway' "$ci_render" +grep -q 'ci-vanity.localhost' "$ci_render" +grep -q 'name: ci-vanity-backend' "$ci_render" +grep -q 'namespace: ci-vanity' "$ci_render" +grep -q 'port: 9090' "$ci_render" +grep -q 'ci.nvidia.com/validated: enabled' "$ci_render" + +helm template nvcf-gateway-routes "$repo_root/chart" \ + --set nvcfGatewayRoutes.routes.vanityGateway.enabled=true \ + > "$default_render" + +grep -q 'vanity.localhost' "$default_render" + +echo "Vanity Gateway route render checks passed." diff --git a/deploy/helm/grpc-proxy/Makefile b/deploy/helm/grpc-proxy/Makefile new file mode 100644 index 000000000..628ee85c2 --- /dev/null +++ b/deploy/helm/grpc-proxy/Makefile @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variables +release ?= grpc-proxy +namespace ?= nvcf +helm_dir ?= ./grpc-proxy +values := $(helm_dir)/values.yaml + +# OPTIONAL for deploy target: Path to an additional Helm values file. +# Example: make deploy values=my-values.yaml additional_values=override.yaml +additional_values ?= + +# OCI Registry details — set these to your own OCI-compatible registry +OCI_REGISTRY_HOST ?= registry.example.com +OCI_REGISTRY_NAMESPACE ?= your-org/your-namespace + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +.PHONY: deploy delete status lint template + +install: +ifndef values + $(error "values" variable is not set. Please specify with 'make deploy values=') +endif + @echo "Deploying $(release) to namespace $(namespace) using values file '$(values)'..." + @echo "Additional values file: '$(if $(additional_values),$(additional_values),N/A)'" + helm install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +uninstall: + @echo "Deleting $(release) from namespace $(namespace)..." + helm uninstall $(release) --namespace $(namespace) + +status: + @echo "Checking status of $(release) in namespace $(namespace)..." + helm status $(release) --namespace $(namespace) + +lint: + @echo "Linting chart $(helm_dir)..." + helm lint $(helm_dir) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) + +template: + @echo "Templating chart $(helm_dir)..." + @mkdir -p bin + helm template $(release) $(helm_dir) \ + --namespace $(namespace) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + > bin/manifest.yaml + @echo "Rendered manifest to bin/manifest.yaml" + +validate: template + @echo "Validating manifest with kubeconform..." + @kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml + +# Publish Chart +# NOTE: this is manual until the CI pipeline is updated to push the chart to the NVCR OCI registry +clean: + rm -rf $(charts_dir)/*.tgz + rm -rf ./packaged-charts + +package: clean lint + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(helm_dir) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." diff --git a/deploy/helm/grpc-proxy/values.additionalvolumes.yaml b/deploy/helm/grpc-proxy/values.additionalvolumes.yaml new file mode 100644 index 000000000..2ab020a30 --- /dev/null +++ b/deploy/helm/grpc-proxy/values.additionalvolumes.yaml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sample values overlay for additional volumes and volume mounts +# Usage: helm install grpc-proxy ./grpc-proxy -f values.deployment.yaml -f values.additionalvolumes.yaml + +grpcproxy: + # Required image configuration + image: + registry: "" + repository: "" + + # Additional volumes to be mounted in the containers + volumes: + # ConfigMap volume for additional configuration + - name: app-config + configMap: + name: grpc-proxy-config + items: + - key: config.yaml + path: config.yaml + + # Additional volume mounts in the container + volumeMounts: + # Mount app configuration + - name: app-config + mountPath: /etc/grpc-proxy + readOnly: true diff --git a/deploy/helm/grpc-proxy/values.daemonset.yaml b/deploy/helm/grpc-proxy/values.daemonset.yaml new file mode 100644 index 000000000..c720a3972 --- /dev/null +++ b/deploy/helm/grpc-proxy/values.daemonset.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sample values overlay for DaemonSet-only deployment +# Usage: helm install grpc-proxy ./grpc-proxy -f values.daemonset.yaml + +grpcproxy: + # Required image configuration + image: + registry: "" + repository: "" + tag: "latest" + + # Set deployment type to DaemonSet + deploymentType: "daemonset" + + # DaemonSet-specific configuration + daemonset: + # DaemonSet-specific tolerations to run on all nodes + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + # Node selector for specific node types if needed + nodeSelector: {} + # Update strategy for DaemonSet + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # DaemonSet typically uses higher resource requests + resources: + limits: + cpu: 2000m + memory: 2Gi + requests: + cpu: 1000m + memory: 1Gi diff --git a/deploy/helm/grpc-proxy/values.deployment.yaml b/deploy/helm/grpc-proxy/values.deployment.yaml new file mode 100644 index 000000000..21b84e60e --- /dev/null +++ b/deploy/helm/grpc-proxy/values.deployment.yaml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sample values overlay for Deployment-only deployment with autoscaling +# Usage: helm install grpc-proxy ./grpc-proxy -f values.deployment.yaml + +grpcproxy: + # Required image configuration — replace with your registry and image + image: + registry: "registry.example.com" + repository: "your-org/grpc-proxy" + tag: "v1.23.4" + + # Set deployment type to Deployment + deploymentType: "deployment" + + # Deployment-specific configuration + deployment: + # Enable autoscaling for deployment + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + # Node affinity for deployment pods + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - grpc-proxy + topologyKey: kubernetes.io/hostname + # Node selector for deployment + nodeSelector: + node-type: worker + # Tolerations for deployment + tolerations: + - key: dedicated + operator: Equal + value: grpc-proxy + effect: NoSchedule + + # Set replica count for non-autoscaling scenarios + replicaCount: 3 + + # Deployment typically uses moderate resources + resources: + limits: + cpu: 1500m + memory: 1.5Gi + requests: + cpu: 500m + memory: 512Mi diff --git a/deploy/helm/grpc-proxy/values.extraenv.yaml b/deploy/helm/grpc-proxy/values.extraenv.yaml new file mode 100644 index 000000000..80194c60e --- /dev/null +++ b/deploy/helm/grpc-proxy/values.extraenv.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sample values overlay for additional environment variables +# Usage: helm install grpc-proxy ./grpc-proxy -f values.deployment.yaml -f values.extraenv.yaml + +grpcproxy: + # Required image configuration + image: + registry: "" + repository: "" + + # Additional environment variables + # These will be merged with the base env vars in values.yaml + env: + ADDITIONAL_ENV_VAR: "additional-env-var" diff --git a/deploy/helm/grpc-proxy/values.local.yaml b/deploy/helm/grpc-proxy/values.local.yaml new file mode 100644 index 000000000..cc4254648 --- /dev/null +++ b/deploy/helm/grpc-proxy/values.local.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +grpcproxy: + image: + registry: "" + repository: "" + tag: 1.29.1 + + replicaCount: 1 + deploymentType: "deployment" + deployment: + # Enable autoscaling for deployment + autoscaling: + enabled: false diff --git a/deploy/helm/http-invocation/files/values-nvks.yaml b/deploy/helm/http-invocation/files/values-nvks.yaml new file mode 100644 index 000000000..0e333fc15 --- /dev/null +++ b/deploy/helm/http-invocation/files/values-nvks.yaml @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +replicaCount: 1 + +nameOverride: nvcf-invocation-service + +image: + repository: "" + pullPolicy: IfNotPresent + tag: + +entrypoint: + command: /bin/server + args: [] + +podAnnotations: + +podLabels: + kratos_logging: "true" + kratos_metrics: "true" + +service: + host: "nvcf-invocation-service.service.odp.nvidia.com" + port: 50051 + tls_secret: "odp-common-dns" + annotations: + 'consul.hashicorp.com/service-name': 'nvcf-invocation-service' + 'consul.hashicorp.com/service-sync': 'true' + 'consul.hashicorp.com/service-tags': 'omni-backstage' + 'consul.hashicorp.com/service-port': 'grpc' + livenessProbe: + # The deployment template reads `delaySeconds` (always did, including + # under the original rsf-template upstream). The previous `grpcDelay` + # spelling here was never read; renaming so the nvks-specific 30s + # initial delay actually applies (it was silently falling back to the + # 10s default in values.yaml). + delaySeconds: 30 + +ingress: + enabled: true + +settings: + server: + metrics_server: + addr: 127.0.0.1:6931 + +# OpenTelemetry collector sidecar for processing logs, metrics, and traces +otelCollector: + enabled: true + exporters: + debug: {} + otlp: + endpoint: "http://nvks-opentelemetry-agent-collector.otel-collector.svc.cluster.local:4317" + tls: + insecure: true + service: + pipelines: + traces: + exporters: [otlp, debug] + metrics: + exporters: [otlp, debug] diff --git a/deploy/helm/llm-api-gateway/Makefile b/deploy/helm/llm-api-gateway/Makefile new file mode 100644 index 000000000..00cca6636 --- /dev/null +++ b/deploy/helm/llm-api-gateway/Makefile @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variables +release ?= llm-api-gateway +namespace ?= nvcf +helm_dir ?= ./llm-api-gateway +values := $(helm_dir)/values.yaml + +# OPTIONAL for deploy target: Path to an additional Helm values file. +additional_values ?= + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +# Automatically determine chart name and version from Chart.yaml +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +.PHONY: install uninstall status lint template validate clean package push-oci + +install: +ifndef values + $(error "values" variable is not set. Please specify with 'make install values='") +endif + helm install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +uninstall: + helm uninstall $(release) --namespace $(namespace) + +status: + helm status $(release) --namespace $(namespace) + +lint: + helm lint $(helm_dir) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) + +template: + @mkdir -p bin + helm template $(release) $(helm_dir) \ + --namespace $(namespace) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + > bin/manifest.yaml + +validate: template + kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml + +clean: + rm -rf ./packaged-charts bin + +package: clean lint + @mkdir -p ./packaged-charts + helm package $(helm_dir) -d ./packaged-charts/ + +push-oci: + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) diff --git a/deploy/helm/llm-api-gateway/values.local.yaml b/deploy/helm/llm-api-gateway/values.local.yaml new file mode 100644 index 000000000..23cf705ab --- /dev/null +++ b/deploy/helm/llm-api-gateway/values.local.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +llmApiGateway: + image: + repository: "" + config: + requestRouterUrl: http://llm-request-router-headless.nvcf.svc.cluster.local:8000 diff --git a/deploy/helm/llm-request-router/Makefile b/deploy/helm/llm-request-router/Makefile new file mode 100644 index 000000000..5e14e15d9 --- /dev/null +++ b/deploy/helm/llm-request-router/Makefile @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variables +release ?= llm-request-router +namespace ?= nvcf +helm_dir ?= ./llm-request-router +values := $(helm_dir)/values.yaml + +additional_values ?= + +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +.PHONY: install uninstall status lint template validate clean package push-oci test check-pki-render check-multi-replica-render + +install: +ifndef values + $(error "values" variable is not set. Please specify with 'make install values='") +endif + helm install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +uninstall: + helm uninstall $(release) --namespace $(namespace) + +status: + helm status $(release) --namespace $(namespace) + +lint: + helm lint $(helm_dir) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) + +template: + @mkdir -p bin + helm template $(release) $(helm_dir) \ + --namespace $(namespace) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + > bin/manifest.yaml + +validate: template + kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml + +test: check-pki-render check-multi-replica-render + +check-pki-render: + sh ./scripts/check-pki-render.sh + +check-multi-replica-render: + sh ./scripts/check-multi-replica-render.sh + +clean: + rm -rf ./packaged-charts bin + +package: clean lint + @mkdir -p ./packaged-charts + helm package $(helm_dir) -d ./packaged-charts/ + +push-oci: + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) diff --git a/deploy/helm/llm-request-router/bin/manifest.yaml b/deploy/helm/llm-request-router/bin/manifest.yaml new file mode 100644 index 000000000..aef26d0d0 --- /dev/null +++ b/deploy/helm/llm-request-router/bin/manifest.yaml @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Source: llm-request-router/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: llm-request-router + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: llm-request-router/templates/configmap-vault-agent-template.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-request-router-vault-agent-tpl + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +data: + secrets.json.tmpl: |- + { + {{- with secret `services/nvcf-api/jwt/sign/llm-request-router` }} + "nvcfApiToken": "{{ .Data.token }}" + {{- end }} + } +--- +# Source: llm-request-router/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: llm-request-router-pod-reader + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] +--- +# Source: llm-request-router/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: llm-request-router-pod-reader + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: llm-request-router-pod-reader +subjects: + - kind: ServiceAccount + name: llm-request-router +--- +# Source: llm-request-router/templates/headless-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: llm-request-router-headless + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + ports: + - name: http + port: 8000 + targetPort: http + protocol: TCP + - name: grpc + port: 50071 + targetPort: grpc + protocol: TCP +--- +# Source: llm-request-router/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: llm-request-router + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + ports: + - name: grpc + port: 50071 + targetPort: grpc + protocol: TCP +--- +# Source: llm-request-router/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-request-router + labels: + helm.sh/chart: llm-request-router-0.0.0 + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + template: + metadata: + labels: + app.kubernetes.io/name: llm-request-router + app.kubernetes.io/instance: llm-request-router + annotations: + vault.hashicorp.com/agent-copy-volume-mounts: llm-request-router + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-template-file-secrets.json: /vault/config/templates/secrets.json.tmpl + vault.hashicorp.com/agent-run-as-same-user: "true" + vault.hashicorp.com/auth-path: auth/jwt + vault.hashicorp.com/role: llm-request-router + vault.hashicorp.com/secret-volume-path: /vault/secrets + spec: + serviceAccountName: llm-request-router + automountServiceAccountToken: false + securityContext: + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 + containers: + - name: llm-request-router + image: :0.1.0 + imagePullPolicy: IfNotPresent + args: + - --stargate-id=$(POD_NAME) + - --listen-addr=0.0.0.0:50071 + - --http-listen-addr=0.0.0.0:8000 + - --advertise-addr=$(POD_IP):50071 + - --stargate-discovery-dns-name=llm-request-router-headless.default.svc.cluster.local + - --pod-name=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + - --pod-label-selector=app.kubernetes.io/name=llm-request-router + - --dns-poll-ms=1000 + - --dns-resolver-ttl-ms=1000 + - --watch-heartbeat-ms=5000 + - --shutdown-drain-timeout-ms=30000 + - --quic-connect-timeout-ms=2000 + - --quic-request-timeout-ms=30000 + - --metrics-port=9090 + - --reverse-tunnel-connect-timeout-ms=10000 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - name: http + containerPort: 8000 + protocol: TCP + - name: grpc + containerPort: 50071 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 2 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 5 + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - name: token + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + - name: vault-config-templates + mountPath: /vault/config/templates + readOnly: true + volumes: + - name: token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + audience: http://openbao-server.vault-system.svc.cluster.local:8200 + - name: vault-config-templates + configMap: + name: llm-request-router-vault-agent-tpl + items: + - key: secrets.json.tmpl + path: secrets.json.tmpl diff --git a/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh b/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh new file mode 100755 index 000000000..08f5d4fb9 --- /dev/null +++ b/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +chart_dir="${1:-./llm-request-router}" +release="${RELEASE:-llm-request-router}" +namespace="${NAMESPACE:-nvcf}" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +render() { + local output="$1" + shift + helm template "${release}" "${chart_dir}" \ + --namespace "${namespace}" \ + --values "${chart_dir}/values.yaml" \ + "$@" \ + > "${output}" +} + +statefulset_field() { + local manifest="$1" + local expression="$2" + yq -r "select(.kind == \"StatefulSet\" and .metadata.name == \"llm-request-router\") | ${expression}" "${manifest}" | head -n1 +} + +statefulset_args() { + local manifest="$1" + yq -r 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${manifest}" +} + +default_manifest="${tmp_dir}/default.yaml" +render "${default_manifest}" + +[ "$(statefulset_field "${default_manifest}" ".kind")" = "StatefulSet" ] || fail "default render did not create StatefulSet" +[ "$(statefulset_field "${default_manifest}" ".spec.serviceName")" = "llm-request-router-headless" ] || fail "default StatefulSet serviceName is not llm-request-router-headless" +[ "$(statefulset_field "${default_manifest}" ".spec.replicas")" = "3" ] || fail "default replica count is not 3" +[ "$(statefulset_field "${default_manifest}" ".spec.podManagementPolicy")" = "Parallel" ] || fail "default StatefulSet podManagementPolicy is not Parallel" + +default_args="$(statefulset_args "${default_manifest}")" +printf '%s\n' "${default_args}" | grep -qx -- "--stargate-discovery-dns-name=llm-request-router-headless.${namespace}.svc.cluster.local" || fail "default render missing headless discovery DNS arg" +printf '%s\n' "${default_args}" | grep -qx -- "--advertised-hostname-template={pod_name}.llm-request-router-headless.${namespace}.svc.cluster.local" || fail "default render missing per-pod advertised hostname template" +printf '%s\n' "${default_args}" | grep -qx -- '--reverse-tunnel-pylon-dial-addr=$(POD_IP):50072' || fail "default render missing reverse tunnel pylon dial addr" +if printf '%s\n' "${default_args}" | grep -qx -- "--disable-dns-discovery"; then + fail "default multi-replica render must not disable DNS discovery" +fi + +invalid_error="${tmp_dir}/invalid.err" +if helm template "${release}" "${chart_dir}" \ + --namespace "${namespace}" \ + --values "${chart_dir}/values.yaml" \ + --set llmRequestRouter.replicaCount=3 \ + --set llmRequestRouter.discovery.disableDnsDiscovery=true \ + > "${tmp_dir}/invalid.yaml" 2> "${invalid_error}"; then + fail "multi-replica render with disabled DNS discovery unexpectedly succeeded" +fi +grep -Fq "llmRequestRouter.discovery.disableDnsDiscovery cannot be true when llmRequestRouter.replicaCount is greater than 1; multi-replica routers require DNS discovery" "${invalid_error}" || fail "invalid render did not return the expected guard message" + +single_manifest="${tmp_dir}/single.yaml" +render "${single_manifest}" \ + --set llmRequestRouter.replicaCount=1 \ + --set llmRequestRouter.discovery.disableDnsDiscovery=true + +single_args="$(statefulset_args "${single_manifest}")" +printf '%s\n' "${single_args}" | grep -qx -- "--disable-dns-discovery" || fail "single-replica self-only render missing --disable-dns-discovery" + +custom_template_manifest="${tmp_dir}/custom-template.yaml" +render "${custom_template_manifest}" \ + --set llmRequestRouter.replicaCount=3 \ + --set-string llmRequestRouter.kubernetes.advertisedHostnameTemplate=router.example.internal + +custom_template_args="$(statefulset_args "${custom_template_manifest}")" +printf '%s\n' "${custom_template_args}" | grep -qx -- '--reverse-tunnel-pylon-dial-addr=$(POD_IP):50072' || fail "custom multi-replica advertised hostname template missing reverse tunnel pylon dial addr" + +echo "multi-replica render checks passed" diff --git a/deploy/helm/llm-request-router/scripts/check-pki-render.sh b/deploy/helm/llm-request-router/scripts/check-pki-render.sh new file mode 100644 index 000000000..66ecee294 --- /dev/null +++ b/deploy/helm/llm-request-router/scripts/check-pki-render.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +manifest="$(mktemp)" +defaults_manifest="$(mktemp)" +trap 'rm -f "${manifest}" "${defaults_manifest}"' EXIT + +# Pass 1: defaults — pki/certificate/tls all off. Assert the chart does NOT +# emit any of the optional PKI resources so a regression that accidentally +# turns them on (or fails to gate them properly) is caught. +helm template llm-request-router ./llm-request-router \ + --namespace nvcf \ + --values ./llm-request-router/values.yaml \ + --set llmRequestRouter.image.repository=stargate \ + > "${defaults_manifest}" + +# No Certificate resource should render at the chart's defaults. +default_cert="$(yq -rN 'select(.kind == "Certificate") | .metadata.name' "${defaults_manifest}" | head -n1)" +[ -z "${default_cert}" ] || { echo "FAIL: Certificate rendered with default values: ${default_cert}" >&2; exit 1; } + +# No PKI provisioning Job should render at the chart's defaults. +default_job="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .metadata.name' "${defaults_manifest}" | head -n1)" +[ -z "${default_job}" ] || { echo "FAIL: addons-llm-migrations Job rendered with default values" >&2; exit 1; } + +# StatefulSet should still render (chart's primary purpose) but with no +# stargate-tls volume or volumeMount. +default_workload="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .metadata.name' "${defaults_manifest}" | head -n1)" +[ "${default_workload}" = "llm-request-router" ] || { echo "FAIL: llm-request-router StatefulSet did not render at defaults" >&2; exit 1; } + +default_workload_args="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${defaults_manifest}")" +if printf '%s\n' "${default_workload_args}" | grep -qx -- "--metrics-prefix=llm_request_router_"; then + echo "FAIL: --metrics-prefix is not supported by the pinned stargate 0.3.0 image" >&2 + exit 1 +fi +if printf '%s\n' "${default_workload_args}" | grep -qx -- "--otel-service-name=llm-request-router"; then + echo "FAIL: --otel-service-name is not supported by the pinned stargate 0.3.0 image" >&2 + exit 1 +fi + +default_tls_mount="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].volumeMounts[]? | select(.name == "stargate-tls") | .name' "${defaults_manifest}" | head -n1)" +[ -z "${default_tls_mount}" ] || { echo "FAIL: stargate-tls volumeMount rendered with default values" >&2; exit 1; } + +default_tls_volume="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.volumes[]? | select(.name == "stargate-tls") | .name' "${defaults_manifest}" | head -n1)" +[ -z "${default_tls_volume}" ] || { echo "FAIL: stargate-tls volume rendered with default values" >&2; exit 1; } + +# Pass 2: fully enabled — pki + certificate + tls. Assert that every +# expected resource and wiring is in place. +helm template llm-request-router ./llm-request-router \ + --namespace nvcf \ + --values ./llm-request-router/values.yaml \ + --set llmRequestRouter.image.repository=stargate \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.secretName=stargate-quic-tls \ + --set llmRequestRouter.certificate.issuerRef.kind=ClusterIssuer \ + --set llmRequestRouter.certificate.issuerRef.name=nvcf-openbao-pki \ + --set-string 'llmRequestRouter.certificate.dnsNames[0]=*.stargate.localhost' \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + --set llmRequestRouter.tls.quicInsecure=false \ + --set llmRequestRouter.pki.enabled=true \ + --set-string 'llmRequestRouter.pki.allowedDomains=stargate.localhost\,cluster.local' \ + --set llmRequestRouter.pki.image.registry=nvcr.io \ + --set llmRequestRouter.pki.image.repository=/nvcf-openbao-migrations \ + --set llmRequestRouter.pki.image.tag=0.12.1 \ + > "${manifest}" + +cert_secret="$(yq -rN 'select(.kind == "Certificate" and .metadata.name == "stargate-quic-tls") | .spec.secretName' "${manifest}")" +cert_issuer_kind="$(yq -rN 'select(.kind == "Certificate" and .metadata.name == "stargate-quic-tls") | .spec.issuerRef.kind' "${manifest}")" +cert_issuer_name="$(yq -rN 'select(.kind == "Certificate" and .metadata.name == "stargate-quic-tls") | .spec.issuerRef.name' "${manifest}")" +cert_dns_name="$(yq -rN 'select(.kind == "Certificate" and .metadata.name == "stargate-quic-tls") | .spec.dnsNames[0]' "${manifest}")" + +[ "${cert_secret}" = "stargate-quic-tls" ] +[ "${cert_issuer_kind}" = "ClusterIssuer" ] +[ "${cert_issuer_name}" = "nvcf-openbao-pki" ] +[ "${cert_dns_name}" = "*.stargate.localhost" ] + +workload_args="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${manifest}")" +printf '%s\n' "${workload_args}" | grep -qx -- "--tls-cert-path=/etc/stargate/tls/tls.crt" +printf '%s\n' "${workload_args}" | grep -qx -- "--tls-key-path=/etc/stargate/tls/tls.key" +if printf '%s\n' "${workload_args}" | grep -qx -- "--quic-insecure"; then + echo "unexpected --quic-insecure flag rendered" >&2 + exit 1 +fi + +tls_mount_name="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].volumeMounts[] | select(.name == "stargate-tls" and .mountPath == "/etc/stargate/tls" and .readOnly == true) | .name' "${manifest}")" +tls_volume_name="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.volumes[] | select(.name == "stargate-tls" and .secret.secretName == "stargate-quic-tls") | .name' "${manifest}")" + +[ "${tls_mount_name}" = "stargate-tls" ] +[ "${tls_volume_name}" = "stargate-tls" ] + +# PKI provisioning hook: Helm hook Job rendered with the right env, image, and root-token mount. +hook_job_name="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .metadata.name' "${manifest}")" +hook_helm_hook="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .metadata.annotations."helm.sh/hook"' "${manifest}")" +hook_image="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].image' "${manifest}")" +hook_addons_llm="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "ADDONS_LLM_ENABLED") | .value' "${manifest}")" +hook_core_off="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "CORE_MIGRATIONS_ENABLED") | .value' "${manifest}")" +hook_allowed_domains="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "NVCF_SERVICE_PKI_ALLOWED_DOMAINS") | .value' "${manifest}")" + +[ "${hook_job_name}" = "addons-llm-migrations" ] +[ "${hook_helm_hook}" = "pre-install,pre-upgrade" ] +[ "${hook_image}" = "//nvcf-openbao-migrations:0.12.1" ] +[ "${hook_addons_llm}" = "true" ] +[ "${hook_core_off}" = "false" ] +[ "${hook_allowed_domains}" = "stargate.localhost,cluster.local" ] diff --git a/deploy/helm/llm-request-router/values.local.yaml b/deploy/helm/llm-request-router/values.local.yaml new file mode 100644 index 000000000..2297d4a73 --- /dev/null +++ b/deploy/helm/llm-request-router/values.local.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +llmRequestRouter: + image: + repository: "" + discovery: + dnsName: llm-request-router-headless.nvcf.svc.cluster.local diff --git a/deploy/helm/nats-auth-callout/README.md b/deploy/helm/nats-auth-callout/README.md new file mode 100644 index 000000000..cb802846d --- /dev/null +++ b/deploy/helm/nats-auth-callout/README.md @@ -0,0 +1,278 @@ +# nvcf-nats-auth-callout-service Service Helm Chart + +This Helm chart deploys the nvcf-nats-auth-callout-service service with comprehensive configuration validation. + +## Configuration Validation + +The chart includes built-in validation to ensure proper configuration before deployment. The validation checks are automatically executed during `helm install` or `helm template` operations. + +### Database Configuration Validation + +When using PostgreSQL as the database provider, you must provide authentication credentials: + +```yaml +# Option 1: Use existing secret +serviceConfig: + database: + provider: postgres +postgres: + existingSecret: my-postgres-secret + +# Option 2: Provide username and password (will create secret) +serviceConfig: + database: + provider: postgres +postgres: + username: myuser + password: mypass +``` + +**Validation Rules:** +- If `database.provider` is `postgres` or `postgresql`, either: + - `postgres.existingSecret` must be set, OR + - Both `postgres.username` AND `postgres.password` must be provided +- Supported database providers: `postgres`, `postgresql`, `sqlite`, `sqlite3` + +### Metrics Configuration Validation + +When metrics are enabled, a port must be specified: + +```yaml +metrics: + enabled: true + port: "9090" # Required when enabled=true +``` + +**ServiceMonitor Validation:** +ServiceMonitor can only be enabled when metrics are enabled: + +```yaml +# Valid: Both metrics and ServiceMonitor enabled +metrics: + enabled: true + port: "9090" + serviceMonitor: + enabled: true + +# Valid: Both metrics and ServiceMonitor disabled +metrics: + enabled: false + serviceMonitor: + enabled: false + +# Invalid: ServiceMonitor enabled but metrics disabled +metrics: + enabled: false + serviceMonitor: + enabled: true # This will fail validation +``` + +**Dashboard Validation:** +Grafana dashboard can only be enabled when metrics are enabled: + +```yaml +# Valid: Both metrics and dashboard enabled +metrics: + enabled: true + port: "9090" +dashboard: + enabled: true + +# Valid: Both metrics and dashboard disabled +metrics: + enabled: false +dashboard: + enabled: false + +# Invalid: Dashboard enabled but metrics disabled +metrics: + enabled: false +dashboard: + enabled: true # This will fail validation +``` + +The dashboard configuration supports additional options: + +```yaml +dashboard: + enabled: true + grafana: + # Labels for dashboard discovery by Grafana operator + labels: + dashboards: "grafana" + # Additional labels for the dashboard + additionalLabels: {} + # Allow cross-namespace import of the dashboard + allowCrossNamespaceImport: true + # Resync period for the dashboard + resyncPeriod: "30s" +``` + +### Health Checks Configuration + +The chart includes comprehensive health check probe configuration: + +```yaml +healthChecks: + # Liveness probe - determines when to restart container + livenessProbe: + enabled: true + httpGet: + path: "/healthz" + port: "http" + scheme: "HTTP" + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + + # Readiness probe - determines when container is ready to serve traffic + readinessProbe: + enabled: true + httpGet: + path: "/healthz" + port: "http" + scheme: "HTTP" + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + successThreshold: 1 + + # Startup probe - for slow-starting containers + startupProbe: + enabled: false # Disabled by default + httpGet: + path: "/healthz" + port: "http" + scheme: "HTTP" + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 # Allow up to 5 minutes for startup + successThreshold: 1 +``` + +**Available Health Endpoints:** +- `/healthz` - Primary health check endpoint (recommended) +- `/health` - Alternative health check endpoint +- `/v1/ping` - Basic connectivity test + +### Tracing Configuration Validation + +When tracing is enabled, authentication credentials must be provided: + +```yaml +# Option 1: Use existing secret +tracing: + enabled: true + existingSecret: my-tracing-secret + +# Option 2: Provide token or API key +tracing: + enabled: true + token: my-token + # OR + api_key: my-api-key +``` + +## Testing Validation + +You can test the validation rules using the provided test script: + +```bash +./scripts/test-helm-validation.sh +``` + +Or manually test specific configurations: + +```bash +# Test valid configuration +helm template test-release ./deploy --set serviceConfig.database.provider=sqlite + +# Test invalid configuration (should fail) +helm template test-release ./deploy \ + --set serviceConfig.database.provider=postgres \ + --set postgres.username="" \ + --set postgres.password="" \ + --set postgres.existingSecret="" +``` + +## Installation Examples + +### SQLite (Default) +```bash +helm install my-release ./deploy +``` + +### PostgreSQL with credentials +```bash +helm install my-release ./deploy \ + --set serviceConfig.database.provider=postgres \ + --set postgres.username=myuser \ + --set postgres.password=mypass +``` + +### PostgreSQL with existing secret +```bash +# Create secret first +kubectl create secret generic my-postgres-secret \ + --from-literal=postgres_username=myuser \ + --from-literal=postgres_password=mypass + +# Install with existing secret +helm install my-release ./deploy \ + --set serviceConfig.database.provider=postgres \ + --set postgres.existingSecret=my-postgres-secret +``` + +### With metrics enabled +```bash +helm install my-release ./deploy \ + --set metrics.enabled=true \ + --set metrics.port=9090 +``` + +### With metrics and Grafana dashboard enabled +```bash +helm install my-release ./deploy \ + --set metrics.enabled=true \ + --set metrics.port=9090 \ + --set dashboard.enabled=true +``` + +### With custom health check configuration +```bash +helm install my-release ./deploy \ + --set healthChecks.livenessProbe.initialDelaySeconds=15 \ + --set healthChecks.readinessProbe.periodSeconds=10 \ + --set healthChecks.startupProbe.enabled=true +``` + +### With health checks disabled +```bash +helm install my-release ./deploy \ + --set healthChecks.livenessProbe.enabled=false \ + --set healthChecks.readinessProbe.enabled=false +``` + +## Validation Error Examples + +The chart will fail with helpful error messages for invalid configurations: + +``` +Error: When database.provider is 'postgres', either postgres.existingSecret must be set OR both postgres.username and postgres.password must be provided + +Error: Unsupported database provider 'mysql'. Supported providers are: postgres, postgresql, sqlite, sqlite3 + +Error: When metrics.enabled is true, metrics.port must be specified + +Error: When metrics.enabled is false, metrics.serviceMonitor.enabled cannot be true. ServiceMonitor requires metrics to be enabled. + +Error: When metrics.enabled is false, dashboard.enabled cannot be true. Dashboard requires metrics to be enabled. +``` + +## Values File Structure + +See `values.yaml` for the complete configuration structure and all available options. \ No newline at end of file diff --git a/deploy/helm/nats/Makefile b/deploy/helm/nats/Makefile new file mode 100644 index 000000000..30b8f6d79 --- /dev/null +++ b/deploy/helm/nats/Makefile @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all lint deps build template test test-render install uninstall clean ensure-helmignore package push-oci bump-version release + +# Variables +CHART_DIR := . +CHARTS_SUBDIR := $(CHART_DIR)/charts # Subcharts are downloaded here + +# Helm release configuration +RELEASE_NAME ?= nvcf-nats +NAMESPACE ?= nats-system +VALUES_FILE := $(CHART_DIR)/values.yaml + +# Optional: specify additional values files, e.g., make install additional_values=my-custom-values.yaml +ADDITIONAL_VALUES ?= +ADDITIONAL_VALUES_CMD := $(if $(ADDITIONAL_VALUES),--values $(ADDITIONAL_VALUES),) + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(CHART_DIR)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(CHART_DIR)/Chart.yaml) + +# Variable for the new version, to be passed as an argument to make +# Example: make bump-version NEW_VERSION=0.2.0 +NEW_VERSION ?= + +# Default target +all: install + +lint: + helm lint $(CHART_DIR) + +deps: + @echo "Building dependencies for chart in $(CHART_DIR)..." + helm dependency build $(CHART_DIR) + +# The build target might be used if you have pre-packaging steps for your scripts or other assets. +# For now, it ensures the scripts directory exists. The generate-nkey.sh is assumed to be in place. +build: deps + @echo "[build] nothing to do" + +template: build + @echo "Templating chart $(CHART_DIR) for release $(RELEASE_NAME) in namespace $(NAMESPACE)..." + helm template $(RELEASE_NAME) $(CHART_DIR) --namespace $(NAMESPACE) --values $(VALUES_FILE) $(ADDITIONAL_VALUES_CMD) + +test: lint test-render + +test-render: deps + @echo "[test-render] Verifying rendered auth-callout secret wiring..." + @CHART_DIR=$(CHART_DIR) RELEASE_NAME=$(RELEASE_NAME) NAMESPACE=$(NAMESPACE) sh ./scripts/test-render-auth-callout.sh + +install: build + @echo "Installing chart $(CHART_DIR) as release $(RELEASE_NAME) in namespace $(NAMESPACE)..." + helm install $(RELEASE_NAME) $(CHART_DIR) \ + --namespace $(NAMESPACE) \ + --create-namespace \ + --values $(VALUES_FILE) \ + $(ADDITIONAL_VALUES_CMD) \ + --atomic \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +uninstall: + @echo "Uninstalling release $(RELEASE_NAME) from namespace $(NAMESPACE)..." + helm uninstall $(RELEASE_NAME) --namespace $(NAMESPACE) || true + # Consider if deleting the namespace is always desired. It can be disruptive. + # kubectl delete ns $(NAMESPACE) + +clean: + @echo "Cleaning up downloaded chart dependencies..." + rm -rf $(CHARTS_SUBDIR)/*.tgz + rm -f helm-nvcf-nats-*.tgz # Remove packaged chart + +# Ensure .helmignore is set up to exclude dependencies and secrets +ensure-helmignore: + # Do not ignore the charts/ directory itself, as helm package needs to access + # dependencies placed there by 'helm dependency build'. + @echo "secrets/" > $(CHART_DIR)/.helmignore + @echo ".DS_Store" >> $(CHART_DIR)/.helmignore + @echo "ci/" >> $(CHART_DIR)/.helmignore + @echo ".git/" >> $(CHART_DIR)/.helmignore + @echo "[ensure-helmignore] .helmignore configured in $(CHART_DIR)" + +package: deps ensure-helmignore + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(CHART_DIR) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: +ifndef OCI_REGISTRY_HOST + $(error OCI_REGISTRY_HOST is not set. Usage: make push-oci OCI_REGISTRY_HOST=example.com OCI_REGISTRY_NAMESPACE=charts) +endif +ifndef OCI_REGISTRY_NAMESPACE + $(error OCI_REGISTRY_NAMESPACE is not set. Usage: make push-oci OCI_REGISTRY_HOST=example.com OCI_REGISTRY_NAMESPACE=charts) +endif + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." + +bump-version: +ifndef NEW_VERSION + $(error NEW_VERSION is not set. Usage: make bump-version NEW_VERSION=x.y.z) +endif + @echo "[bump-version] Current chart version in $(CHART_DIR)/Chart.yaml (name: $(CHART_NAME)): $(CHART_VERSION)" + @echo "[bump-version] Bumping chart version to: $(NEW_VERSION)" + @yq e '.version = "$(NEW_VERSION)"' -i $(CHART_DIR)/Chart.yaml + $(eval CHART_VERSION := $(NEW_VERSION)) # Update CHART_VERSION for subsequent messages + @echo "[bump-version] Successfully bumped chart version in $(CHART_DIR)/Chart.yaml to $(CHART_VERSION)" + @echo "[bump-version] You might want to commit this change." + +release: bump-version package push-oci + @echo "[release] Successfully released chart $(CHART_NAME) version $(CHART_VERSION) to OCI registry." + @echo "[release] You might want to commit this change." diff --git a/deploy/helm/nvca-operator/Makefile b/deploy/helm/nvca-operator/Makefile new file mode 100644 index 000000000..80a71e9bd --- /dev/null +++ b/deploy/helm/nvca-operator/Makefile @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variables +release ?= nvca-operator +namespace ?= nvca-operator +helm_dir ?= ./nvca-operator +values := $(helm_dir)/values.yaml +stack_repo ?= ../../nvcf-self-managed-stack +stack_env ?= local +stack_env_file ?= $(stack_repo)/environments/$(stack_env).yaml +generated_values ?= ./bin/nvca-operator-values.$(stack_env).yaml + +# Load environment variables from .env file if it exists +-include .env + +# OPTIONAL for install targets: Path to an additional Helm values file. +# Example: make install values=my-values.yaml additional_values=override.yaml +additional_values ?= +self_managed_endpoint_values ?= \ + --set-string selfManaged.icmsServiceURL=http://icms.example.invalid:8080 \ + --set-string selfManaged.revalServiceURL=http://reval.example.invalid:8080 \ + --set-string selfManaged.natsURL=nats://nats.example.invalid:4222 + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +.PHONY: install uninstall status lint template validate clean package push-oci sync-chart check-synced-chart render-values-from-stack install-from-stack test-render-values test-build-release-assets test-release-image-manifest test-release-artifact-permissions test-package-release-assets test-attach-release-assets test-release-sbom-wrapper test-self-managed-nvca-image-reference + +install: +ifndef values + $(error "values" variable is not set. Please specify with 'make install values=') +endif + @echo "Deploying $(release) to namespace $(namespace) using values file '$(values)'..." + @echo "Additional values file: '$(if $(additional_values),$(additional_values),N/A)'" + helm install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +render-values-from-stack: + @echo "Rendering $(generated_values) from $(stack_env_file)..." + @mkdir -p "$(dir $(generated_values))" + @STACK_ENV_FILE="$(stack_env_file)" \ + OUTPUT_FILE="$(generated_values)" \ + RELEASE="$(release)" \ + NAMESPACE="$(namespace)" \ + NCA_ID="$(NCA_ID)" \ + CLUSTER_NAME="$(CLUSTER_NAME)" \ + CLUSTER_ID="$(CLUSTER_ID)" \ + IMAGE_PULL_SECRET_NAME="$(IMAGE_PULL_SECRET_NAME)" \ + ./scripts/render_values_from_stack_env.sh + @echo "Rendered values to $(generated_values)" + +install-from-stack: render-values-from-stack + @echo "Deploying $(release) to namespace $(namespace) using generated values '$(generated_values)'..." + @echo "Additional values file: '$(if $(additional_values),$(additional_values),N/A)'" + helm upgrade --install $(release) $(helm_dir) \ + --namespace $(namespace) \ + --values $(generated_values) \ + $(if $(additional_values),--values $(additional_values),) \ + --atomic \ + --create-namespace \ + --wait \ + --wait-for-jobs \ + --timeout 20m + +test-render-values: + @bash ./tests/render_values_from_stack_env_test.sh + +test-build-release-assets: + @bash ./tests/build_release_assets_test.sh + +test-release-image-manifest: + @bash ./tests/release_image_manifest_test.sh + +test-release-artifact-permissions: + @bash ./tests/release_asset_permissions_test.sh + +test-package-release-assets: + @bash ./tests/package_release_assets_test.sh + +test-attach-release-assets: + @bash ./tests/attach_release_assets_test.sh + +test-release-sbom-wrapper: + @bash ./tests/generate_release_sbom_test.sh + +test-self-managed-nvca-image-reference: + @bash ./tests/self_managed_nvca_image_reference_test.sh + +uninstall: + @echo "Deleting $(release) from namespace $(namespace)..." + helm uninstall $(release) --namespace $(namespace) + +status: + @echo "Checking status of $(release) in namespace $(namespace)..." + helm status $(release) --namespace $(namespace) + +lint: + @echo "Linting chart $(helm_dir)..." + helm lint $(helm_dir) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + $(self_managed_endpoint_values) + +template: + @echo "Templating chart $(helm_dir)..." + @mkdir -p bin + helm template $(release) $(helm_dir) \ + --namespace $(namespace) \ + $(if $(values),--values $(values),) \ + $(if $(additional_values),--values $(additional_values),) \ + $(self_managed_endpoint_values) \ + > bin/manifest.yaml + @echo "Rendered manifest to bin/manifest.yaml" + +validate: template + @echo "Validating manifest with kubeconform..." + @kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 -ignore-missing-schemas bin/manifest.yaml + +# Publish Chart +# NOTE: this is manual until the CI pipeline is updated to push the chart to the NVCR OCI registry +clean: + rm -rf $(charts_dir)/*.tgz + rm -rf ./packaged-charts + +package: clean lint + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(helm_dir) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." + +# Sync NVCA Operator chart from monorepo source chart +vendor-chart: + @echo "🚀 [vendor-chart] Vendor NVCA Operator chart from local monorepo source..." + @./scripts/ci_vendor_nvca_operator_chart + @echo "🎉 [vendor-chart] Vendor complete. Deployments available in $(helm_dir)/" + +check-vendor-chart: vendor-chart + @echo "🚀 [check-vendor-chart] Checking if chart is synced..." + @if ! git diff --ignore-space-at-eol --exit-code -- .; then \ + echo ""; \ + echo "❌ [check-vendor-chart] Chart is NOT synced with source chart!"; \ + echo ""; \ + echo "📋 To fix this issue:"; \ + echo " 1. Run: make vendor-chart"; \ + echo " 2. Review and commit the changes"; \ + echo " 3. Update your merge request"; \ + echo ""; \ + echo "💡 This ensures your deployment matches the monorepo source NVCA Operator chart."; \ + exit 1; \ + fi + @echo "🎉 [check-vendor-chart] Chart is synced." diff --git a/deploy/helm/nvca-operator/scripts/attach_release_assets.sh b/deploy/helm/nvca-operator/scripts/attach_release_assets.sh new file mode 100755 index 000000000..60ea1b652 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/attach_release_assets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +version="${CI_COMMIT_TAG:?CI_COMMIT_TAG is required}" +ci_api_v4_url="${CI_API_V4_URL:?CI_API_V4_URL is required}" +ci_project_id="${CI_PROJECT_ID:?CI_PROJECT_ID is required}" +ci_job_token="${CI_JOB_TOKEN:?CI_JOB_TOKEN is required}" +package_name="${RELEASE_ASSET_PACKAGE_NAME:?RELEASE_ASSET_PACKAGE_NAME is required}" + +package_url="${ci_api_v4_url}/projects/${ci_project_id}/packages/generic/${package_name}/${version}" +release_api="${ci_api_v4_url}/projects/${ci_project_id}/releases/${version}" + +echo "Waiting for release ${version} to be created..." +for i in $(seq 1 30); do + if curl --silent --fail --header "JOB-TOKEN: ${ci_job_token}" "${release_api}" > /dev/null 2>&1; then + echo "Release found" + break + fi + if [ "${i}" -eq 30 ]; then + echo "ERROR: Release not found after 30 attempts" >&2 + exit 1 + fi + sleep 10 +done + +existing_links="$(curl --silent --header "JOB-TOKEN: ${ci_job_token}" "${release_api}/assets/links")" +echo "Existing release links:" +echo "${existing_links}" | jq -r '.[]?.name' + +for path in release-artifacts/*; do + asset="$(basename "${path}")" + link_id="$(echo "${existing_links}" | jq -r ".[] | select(.name == \"${asset}\") | .id")" + if [ -n "${link_id}" ] && [ "${link_id}" != "null" ]; then + echo "Updating existing link for ${asset} (id: ${link_id})..." + curl --fail --request PUT \ + --header "JOB-TOKEN: ${ci_job_token}" \ + --header "Content-Type: application/json" \ + --data "{\"name\":\"${asset}\",\"url\":\"${package_url}/${asset}\",\"link_type\":\"package\"}" \ + "${release_api}/assets/links/${link_id}" + else + echo "Creating release link for ${asset}..." + curl --fail --request POST \ + --header "JOB-TOKEN: ${ci_job_token}" \ + --header "Content-Type: application/json" \ + --data "{\"name\":\"${asset}\",\"url\":\"${package_url}/${asset}\",\"link_type\":\"package\"}" \ + "${release_api}/assets/links" + fi + echo +done diff --git a/deploy/helm/nvca-operator/scripts/build_release_assets.sh b/deploy/helm/nvca-operator/scripts/build_release_assets.sh new file mode 100755 index 000000000..38b949c67 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/build_release_assets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +version="${CI_COMMIT_TAG:?CI_COMMIT_TAG is required}" +chart_file="nvca-operator/Chart.yaml" +chart_name="$(sed -n 's/^name:[[:space:]]*//p' "${chart_file}" | head -n1)" +chart_version="${version}" +image_manifest="sbom-inputs/artifacts-${version}.txt" +chart_package_dir="sbom-inputs/package" + +mkdir -p sbom-inputs +mkdir -p "${chart_package_dir}" + +helm template nvca-operator ./nvca-operator \ + --namespace nvca-operator \ + --values values.local.yml \ + --values values.release-sbom.yaml \ + > sbom-inputs/rendered.yaml + +helm package ./nvca-operator \ + -d "${chart_package_dir}" \ + --version "${chart_version}" \ + >/dev/null + +chart_package="$(find "${chart_package_dir}" -maxdepth 1 -type f -name '*.tgz' | sort | head -n1)" +if [[ -z "${chart_package}" ]]; then + echo "failed to package helm chart" >&2 + exit 1 +fi + +./scripts/render_release_supplemental_images.sh \ + sbom-inputs/rendered.yaml \ + sbom-inputs/supplemental-images.yaml + +./scripts/build_release_image_manifest.sh \ + sbom-inputs/rendered.yaml \ + sbom-inputs/supplemental-images.yaml \ + "${image_manifest}" + +./scripts/normalize_release_artifact_permissions.sh sbom-inputs + +test -s "${image_manifest}" + +echo "Release chart: ${chart_name}" +echo "Chart version: ${chart_version}" +echo "Release tag: ${version}" +echo "Release chart package: ${chart_package}" +echo "Image manifest: ${image_manifest}" +cat "${image_manifest}" diff --git a/deploy/helm/nvca-operator/scripts/build_release_image_manifest.sh b/deploy/helm/nvca-operator/scripts/build_release_image_manifest.sh new file mode 100755 index 000000000..1c8f4096f --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/build_release_image_manifest.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +rendered_manifest="$1" +supplemental_manifest="$2" +output_manifest="$3" + +for input in "${rendered_manifest}" "${supplemental_manifest}"; do + if [[ ! -f "${input}" ]]; then + echo "manifest not found: ${input}" >&2 + exit 1 + fi +done + +tmp_images="$(mktemp)" +cleanup() { + rm -f "${tmp_images}" +} +trap cleanup EXIT + +awk ' +function valid_image(value) { + if (value == "") { + return 0 + } + if (value ~ /:\/\//) { + return 0 + } + if (index(value, "{{") > 0 || index(value, "${") > 0) { + return 0 + } + if (value !~ /\//) { + return 0 + } + if (value !~ /[:@]/) { + return 0 + } + return 1 +} +/^[[:space:]]*image:[[:space:]]*/ { + image = $0 + sub(/^[[:space:]]*image:[[:space:]]*/, "", image) + sub(/[[:space:]]*#.*/, "", image) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", image) + gsub(/^["'\'']/, "", image) + gsub(/["'\'']$/, "", image) + if (valid_image(image)) { + print image + } +} +' "${rendered_manifest}" "${supplemental_manifest}" | sort -u > "${tmp_images}" + +cp "${tmp_images}" "${output_manifest}" diff --git a/deploy/helm/nvca-operator/scripts/ci_vendor_nvca_operator_chart b/deploy/helm/nvca-operator/scripts/ci_vendor_nvca_operator_chart new file mode 100755 index 000000000..ed44d006a --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/ci_vendor_nvca_operator_chart @@ -0,0 +1,292 @@ +#!/bin/bash +set -euo pipefail +root_dir="$(cd "$(dirname "$0")/.." && pwd)" + +update_yaml_key() { + local yaml_update_expr yaml_file tmp_file + yaml_update_expr="$1" + yaml_file="$2" + tmp_file="$(mktemp)" + echo "Updating $yaml_file with expression '$yaml_update_expr'" + yq eval "${yaml_update_expr}" "$yaml_file" > "$tmp_file" + mv "$tmp_file" "$yaml_file" +} + +update_json_key() { + local json_update_expr json_file tmp_file + json_update_expr="$1" + json_file="$2" + tmp_file="$(mktemp)" + echo "Updating $json_file with expression '$json_update_expr'" + yq eval --output-format=json --indent 4 "${json_update_expr}" "$json_file" > "$tmp_file" + mv "$tmp_file" "$json_file" +} + +replace_text() { + local sed_expr target_file tmp_file + sed_expr="$1" + target_file="$2" + tmp_file="$(mktemp)" + sed "${sed_expr}" "$target_file" > "$tmp_file" + mv "$tmp_file" "$target_file" +} + +insert_after_line_once() { + local target_line inserted_text target_file inserted_first_line tmp_file + target_line="$1" + inserted_text="$2" + target_file="$3" + inserted_first_line="$(printf '%s' "$inserted_text" | sed 's/\\n.*//')" + + if grep -Fxq "$inserted_first_line" "$target_file"; then + return + fi + + tmp_file="$(mktemp)" + awk -v target_line="$target_line" -v inserted_text="$inserted_text" ' + BEGIN { + gsub(/\\n/, "\n", inserted_text) + } + { + print + if (!inserted && $0 == target_line) { + print inserted_text + inserted = 1 + } + } + END { + if (!inserted) { + exit 1 + } + } + ' "$target_file" > "$tmp_file" || { + rm -f "$tmp_file" + echo "Failed to insert text after '${target_line}' in ${target_file}" >&2 + exit 1 + } + mv "$tmp_file" "$target_file" +} + +insert_release_artifact_image_annotations() { + local target_file tmp_file + local nvca_annotation_line image_credential_helper_annotation_line samba_annotation_line + local has_nvca_annotation has_image_credential_helper_annotation has_samba_annotation + target_file="$1" + nvca_annotation_line=' release-artifact-nvca-image: {{ printf "%s:%s" .Values.nvcaImage.repositoryOverride .Values.selfManaged.nvcaVersion | quote }}' + image_credential_helper_annotation_line=' release-artifact-nvcf-image-credential-helper-image: {{ printf "%s:%s" (include "nvcaop.imageCredHelperRepository" (dict "imageRepository" .Values.selfManaged.imageCredHelper.imageRepository "defaultRepository" .Values.image.repository)) .Values.selfManaged.imageCredHelper.imageTag | quote }}' + samba_annotation_line=' release-artifact-samba-image: {{ printf "%s:%s" .Values.selfManaged.sharedStorage.imageRepository .Values.selfManaged.sharedStorage.imageTag | quote }}' + + has_nvca_annotation=0 + has_image_credential_helper_annotation=0 + has_samba_annotation=0 + + if grep -Fq "release-artifact-nvca-image" "$target_file"; then + has_nvca_annotation=1 + fi + + if grep -Fq "release-artifact-nvcf-image-credential-helper-image" "$target_file"; then + has_image_credential_helper_annotation=1 + fi + if grep -Fq "release-artifact-samba-image" "$target_file"; then + has_samba_annotation=1 + fi + + if [ "$has_nvca_annotation" -eq 0 ] && [ "$has_image_credential_helper_annotation" -eq 0 ] && [ "$has_samba_annotation" -eq 0 ]; then + tmp_file="$(mktemp)" + awk ' + { + print + if ($0 == " namespace: {{ .Release.Namespace }}") { + print " {{- $releaseArtifactNvcaImageRepository := .Values.nvcaImage.repositoryOverride }}" + print " {{- $releaseArtifactImageCredentialHelperRepository := .Values.selfManaged.imageCredHelper.imageRepository }}" + print " {{- $releaseArtifactSambaRepository := .Values.selfManaged.sharedStorage.imageRepository }}" + print " {{- if or $releaseArtifactNvcaImageRepository $releaseArtifactImageCredentialHelperRepository $releaseArtifactSambaRepository }}" + print " annotations:" + print " {{- if $releaseArtifactNvcaImageRepository }}" + print " release-artifact-nvca-image: {{ printf \"%s:%s\" .Values.nvcaImage.repositoryOverride .Values.selfManaged.nvcaVersion | quote }}" + print " {{- end }}" + print " {{- if $releaseArtifactImageCredentialHelperRepository }}" + print " release-artifact-nvcf-image-credential-helper-image: {{ printf \"%s:%s\" (include \"nvcaop.imageCredHelperRepository\" (dict \"imageRepository\" .Values.selfManaged.imageCredHelper.imageRepository \"defaultRepository\" .Values.image.repository)) .Values.selfManaged.imageCredHelper.imageTag | quote }}" + print " {{- end }}" + print " {{- if $releaseArtifactSambaRepository }}" + print " release-artifact-samba-image: {{ printf \"%s:%s\" .Values.selfManaged.sharedStorage.imageRepository .Values.selfManaged.sharedStorage.imageTag | quote }}" + print " {{- end }}" + print " {{- end }}" + inserted = 1 + } + } + END { + if (!inserted) { + exit 1 + } + } + ' "$target_file" > "$tmp_file" || { + rm -f "$tmp_file" + echo "Failed to insert release-artifact image annotations into $target_file" >&2 + exit 1 + } + mv "$tmp_file" "$target_file" + return + fi + + if [ "$has_nvca_annotation" -eq 0 ]; then + insert_after_line_once " annotations:" "${nvca_annotation_line}" "$target_file" + fi + + if [ "$has_image_credential_helper_annotation" -eq 0 ]; then + if grep -Fq "${nvca_annotation_line}" "$target_file"; then + insert_after_line_once "${nvca_annotation_line}" "${image_credential_helper_annotation_line}" "$target_file" + else + insert_after_line_once " annotations:" "${image_credential_helper_annotation_line}" "$target_file" + fi + fi + + if [ "$has_samba_annotation" -eq 0 ]; then + if grep -Fq "${image_credential_helper_annotation_line}" "$target_file"; then + insert_after_line_once "${image_credential_helper_annotation_line}" "${samba_annotation_line}" "$target_file" + elif grep -Fq "${nvca_annotation_line}" "$target_file"; then + insert_after_line_once "${nvca_annotation_line}" "${samba_annotation_line}" "$target_file" + else + insert_after_line_once " annotations:" "${samba_annotation_line}" "$target_file" + fi + fi +} + +apply_license_headers_if_missing() { + local target_dir header_file file + target_dir="$1" + header_file="${root_dir}/.license-header.txt" + + while IFS= read -r -d '' file; do + # Some source templates already carry SPDX headers below Helm guard lines. + # Do not prepend another header when an SPDX marker is already present. + if grep -Fq "SPDX-FileCopyrightText:" "$file"; then + continue + fi + sh "${root_dir}/scripts/apply-license-header.sh" "${header_file}" "${file}" + done < <(find "${target_dir}" -type f \( -name '*.yaml' -o -name '*.yml' \) -print0) +} + +# Vendor NVCA Operator chart from the monorepo source chart +main() { + # Load local defaults when present; CI supplies these values as variables. + if [ -f "${root_dir}/.env" ]; then + source "${root_dir}/.env" + fi + + TARGET_DIR="${root_dir}/nvca-operator" + SOURCE_DIR="${root_dir}/../../../src/compute-plane-services/nvca/deployments/nvca-operator" + + if [ ! -d "${SOURCE_DIR}" ]; then + echo "❌ [vendor] Error: monorepo source chart not found at ${SOURCE_DIR}" + exit 1 + fi + + echo "📂 [vendor] Copying monorepo source chart from ${SOURCE_DIR}..." + SUBDIR_PATH="${SOURCE_DIR}" + SOURCE_LABEL="monorepo source chart" + + # Remove the existing chart only after the replacement source is available. + if [ -d "${TARGET_DIR}" ]; then + echo "🗑️ [vendor] Removing existing ${TARGET_DIR} directory..." + rm -rf "${TARGET_DIR}" + fi + + # Copy only the deployments/nvca-operator subdirectory to target + echo "📂 [vendor] Copying deployments/nvca-operator subdirectory..." + mkdir -p "${TARGET_DIR}" + cp -r "${SUBDIR_PATH}"/* "${TARGET_DIR}/" + + # Default various values so chart works OOTB + echo "🔧 [vendor] Defaulting various values so chart works OOTB..." + update_yaml_key ".ngcConfig.clusterSource = \"self-managed\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".ngcConfig.serviceKey = \"dummy-api-key\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".image.tag = \"${NVCA_OPERATOR_VERSION:?"NVCA_OPERATOR_VERSION is not set"}\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".selfManaged.nvcaVersion = \"${NVCA_VERSION:?"NVCA_VERSION is not set"}\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".selfManaged.icmsServiceURL = \"http://icms.example.invalid:8080\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".selfManaged.revalServiceURL = \"http://reval.example.invalid:8080\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".selfManaged.natsURL = \"nats://nats.example.invalid:4222\"" "${TARGET_DIR}/values.yaml" + insert_after_line_once \ + ' icmsServiceURL: "http://icms.example.invalid:8080"' \ + ' ## @param selfManaged.icmsServiceHostHeaderOverride Optional Host header override for selfManaged.icmsServiceURL.\n icmsServiceHostHeaderOverride: ""' \ + "${TARGET_DIR}/values.yaml" + insert_after_line_once \ + ' revalServiceURL: "http://reval.example.invalid:8080"' \ + ' ## @param selfManaged.revalServiceHostHeaderOverride Optional Host header override for selfManaged.revalServiceURL.\n revalServiceHostHeaderOverride: ""' \ + "${TARGET_DIR}/values.yaml" + insert_after_line_once \ + ' natsURL: "nats://nats.example.invalid:4222"' \ + ' ## @param selfManaged.natsHostOverride Optional TLS SNI host override for selfManaged.natsURL when using a tls or wss NATS URL.\n natsHostOverride: ""' \ + "${TARGET_DIR}/values.yaml" + replace_text 's|^ ## @param selfManaged\.icmsServiceURL .*| ## @param selfManaged.icmsServiceURL URL of the ICMS service for self-managed clusters. Override with the endpoint generated during cluster registration.|' "${TARGET_DIR}/values.yaml" + replace_text 's#URL of the ReVal service for self-managed clusters\. Required when ngcConfig\.clusterSource is "self-managed"\.#URL of the ReVal service for self-managed clusters. Override with the endpoint generated during cluster registration.#' "${TARGET_DIR}/values.yaml" + replace_text 's#URL of the NATS service for self-managed clusters\. Required when ngcConfig\.clusterSource is "self-managed"\.#URL of the NATS service for self-managed clusters. Override with the endpoint generated during cluster registration.#' "${TARGET_DIR}/values.yaml" + replace_text '/"icmsServiceURL": {/,/}/ s#"description": ".*"#"description": "URL of the ICMS service for self-managed clusters. Override with the endpoint generated during cluster registration."#' "${TARGET_DIR}/values.schema.json" + replace_text 's#URL of the ReVal service for self-managed clusters\. Required when ngcConfig\.clusterSource is \\"self-managed\\"\.#URL of the ReVal service for self-managed clusters. Override with the endpoint generated during cluster registration.#' "${TARGET_DIR}/values.schema.json" + replace_text 's#URL of the NATS service for self-managed clusters\. Required when ngcConfig\.clusterSource is \\"self-managed\\"\.#URL of the NATS service for self-managed clusters. Override with the endpoint generated during cluster registration.#' "${TARGET_DIR}/values.schema.json" + replace_text '/"icmsServiceURL": {/,/}/ s#"default": ""#"default": "http://icms.example.invalid:8080"#' "${TARGET_DIR}/values.schema.json" + replace_text '/"revalServiceURL": {/,/}/ s#"default": ""#"default": "http://reval.example.invalid:8080"#' "${TARGET_DIR}/values.schema.json" + replace_text '/"natsURL": {/,/}/ s#"default": ""#"default": "nats://nats.example.invalid:4222"#' "${TARGET_DIR}/values.schema.json" + update_json_key '.properties.selfManaged.properties.icmsServiceHostHeaderOverride = {"type": "string", "description": "Optional Host header override for selfManaged.icmsServiceURL.", "default": ""}' "${TARGET_DIR}/values.schema.json" + update_json_key '.properties.selfManaged.properties.revalServiceHostHeaderOverride = {"type": "string", "description": "Optional Host header override for selfManaged.revalServiceURL.", "default": ""}' "${TARGET_DIR}/values.schema.json" + update_json_key '.properties.selfManaged.properties.natsHostOverride = {"type": "string", "description": "Optional TLS SNI host override for selfManaged.natsURL when using a tls or wss NATS URL.", "default": ""}' "${TARGET_DIR}/values.schema.json" + replace_text 's#^| `selfManaged.icmsServiceURL`.*#| `selfManaged.icmsServiceURL` | URL of the ICMS service for self-managed clusters. Override with the endpoint generated during cluster registration. | `http://icms.example.invalid:8080` |#' "${TARGET_DIR}/README.md" + replace_text 's#| `selfManaged.revalServiceURL` | URL of the ReVal service for self-managed clusters. Required when ngcConfig.clusterSource is "self-managed". | `""` |#| `selfManaged.revalServiceURL` | URL of the ReVal service for self-managed clusters. Override with the endpoint generated during cluster registration. | `http://reval.example.invalid:8080` |#' "${TARGET_DIR}/README.md" + replace_text 's#| `selfManaged.natsURL` | URL of the NATS service for self-managed clusters. Required when ngcConfig.clusterSource is "self-managed". | `""` |#| `selfManaged.natsURL` | URL of the NATS service for self-managed clusters. Override with the endpoint generated during cluster registration. | `nats://nats.example.invalid:4222` |#' "${TARGET_DIR}/README.md" + insert_after_line_once \ + '| `selfManaged.icmsServiceURL` | URL of the ICMS service for self-managed clusters. Override with the endpoint generated during cluster registration. | `http://icms.example.invalid:8080` |' \ + '| `selfManaged.icmsServiceHostHeaderOverride` | Optional Host header override for selfManaged.icmsServiceURL. | `""` |' \ + "${TARGET_DIR}/README.md" + insert_after_line_once \ + '| `selfManaged.revalServiceURL` | URL of the ReVal service for self-managed clusters. Override with the endpoint generated during cluster registration. | `http://reval.example.invalid:8080` |' \ + '| `selfManaged.revalServiceHostHeaderOverride` | Optional Host header override for selfManaged.revalServiceURL. | `""` |' \ + "${TARGET_DIR}/README.md" + insert_after_line_once \ + '| `selfManaged.natsURL` | URL of the NATS service for self-managed clusters. Override with the endpoint generated during cluster registration. | `nats://nats.example.invalid:4222` |' \ + '| `selfManaged.natsHostOverride` | Optional TLS SNI host override for selfManaged.natsURL when using a tls or wss NATS URL. | `""` |' \ + "${TARGET_DIR}/README.md" + insert_after_line_once \ + ' icmsServiceURL: {{ .Values.selfManaged.icmsServiceURL | quote }}' \ + ' {{- with .Values.selfManaged.icmsServiceHostHeaderOverride }}\n icmsServiceHostHeaderOverride: {{ . | quote }}\n {{- end }}' \ + "${TARGET_DIR}/templates/self-managed-nvcfbackend-cm.yaml" + insert_after_line_once \ + ' helmReValServiceURL: {{ .Values.selfManaged.revalServiceURL | quote }}' \ + ' {{- with .Values.selfManaged.revalServiceHostHeaderOverride }}\n helmReValServiceHostHeaderOverride: {{ . | quote }}\n {{- end }}' \ + "${TARGET_DIR}/templates/self-managed-nvcfbackend-cm.yaml" + insert_after_line_once \ + ' natsURL: {{ .Values.selfManaged.natsURL | quote }}' \ + ' {{- with .Values.selfManaged.natsHostOverride }}\n natsHostOverride: {{ . | quote }}\n {{- end }}' \ + "${TARGET_DIR}/templates/self-managed-nvcfbackend-cm.yaml" + insert_release_artifact_image_annotations "${TARGET_DIR}/templates/self-managed-nvcfbackend-cm.yaml" + update_yaml_key ".version = \"0.0.0\"" "${TARGET_DIR}/Chart.yaml" + update_yaml_key ".name = \"helm-nvca-operator\"" "${TARGET_DIR}/Chart.yaml" + # appVersion tracks the NVCA Operator binary version this chart deploys, + # not the chart's published semver (which CI sets from the git tag at + # release time). Without this overlay the upstream's stale "1.0" + # placeholder leaks through and helm releases report appVersion="1.0" + # regardless of which NVCA Operator the chart is actually wired to. + update_yaml_key ".appVersion = \"${NVCA_OPERATOR_VERSION:?"NVCA_OPERATOR_VERSION is not set"}\"" "${TARGET_DIR}/Chart.yaml" + update_yaml_key ".generateImagePullSecret = false" "${TARGET_DIR}/values.yaml" + update_yaml_key ".selfManaged.sharedStorage.imageTag = \"${NVCA_SHARED_STORAGE_IMAGE_TAG:?"NVCA_SHARED_STORAGE_IMAGE_TAG is not set"}\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".nameOverride = \"nvca-operator\"" "${TARGET_DIR}/values.yaml" + update_yaml_key ".fullnameOverride = \"nvca-operator\"" "${TARGET_DIR}/values.yaml" + + # Set default validation policy to Unrestricted for self-managed clusters + yq eval -i '.agentConfig.mergeConfig = "cluster:\n validationPolicy:\n name: Unrestricted" | .agentConfig.mergeConfig style="literal"' "${TARGET_DIR}/values.yaml" + + # OSS scrub: clear the upstream's hardcoded image.repository so OSS + # consumers must supply it via an override values file at install time. + update_yaml_key ".image.repository = \"\"" "${TARGET_DIR}/values.yaml" + + # Apply Apache-2.0 / NVIDIA copyright headers to all yaml files in the + # vendored chart. Skip files that already carry an SPDX marker so this + # step stays idempotent and does not duplicate headers. + echo "📝 [vendor] Applying license headers..." + apply_license_headers_if_missing "${TARGET_DIR}" + + echo "✅ [vendor] Successfully vendored NVCA Operator deployments (${SOURCE_LABEL}) to ${TARGET_DIR}/" + echo "📁 [vendor] Contents:" + ls -la "${TARGET_DIR}/" +} + +main "$@" diff --git a/deploy/helm/nvca-operator/scripts/generate_release_sbom.sh b/deploy/helm/nvca-operator/scripts/generate_release_sbom.sh new file mode 100755 index 000000000..665822904 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/generate_release_sbom.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +manifest_dir="$1" +release_version="$2" +output_file="$3" + +if [[ ! -d "${manifest_dir}" ]]; then + echo "manifest directory not found: ${manifest_dir}" >&2 + exit 1 +fi + +if ! command -v generate-sbom >/dev/null 2>&1; then + echo "generate-sbom binary is required on PATH" >&2 + exit 1 +fi + +tmp_dir="$(mktemp -d)" +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +workspace_root="${tmp_dir}/workspace" +render_dir="${workspace_root}/out/00-release.yaml-deadbeef-helm-nvca-operator/templates" +artifact_dir="${tmp_dir}/artifacts" + +mkdir -p "${workspace_root}/helmfile.d" "${render_dir}" "${artifact_dir}" + +cat > "${workspace_root}/helmfile.d/00-release.yaml.gotmpl" <&2 + exit 1 +fi + +chmod -R a+rX "${artifact_dir}" diff --git a/deploy/helm/nvca-operator/scripts/package_release_assets.sh b/deploy/helm/nvca-operator/scripts/package_release_assets.sh new file mode 100755 index 000000000..abeea9351 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/package_release_assets.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +version="${CI_COMMIT_TAG:?CI_COMMIT_TAG is required}" +ci_api_v4_url="${CI_API_V4_URL:?CI_API_V4_URL is required}" +ci_project_id="${CI_PROJECT_ID:?CI_PROJECT_ID is required}" +ci_job_token="${CI_JOB_TOKEN:?CI_JOB_TOKEN is required}" +package_name="${RELEASE_ASSET_PACKAGE_NAME:?RELEASE_ASSET_PACKAGE_NAME is required}" + +mkdir -p release-artifacts + +./scripts/generate_release_sbom.sh \ + ./sbom-inputs \ + "${version}" \ + "release-artifacts/sbom-${version}.spdx.json" + +cp "sbom-inputs/artifacts-${version}.txt" "release-artifacts/artifacts-${version}.txt" +cp ./sbom-inputs/package/*.tgz release-artifacts/ + +artifact_paths=(release-artifacts/*) +if [[ ${#artifact_paths[@]} -eq 0 || ! -e "${artifact_paths[0]}" ]]; then + echo "release artifacts were not created" >&2 + exit 1 +fi + +artifact_files=() +for path in "${artifact_paths[@]}"; do + artifact_files+=("$(basename "${path}")") +done + +sorted_artifact_files=() +while IFS= read -r file; do + sorted_artifact_files+=("${file}") +done < <(printf '%s\n' "${artifact_files[@]}" | sort) +artifact_files=("${sorted_artifact_files[@]}") + +( + cd release-artifacts + sha256sum "${artifact_files[@]}" > "checksums-${version}.sha256" +) + +for path in release-artifacts/*; do + file="$(basename "${path}")" + echo "Uploading ${file} to package registry..." + curl --fail --header "JOB-TOKEN: ${ci_job_token}" \ + --upload-file "${path}" \ + "${ci_api_v4_url}/projects/${ci_project_id}/packages/generic/${package_name}/${version}/${file}" +done diff --git a/deploy/helm/nvca-operator/scripts/render_release_supplemental_images.sh b/deploy/helm/nvca-operator/scripts/render_release_supplemental_images.sh new file mode 100755 index 000000000..ef174c4e7 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/render_release_supplemental_images.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +input_manifest="$1" +output_manifest="$2" + +if [[ ! -f "${input_manifest}" ]]; then + echo "rendered manifest not found: ${input_manifest}" >&2 + exit 1 +fi + +tmp_images="$(mktemp)" +cleanup() { + rm -f "${tmp_images}" +} +trap cleanup EXIT + +awk ' +function emit(repo, tag, image) { + repo = trim(repo) + tag = trim(tag) + if (repo == "" || tag == "") { + return + } + if (repo ~ /^ 0 || index(image, "${") > 0) { + return + } + seen[image] = 1 +} +function trim(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^"/, "", value) + gsub(/"$/, "", value) + gsub(/^'\''/, "", value) + gsub(/'\''$/, "", value) + return value +} +/name:[[:space:]]+NVCA_IMAGE_REPO$/ { + expect_nvca_repo = 1 + next +} +expect_nvca_repo && /^[[:space:]]*value:[[:space:]]*/ { + nvca_repo = $0 + sub(/^[[:space:]]*value:[[:space:]]*/, "", nvca_repo) + nvca_repo = trim(nvca_repo) + expect_nvca_repo = 0 + if (nvca_version != "") { + emit(nvca_repo, nvca_version) + } + next +} +/name:[[:space:]]+OTEL_COLLECTOR_IMAGE_REPO$/ { + expect_otel_repo = 1 + next +} +expect_otel_repo && /^[[:space:]]*value:[[:space:]]*/ { + otel_repo = $0 + sub(/^[[:space:]]*value:[[:space:]]*/, "", otel_repo) + otel_repo = trim(otel_repo) + expect_otel_repo = 0 + if (otel_tag != "") { + emit(otel_repo, otel_tag) + } + next +} +/name:[[:space:]]+OTEL_COLLECTOR_IMAGE_TAG$/ { + expect_otel_tag = 1 + next +} +expect_otel_tag && /^[[:space:]]*value:[[:space:]]*/ { + otel_tag = $0 + sub(/^[[:space:]]*value:[[:space:]]*/, "", otel_tag) + otel_tag = trim(otel_tag) + expect_otel_tag = 0 + if (otel_repo != "") { + emit(otel_repo, otel_tag) + } + next +} +/nvcaVersion:[[:space:]]*/ { + nvca_version = $0 + sub(/.*nvcaVersion:[[:space:]]*/, "", nvca_version) + nvca_version = trim(nvca_version) + if (nvca_repo != "") { + emit(nvca_repo, nvca_version) + } + next +} +/repository:[[:space:]]*/ || /imageRepository:[[:space:]]*/ { + pending_repo = $0 + sub(/.*(repository|imageRepository):[[:space:]]*/, "", pending_repo) + pending_repo = trim(pending_repo) + next +} +pending_repo != "" && (/tag:[[:space:]]*/ || /imageTag:[[:space:]]*/) { + pending_tag = $0 + sub(/.*(tag|imageTag):[[:space:]]*/, "", pending_tag) + pending_tag = trim(pending_tag) + emit(pending_repo, pending_tag) + pending_repo = "" + next +} +END { + for (image in seen) { + print image + } +} +' "${input_manifest}" | sort -u > "${tmp_images}" + +if [[ ! -s "${tmp_images}" ]]; then + cat > "${output_manifest}" <<'EOF' +apiVersion: v1 +kind: List +items: [] +EOF + exit 0 +fi + +{ + echo "apiVersion: v1" + echo "kind: Pod" + echo "metadata:" + echo " name: nvca-release-supplemental-images" + echo "spec:" + echo " restartPolicy: Never" + echo " containers:" + index=0 + while IFS= read -r image; do + [[ -z "${image}" ]] && continue + printf " - name: image-%02d\n" "${index}" + printf " image: %s\n" "${image}" + index=$((index + 1)) + done < "${tmp_images}" +} > "${output_manifest}" diff --git a/deploy/helm/nvca-operator/scripts/render_values_from_stack_env.sh b/deploy/helm/nvca-operator/scripts/render_values_from_stack_env.sh new file mode 100755 index 000000000..7ecd58d78 --- /dev/null +++ b/deploy/helm/nvca-operator/scripts/render_values_from_stack_env.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +root_dir="$(cd "$(dirname "$0")/.." && pwd)" +stack_env_file="${STACK_ENV_FILE:?STACK_ENV_FILE is required}" +output_file="${OUTPUT_FILE:?OUTPUT_FILE is required}" +release="${RELEASE:-nvca-operator}" +namespace="${NAMESPACE:-nvca-operator}" +env_nvca_operator_version="${NVCA_OPERATOR_VERSION:-}" +env_nvca_version="${NVCA_VERSION:-}" + +if [ -f "${root_dir}/.env" ]; then + # Keep stack-derived rendering aligned with the currently vendored chart version. + # Explicit environment variables still win over values sourced from .env. + # shellcheck disable=SC1091 + source "${root_dir}/.env" +fi + +if [ -n "${env_nvca_operator_version}" ]; then + NVCA_OPERATOR_VERSION="${env_nvca_operator_version}" +fi + +if [ -n "${env_nvca_version}" ]; then + NVCA_VERSION="${env_nvca_version}" +fi + +for cmd in yq helm; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Error: '${cmd}' command not found" >&2 + exit 1 + fi +done + +if [ ! -f "${stack_env_file}" ]; then + echo "Error: stack environment file not found: ${stack_env_file}" >&2 + exit 1 +fi + +stack_image_registry="$(yq -r '.global.image.registry // ""' "${stack_env_file}")" +stack_image_repository="$(yq -r '.global.image.repository // ""' "${stack_env_file}")" + +if [ -z "${stack_image_registry}" ] || [ -z "${stack_image_repository}" ]; then + echo "Error: ${stack_env_file} is missing global.image.registry or global.image.repository" >&2 + exit 1 +fi + +repo_prefix="${stack_image_registry}/${stack_image_repository}" +tmp_dir="$(mktemp -d)" +tmp_existing="${tmp_dir}/existing-values.yaml" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +cp "${root_dir}/values.local.yml" "${output_file}" + +if helm status "${release}" --namespace "${namespace}" >/dev/null 2>&1; then + echo "Found existing release ${release} in namespace ${namespace}, reusing its current values when present." + helm get values "${release}" --namespace "${namespace}" -o yaml > "${tmp_existing}" + yq eval-all -i '. as $item ireduce ({}; . *+ $item )' "${output_file}" "${tmp_existing}" +fi + +export REPO_PREFIX="${repo_prefix}" +yq eval -i ' + .image.repository = strenv(REPO_PREFIX) + "/nvca-operator" | + .nvcaImage.repositoryOverride = strenv(REPO_PREFIX) + "/nvca" | + .ngcConfig.clusterSource = "self-managed" | + .ngcConfig.serviceKey = "not-used" | + .generateImagePullSecret = false | + .selfManaged.featureGateValues = ["DynamicGPUDiscovery", "SelfHosted"] | + .selfManaged.imageCredHelper.imageRepository = strenv(REPO_PREFIX) + "/nvcf-image-credential-helper" | + .selfManaged.sharedStorage.imageRepository = strenv(REPO_PREFIX) + "/samba" | + .nameOverride = "nvca-operator" | + .fullnameOverride = "nvca-operator" +' "${output_file}" + +if [ -n "${NVCA_OPERATOR_VERSION:-}" ]; then + export NVCA_OPERATOR_VERSION + yq eval -i '.image.tag = strenv(NVCA_OPERATOR_VERSION)' "${output_file}" +fi + +if [ -n "${NVCA_VERSION:-}" ]; then + export NVCA_VERSION + yq eval -i '.selfManaged.nvcaVersion = strenv(NVCA_VERSION)' "${output_file}" +fi + +if [ -n "${NCA_ID:-}" ]; then + export NCA_ID + yq eval -i '.ncaID = strenv(NCA_ID)' "${output_file}" +fi + +if [ -n "${CLUSTER_NAME:-}" ]; then + export CLUSTER_NAME + yq eval -i '.clusterName = strenv(CLUSTER_NAME)' "${output_file}" +fi + +if [ -n "${CLUSTER_ID:-}" ]; then + export CLUSTER_ID + yq eval -i '.clusterID = strenv(CLUSTER_ID)' "${output_file}" +fi + +if [ -n "${IMAGE_PULL_SECRET_NAME:-}" ]; then + export IMAGE_PULL_SECRET_NAME + yq eval -i '.imagePullSecrets = [{"name": strenv(IMAGE_PULL_SECRET_NAME)}]' "${output_file}" +fi + +echo "Generated ${output_file} from ${stack_env_file}" +echo " image repository prefix: ${repo_prefix}" +echo " release namespace: ${namespace}" diff --git a/deploy/helm/nvca-operator/tests/attach_release_assets_test.sh b/deploy/helm/nvca-operator/tests/attach_release_assets_test.sh new file mode 100644 index 000000000..3a193bc69 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/attach_release_assets_test.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +mkdir -p "${tmp_dir}/scripts" "${tmp_dir}/release-artifacts" "${tmp_dir}/bin" + +cp "${repo_root}/scripts/attach_release_assets.sh" "${tmp_dir}/scripts/attach_release_assets.sh" +chmod +x "${tmp_dir}/scripts/attach_release_assets.sh" + +printf 'image\n' > "${tmp_dir}/release-artifacts/artifacts-9.9.9.txt" +printf '{"spdxVersion":"SPDX-2.3"}\n' > "${tmp_dir}/release-artifacts/sbom-9.9.9.spdx.json" +printf 'chart-bytes\n' > "${tmp_dir}/release-artifacts/helm-nvca-operator-9.9.9.tgz" +printf 'deadbeef artifacts-9.9.9.txt\n' > "${tmp_dir}/release-artifacts/checksums-9.9.9.sha256" + +cat > "${tmp_dir}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +method="GET" +url="" +payload="" +while [[ $# -gt 0 ]]; do + case "$1" in + --request) + method="$2" + shift 2 + ;; + --data) + payload="$2" + shift 2 + ;; + --header) + shift 2 + ;; + --silent|--fail) + shift + ;; + http*) + url="$1" + shift + ;; + *) + shift + ;; + esac +done + +case "${method} ${url}" in + "GET https://gitlab.example.com/api/v4/projects/123/releases/9.9.9") + printf '{"tag_name":"9.9.9"}\n' + ;; + "GET https://gitlab.example.com/api/v4/projects/123/releases/9.9.9/assets/links") + printf '[]\n' + ;; + "POST https://gitlab.example.com/api/v4/projects/123/releases/9.9.9/assets/links") + printf '%s\n' "${payload}" >> "${POST_LOG:?POST_LOG is required}" + printf '{"id":1}\n' + ;; + *) + echo "unexpected curl invocation: ${method} ${url}" >&2 + exit 1 + ;; +esac +EOF +chmod +x "${tmp_dir}/bin/curl" + +post_log="${tmp_dir}/post.log" + +( + cd "${tmp_dir}" + PATH="${tmp_dir}/bin:${PATH}" \ + CI_COMMIT_TAG=9.9.9 \ + CI_API_V4_URL=https://gitlab.example.com/api/v4 \ + CI_PROJECT_ID=123 \ + CI_JOB_TOKEN=token \ + RELEASE_ASSET_PACKAGE_NAME=nvca-operator-release-assets \ + POST_LOG="${post_log}" \ + ./scripts/attach_release_assets.sh +) + +for expected in \ + 'artifacts-9.9.9.txt' \ + 'sbom-9.9.9.spdx.json' \ + 'helm-nvca-operator-9.9.9.tgz' \ + 'checksums-9.9.9.sha256' +do + if ! grep -Fq "${expected}" "${post_log}"; then + echo "expected release link creation for ${expected}" >&2 + cat "${post_log}" >&2 + exit 1 + fi +done + +echo "attach_release_assets.sh links every packaged release artifact including checksums" diff --git a/deploy/helm/nvca-operator/tests/build_release_assets_test.sh b/deploy/helm/nvca-operator/tests/build_release_assets_test.sh new file mode 100644 index 000000000..46938d263 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/build_release_assets_test.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +mkdir -p "${tmp_dir}/scripts" "${tmp_dir}/nvca-operator" "${tmp_dir}/bin" + +cp "${repo_root}/scripts/build_release_assets.sh" "${tmp_dir}/scripts/build_release_assets.sh" +chmod +x "${tmp_dir}/scripts/build_release_assets.sh" + +cat > "${tmp_dir}/nvca-operator/Chart.yaml" <<'EOF' +apiVersion: v2 +name: helm-nvca-operator +version: 0.0.0 +EOF + +cat > "${tmp_dir}/values.local.yml" <<'EOF' +{} +EOF + +cat > "${tmp_dir}/values.release-sbom.yaml" <<'EOF' +{} +EOF + +cat > "${tmp_dir}/scripts/render_release_supplemental_images.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'apiVersion: v1\nkind: Pod\nmetadata:\n name: supplemental\n' > "$2" +EOF +chmod +x "${tmp_dir}/scripts/render_release_supplemental_images.sh" + +cat > "${tmp_dir}/scripts/build_release_image_manifest.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'nvcr.io/example/image:9.9.9\n' > "$3" +EOF +chmod +x "${tmp_dir}/scripts/build_release_image_manifest.sh" + +cat > "${tmp_dir}/scripts/normalize_release_artifact_permissions.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +test -d "$1" +EOF +chmod +x "${tmp_dir}/scripts/normalize_release_artifact_permissions.sh" + +cat > "${tmp_dir}/bin/helm" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >> "${HELM_LOG:?HELM_LOG is required}" + +case "$1" in + template) + printf 'apiVersion: v1\nkind: Pod\nmetadata:\n name: rendered\n' + ;; + package) + chart_dir="$2" + shift 2 + output_dir="" + package_version="" + while [[ $# -gt 0 ]]; do + case "$1" in + -d) + output_dir="$2" + shift 2 + ;; + --version) + package_version="$2" + shift 2 + ;; + *) + shift + ;; + esac + done + if [[ -z "${output_dir}" || -z "${package_version}" ]]; then + echo "expected output dir and package version" >&2 + exit 1 + fi + chart_name="$(sed -n 's/^name:[[:space:]]*//p' "${chart_dir}/Chart.yaml" | head -n1)" + touch "${output_dir}/${chart_name}-${package_version}.tgz" + ;; + *) + echo "unexpected helm invocation: $*" >&2 + exit 1 + ;; +esac +EOF +chmod +x "${tmp_dir}/bin/helm" + +helm_log="${tmp_dir}/helm.log" +output_log="${tmp_dir}/output.log" + +( + cd "${tmp_dir}" + PATH="${tmp_dir}/bin:${PATH}" \ + CI_COMMIT_TAG=9.9.9 \ + HELM_LOG="${helm_log}" \ + ./scripts/build_release_assets.sh +) | tee "${output_log}" >/dev/null + +if ! grep -Fqx 'package ./nvca-operator -d sbom-inputs/package --version 9.9.9' "${helm_log}"; then + echo "expected helm package invocation to override chart version with release tag" >&2 + cat "${helm_log}" >&2 + exit 1 +fi + +if [[ ! -f "${tmp_dir}/sbom-inputs/package/helm-nvca-operator-9.9.9.tgz" ]]; then + echo "expected chart package to use release tag version" >&2 + ls -R "${tmp_dir}/sbom-inputs" >&2 + exit 1 +fi + +if ! grep -Fq 'Chart version: 9.9.9' "${output_log}"; then + echo "expected build script output to log release tag as chart version" >&2 + cat "${output_log}" >&2 + exit 1 +fi + +echo "build_release_assets.sh packages the chart with the release tag version" diff --git a/deploy/helm/nvca-operator/tests/generate_release_sbom_test.sh b/deploy/helm/nvca-operator/tests/generate_release_sbom_test.sh new file mode 100644 index 000000000..3fbce7ace --- /dev/null +++ b/deploy/helm/nvca-operator/tests/generate_release_sbom_test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +manifest_dir="${tmp_dir}/manifests" +output_file="${tmp_dir}/sbom-1.4.7.spdx.json" +stub_bin_dir="${tmp_dir}/bin" +mkdir -p "${manifest_dir}" "${stub_bin_dir}" + +cat > "${manifest_dir}/rendered.yaml" <<'EOF' +apiVersion: v1 +kind: Pod +metadata: + name: nvca-operator +spec: + containers: + - name: operator + image: nvcr.io/nvidia/nvcf-byoc/nvca-operator:2.52.0-rc.5 +EOF + +cat > "${manifest_dir}/supplemental-images.yaml" <<'EOF' +apiVersion: v1 +kind: Pod +metadata: + name: nvca-release-supplemental-images +spec: + containers: + - name: image-00 + image: nvcr.io/nvidia/nvcf-byoc/nvca:2.52.0-rc.5 +EOF + +cat > "${stub_bin_dir}/generate-sbom" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +if [[ ! -f "helmfile.d/00-release.yaml.gotmpl" ]]; then + echo "expected synthetic helmfile workspace" >&2 + exit 1 +fi + +if ! grep -Fq "version: 1.4.7" "helmfile.d/00-release.yaml.gotmpl"; then + echo "expected release version in synthetic helmfile" >&2 + exit 1 +fi + +if [[ ! -f "out/00-release.yaml-deadbeef-helm-nvca-operator/templates/rendered.yaml" ]]; then + echo "expected rendered manifest in synthetic out dir" >&2 + exit 1 +fi + +if [[ ! -f "out/00-release.yaml-deadbeef-helm-nvca-operator/templates/supplemental-images.yaml" ]]; then + echo "expected supplemental manifest in synthetic out dir" >&2 + exit 1 +fi + +output_dir="" +while [[ $# -gt 0 ]]; do + case "$1" in + -o|--output-dir) + output_dir="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +mkdir -p "${output_dir}" +printf '{"spdxVersion":"SPDX-2.3"}\n' > "${output_dir}/sbom-release.spdx.json" +EOF +chmod +x "${stub_bin_dir}/generate-sbom" + +PATH="${stub_bin_dir}:$PATH" \ + "${repo_root}/scripts/generate_release_sbom.sh" \ + "${manifest_dir}" \ + "1.4.7" \ + "${output_file}" + +if [[ ! -f "${output_file}" ]]; then + echo "expected output file to be created" >&2 + exit 1 +fi + +if ! grep -Fq '"spdxVersion":"SPDX-2.3"' "${output_file}"; then + echo "expected wrapper to preserve generated sbom output" >&2 + exit 1 +fi + +echo "generate_release_sbom.sh prepares a synthetic workspace for generate-sbom" diff --git a/deploy/helm/nvca-operator/tests/package_release_assets_test.sh b/deploy/helm/nvca-operator/tests/package_release_assets_test.sh new file mode 100644 index 000000000..85b3a6c81 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/package_release_assets_test.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +mkdir -p "${tmp_dir}/scripts" "${tmp_dir}/sbom-inputs/package" "${tmp_dir}/bin" + +cp "${repo_root}/scripts/package_release_assets.sh" "${tmp_dir}/scripts/package_release_assets.sh" +chmod +x "${tmp_dir}/scripts/package_release_assets.sh" + +cat > "${tmp_dir}/scripts/generate_release_sbom.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '{"spdxVersion":"SPDX-2.3"}\n' > "$3" +EOF +chmod +x "${tmp_dir}/scripts/generate_release_sbom.sh" + +cat > "${tmp_dir}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +upload_file="" +url="" +while [[ $# -gt 0 ]]; do + case "$1" in + --upload-file) + upload_file="$2" + shift 2 + ;; + --header) + shift 2 + ;; + --fail) + shift + ;; + http*) + url="$1" + shift + ;; + *) + shift + ;; + esac +done + +printf '%s\t%s\n' "${upload_file}" "${url}" >> "${UPLOAD_LOG:?UPLOAD_LOG is required}" +EOF +chmod +x "${tmp_dir}/bin/curl" + +printf 'image\n' > "${tmp_dir}/sbom-inputs/artifacts-9.9.9.txt" +printf 'chart-bytes\n' > "${tmp_dir}/sbom-inputs/package/helm-nvca-operator-9.9.9.tgz" + +upload_log="${tmp_dir}/upload.log" + +( + cd "${tmp_dir}" + PATH="${tmp_dir}/bin:${PATH}" \ + CI_COMMIT_TAG=9.9.9 \ + CI_API_V4_URL=https://gitlab.example.com/api/v4 \ + CI_PROJECT_ID=123 \ + CI_JOB_TOKEN=token \ + RELEASE_ASSET_PACKAGE_NAME=nvca-operator-release-assets \ + UPLOAD_LOG="${upload_log}" \ + ./scripts/package_release_assets.sh +) + +for expected in \ + "${tmp_dir}/release-artifacts/artifacts-9.9.9.txt" \ + "${tmp_dir}/release-artifacts/sbom-9.9.9.spdx.json" \ + "${tmp_dir}/release-artifacts/helm-nvca-operator-9.9.9.tgz" \ + "${tmp_dir}/release-artifacts/checksums-9.9.9.sha256" +do + if [[ ! -f "${expected}" ]]; then + echo "expected release artifact missing: ${expected}" >&2 + exit 1 + fi +done + +checksum_file="${tmp_dir}/release-artifacts/checksums-9.9.9.sha256" +for checksummed in \ + 'artifacts-9.9.9.txt' \ + 'sbom-9.9.9.spdx.json' \ + 'helm-nvca-operator-9.9.9.tgz' +do + if ! grep -Fq " ${checksummed}" "${checksum_file}"; then + echo "expected checksum file to include ${checksummed}" >&2 + cat "${checksum_file}" >&2 + exit 1 + fi +done + +for uploaded in \ + "release-artifacts/artifacts-9.9.9.txt" \ + "release-artifacts/sbom-9.9.9.spdx.json" \ + "release-artifacts/helm-nvca-operator-9.9.9.tgz" \ + "release-artifacts/checksums-9.9.9.sha256" +do + if ! grep -Fq "${uploaded}" "${upload_log}"; then + echo "expected upload log to include ${uploaded}" >&2 + cat "${upload_log}" >&2 + exit 1 + fi +done + +echo "package_release_assets.sh uploads the chart package and checksum alongside manifest assets" diff --git a/deploy/helm/nvca-operator/tests/release_asset_permissions_test.sh b/deploy/helm/nvca-operator/tests/release_asset_permissions_test.sh new file mode 100644 index 000000000..063f2a7d2 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/release_asset_permissions_test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +artifact_dir="${tmp_dir}/sbom-inputs" +mkdir -p "${artifact_dir}/nested" + +umask 077 +printf 'image\n' > "${artifact_dir}/artifacts-test.txt" +printf 'kind: List\n' > "${artifact_dir}/nested/rendered.yaml" +umask 022 + +"${repo_root}/scripts/normalize_release_artifact_permissions.sh" "${artifact_dir}" + +mode_of() { + if stat -c '%a' "$1" >/dev/null 2>&1; then + stat -c '%a' "$1" + else + stat -f '%Lp' "$1" + fi +} + +file_mode="$(mode_of "${artifact_dir}/artifacts-test.txt")" +dir_mode="$(mode_of "${artifact_dir}")" +nested_dir_mode="$(mode_of "${artifact_dir}/nested")" + +if [[ "${file_mode}" != "644" ]]; then + echo "expected artifact file mode 644, got ${file_mode}" >&2 + exit 1 +fi + +if [[ "${dir_mode}" != "755" ]]; then + echo "expected artifact directory mode 755, got ${dir_mode}" >&2 + exit 1 +fi + +if [[ "${nested_dir_mode}" != "755" ]]; then + echo "expected nested artifact directory mode 755, got ${nested_dir_mode}" >&2 + exit 1 +fi + +echo "release artifact permissions are normalized for cross-job access" diff --git a/deploy/helm/nvca-operator/tests/release_image_manifest_test.sh b/deploy/helm/nvca-operator/tests/release_image_manifest_test.sh new file mode 100755 index 000000000..e3e845e49 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/release_image_manifest_test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +rendered_manifest="${tmp_dir}/rendered.yaml" +supplemental_manifest="${tmp_dir}/supplemental.yaml" +image_manifest="${tmp_dir}/artifacts.txt" + +cat > "${rendered_manifest}" <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nvca-operator +spec: + template: + spec: + containers: + - name: operator + image: nvcr.io/nvidia/nvcf-byoc/nvca-operator:2.52.0-rc.5 + env: + - name: NVCA_IMAGE_REPO + value: "nvcr.io/nvidia/nvcf-byoc/nvca" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcfbackend-self-managed +data: + cluster-dto.yaml: | + nvcaVersion: "2.52.0-rc.5" + imageCredentialHelper: + imageConfig: + repository: "nvcr.io/nvidia/nvcf-byoc/nvcf-image-credential-helper" + tag: "0.5.0" + sharedStorage: + imageRepository: "nvcr.io/nvidia/nvcf-byoc/samba" + imageTag: "1.0.5" + otelCollector: + enabled: false + imageConfig: + repository: "nvcr.io/nvidia/nvcf-byoc/nvcf-otel-collector" + tag: "0.143.2" + placeholder: + imageRepository: "" + imageTag: "ignored" + duplicate: + imageConfig: + repository: "nvcr.io/nvidia/nvcf-byoc/nvcf-image-credential-helper" + tag: "0.5.0" +EOF + +"${repo_root}/scripts/render_release_supplemental_images.sh" \ + "${rendered_manifest}" \ + "${supplemental_manifest}" + +"${repo_root}/scripts/build_release_image_manifest.sh" \ + "${rendered_manifest}" \ + "${supplemental_manifest}" \ + "${image_manifest}" + +expected_images=( + "nvcr.io/nvidia/nvcf-byoc/nvca-operator:2.52.0-rc.5" + "nvcr.io/nvidia/nvcf-byoc/nvca:2.52.0-rc.5" + "nvcr.io/nvidia/nvcf-byoc/nvcf-image-credential-helper:0.5.0" + "nvcr.io/nvidia/nvcf-byoc/nvcf-otel-collector:0.143.2" + "nvcr.io/nvidia/nvcf-byoc/samba:1.0.5" +) + +for image in "${expected_images[@]}"; do + if ! grep -Fxq "${image}" "${image_manifest}"; then + echo "expected image manifest to include ${image}" >&2 + exit 1 + fi +done + +line_count="$(wc -l < "${image_manifest}" | tr -d '[:space:]')" +if [[ "${line_count}" != "5" ]]; then + echo "expected exactly 5 unique images, got ${line_count}" >&2 + cat "${image_manifest}" >&2 + exit 1 +fi + +if ! grep -Fq "nvca-release-supplemental-images" "${supplemental_manifest}"; then + echo "expected supplemental manifest to contain the synthetic Pod" >&2 + exit 1 +fi + +echo "validated image manifest:" +cat "${image_manifest}" + +echo "release image manifest generation captures direct and split image references" diff --git a/deploy/helm/nvca-operator/tests/render_values_from_stack_env_test.sh b/deploy/helm/nvca-operator/tests/render_values_from_stack_env_test.sh new file mode 100644 index 000000000..a277ed9f8 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/render_values_from_stack_env_test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +stack_env_file="${tmp_dir}/stack-env.yaml" +output_file="${tmp_dir}/rendered-values.yaml" +stub_bin_dir="${tmp_dir}/bin" +mkdir -p "${stub_bin_dir}" + +cat > "${stack_env_file}" <<'EOF' +global: + image: + registry: nvcr.io + repository: +EOF + +cat > "${stub_bin_dir}/helm" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$1" == "status" ]]; then + exit 0 +fi + +if [[ "$1" == "get" && "$2" == "values" ]]; then + cat <<'YAML' +image: + tag: 2.52.0-rc.1 +selfManaged: + nvcaVersion: 2.52.0-rc.1 +YAML + exit 0 +fi + +echo "unexpected helm invocation: $*" >&2 +exit 1 +EOF +chmod +x "${stub_bin_dir}/helm" + +PATH="${stub_bin_dir}:$PATH" \ +STACK_ENV_FILE="${stack_env_file}" \ +OUTPUT_FILE="${output_file}" \ +RELEASE="nvca-operator" \ +NAMESPACE="nvca-operator" \ +NVCA_OPERATOR_VERSION="2.52.0-rc.5" \ +NVCA_VERSION="2.52.0-rc.5" \ +"${repo_root}/scripts/render_values_from_stack_env.sh" + +actual_image_tag="$(yq -r '.image.tag' "${output_file}")" +actual_nvca_version="$(yq -r '.selfManaged.nvcaVersion' "${output_file}")" + +if [[ "${actual_image_tag}" != "2.52.0-rc.5" ]]; then + echo "expected image.tag to stay aligned with vendored chart defaults, got ${actual_image_tag}" >&2 + exit 1 +fi + +if [[ "${actual_nvca_version}" != "2.52.0-rc.5" ]]; then + echo "expected selfManaged.nvcaVersion to stay aligned with vendored chart defaults, got ${actual_nvca_version}" >&2 + exit 1 +fi + +echo "render_values_from_stack_env.sh keeps upgrade version fields aligned with vendored defaults" diff --git a/deploy/helm/nvca-operator/tests/self_managed_nvca_image_reference_test.sh b/deploy/helm/nvca-operator/tests/self_managed_nvca_image_reference_test.sh new file mode 100755 index 000000000..80cf8cfbb --- /dev/null +++ b/deploy/helm/nvca-operator/tests/self_managed_nvca_image_reference_test.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +manifest="${tmp_dir}/manifest.yaml" +manifest_without_nvca_image_override="${tmp_dir}/manifest-without-nvca-image-override.yaml" +manifest_with_transport_tls_bundle="${tmp_dir}/manifest-with-transport-tls-bundle.yaml" +transport_tls_merge_config="${tmp_dir}/transport-tls-merge-config.yaml" + +cat > "${transport_tls_merge_config}" <<'EOF' +workload: + transportTLS: + trustMode: bundle + trustBundleFingerprint: sha256:9a7814909424061a68756ee5c26aa1a1491b8d20a7b813fb24fa7e73b2fa1c93 + trustBundlePem: test-ca-bundle + installerImage: nvcr.io/nvidia/nvcf-byoc/nvca:test +EOF + +helm template nvca-operator "${repo_root}/nvca-operator" \ + --namespace nvca-operator \ + --values "${repo_root}/nvca-operator/values.yaml" \ + --values "${repo_root}/values.release-sbom.yaml" \ + --set-string selfManaged.icmsServiceURL=http://sis.example.invalid:8080 \ + --set-string selfManaged.icmsServiceHostHeaderOverride=sis.gateway.example.invalid \ + --set-string selfManaged.revalServiceURL=http://reval.example.invalid:8080 \ + --set-string selfManaged.revalServiceHostHeaderOverride=reval.gateway.example.invalid \ + --set-string selfManaged.natsURL=nats://nats.example.invalid:4222 \ + --set-string selfManaged.natsHostOverride=nats.gateway.example.invalid \ + > "${manifest}" + +helm template nvca-operator "${repo_root}/nvca-operator" \ + --namespace nvca-operator \ + --values "${repo_root}/nvca-operator/values.yaml" \ + --values "${repo_root}/values.release-sbom.yaml" \ + --set-string nvcaImage.repositoryOverride= \ + --set-string selfManaged.icmsServiceURL=http://sis.example.invalid:8080 \ + --set-string selfManaged.icmsServiceHostHeaderOverride=sis.gateway.example.invalid \ + --set-string selfManaged.revalServiceURL=http://reval.example.invalid:8080 \ + --set-string selfManaged.revalServiceHostHeaderOverride=reval.gateway.example.invalid \ + --set-string selfManaged.natsURL=nats://nats.example.invalid:4222 \ + --set-string selfManaged.natsHostOverride=nats.gateway.example.invalid \ + > "${manifest_without_nvca_image_override}" + +helm template nvca-operator "${repo_root}/nvca-operator" \ + --namespace nvca-operator \ + --values "${repo_root}/nvca-operator/values.yaml" \ + --values "${repo_root}/values.release-sbom.yaml" \ + --set-string selfManaged.icmsServiceURL=http://sis.example.invalid:8080 \ + --set-string selfManaged.revalServiceURL=http://reval.example.invalid:8080 \ + --set-string selfManaged.natsURL=nats://nats.example.invalid:4222 \ + --set-file agentConfig.mergeConfig="${transport_tls_merge_config}" \ + > "${manifest_with_transport_tls_bundle}" + +nvca_image_repository="$(yq -r '.nvcaImage.repositoryOverride' "${repo_root}/values.release-sbom.yaml")" +nvca_version="$(yq -r '.selfManaged.nvcaVersion' "${repo_root}/nvca-operator/values.yaml")" +image_credential_helper_repository="$(yq -r '.selfManaged.imageCredHelper.imageRepository' "${repo_root}/values.release-sbom.yaml")" +image_credential_helper_tag="$(yq -r '.selfManaged.imageCredHelper.imageTag' "${repo_root}/nvca-operator/values.yaml")" +samba_repository="$(yq -r '.selfManaged.sharedStorage.imageRepository' "${repo_root}/values.release-sbom.yaml")" +samba_tag="$(yq -r '.selfManaged.sharedStorage.imageTag' "${repo_root}/nvca-operator/values.yaml")" + +expected_annotations=( + "release-artifact-nvca-image: \"${nvca_image_repository}:${nvca_version}\"" + "release-artifact-nvcf-image-credential-helper-image: \"${image_credential_helper_repository}:${image_credential_helper_tag}\"" + "release-artifact-samba-image: \"${samba_repository}:${samba_tag}\"" +) + +for expected_annotation in "${expected_annotations[@]}"; do + if ! grep -Fq "${expected_annotation}" "${manifest}"; then + echo "expected rendered self-managed manifest to expose ${expected_annotation}" >&2 + exit 1 + fi +done + +if grep -Fq "release-artifact-nvca-image:" "${manifest_without_nvca_image_override}"; then + echo "expected rendered self-managed manifest without nvcaImage.repositoryOverride to omit release-artifact-nvca-image" >&2 + exit 1 +fi + +for expected_annotation in \ + "release-artifact-nvcf-image-credential-helper-image: \"${image_credential_helper_repository}:${image_credential_helper_tag}\"" \ + "release-artifact-samba-image: \"${samba_repository}:${samba_tag}\"" +do + if ! grep -Fq "${expected_annotation}" "${manifest_without_nvca_image_override}"; then + echo "expected rendered self-managed manifest without nvcaImage.repositoryOverride to expose ${expected_annotation}" >&2 + exit 1 + fi +done + +for expected_host in \ + 'icmsServiceHostHeaderOverride: "sis.gateway.example.invalid"' \ + 'helmReValServiceHostHeaderOverride: "reval.gateway.example.invalid"' \ + 'natsHostOverride: "nats.gateway.example.invalid"' +do + if ! grep -Fq "${expected_host}" "${manifest}"; then + echo "expected rendered self-managed manifest to expose ${expected_host}" >&2 + exit 1 + fi +done + +if grep -Fq 'trustBundlePem:' "${manifest}"; then + echo "expected default self-managed manifest not to render trustBundlePem when unset" >&2 + exit 1 +fi + +if grep -Fq ' transportTls:' "${manifest}"; then + echo "expected default self-managed manifest not to render transportTls when only defaults are set" >&2 + exit 1 +fi + +cluster_dto_has_transport_tls="$( + yq -r 'select(.kind == "ConfigMap" and .metadata.name == "nvcfbackend-self-managed") | .data."cluster-dto.yaml" | from_yaml | has("transportTls")' \ + "${manifest_with_transport_tls_bundle}" +)" +if [[ "${cluster_dto_has_transport_tls}" != "false" ]]; then + echo "expected bundle-mode self-managed manifest not to render transportTls in cluster-dto.yaml" >&2 + exit 1 +fi + +agent_config_transport_tls_trust_mode="$( + yq -r 'select(.kind == "ConfigMap" and .metadata.name == "agent-config-merge") | .data."config.yaml" | from_yaml | .workload.transportTLS.trustMode' \ + "${manifest_with_transport_tls_bundle}" +)" +if [[ "${agent_config_transport_tls_trust_mode}" != "bundle" ]]; then + echo "expected bundle-mode self-managed manifest to render workload.transportTLS.trustMode in agent-config-merge" >&2 + exit 1 +fi + +agent_config_transport_tls_installer_image="$( + yq -r 'select(.kind == "ConfigMap" and .metadata.name == "agent-config-merge") | .data."config.yaml" | from_yaml | .workload.transportTLS.installerImage' \ + "${manifest_with_transport_tls_bundle}" +)" +if [[ "${agent_config_transport_tls_installer_image}" != "nvcr.io/nvidia/nvcf-byoc/nvca:test" ]]; then + echo "expected bundle-mode self-managed manifest to render workload.transportTLS.installerImage from agentConfig.mergeConfig" >&2 + exit 1 +fi + +echo "self-managed chart render exposes release-artifact image references and service host overrides" diff --git a/deploy/helm/nvca-operator/values.local.yml b/deploy/helm/nvca-operator/values.local.yml new file mode 100644 index 000000000..d5019f5bf --- /dev/null +++ b/deploy/helm/nvca-operator/values.local.yml @@ -0,0 +1,41 @@ +# Baseline values for standalone self-managed NVCA installation. +# The `render-values-from-stack` and `install-from-stack` targets replace the +# placeholder repositories below with values derived from the control-plane +# stack environment file. + +# Optional for self-managed mode. Leave blank to use the chart defaults. +ncaID: "" +clusterName: "" +clusterID: "" + +image: + repository: "//nvca-operator" + tag: "3.0.4" + +nvcaImage: + repositoryOverride: "//nvca" + +ngcConfig: + clusterSource: self-managed + serviceKey: "not-used" + +selfManaged: + icmsServiceURL: "http://icms.example.invalid:8080" + icmsServiceHostHeaderOverride: "" + revalServiceURL: "http://reval.example.invalid:8080" + revalServiceHostHeaderOverride: "" + natsURL: "nats://nats.example.invalid:4222" + natsHostOverride: "" + nvcaVersion: "3.0.4" + featureGateValues: ["DynamicGPUDiscovery", "SelfHosted"] + imageCredHelper: + imageRepository: "//nvcf-image-credential-helper" + imageTag: "0.5.0" + sharedStorage: + imageRepository: "//samba" + imageTag: "1.0.5" + +generateImagePullSecret: false +imagePullSecrets: [] +nameOverride: "nvca-operator" +fullnameOverride: "nvca-operator" diff --git a/deploy/helm/nvca-operator/values.release-sbom.yaml b/deploy/helm/nvca-operator/values.release-sbom.yaml new file mode 100644 index 000000000..1ea3f3f3a --- /dev/null +++ b/deploy/helm/nvca-operator/values.release-sbom.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: + repository: "nvcr.io/nvidia/nvcf-byoc/nvca-operator" + +nvcaImage: + repositoryOverride: "nvcr.io/nvidia/nvcf-byoc/nvca" + +selfManaged: + imageCredHelper: + imageRepository: "nvcr.io/nvidia/nvcf-byoc/nvcf-image-credential-helper" + sharedStorage: + imageRepository: "nvcr.io/nvidia/nvcf-byoc/samba" diff --git a/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-cert-mount.yaml b/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-cert-mount.yaml new file mode 100644 index 000000000..1243d49bb --- /dev/null +++ b/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-cert-mount.yaml @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if and .Values.kyverno.enabled (not .Values.webhook.enabled) }} +{{- $oc := "{{" }} +{{- $cc := "}}" }} +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-certificates-volume-combined + annotations: + pod-policies.kyverno.io/autogen-controllers: none + policies.kyverno.io/title: Add Certificates as a Volume - Combined Single Rule + policies.kyverno.io/category: NVCFCertBundle + policies.kyverno.io/subject: Pod,Volume + kyverno.io/kyverno-version: 1.6.0 + kyverno.io/kubernetes-version: "1.21" + policies.kyverno.io/minversion: 1.5.0 + policies.kyverno.io/description: NVCF Certificate Bundle - Single Comprehensive Rule +spec: + background: false + + rules: + # -- Add volumes (autogen disabled, mutations applied at Pod level only) -- + - name: base-setup + match: + any: + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/spot-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/workload-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + mutate: + patchStrategicMerge: + spec: + volumes: + - name: tools + emptyDir: + medium: Memory + sizeLimit: 25Mi + - name: extracted + emptyDir: + medium: Memory + sizeLimit: 25Mi + - name: merged-certs + emptyDir: + medium: Memory + sizeLimit: 25Mi + + # -- Single comprehensive rule for all init containers -- + - name: setup-all-init-containers + match: + any: + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/spot-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/workload-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + mutate: + patchStrategicMerge: + spec: + initContainers: + # 1. Toolbox - copies NVCF certificates and provides busybox (runs early) + - name: a-toolbox + image: {{ required "Set .Values.images.certificates" .Values.images.certificates | quote }} + command: + - sh + - -c + - | + # Copy busybox for other containers + cp /bin/busybox /tools/busybox && chmod +x /tools/busybox + + # Simple copy of NVCF certificates (already merged) + cp -r /etc/ssl/certs/* /merged-certs/ + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: { type: RuntimeDefault } + capabilities: { drop: ["ALL"] } + resources: + requests: { cpu: "50m", memory: "128Mi" } + limits: { cpu: "50m", memory: "128Mi" } + volumeMounts: + - name: tools + mountPath: /tools + - name: merged-certs + mountPath: /merged-certs + + # 2. Fast-merge - handles complex certificate merging (if needed) + - name: fast-merge-certs + image: {{ required "Set .Values.images.certificates" .Values.images.certificates | quote }} + command: + - sh + - -c + - | + set -e + echo "[FAST-MERGE] Starting fast certificate merge operation..." + + # Step 1: Check if inference certificates exist + INFERENCE_CERTS_FOUND=false + echo "[FAST-MERGE] Step 1: Checking for inference certificates..." + if [ -d "/extracted/inference" ]; then + CERT_COUNT=$(find /extracted/inference -name "*.crt" -o -name "*.pem" -o -name "*.key" 2>/dev/null | wc -l || echo 0) + if [ "$CERT_COUNT" -gt 0 ]; then + echo "[FAST-MERGE] Found $CERT_COUNT certificate files from inference container" + INFERENCE_CERTS_FOUND=true + fi + fi + + if [ "$INFERENCE_CERTS_FOUND" = "false" ]; then + echo "[FAST-MERGE] No inference certificates found - NVCF certificates already available from toolbox" + else + echo "[FAST-MERGE] Inference certificates found - merging with NVCF certificates..." + + # Inference certificate processing + if [ -d "/extracted/inference" ]; then + echo "[FAST-MERGE] Processing inference certificates..." + + # Copy all inference certificate files (including server.crt/server.key) + find "/extracted/inference" \( -name "*.crt" -o -name "*.pem" -o -name "*.key" -o -name "*.0" \) | while read cert_file; do + if [ -f "$cert_file" ]; then + cert_basename=$(basename "$cert_file") + echo "[FAST-MERGE] Adding inference cert: $cert_basename" + cp "$cert_file" "/merged-certs/$cert_basename" 2>/dev/null || true + + # If it's a CA certificate, also append to bundle (but not server.crt) + if grep -q "BEGIN CERTIFICATE" "$cert_file" 2>/dev/null && [ "$cert_basename" != "ca-certificates.crt" ] && [ "$cert_basename" != "server.crt" ]; then + cat "$cert_file" >> /merged-certs/ca-certificates.crt 2>/dev/null || true + fi + fi + done + echo "[FAST-MERGE] Inference certificates processed" + fi + + # Ensure NVCF certs are in the bundle + echo "[FAST-MERGE] Ensuring NVCF certificates are in bundle..." + find /etc/ssl/certs \( -name "*.crt" -o -name "*.pem" \) ! -name "ca-certificates.crt" | while read cert_path; do + if [ -f "$cert_path" ]; then + cert_name=$(basename "$cert_path") + echo "[FAST-MERGE] Merging NVCF cert: $cert_name" + cat "$cert_path" >> /merged-certs/ca-certificates.crt 2>/dev/null || true + fi + done + fi + + FINAL_SIZE=$(wc -c < /merged-certs/ca-certificates.crt 2>/dev/null || echo 0) + CERT_COUNT=$(grep -c "BEGIN CERTIFICATE" /merged-certs/ca-certificates.crt 2>/dev/null || echo 0) + echo "[FAST-MERGE] *** MERGE COMPLETE *** Size: $FINAL_SIZE bytes, Certs: $CERT_COUNT" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: { type: RuntimeDefault } + capabilities: { drop: ["ALL"] } + resources: + requests: { cpu: "100m", memory: "128Mi" } + limits: { cpu: "100m", memory: "128Mi" } + volumeMounts: + - name: merged-certs + mountPath: /merged-certs + - name: extracted + mountPath: /extracted + + # 3. Mount certificates into consumer init containers (*init* or + # *download* in the name). Add more patterns here for new consumers. + - (name): "*init*|*download*" + env: + - name: REQUESTS_CA_BUNDLE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: SSL_CERT_FILE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: AWS_CA_BUNDLE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: NIM_SDK_USE_NATIVE_TLS + value: "1" + - name: NIM_SDK_DOWNLOAD_MAX_RETRY_COUNT + value: "8" + - name: NIM_SDK_DOWNLOAD_BACKOFF_INTERVAL_MS + value: "500" + volumeMounts: + - name: merged-certs + mountPath: /etc/ssl/certs + readOnly: true + + # -- Extract certificates from customer containers (dynamic rule) -- + # DISABLED: extract-customer-certificates (moved to combined rule to fix ordering) + # This entire rule is commented out to prevent BusyBox dependency issues + # + # -- Extract certificates ONLY from inference container (if it exists) -- + - name: b-extract-inference-certificates + match: + any: + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/spot-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/workload-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + mutate: + foreach: + - list: request.object.spec.containers + preconditions: + all: + - key: "{{$oc}} element.name {{$cc}}" + operator: Equals + value: "inference" + patchesJson6902: |- + - op: add + path: /spec/initContainers/1 + value: + name: "b-extract-inference" + image: "{{$oc}} element.image {{$cc}}" + command: + - /tools/busybox + - sh + - -c + - | + BB=/tools/busybox + DEST="/extracted/inference" + echo "[B-EXTRACT-INFERENCE] Starting certificate extraction from inference container..." + $BB mkdir -p "$DEST" + + FOUND_CERTS=0 + for d in /etc/ssl/certs /etc/pki /etc/certs /app/certs /opt/certs; do + if [ -d "$d" ]; then + echo "[B-EXTRACT-INFERENCE] Checking directory: $d" + COUNT=$($BB find "$d" -maxdepth 2 \( -name "*.crt" -o -name "*.pem" -o -name "*.key" -o -name "*.0" \) 2>/dev/null | $BB wc -l || echo 0) + if [ "$COUNT" -gt 0 ]; then + echo "[B-EXTRACT-INFERENCE] Found $COUNT certificate files in $d" + $BB find "$d" -maxdepth 2 \( -name "*.crt" -o -name "*.pem" -o -name "*.key" -o -name "*.0" \) -exec $BB cp -v {} "$DEST"/ \; 2>/dev/null && FOUND_CERTS=$((FOUND_CERTS + COUNT)) || true + fi + else + echo "[B-EXTRACT-INFERENCE] Directory $d does not exist" + fi + done + + echo "[B-EXTRACT-INFERENCE] Total certificates found: $FOUND_CERTS" + $BB ls -la "$DEST" 2>/dev/null || echo "[B-EXTRACT-INFERENCE] No files in destination" + exit 0 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: { type: RuntimeDefault } + capabilities: { drop: ["ALL"] } + resources: + requests: { cpu: "5m", memory: "128Mi" } + limits: { cpu: "50m", memory: "128Mi" } + volumeMounts: + - name: tools + mountPath: /tools + - name: extracted + mountPath: /extracted + + # - name: extract-customer-certificates-DISABLED + # match: + # any: + # - resources: + # kinds: ["Pod"] + # namespaceSelector: + # matchExpressions: + # - key: "nvca.nvcf.nvidia.io/spot-instance-type" + # operator: In + # values: [pod_spec, helm_chart, miniservice] + # mutate: + # foreach: + # - list: request.object.spec.containers + # preconditions: + # all: + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "ess" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "utils" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "init" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "sbm-server" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "toolbox" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "ess-init" + # - key: "{{$oc}} element.name {{$cc}}" + # operator: NotEquals + # value: "byoo-otel-collector" + # patchStrategicMerge: + # spec: + # initContainers: + # - name: "z-extract-{{$oc}} element.name {{$cc}}" + # image: "{{$oc}} element.image {{$cc}}" + # command: + # - /tools/busybox + # - sh + # - -c + # - | + # CONTAINER="{{$oc}} element.name {{$cc}}" + # DEST="/extracted/customer-$CONTAINER" + # DLIST="/etc/ssl/certs" + # /tools/busybox echo "[Z-EXTRACT-$CONTAINER] Extracting certificates..." + # /tools/busybox mkdir -p "$DEST" + # + # FOUND_CERTS=0 + # for d in $DLIST; do + # if [ -d "$d" ]; then + # COUNT=$(/tools/busybox find "$d" -maxdepth 2 \( -name "*.crt" -o -name "*.pem" -o -name "*.0" \) 2>/dev/null | /tools/busybox wc -l || /tools/busybox echo 0) + # if [ "$COUNT" -gt 0 ]; then + # /tools/busybox echo "[Z-EXTRACT-$CONTAINER] Found $COUNT certificate files in $d" + # /tools/busybox find "$d" -maxdepth 2 \( -name "*.crt" -o -name "*.pem" -o -name "*.0" \) -exec /tools/busybox cp {} "$DEST"/ \; 2>/dev/null && FOUND_CERTS=$((FOUND_CERTS + COUNT)) || true + # fi + # fi + # done + # + # /tools/busybox echo "[Z-EXTRACT-$CONTAINER] Total certificates found: $FOUND_CERTS" + # securityContext: + # allowPrivilegeEscalation: false + # readOnlyRootFilesystem: true + # seccompProfile: { type: RuntimeDefault } + # capabilities: { drop: ["ALL"] } + # resources: + # requests: { cpu: "50m", memory: "32Mi" } + # limits: { cpu: "50m", memory: "32Mi" } + # volumeMounts: + # - name: tools + # mountPath: /tools + # - name: extracted + # mountPath: /extracted + + # -- Mount certificate bundle into all regular containers -- + - name: mount-bundle + match: + any: + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/spot-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + - resources: + kinds: ["Pod"] + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/workload-instance-type" + operator: In + values: [pod_spec, helm_chart, miniservice] + mutate: + patchStrategicMerge: + spec: + containers: + - (name): "*" + env: + - name: REQUESTS_CA_BUNDLE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: SSL_CERT_FILE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: AWS_CA_BUNDLE + value: "/etc/ssl/certs/ca-certificates.crt" + - name: NIM_SDK_USE_NATIVE_TLS + value: "1" + - name: NIM_SDK_DOWNLOAD_MAX_RETRY_COUNT + value: "8" + - name: NIM_SDK_DOWNLOAD_BACKOFF_INTERVAL_MS + value: "500" + volumeMounts: + - name: merged-certs + mountPath: /etc/ssl/certs + readOnly: true +{{- end }} \ No newline at end of file diff --git a/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-dns.yaml b/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-dns.yaml new file mode 100644 index 000000000..0dcefb2cf --- /dev/null +++ b/deploy/helm/nvcf-unbound/templates/kyverno/kyverno-dns.yaml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if and .Values.kyverno.enabled (not .Values.webhook.enabled) }} +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-unbound-dns + annotations: + pod-policies.kyverno.io/autogen-controllers: none + policies.kyverno.io/title: Add Unbound DNS to all NVCF pods/helms + policies.kyverno.io/category: Sample + policies.kyverno.io/subject: Pod,Volume + kyverno.io/kyverno-version: 1.6.0 + kyverno.io/kubernetes-version: "1.21" + policies.kyverno.io/minversion: 1.5.0 + policies.kyverno.io/description: Add Unbound DNS to all NVCF pods/helms +spec: + background: false + rules: + - name: add-unbound-dns + match: + any: + - resources: + kinds: + - Pod + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/spot-instance-type" + operator: In + values: + - pod_spec + - helm_chart + - miniservice + - resources: + kinds: + - Pod + operations: ["CREATE"] + namespaceSelector: + matchExpressions: + - key: "nvca.nvcf.nvidia.io/workload-instance-type" + operator: In + values: + - pod_spec + - helm_chart + - miniservice + preconditions: + # Per-function opt-out: skip the DNS redirect when any container or init + # container sets NVCF_DISABLE_PROXY_CACHE=true, so the pod bypasses the + # proxy cache. The trailing [] flattens the per-container projection to a + # flat list of values; without it AllNotIn compares against sub-arrays and + # never matches. Kept in sync with the webhook cacheOptedOut check. + all: + - key: "true" + operator: AllNotIn + value: "{{ `{{` }} request.object.spec.containers[].env[?name=='NVCF_DISABLE_PROXY_CACHE'].value[] {{ `}}` }}" + - key: "true" + operator: AllNotIn + value: "{{ `{{` }} request.object.spec.initContainers[].env[?name=='NVCF_DISABLE_PROXY_CACHE'].value[] || `[]` {{ `}}` }}" + mutate: + patchStrategicMerge: + spec: + dnsPolicy: None + dnsConfig: + nameservers: + - {{ $.Values.unbound.cluster_ip }} # IP of UnboundDNS + - {{ $.Values.unbound.stub_nameserver }} # IP of coreDNS + searches: + - "{{ `{{` }} request.namespace {{ `}}` }}.svc.cluster.local" + options: + - name: ndots + value: "5" + - name: timeout + value: "1" +{{- end }} diff --git a/deploy/helm/openbao/Makefile b/deploy/helm/openbao/Makefile new file mode 100644 index 000000000..17845d647 --- /dev/null +++ b/deploy/helm/openbao/Makefile @@ -0,0 +1,110 @@ +.PHONY: all deps build install uninstall clean package push-oci bump-version release + +# Directory variables +helm_dir := helm +charts_dir := $(helm_dir)/charts + +# Chart and namespace +release := openbao-server +namespace := vault-system +values := $(helm_dir)/values.yaml + +# Optional additional values files +additional_values ?= + +# OCI Registry details +OCI_REGISTRY_HOST ?= +OCI_REGISTRY_NAMESPACE ?= + + +# Automatically determine chart name and version from Chart.yaml +# IMPORTANT: For this setup, CHART_NAME is expected to include any desired OCI prefix (e.g., "helm-yourchart") +# as defined in helm/Chart.yaml's 'name' field. +CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) +CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) + +# Variable for the new version, to be passed as an argument to make +# Example: make bump-version NEW_VERSION=0.2.0 +NEW_VERSION ?= + +# used for local dev on k3d based clusters +K3D_CLUSTER_NAME ?= ncp-local + +all: install + +deps: + @if ! ls $(charts_dir)/*.tgz 1>/dev/null 2>&1; then \ + helm dependency build $(helm_dir); \ + fi + +build: deps + +install: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + helm install $(release) $(helm_dir) -n $(namespace) --values $(values) $(if $(additional_values),--values $(additional_values),) --rollback-on-failure --create-namespace --wait --wait-for-jobs --timeout 20m + +install-local: build + @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - + @helm install $(release) $(helm_dir) -n $(namespace) --values $(values) --values ./values.local.yaml $(if $(additional_values),--values $(additional_values),) $(if $(atomic),--atomic,) --create-namespace --wait --wait-for-jobs --timeout 20m + +uninstall: + @helm uninstall $(release) -n $(namespace) 2>/dev/null || true + @kubectl delete ns ${namespace} --ignore-not-found + +# Publish Chart +clean: + rm -rf $(charts_dir)/*.tgz + rm -rf ./packaged-charts + +# Ensure .helmignore is set up to exclude dependencies and secrets +ensure-helmignore: + @echo "secrets/" > $(helm_dir)/.helmignore + @echo ".DS_Store" >> $(helm_dir)/.helmignore + @echo "ci/" >> $(helm_dir)/.helmignore + @echo "[ensure-helmignore] .helmignore configured in $(helm_dir)" + +package: build ensure-helmignore + @echo "[package] Packaging Helm chart $(CHART_NAME) version $(CHART_VERSION)..." + @mkdir -p ./packaged-charts + @helm package $(helm_dir) -d ./packaged-charts/ + @echo "[package] Packaged chart to ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz" + +push-oci: + @echo "[push-oci] Pushing chart $(CHART_NAME) version $(CHART_VERSION) to oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE)/$(CHART_NAME):$(CHART_VERSION)" + @echo "[push-oci] Note: You must be logged into oci://$(OCI_REGISTRY_HOST) for the push to succeed." + @if [ ! -f ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz ]; then \ + echo "[push-oci] Error: Packaged chart ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz not found. Run 'make package' first."; \ + exit 1; \ + fi + @helm push ./packaged-charts/$(CHART_NAME)-$(CHART_VERSION).tgz oci://$(OCI_REGISTRY_HOST)/$(OCI_REGISTRY_NAMESPACE) + @echo "[push-oci] Successfully pushed chart to OCI registry." + @echo "[push-oci] Cleaning up temporary package directory..." + @rm -rf ./packaged-charts + @echo "[push-oci] Cleanup complete." + +bump-version: +ifndef NEW_VERSION + $(error NEW_VERSION is not set. Usage: make bump-version NEW_VERSION=x.y.z) +endif + @echo "[bump-version] Current chart version in $(helm_dir)/Chart.yaml (name: $(CHART_NAME)): $(CHART_VERSION)" + @echo "[bump-version] Bumping chart version to: $(NEW_VERSION)" + @yq e '.version = "$(NEW_VERSION)"' -i $(helm_dir)/Chart.yaml + $(eval CHART_VERSION := $(NEW_VERSION)) # Update CHART_VERSION for subsequent messages + @echo "[bump-version] Successfully bumped chart version in $(helm_dir)/Chart.yaml to $(CHART_VERSION)" + @echo "[bump-version] You might want to commit this change." + +release: bump-version package push-oci + @echo "[release] Successfully released chart $(CHART_NAME) version $(CHART_VERSION) to OCI registry." + @echo "[release] You might want to commit this change." + +DOCKER_CONFIG_JSON := ./helm/secrets/.dockerconfigjson +prepare-local: deps + @echo "[prepare-local] Preparing local environment" + @if [ -f ${DOCKER_CONFIG_JSON} ]; then \ + echo "[prepare-local] ${DOCKER_CONFIG_JSON} file exists, skipping creation"; \ + else \ + echo "[prepare-local] ${DOCKER_CONFIG_JSON} does not exist, creating it. please update the file with your auth token for nvcr.io"; \ + echo '{"auths":{"nvcr.io":{"auth":""}}}' > ${DOCKER_CONFIG_JSON}; \ + fi + @echo "[prepare-local] Successfully prepared local environment" + diff --git a/deploy/helm/openbao/README.md b/deploy/helm/openbao/README.md index e50181d4b..780e925ff 100644 --- a/deploy/helm/openbao/README.md +++ b/deploy/helm/openbao/README.md @@ -39,18 +39,18 @@ openbao: migrations: image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/nvcf-openbao-migrations + repository: /nvcf-openbao-migrations server: image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/nvcf-openbao + repository: /nvcf-openbao injector: image: registry: nvcr.io - repository: 0651155215864979/ncp-dev/oss-vault-k8s + repository: /oss-vault-k8s agentImage: registry: nvcr.io - repository: 0651155215864979/ncp-dev/nvcf-openbao + repository: /nvcf-openbao ``` ## Installation diff --git a/deploy/helm/openbao/README.oidc.md b/deploy/helm/openbao/README.oidc.md new file mode 100644 index 000000000..423f955f4 --- /dev/null +++ b/deploy/helm/openbao/README.oidc.md @@ -0,0 +1,38 @@ +# Dynamic OIDC Authentication Setup + +## 1. Background and Justification + +The OpenBao JWT authentication method requires configuration with the OIDC issuer URL specific to the Kubernetes cluster it runs on. This issuer URL is not constant; it varies significantly between standard distributions (like k3d or KinD) and managed cloud provider services (like Amazon EKS, GKE, or AKS). + +A static configuration that works for a local k3d cluster (`https://kubernetes.default.svc.cluster.local`) would fail in an EKS environment, which uses an external public URL for its OIDC issuer to support features like IAM Roles for Service Accounts (IRSA). A portable and reliable solution must be able to adapt to its environment at runtime. + +To solve this, we implemented a dynamic discovery mechanism that runs during the Helm deployment as part of the `openbao-server-migrations` job. This process queries the cluster's OIDC discovery endpoint, determines the correct issuer for the current environment, and uses it to configure the OpenBao JWT auth method. + +## 2. Implementation Details + +The core of the implementation resides in two utility scripts and the Helm chart templates. + +### `issuer_discovery.sh` + +This script is the primary driver of the discovery process. It is sourced at the beginning of the migrations job and is responsible for: + +1. Securely connecting to the cluster's OIDC discovery endpoint (`/.well-known/openid-configuration`). It uses the pod's automatically mounted Service Account token and CA certificate to authenticate and establish a trusted TLS connection. +2. Parsing the JSON response to extract the `issuer` and `jwks_uri` values. +3. Handling the EKS-specific networking architecture. The EKS discovery endpoint returns a `jwks_uri` with an internal, unreachable VPC hostname. The script detects an EKS issuer and reconstructs the correct, publicly-accessible JWKS URL by appending `/keys` to the public issuer URL, as per official AWS documentation. +4. Exporting the final `OPENBAO_JWT_ISSUER` and `OPENBAO_JWT_JWKS_URL` as environment variables for use by other migration scripts. + +### `functions.sh` (`configure_auth_jwt`) + +This script contains the `configure_auth_jwt` function, which consumes the environment variables and executes the `bao write` command. A key design decision was to implement two distinct configuration paths to maximize robustness: + +* **Default Issuer Path**: If the discovered issuer is the standard in-cluster one (`https://kubernetes.default.svc.cluster.local`), the function configures OpenBao using the `jwt_validation_pubkeys` parameter. It reads the public key directly from a secret that is pre-mounted into the pod by the Helm chart. This approach is more resilient for standard clusters as it removes any dependency on the OpenBao server pod having network access to the JWKS endpoint. +* **Custom Issuer Path**: For any other issuer (e.g., EKS, GKE), the function configures OpenBao using the discovered `jwks_url`. This is the correct approach for public OIDC providers as it delegates the responsibility of fetching, caching, and rotating the signing keys to the OpenBao server, ensuring long-term operational stability. + +### Helm Chart (`values.yaml` and Templates) + +The feature is controlled via the `openbao.migrations.issuerDiscovery` section in `values.yaml`. + +* `enabled`: A boolean to enable or disable the discovery feature. If `false`, the system gracefully falls back to the robust "Default Issuer Path". +* `urlOverride`: Allows operators to point the discovery script at a non-standard or private OIDC provider. +* `caBundleSecretName`: Provides a mechanism for the discovery script to trust private CAs, which is a critical feature for enterprise environments that use TLS-intercepting proxies or internal identity providers. +* `insecure`: A flag to disable TLS verification for development and testing purposes only. diff --git a/deploy/helm/openbao/cleanup.sh b/deploy/helm/openbao/cleanup.sh new file mode 100755 index 000000000..caa2f1f15 --- /dev/null +++ b/deploy/helm/openbao/cleanup.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/utils" + exit 1 +fi +source "$SCRIPT_DIR/utils/utils.sh" + +if ! check_kubernetes; then + log_error "kubernetes check failed" + exit 1 +fi + +if ! check_helm; then + log_error "helm check failed" + exit 1 +fi + +cleanup_cluster() { + local namespace=$1 + local statefulset=$2 + + log_section "Starting OpenBao cleanup" + + # Delete the Helm release + log_info "Uninstalling OpenBao Helm release in namespace: $namespace" + if ! helm uninstall $statefulset -n $namespace --ignore-not-found > /dev/null 2>&1; then + log_error "Failed to uninstall Helm release (exit code: $?): $?" + exit 1 + fi + log_success "Cleaned up OpenBao Helm release in namespace: $namespace" + + # Delete PVCs used by OpenBao + log_info "Deleting OpenBao PVCs in namespace: $namespace" + if ! kubectl delete pvc -l app.kubernetes.io/name=openbao -n $namespace > /dev/null 2>&1; then + log_error "Failed to delete PVCs (exit code: $?): $?" + exit 1 + fi + log_success "Cleaned up OpenBao PVCs in namespace: $namespace" + + # Delete the unseal key and root token secret + log_info "Deleting secrets in namespace: $namespace" + if ! kubectl delete secret $statefulset-unseal -n $namespace --ignore-not-found=true > /dev/null 2>&1; then + log_error "Failed to delete unseal secret (exit code: $?): $?" + exit 1 + fi + if ! kubectl delete secret $statefulset-root-token -n $namespace --ignore-not-found=true > /dev/null 2>&1; then + log_error "Failed to delete root token secret (exit code: $?): $?" + exit 1 + fi + if ! kubectl delete secret cluster-jwt -n $namespace --ignore-not-found=true > /dev/null 2>&1; then + log_error "Failed to delete cluster-jwt secret (exit code: $?): $?" + exit 1 + fi + log_success "Cleaned up secrets in namespace: $namespace" + + # Delete test-nginx namespace if available + log_info "Deleting k8s test-nginx namespace (if present)" + if ! kubectl delete namespace test-nginx --ignore-not-found=true > /dev/null 2>&1; then + log_error "Failed to delete namespace 'test-nginx' (exit code: $?): $?" + exit 1 + fi + log_success "Namespace 'test-nginx' deleted successfully (or it didn't exist)." +} + + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! cleanup_cluster "$namespace" "$statefulset"; then + echo "Failed to cleanup cluster" + exit 1 +fi + diff --git a/deploy/helm/openbao/deploy.sh b/deploy/helm/openbao/deploy.sh new file mode 100755 index 000000000..328884017 --- /dev/null +++ b/deploy/helm/openbao/deploy.sh @@ -0,0 +1,375 @@ +#!/bin/bash + +# Usage +namespace="${1:-vault-system}" +statefulset="${2:-openbao-server}" + +# Valid install methods +readonly INSTALL_METHOD_SCRIPT="script" +readonly INSTALL_METHOD_HELM="helm" + +# Default to local if not specified +install_method="${3:-$INSTALL_METHOD_SCRIPT}" + +# Validate install method +case "$install_method" in +"$INSTALL_METHOD_SCRIPT" | "$INSTALL_METHOD_HELM") ;; +*) + log_error "Invalid install method: $install_method. Must be one of: $INSTALL_METHOD_SCRIPT, $INSTALL_METHOD_HELM" + exit 1 + ;; +esac + +DEPLOY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ ! -f "${DEPLOY_SCRIPT_DIR}/utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${DEPLOY_SCRIPT_DIR}/utils" + exit 1 +fi + +source "$DEPLOY_SCRIPT_DIR/utils/utils.sh" + +if ! check_kubernetes; then + log_error "kubernetes check failed" + exit 1 +fi + +if ! check_helm; then + log_error "helm check failed" + exit 1 +fi + +if [ "${install_method}" = "$INSTALL_METHOD_HELM" ]; then + log_info "Checking jwker availability for helm deployment..." + jwker_path="$(command -v jwker)" + if [ -z "$jwker_path" ]; then + log_error "jwker is not available in the init image. Use an nvcf-openbao-migrations image that includes jwker." + exit 1 + fi + log_success "jwker available at $jwker_path" +fi + +# Helper function to get root token +get_root_token() { + local namespace=$1 + local statefulset=$2 + kubectl get secret ${statefulset}-root-token -n ${namespace} -o jsonpath='{.data.root_token}' | base64 -d +} + +# Step 0: Pre-checks +pre_checks() { + local namespace=$1 + local statefulset=$2 + + log_section "Pre-checks" + + # Check if pod exists + if ! kubectl get pod ${statefulset}-0 -n ${namespace} >/dev/null 2>&1; then + log_info "Pod '${statefulset}-0' not found. Checking for existing PVCs..." + # If pod doesn't exist, check for leftover PVCs + # Capture the output and check exit code + local PVC_OUTPUT=$(kubectl get pvc -l app.kubernetes.io/name=openbao -n ${namespace} 2>/dev/null) + if [ $? -ne 0 ]; then + log_error "Found existing PVCs but no pods. Please run ./cleanup.sh script first." + return 1 + fi + fi + + # If pod exists, check initialization status + local init_status=$(kubectl exec ${statefulset}-0 -n ${namespace} -- \ + bao status -format=json | jq -r '.initialized') + if [ "$init_status" = "true" ]; then + log_error "OpenBao is already initialized. Please run ./cleanup.sh script first." + return 1 + fi +} + +# Step 1: Create empty secret +create_empty_secret() { + local namespace=$1 + local statefulset=$2 + + log_section "Setting up kubernetes secrets in namespace '${namespace}' and statefulset '${statefulset}'" + log_info "Creating empty unseal secret..." + kubectl create secret generic ${statefulset}-unseal \ + --from-literal=unseal_key="" \ + -n ${namespace} +} + +# Step 2: Install OpenBao with Helm +install_openbao() { + local namespace=$1 + local statefulset=$2 + + log_section "Setting up OpenBao nodes" + log_info "Installing OpenBao via Helm..." + helm repo add openbao https://openbao.github.io/openbao-helm + helm repo update openbao + helm install -n ${namespace} ${statefulset} openbao/openbao --values helm/values.yaml --debug --set='global.imagePullSecrets[0].name=nvcr-secret' +} + +# Step 3: Initialize the cluster +initialize_cluster() { + local namespace=$1 + local statefulset=$2 + + log_section "Initializing OpenBao nodes" + + # Wait for pods to be ready with 2-minute timeout + log_info "Waiting for OpenBao pods to be ready (timeout: 2 minutes)..." + local end=$((SECONDS + 120)) # 120 seconds = 2 minutes + + for i in {0..2}; do + while [[ $(kubectl get pod $statefulset-${i} -n ${namespace} -o jsonpath='{.status.containerStatuses[0].ready}') != "true" ]]; do + if [ $SECONDS -gt $end ]; then + log_error "Timeout waiting for pod ${statefulset}-${i} to be ready" + return 1 + fi + log_info "Waiting for pod ${statefulset}-${i} to be ready..." + sleep 5 + done + done + log_info "All OpenBao pods are ready" + + log_info "Initializing OpenBao cluster" + local init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + bao operator init \ + -key-shares=1 \ + -key-threshold=1 \ + -format=json) + + # Extract keys + local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]') + local root_token=$(echo ${init_output} | jq -r '.root_token') + + # Check if unseal key is empty + if [ -z "${unseal_key}" ]; then + log_error "Failed to get unseal key from initialization output" + return 1 + fi + + # Update the secret with the new unseal key + kubectl patch secret ${statefulset}-unseal \ + --patch "data: + unseal_key: $(echo -n "${unseal_key}" | base64)" \ + -n ${namespace} + + log_info "Updated Kubernetes secret '${statefulset}-unseal' with unseal key" + + # Store root token in a new secret + log_info "Creating secret '${statefulset}-root-token' with root token..." + kubectl create secret generic ${statefulset}-root-token \ + -n ${namespace} \ + --from-literal=root_token=${root_token} +} + +get_unseal_key() { + local namespace=$1 + local statefulset=$2 + kubectl get secret ${statefulset}-unseal -n ${namespace} -o jsonpath='{.data.unseal_key}' | base64 -d +} + +# Step 4: Unseal the cluster +unseal_cluster() { + local namespace=$1 + local statefulset=$2 + local unseal_key=$(get_unseal_key "${namespace}" "${statefulset}") + + log_section "Unsealing OpenBao cluster" + + # First unseal the primary node (pod 0) + log_info "Unsealing primary pod ${statefulset}-0" + if ! kubectl exec ${statefulset}-0 -n ${namespace} -- \ + bao operator unseal ${unseal_key}; then + log_error "Failed to unseal primary pod ${statefulset}-0" + return 1 + fi + + # Wait a moment for the primary to be ready + sleep 5 + + # Join and unseal remaining pods + for i in {1..2}; do + log_info "Joining pod ${statefulset}-${i} to Raft cluster" + if ! kubectl exec ${statefulset}-${i} -c openbao -n ${namespace} -- \ + bao operator raft join http://${statefulset}-0.${statefulset}-internal:8200; then + log_error "Failed to join pod ${statefulset}-${i} to Raft cluster" + return 1 + fi + + log_info "Unsealing pod ${statefulset}-${i}" + if ! kubectl exec ${statefulset}-${i} -c openbao -n ${namespace} -- \ + bao operator unseal ${unseal_key}; then + log_error "Failed to unseal pod ${statefulset}-${i}" + return 1 + fi + done + log_info "All pods joined Raft cluster and unsealed successfully" +} + +get_and_save_jwt_signing_key() { + local namespace=$1 + local statefulset=$2 + local jwt_pem_secret_name="cluster-jwt" + + log_section "Getting and saving Kubernetes JWT Signing key" + + log_info "Fetching JWT signing key from Kubernetes API" + + # Get the kubernetes service host ip from primary pod + local kub_api_ip=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + printenv KUBERNETES_SERVICE_HOST) + + # get svc account token to access kubernetes api + local svc_token=$(kubectl exec openbao-server-0 -c openbao -n ${namespace} -- \ + cat /var/run/secrets/kubernetes.io/serviceaccount/token) + + # call kubernetes api to get the jwt signing key and decode it to a pem + local jwt_pem=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + curl -s --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer ${svc_token}" \ + "https://${kub_api_ip}/openid/v1/jwks" | jq ".keys[0]" > /tmp/cluster.jwks && jwker /tmp/cluster.jwks | base64) + + # Check if jwt_pem is empty + if [ -z "${jwt_pem}" ]; then + log_error "Failed to get JWT signing key from Kubernetes API, or there was a problem with the jwker tool" + return 1 + fi + + # write to file to avoid string errors when creating secret + echo "${jwt_pem}" > /tmp/jwt.pem + trap 'rm -f /tmp/jwt.pem /tmp/cluster.jwks' EXIT + + log_info "Creating/Updating JWT signing key in kubernetes secrets" + + # Check if secret exists first + if ! kubectl get secret "${jwt_pem_secret_name}" -n ${namespace} >/dev/null 2>&1; then + log_info "Saving JWT signing key to secret '${jwt_pem_secret_name}'" + if ! kubectl create secret generic ${jwt_pem_secret_name} \ + -n ${namespace} \ + --from-file=pem=/tmp/jwt.pem; then + log_error "Failed to create secret '${jwt_pem_secret_name}'" + return 1 + fi + log_success "JWT signing key saved to secret '${jwt_pem_secret_name}'" + return 0 + else + log_info "Kubernetes JWT Signing key already exists in secret '${jwt_pem_secret_name}', patching..." + if ! kubectl patch secret "${jwt_pem_secret_name}" \ + -n ${namespace} \ + --patch "{\"data\":{\"pem\":\"${jwt_pem}\"}}"; then + log_error "Failed to patch secret at '${jwt_pem_secret_name}'" + return 1 + fi + log_success "JWT signing key updated at '${jwt_pem_secret_name}'" + return 0 + fi +} + +register_and_enable_jwt_plugin() { + local namespace=$1 + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + local plugin_dir="/openbao/plugins" + local plugin_name="vault-plugin-secrets-jwt" + local pod_name="${statefulset}-0" + + log_section "Enabling JWT Secret Engine" + + if kubectl exec -n $namespace $pod_name -c openbao -- \ + env BAO_TOKEN="$root_token" \ + bao plugin list | grep "$plugin_name"; then + log_success "Plugin '$plugin_name' already registered" + return 0 + fi + + # Step 1: Verify Plugin Binary Exists + log_step "Verifying if plugin binary exists at $plugin_dir/$plugin_name..." + if ! kubectl exec -n $namespace $pod_name -c openbao -- test -f "$plugin_dir/$plugin_name"; then + log_error "Plugin binary not found at $plugin_dir/$plugin_name. Check your container image tag." + return 1 + fi + + # Step 2: Calculate SHA256 Checksum of the Plugin Binary + log_step "Calculating SHA256 checksum for $plugin_name..." + local plugin_sha256=$(kubectl exec -n $namespace $pod_name -c openbao -- sha256sum "$plugin_dir/$plugin_name" | awk '{print $1}') + if [[ -z "$plugin_sha256" ]]; then + log_error "Failed to calculate SHA256 checksum for $plugin_name." + return 1 + fi + log_success "JWT Plugin SHA256 checksum: $plugin_sha256" + + # Step 3: Register the Plugin + log_step "Registering plugin '$plugin_name'..." + kubectl exec -n $namespace $pod_name -c openbao -- \ + env BAO_TOKEN="$root_token" \ + bao plugin register \ + -sha256="$plugin_sha256" \ + -command="$plugin_name" \ + secret "$plugin_name" + if [[ $? -ne 0 ]]; then + log_error "Failed to register plugin '$plugin_name'." + return 1 + fi + log_success "Plugin '$plugin_name' registered successfully." +} + +log_section "Deploying OpenBao cluster in namespace '${namespace}' and statefulset '${statefulset}' using ${install_method} method" + +if [ "${install_method}" = "script" ]; then + if ! pre_checks ${namespace} ${statefulset}; then + log_error "Failed pre-checks" + exit 1 + else + log_success "pre-checks completed" + fi +else + log_info "Skipping pre-checks as install_method is not 'script'" +fi + +if [ "${install_method}" = "script" ]; then + if ! create_empty_secret ${namespace} ${statefulset}; then + exit 1 + else + log_success "created empty unseal secret on k8s" + fi +else + log_info "Skipping local installation as install_method is not 'script'" +fi + +if [ "${install_method}" = "script" ]; then + if ! install_openbao ${namespace} ${statefulset}; then + exit 1 + else + log_success "Successfully deployed the cluster" + fi +else + log_info "Skipping local installation as install_method is not 'script'" +fi + +if ! initialize_cluster ${namespace} ${statefulset}; then + exit 1 +else + log_success "Successfully initialized the cluster" +fi + +if ! unseal_cluster ${namespace} ${statefulset}; then + exit 1 +else + log_success "Successfully unsealed the cluster" +fi + +if ! get_and_save_jwt_signing_key ${namespace} ${statefulset}; then + exit 1 +else + log_success "Successfully saved JWT signing key to kubernetes secrets" +fi + +if ! register_and_enable_jwt_plugin ${namespace} ${statefulset}; then + exit 1 +else + log_success "Successfully enabled JWT plugin" +fi + +log_section "Install Successful" + +log_success "Successfully deployed the OpenBao cluster to ${namespace} namespace" diff --git a/deploy/helm/openbao/dev.md b/deploy/helm/openbao/dev.md new file mode 100644 index 000000000..9f82cb47c --- /dev/null +++ b/deploy/helm/openbao/dev.md @@ -0,0 +1,38 @@ +## Commands + +### Deployment +* `deploy.sh`: + +* `cleanup.sh`: + +### Admin Setup +* `admin/setup.sh`: + +* `admin/setup.sh --cleanup`: + +### Tool +`tool/` directory contains useful wrappers to interact with OpenBao via kubectl + +* `tool/raft-list-peers.sh` + +### Tests +`test/` directory contains various test scripts written in Bash + +### Useful Debug Commands + +``` +# MetaLLB loadbalancer related commands +kubectl logs -n metallb-system -l app=metallb,component=speaker +kubectl logs -n metallb-system -l app=metallb,component=controller +kubectl get ipaddresspool -n metallb-system +kubectl get l2advertisement -n metallb-system +kubectl get svc -n vault-server +kubectl get endpoints openbao-server -n vault-server -o yaml + +# kubernetes secret read +kubectl get secret openbao-server-unseal -n vault-server -o jsonpath='{.data.unseal_key}' | xargs | base64 -d +kubectl get secret openbao-server-root-token -n vault-server -o jsonpath='{.data.root_token}' | xargs | base64 -d + +# force delete pod +kubectl delete pod openbao-server-<> -n vault-server --force --grace-period=0 +``` \ No newline at end of file diff --git a/deploy/helm/openbao/docs/bug-report-notary-signing-key-corruption.md b/deploy/helm/openbao/docs/bug-report-notary-signing-key-corruption.md new file mode 100644 index 000000000..8ab44586f --- /dev/null +++ b/deploy/helm/openbao/docs/bug-report-notary-signing-key-corruption.md @@ -0,0 +1,441 @@ +# Bug Report: Intermittent Notary Signing Key Corruption + +> **Status**: ✅ Root Cause Identified, Fix Applied, **Fix Validated** +> **Severity**: High +> **Affected Component**: `service-migrations/migrations/utils/encryption_setup.sh` +> **Affected Service**: `nvcf-notary` (JWT signing) +> **Date Identified**: December 2024 +> **Date Validated**: December 2024 + +--- + +## 1. Executive Summary + +An intermittent bug in the EC P-256 signing key generation function causes approximately **5.6% of generated keys to be corrupted** (higher than initially estimated). When affected, the notary service produces JWTs with invalid signatures that fail verification by downstream services. + +**Key Finding**: The bug only manifests on **Linux** (production environment) due to differences in how `xxd` handles invalid characters. macOS `xxd` behaves differently, making local testing unreliable without containerization. + +--- + +## 2. Symptoms + +- JWT signature verification fails on jwt.io and in consuming services +- The JWK's `d` (private key) parameter is 42 base64url characters instead of the required 43 +- Occurs randomly on new cluster deployments +- Once a corrupted key is generated, all JWTs signed by that key are invalid + +--- + +## 3. Root Cause Analysis + +### 3.1 The Bug Location + +**File**: `service-migrations/migrations/utils/encryption_setup.sh` +**Function**: `generate_asymmetric_signing_key()` +**Lines**: 112-149 + +### 3.2 The Flawed Code + +```bash +# Line 112-115: Extract private key value +D_VALUE=$(openssl ec -in "$TEMP_KEY" -text -noout 2>/dev/null | \ + awk '/priv:/{flag=1; next} /pub:/{flag=0} flag{print}' | \ + tr -d ' \n:' | \ + sed 's/^00*//') # <-- PROBLEM 1: Strips leading zeros + +# Line 145-146: Pad to 64 hex characters +D_PADDED=$(printf "%064s" "$D_VALUE") # <-- PROBLEM 2: Pads with spaces, not zeros +``` + +### 3.3 Problem 1: Leading Zero Stripping + +The `sed 's/^00*//'` command removes leading zeros from the hex representation of the private key. This is incorrect because: + +- EC private keys are fixed-length 256-bit integers +- When the integer's high bytes are `0x00`, they MUST be preserved +- Stripping them changes the mathematical value of the key + +**Example:** +``` +Original hex: 00a1b2c3d4... (64 chars, starts with 00) +After sed: a1b2c3d4... (62 chars, leading 00 removed) +``` + +### 3.4 Problem 2: Space Padding Instead of Zero Padding + +The `printf "%064s"` format specifier pads strings with **spaces** on the left, not zeros: + +```bash +$ printf "%064s" "abc" + abc +# ^^^ +# 61 spaces followed by "abc" - NOT valid hex! +``` + +When this space-padded string is passed to `hex_to_base64url()`: + +```bash +hex_to_base64url() { + echo -n "$hex_input" | xxd -r -p | base64 | tr '+/' '-_' | tr -d '=' +} +``` + +The `xxd -r -p` command silently ignores the invalid space characters, producing a shorter binary output than expected. + +### 3.5 The Chain of Failure + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ 1. OpenSSL generates EC private key with leading zero byte (1/256 chance) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 2. sed strips "00" prefix: "00abc..." → "abc..." (62 chars) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 3. printf pads with spaces: " abc..." (2 spaces + 62 chars) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 4. xxd ignores spaces, converts only 62 hex chars → 31 bytes │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 5. base64url of 31 bytes = 42 chars (should be 43 for 32 bytes) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 6. JWK stored with corrupted 'd' value │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ 7. All JWTs signed with this key have INVALID signatures │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 4. Mathematical Background + +### 4.1 EC P-256 Key Parameters + +| Parameter | Description | Size | Base64URL Length | +|-----------|-------------|------|------------------| +| `d` | Private key (integer) | 32 bytes (256 bits) | 43 chars | +| `x` | Public key X coordinate | 32 bytes | 43 chars | +| `y` | Public key Y coordinate | 32 bytes | 43 chars | + +### 4.2 Probability of Bug Occurrence + +**Initial Estimate** (based on leading zero bytes): + +| Leading Zero Bytes | Probability | Hex Length | Base64URL Length | +|-------------------|-------------|------------|------------------| +| 0 | 255/256 ≈ 99.61% | 64 chars | 43 chars ✓ | +| 1 | 1/256 ≈ 0.39% | 62 chars* | 42 chars ✗ | +| 2 | 1/65536 ≈ 0.0015% | 60 chars* | 41 chars ✗ | + +*After incorrect zero-stripping + +**Actual Observed Rate: ~5.6%** (see Section 8) + +The observed rate is higher because `sed 's/^00*//'` strips **any** leading zeros in the hex string, not just complete `00` byte pairs. This includes cases where the hex starts with a single `0` character. + +### 4.3 Base64URL Encoding Math + +``` +32 bytes = 256 bits +256 bits ÷ 6 bits/char = 42.67 → 43 base64url characters + +31 bytes = 248 bits +248 bits ÷ 6 bits/char = 41.33 → 42 base64url characters +``` + +--- + +## 5. Affected Code Path + +### 5.1 Call Chain + +``` +05_setup_notary-service.sh + └── generate_asymmetric_signing_key() # encryption_setup.sh:87 + └── D_VALUE extraction # Line 112-115 (BUG) + └── D_PADDED padding # Line 146 (BUG) + └── hex_to_base64url() # Line 149 + └── Returns corrupted 'd' value +``` + +### 5.2 Where the Key is Stored + +``` +OpenBao Path: services/nvcf-notary/kv/keys/signing-key +Fields: + - keys: Base64-encoded JWKS containing the signing key + - kid: Key ID (UUID) + - alg: "ES256" +``` + +### 5.3 Where the Key is Used + +The `nvcf-notary` service: +1. Retrieves the signing key from OpenBao +2. Uses it to sign JWTs for function invocation assertions +3. Downstream services verify these JWTs + +--- + +## 6. Evidence + +### 6.1 Corrupted Key from Cluster + +```json +{ + "kty": "EC", + "d": "qx2F_T5zOu2p6iMsJFGoHSX4VTILl8z_mQbyfuHDqA", // 42 chars ✗ + "crv": "P-256", + "kid": "b640a5a8-d7aa-11f0-871e-e637564d75f3", + "x": "N5mw64h2O96Jw0z0XyCoj4X83zld_dj8dHD3EzEcyX8", // 43 chars ✓ + "y": "Q8nC9HqwEjShMaG73lVNhhpgRJVSoEKws-L6TGxMBO8", // 43 chars ✓ + "alg": "ES256" +} +``` + +### 6.2 Valid Key (After Fix) + +```json +{ + "kty": "EC", + "d": "5xMau19x1ujKI_xdDKKR6g7-MEn7c81Tv7Ea7FlVn9U", // 43 chars ✓ + "crv": "P-256", + "kid": "76a949fb-ce73-45dc-8f76-1c350ff86616", + "x": "gIwtqavcjZ38ievWVpU1glsCqoqyn4AOjD_Rsua0SEE", // 43 chars ✓ + "y": "jx5X9N2P8PtgKudPDaBu8zVHJmNGkP1crLcmUKwr1Nk", // 43 chars ✓ + "alg": "ES256" +} +``` + +--- + +## 7. The Fix + +### 7.1 Applied Fix (Line 146) + +**Before:** +```bash +D_PADDED=$(printf "%064s" "$D_VALUE") +``` + +**After:** +```bash +# Note: printf %s pads with spaces, so we must replace them with zeros +D_PADDED=$(printf "%064s" "$D_VALUE" | tr ' ' '0') +``` + +### 7.2 Why This Works + +The `tr ' ' '0'` converts all space characters to `0` characters, producing proper zero-padding: + +```bash +# Before fix: +printf "%064s" "abc" → " abc" + +# After fix: +printf "%064s" "abc" | tr ' ' '0' → "0000000000000000000000000000000000000000000000000000000000000abc" +``` + +--- + +## 8. Verification - Test Harness Results + +A Go-based test harness was developed to statistically validate the bug and fix. The harness: +1. Calls the bash `generate_asymmetric_signing_key()` function +2. Parses the resulting JWK +3. Signs a JWT using Go's `crypto/ecdsa` +4. Verifies the signature +5. Tracks `d` parameter lengths and pass/fail rates + +### 8.1 Environment Discovery + +**Critical Finding**: The bug only manifests on **Linux**, not macOS. + +``` +=== Testing xxd behavior with spaces (the bug) === +Padded: | abc123| +After xxd round-trip: |abc123| +Result length: 6 (should be 64 if working, less if broken) + +=== Testing with zero padding (the fix) === +Padded: |0000000000000000000000000000000000000000000000000000000000abc123| +After xxd round-trip: |0000000000000000000000000000000000000000000000000000000000abc123| +Result length: 64 +``` + +Linux `xxd -r -p` silently drops space characters (invalid hex). macOS `xxd` handles this differently. + +### 8.2 Test Results - Buggy Code (Linux Container) + +``` +╔══════════════════════════════════════════════════════════════╗ +║ JWT Signing Key Test Harness (Go) ║ +╠══════════════════════════════════════════════════════════════╣ +║ Script: encryption_setup_buggy.sh ║ +║ Version: BUGGY (Linux) ║ +║ Iterations: 1000 ║ +╚══════════════════════════════════════════════════════════════╝ + +═════════════════════════════════════════════════════════════════ + RESULTS +───────────────────────────────────────────────────────────────── + Passed: 944 ( 94.40%) + Failed: 56 ( 5.60%) ← Signature verification failed + Parse Errors: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + 'd' LENGTH DISTRIBUTION +───────────────────────────────────────────────────────────────── + 40 chars: 1 ( 0.10%) ← CORRUPTED (2 leading zero bytes) + 41 chars: 0 ( 0.00%) + 42 chars: 56 ( 5.60%) ← CORRUPTED (1 leading zero byte) + 43 chars: 943 ( 94.30%) ← VALID + +───────────────────────────────────────────────────────────────── + CONCLUSION +───────────────────────────────────────────────────────────────── + Bug Confirmed: YES + Expected Rate: ~0.39% + Observed Rate: 5.60% + Statistically Significant: ✓ +═════════════════════════════════════════════════════════════════ +``` + +### 8.3 Test Results - Fixed Code (Linux Container) + +``` +╔══════════════════════════════════════════════════════════════╗ +║ JWT Signing Key Test Harness (Go) ║ +╠══════════════════════════════════════════════════════════════╣ +║ Script: encryption_setup_fixed.sh ║ +║ Version: FIXED (Linux) ║ +║ Iterations: 1000 ║ +╚══════════════════════════════════════════════════════════════╝ + +═════════════════════════════════════════════════════════════════ + RESULTS +───────────────────────────────────────────────────────────────── + Passed: 1000 (100.00%) + Failed: 0 ( 0.00%) ← Signature verification failed + Parse Errors: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + 'd' LENGTH DISTRIBUTION +───────────────────────────────────────────────────────────────── + 40 chars: 0 ( 0.00%) + 41 chars: 0 ( 0.00%) + 42 chars: 0 ( 0.00%) + 43 chars: 1000 (100.00%) ← VALID + +───────────────────────────────────────────────────────────────── + CONCLUSION +───────────────────────────────────────────────────────────────── + Bug Confirmed: NO + All keys valid: ✓ +═════════════════════════════════════════════════════════════════ +``` + +### 8.4 Summary Comparison + +| Version | Passed | Failed | d=43 (valid) | d<43 (corrupted) | +|---------|--------|--------|--------------|------------------| +| **BUGGY** | 944 (94.4%) | **56 (5.6%)** | 943 | 57 | +| **FIXED** | **1000 (100%)** | 0 (0%) | 1000 | 0 | + +### 8.5 Running the Test Harness + +```bash +# Build the harness for Linux +cd tests/signing-key-harness +GOOS=linux GOARCH=amd64 go build -o signing-key-harness-linux . + +# Run in container (matches production environment) +docker run --rm \ + -v "$(pwd):/harness" \ + --entrypoint /bin/sh \ + openbao/openbao:2.2.2 \ + -c ' + apk add --no-cache openssl uuidgen bash > /dev/null 2>&1 + cd /harness + ./signing-key-harness-linux --script ./testdata/encryption_setup_buggy.sh --version "BUGGY" --n 1000 + ' +``` + +--- + +## 9. Remediation for Affected Clusters + +Clusters that were deployed with the bug and have corrupted notary signing keys will need remediation: + +### Option 1: Re-run Migration (Destructive) + +```bash +# Delete and recreate the signing key +bao kv delete services/nvcf-notary/kv/keys/signing-key + +# Re-run the notary migration +# (will generate a new valid key) +``` + +**Impact**: Existing signed JWTs become invalid immediately. + +### Option 2: Manual Key Replacement + +```bash +# Generate new valid key +NEW_KID=$(uuidgen | tr '[:upper:]' '[:lower:]') +# Use fixed generate_asymmetric_signing_key to create key +# Write to OpenBao with overwrite +``` + +### Option 3: Identify and Remediate + +```bash +# Check if cluster is affected +bao kv get services/nvcf-notary/kv/keys/signing-key + +# Decode the keys field and check 'd' length +# If 'd' is 42 chars, cluster is affected +``` + +--- + +## 10. Prevention + +### 10.1 Code Review Checklist + +- [ ] Cryptographic key generation preserves full key length +- [ ] Hex-to-base64url conversion handles all input lengths +- [ ] Unit tests verify key parameter lengths + +### 10.2 Recommended Additional Fixes + +Consider also reviewing line 115 (`sed 's/^00*//'`) to ensure leading zeros are handled correctly at the source, rather than relying solely on padding to fix truncated values. + +--- + +## 11. Timeline + +| Date | Event | +|------|-------| +| Unknown | Bug introduced in `generate_asymmetric_signing_key()` | +| Dec 2024 | JWT signature verification failure reported | +| Dec 2024 | Root cause identified: `d` value truncation due to space padding | +| Dec 2024 | Fix applied to `encryption_setup.sh` line 146-147 | +| Dec 2024 | Go test harness developed for statistical validation | +| Dec 2024 | Bug confirmed: 5.6% failure rate in Linux container | +| Dec 2024 | Fix validated: 100% pass rate in Linux container | +| Dec 2024 | Environment-specific behavior documented (Linux vs macOS) | + +--- + +## 12. References + +### Standards +- [RFC 7517 - JSON Web Key (JWK)](https://tools.ietf.org/html/rfc7517) +- [RFC 7518 - JSON Web Algorithms (JWA)](https://tools.ietf.org/html/rfc7518) - Section 6.2.2 (EC Private Key) +- [NIST FIPS 186-4](https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf) - P-256 Curve specification + +### Internal Documentation +- Test Harness: `tests/signing-key-harness/` +- Test Harness Design: `docs/signing-key-test-harness-design.md` +- Buggy Script (for testing): `tests/signing-key-harness/testdata/encryption_setup_buggy.sh` +- Fixed Script (for testing): `tests/signing-key-harness/testdata/encryption_setup_fixed.sh` + diff --git a/deploy/helm/openbao/docs/signing-key-test-harness-design.md b/deploy/helm/openbao/docs/signing-key-test-harness-design.md new file mode 100644 index 000000000..6dba9f637 --- /dev/null +++ b/deploy/helm/openbao/docs/signing-key-test-harness-design.md @@ -0,0 +1,922 @@ +# Signing Key Test Harness Design + +> **Purpose**: Statistically prove the signing key corruption bug through brute-force repetition +> **Language**: Go +> **Target**: Run 1000+ iterations, expect ~0.4% failure rate with buggy code, 0% with fix + +--- + +## 1. Overview + +This test harness validates the `generate_asymmetric_signing_key()` function by: + +1. Calling the bash function via subprocess to generate EC P-256 signing keys +2. Parsing the resulting JWK and validating parameter lengths +3. Signing a JWT using Go's `crypto/ecdsa` package +4. Verifying the JWT signature +5. Collecting statistics on pass/fail rates and key parameter distributions + +--- + +## 2. Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ TEST HARNESS (Go) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ keygen │ │ jwt │ │ stats │ │ +│ │ │ │ │ │ │ │ +│ │ - ExecBash │─────▶│ - Sign │─────▶│ - Record │ │ +│ │ - ParseJWK │ │ - Verify │ │ - Summary │ │ +│ │ - Validate │ │ - BuildJWT │ │ - Export │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ │ │ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ main.go │ │ +│ │ │ │ +│ │ for i := 0; i < iterations; i++ { │ │ +│ │ jwk := keygen.Generate() │ │ +│ │ token := jwt.Sign(jwk, payload) │ │ +│ │ valid := jwt.Verify(token, jwk.PublicKey()) │ │ +│ │ stats.Record(i, jwk, valid) │ │ +│ │ } │ │ +│ │ stats.PrintSummary() │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 3. Directory Structure + +``` +tests/ +└── signing-key-harness/ + ├── go.mod # Go module definition + ├── go.sum # Dependency checksums + ├── main.go # Entry point, CLI flags, orchestration + ├── keygen/ + │ └── keygen.go # Bash subprocess, JWK parsing + ├── jwt/ + │ └── jwt.go # JWT signing and verification + ├── stats/ + │ └── stats.go # Results collection and reporting + ├── testdata/ + │ ├── encryption_setup_buggy.sh # Original buggy version + │ └── encryption_setup_fixed.sh # Fixed version (copy from source) + └── README.md # Usage instructions +``` + +--- + +## 4. Implementation Details + +### 4.1 `go.mod` + +```go +module github.com/nvidia/nvcf/signing-key-harness + +go 1.21 + +require ( + // No external dependencies - stdlib only +) +``` + +**Note**: Using only Go standard library for cryptographic operations. + +--- + +### 4.2 `keygen/keygen.go` + +```go +package keygen + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "os" + "os/exec" + "path/filepath" +) + +// JWK represents an EC JSON Web Key +type JWK struct { + Kty string `json:"kty"` + Crv string `json:"crv"` + D string `json:"d"` // Private key (base64url) + X string `json:"x"` // Public key X coordinate (base64url) + Y string `json:"y"` // Public key Y coordinate (base64url) + Kid string `json:"kid"` + Alg string `json:"alg"` + Use string `json:"use"` + Iat int64 `json:"iat"` +} + +// KeyGenResult contains the generated key and metadata +type KeyGenResult struct { + JWK JWK + RawOutput string + DLength int + XLength int + YLength int + ParseError error +} + +// Generator handles key generation via bash subprocess +type Generator struct { + ScriptPath string // Path to encryption_setup.sh +} + +// NewGenerator creates a generator with the specified script +func NewGenerator(scriptPath string) *Generator { + return &Generator{ScriptPath: scriptPath} +} + +// Generate calls bash to generate a signing key and returns the result +func (g *Generator) Generate() (*KeyGenResult, error) { + result := &KeyGenResult{} + + // Build bash command that sources the script and calls the function + // We generate a UUID for the kid using Go to avoid uuidgen portability issues + kid := generateUUID() + + bashScript := fmt.Sprintf(` + source "%s" + generate_asymmetric_signing_key "%s" + `, g.ScriptPath, kid) + + cmd := exec.Command("bash", "-c", bashScript) + cmd.Env = os.Environ() + + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("bash execution failed: %w", err) + } + + result.RawOutput = string(output) + + // Parse the JWK JSON + if err := json.Unmarshal(output, &result.JWK); err != nil { + result.ParseError = err + return result, nil // Return result with parse error for analysis + } + + // Record lengths + result.DLength = len(result.JWK.D) + result.XLength = len(result.JWK.X) + result.YLength = len(result.JWK.Y) + + return result, nil +} + +// ToECDSAPrivateKey converts JWK to *ecdsa.PrivateKey +func (j *JWK) ToECDSAPrivateKey() (*ecdsa.PrivateKey, error) { + // Decode base64url values + dBytes, err := base64URLDecode(j.D) + if err != nil { + return nil, fmt.Errorf("failed to decode d: %w", err) + } + + xBytes, err := base64URLDecode(j.X) + if err != nil { + return nil, fmt.Errorf("failed to decode x: %w", err) + } + + yBytes, err := base64URLDecode(j.Y) + if err != nil { + return nil, fmt.Errorf("failed to decode y: %w", err) + } + + // Construct the private key + privateKey := &ecdsa.PrivateKey{ + PublicKey: ecdsa.PublicKey{ + Curve: elliptic.P256(), + X: new(big.Int).SetBytes(xBytes), + Y: new(big.Int).SetBytes(yBytes), + }, + D: new(big.Int).SetBytes(dBytes), + } + + return privateKey, nil +} + +// ToECDSAPublicKey returns just the public key portion +func (j *JWK) ToECDSAPublicKey() (*ecdsa.PublicKey, error) { + privKey, err := j.ToECDSAPrivateKey() + if err != nil { + return nil, err + } + return &privKey.PublicKey, nil +} + +// base64URLDecode decodes a base64url string (no padding) +func base64URLDecode(s string) ([]byte, error) { + // Add padding if needed + switch len(s) % 4 { + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} + +// generateUUID creates a simple UUID v4 +func generateUUID() string { + // Use crypto/rand for proper randomness + b := make([]byte, 16) + _, _ = rand.Read(b) + + // Set version (4) and variant bits + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} +``` + +--- + +### 4.3 `jwt/jwt.go` + +```go +package jwt + +import ( + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "strings" + "time" +) + +// Header represents a JWT header +type Header struct { + Alg string `json:"alg"` + Typ string `json:"typ"` + Kid string `json:"kid,omitempty"` +} + +// Payload represents a JWT payload +type Payload struct { + Iss string `json:"iss"` + Sub string `json:"sub"` + Aud string `json:"aud"` + Iat int64 `json:"iat"` + Exp int64 `json:"exp"` + Jti string `json:"jti"` +} + +// Sign creates a signed JWT using the given private key +func Sign(privateKey *ecdsa.PrivateKey, kid string) (string, error) { + // Create header + header := Header{ + Alg: "ES256", + Typ: "JWT", + Kid: kid, + } + + // Create payload + now := time.Now().Unix() + payload := Payload{ + Iss: "test-harness", + Sub: "signing-key-test", + Aud: "verification", + Iat: now, + Exp: now + 3600, + Jti: fmt.Sprintf("test-%d", now), + } + + // Encode header and payload + headerJSON, _ := json.Marshal(header) + payloadJSON, _ := json.Marshal(payload) + + headerB64 := base64URLEncode(headerJSON) + payloadB64 := base64URLEncode(payloadJSON) + + // Create signing input + signingInput := headerB64 + "." + payloadB64 + + // Sign with ECDSA + hash := sha256.Sum256([]byte(signingInput)) + r, s, err := ecdsa.Sign(rand.Reader, privateKey, hash[:]) + if err != nil { + return "", fmt.Errorf("signing failed: %w", err) + } + + // Convert r, s to fixed-size byte arrays (32 bytes each for P-256) + rBytes := padToLength(r.Bytes(), 32) + sBytes := padToLength(s.Bytes(), 32) + + // Concatenate r || s + signature := append(rBytes, sBytes...) + signatureB64 := base64URLEncode(signature) + + return signingInput + "." + signatureB64, nil +} + +// Verify validates a JWT signature using the given public key +func Verify(token string, publicKey *ecdsa.PublicKey) (bool, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return false, fmt.Errorf("invalid JWT format: expected 3 parts, got %d", len(parts)) + } + + signingInput := parts[0] + "." + parts[1] + signatureB64 := parts[2] + + // Decode signature + signature, err := base64URLDecode(signatureB64) + if err != nil { + return false, fmt.Errorf("failed to decode signature: %w", err) + } + + // For ES256, signature should be 64 bytes (32 + 32) + if len(signature) != 64 { + return false, fmt.Errorf("invalid signature length: expected 64, got %d", len(signature)) + } + + // Extract r and s + r := new(big.Int).SetBytes(signature[:32]) + s := new(big.Int).SetBytes(signature[32:]) + + // Verify + hash := sha256.Sum256([]byte(signingInput)) + valid := ecdsa.Verify(publicKey, hash[:], r, s) + + return valid, nil +} + +// base64URLEncode encodes bytes to base64url without padding +func base64URLEncode(data []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(data), "=") +} + +// base64URLDecode decodes base64url string +func base64URLDecode(s string) ([]byte, error) { + // Add padding if needed + switch len(s) % 4 { + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} + +// padToLength pads a byte slice to the specified length with leading zeros +func padToLength(b []byte, length int) []byte { + if len(b) >= length { + return b[len(b)-length:] + } + padded := make([]byte, length) + copy(padded[length-len(b):], b) + return padded +} +``` + +--- + +### 4.4 `stats/stats.go` + +```go +package stats + +import ( + "fmt" + "strings" + "sync" +) + +// FailedKey records details of a failed key +type FailedKey struct { + Iteration int + Kid string + DLength int + XLength int + YLength int + Error string +} + +// Stats collects test results +type Stats struct { + mu sync.Mutex + + Total int + Passed int + Failed int + ParseError int + + DLengthDist map[int]int // Distribution of 'd' lengths + FailedKeys []FailedKey +} + +// New creates a new Stats collector +func New() *Stats { + return &Stats{ + DLengthDist: make(map[int]int), + FailedKeys: make([]FailedKey, 0), + } +} + +// Record records the result of a single test iteration +func (s *Stats) Record(iteration int, kid string, dLen, xLen, yLen int, valid bool, errMsg string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.Total++ + s.DLengthDist[dLen]++ + + if errMsg != "" { + s.ParseError++ + s.FailedKeys = append(s.FailedKeys, FailedKey{ + Iteration: iteration, + Kid: kid, + DLength: dLen, + XLength: xLen, + YLength: yLen, + Error: errMsg, + }) + } else if valid { + s.Passed++ + } else { + s.Failed++ + s.FailedKeys = append(s.FailedKeys, FailedKey{ + Iteration: iteration, + Kid: kid, + DLength: dLen, + XLength: xLen, + YLength: yLen, + Error: "signature verification failed", + }) + } +} + +// PrintSummary outputs the test results +func (s *Stats) PrintSummary(codeVersion string) { + s.mu.Lock() + defer s.mu.Unlock() + + width := 65 + + fmt.Println() + fmt.Println(strings.Repeat("═", width)) + fmt.Println(" JWT SIGNING KEY TEST HARNESS - RESULTS") + fmt.Println(strings.Repeat("═", width)) + fmt.Println() + + fmt.Printf(" Code Version: %s\n", codeVersion) + fmt.Printf(" Iterations: %d\n", s.Total) + fmt.Println() + + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" RESULTS") + fmt.Println(strings.Repeat("─", width)) + + passRate := float64(s.Passed) / float64(s.Total) * 100 + failRate := float64(s.Failed) / float64(s.Total) * 100 + parseRate := float64(s.ParseError) / float64(s.Total) * 100 + + fmt.Printf(" Passed: %5d (%6.2f%%)\n", s.Passed, passRate) + fmt.Printf(" Failed: %5d (%6.2f%%) ← Signature verification failed\n", s.Failed, failRate) + fmt.Printf(" Parse Errors: %5d (%6.2f%%)\n", s.ParseError, parseRate) + fmt.Println() + + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" 'd' LENGTH DISTRIBUTION") + fmt.Println(strings.Repeat("─", width)) + + // Show lengths from 40 to 44 + for l := 40; l <= 44; l++ { + count := s.DLengthDist[l] + pct := float64(count) / float64(s.Total) * 100 + + indicator := "" + if l < 43 && count > 0 { + indicator = " ← CORRUPTED" + } else if l == 43 { + indicator = " ← VALID" + } + + fmt.Printf(" %d chars: %5d (%6.2f%%)%s\n", l, count, pct, indicator) + } + fmt.Println() + + // Show failed samples (up to 10) + if len(s.FailedKeys) > 0 { + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" FAILED SAMPLES") + fmt.Println(strings.Repeat("─", width)) + + limit := 10 + if len(s.FailedKeys) < limit { + limit = len(s.FailedKeys) + } + + for i := 0; i < limit; i++ { + f := s.FailedKeys[i] + fmt.Printf(" #%-4d d=%d chars kid=%s\n", f.Iteration, f.DLength, f.Kid[:18]+"...") + } + + if len(s.FailedKeys) > limit { + fmt.Printf(" ... and %d more\n", len(s.FailedKeys)-limit) + } + fmt.Println() + } + + // Conclusion + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" CONCLUSION") + fmt.Println(strings.Repeat("─", width)) + + if s.Failed > 0 || s.ParseError > 0 { + expectedRate := 0.39 + observedRate := float64(s.Failed+s.ParseError) / float64(s.Total) * 100 + + fmt.Println(" Bug Confirmed: YES") + fmt.Printf(" Expected Rate: ~%.2f%%\n", expectedRate) + fmt.Printf(" Observed Rate: %.2f%%\n", observedRate) + + // Chi-squared test approximation for significance + if s.Total >= 100 && (s.Failed+s.ParseError) >= 1 { + fmt.Println(" Statistically Significant: ✓") + } + } else { + fmt.Println(" Bug Confirmed: NO") + fmt.Println(" All keys valid: ✓") + } + + fmt.Println() + fmt.Println(strings.Repeat("═", width)) +} +``` + +--- + +### 4.5 `main.go` + +```go +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/nvidia/nvcf/signing-key-harness/keygen" + "github.com/nvidia/nvcf/signing-key-harness/jwt" + "github.com/nvidia/nvcf/signing-key-harness/stats" +) + +func main() { + // CLI flags + iterations := flag.Int("n", 1000, "Number of iterations to run") + scriptPath := flag.String("script", "", "Path to encryption_setup.sh (required)") + codeVersion := flag.String("version", "unknown", "Code version label (e.g., 'buggy' or 'fixed')") + workers := flag.Int("workers", runtime.NumCPU(), "Number of parallel workers") + flag.Parse() + + if *scriptPath == "" { + fmt.Println("Error: --script flag is required") + fmt.Println("Usage: signing-key-harness --script /path/to/encryption_setup.sh") + os.Exit(1) + } + + // Resolve absolute path + absPath, err := filepath.Abs(*scriptPath) + if err != nil { + fmt.Printf("Error resolving script path: %v\n", err) + os.Exit(1) + } + + // Verify script exists + if _, err := os.Stat(absPath); os.IsNotExist(err) { + fmt.Printf("Error: script not found: %s\n", absPath) + os.Exit(1) + } + + // Print header + fmt.Println() + fmt.Println("╔══════════════════════════════════════════════════════════════╗") + fmt.Println("║ JWT Signing Key Test Harness (Go) ║") + fmt.Println("╠══════════════════════════════════════════════════════════════╣") + fmt.Printf("║ Script: %-47s ║\n", filepath.Base(absPath)) + fmt.Printf("║ Version: %-47s ║\n", *codeVersion) + fmt.Printf("║ Iterations: %-47d ║\n", *iterations) + fmt.Printf("║ Workers: %-47d ║\n", *workers) + fmt.Println("╚══════════════════════════════════════════════════════════════╝") + fmt.Println() + + // Create generator and stats + gen := keygen.NewGenerator(absPath) + s := stats.New() + + // Progress tracking + var completed int64 + total := int64(*iterations) + + // Progress display goroutine + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-done: + return + case <-ticker.C: + current := atomic.LoadInt64(&completed) + pct := float64(current) / float64(total) * 100 + bar := progressBar(int(pct), 40) + fmt.Printf("\rRunning... [%s] %d/%d (%.1f%%)", bar, current, total, pct) + } + } + }() + + // Run tests + startTime := time.Now() + + // Use worker pool for parallel execution + var wg sync.WaitGroup + workChan := make(chan int, *iterations) + + // Start workers + for w := 0; w < *workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for iteration := range workChan { + runSingleTest(gen, s, iteration) + atomic.AddInt64(&completed, 1) + } + }() + } + + // Send work + for i := 0; i < *iterations; i++ { + workChan <- i + } + close(workChan) + + // Wait for completion + wg.Wait() + close(done) + + elapsed := time.Since(startTime) + + // Clear progress line + fmt.Printf("\rRunning... [%s] %d/%d (100.0%%) \n", progressBar(100, 40), total, total) + fmt.Printf("\nCompleted in %v (%.1f keys/sec)\n", elapsed.Round(time.Millisecond), float64(*iterations)/elapsed.Seconds()) + + // Print results + s.PrintSummary(*codeVersion) +} + +// runSingleTest executes one iteration of the test +func runSingleTest(gen *keygen.Generator, s *stats.Stats, iteration int) { + result, err := gen.Generate() + if err != nil { + s.Record(iteration, "unknown", 0, 0, 0, false, err.Error()) + return + } + + if result.ParseError != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, result.ParseError.Error()) + return + } + + // Convert to ECDSA key + privateKey, err := result.JWK.ToECDSAPrivateKey() + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + // Sign JWT + token, err := jwt.Sign(privateKey, result.JWK.Kid) + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + // Verify JWT + valid, err := jwt.Verify(token, &privateKey.PublicKey) + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, valid, "") +} + +// progressBar creates an ASCII progress bar +func progressBar(percent, width int) string { + filled := percent * width / 100 + bar := "" + for i := 0; i < width; i++ { + if i < filled { + bar += "█" + } else { + bar += "░" + } + } + return bar +} +``` + +--- + +## 5. Test Data Scripts + +### 5.1 `testdata/encryption_setup_buggy.sh` + +Copy of the original buggy code with the space-padding issue: + +```bash +# ... (copy original encryption_setup.sh here, BEFORE the fix) +# Key line 146 should be: +# D_PADDED=$(printf "%064s" "$D_VALUE") +``` + +### 5.2 `testdata/encryption_setup_fixed.sh` + +Copy of the fixed code: + +```bash +# ... (copy encryption_setup.sh here, AFTER the fix) +# Key line 146-147 should be: +# # Note: printf %s pads with spaces, so we must replace them with zeros +# D_PADDED=$(printf "%064s" "$D_VALUE" | tr ' ' '0') +``` + +--- + +## 6. Usage + +### Build + +```bash +cd tests/signing-key-harness +go build -o signing-key-harness . +``` + +### Run Against Buggy Code + +```bash +./signing-key-harness \ + --script ./testdata/encryption_setup_buggy.sh \ + --version "BUGGY (before fix)" \ + --n 1000 +``` + +**Expected**: ~4 failures (0.4%) + +### Run Against Fixed Code + +```bash +./signing-key-harness \ + --script ./testdata/encryption_setup_fixed.sh \ + --version "FIXED (after fix)" \ + --n 1000 +``` + +**Expected**: 0 failures + +### Run Against Production Code + +```bash +./signing-key-harness \ + --script ../../service-migrations/migrations/utils/encryption_setup.sh \ + --version "PRODUCTION" \ + --n 1000 +``` + +--- + +## 7. Expected Output + +### Buggy Code (Before Fix) + +``` +═════════════════════════════════════════════════════════════════ + JWT SIGNING KEY TEST HARNESS - RESULTS +═════════════════════════════════════════════════════════════════ + + Code Version: BUGGY (before fix) + Iterations: 1000 + +───────────────────────────────────────────────────────────────── + RESULTS +───────────────────────────────────────────────────────────────── + Passed: 996 ( 99.60%) + Failed: 4 ( 0.40%) ← Signature verification failed + Parse Errors: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + 'd' LENGTH DISTRIBUTION +───────────────────────────────────────────────────────────────── + 40 chars: 0 ( 0.00%) + 41 chars: 0 ( 0.00%) + 42 chars: 4 ( 0.40%) ← CORRUPTED + 43 chars: 996 ( 99.60%) ← VALID + 44 chars: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + FAILED SAMPLES +───────────────────────────────────────────────────────────────── + #127 d=42 chars kid=a1b2c3d4-e5f6-... + #456 d=42 chars kid=78901234-abcd-... + #789 d=42 chars kid=efgh5678-ijkl-... + #901 d=42 chars kid=mnop9012-qrst-... + +───────────────────────────────────────────────────────────────── + CONCLUSION +───────────────────────────────────────────────────────────────── + Bug Confirmed: YES + Expected Rate: ~0.39% + Observed Rate: 0.40% + Statistically Significant: ✓ + +═════════════════════════════════════════════════════════════════ +``` + +### Fixed Code (After Fix) + +``` +═════════════════════════════════════════════════════════════════ + JWT SIGNING KEY TEST HARNESS - RESULTS +═════════════════════════════════════════════════════════════════ + + Code Version: FIXED (after fix) + Iterations: 1000 + +───────────────────────────────────────────────────────────────── + RESULTS +───────────────────────────────────────────────────────────────── + Passed: 1000 (100.00%) + Failed: 0 ( 0.00%) ← Signature verification failed + Parse Errors: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + 'd' LENGTH DISTRIBUTION +───────────────────────────────────────────────────────────────── + 40 chars: 0 ( 0.00%) + 41 chars: 0 ( 0.00%) + 42 chars: 0 ( 0.00%) ← CORRUPTED + 43 chars: 1000 (100.00%) ← VALID + 44 chars: 0 ( 0.00%) + +───────────────────────────────────────────────────────────────── + CONCLUSION +───────────────────────────────────────────────────────────────── + Bug Confirmed: NO + All keys valid: ✓ + +═════════════════════════════════════════════════════════════════ +``` + +--- + +## 8. Validation Checklist + +- [ ] Build harness successfully +- [ ] Run 1000 iterations against buggy code +- [ ] Observe ~0.4% failure rate with 'd' length = 42 +- [ ] Run 1000 iterations against fixed code +- [ ] Observe 0% failure rate, all 'd' lengths = 43 +- [ ] Document results in bug report + +--- + +## 9. Dependencies + +| Dependency | Version | Purpose | +|------------|---------|---------| +| Go | 1.21+ | Runtime | +| Bash | 4.0+ | Key generation subprocess | +| OpenSSL | 1.1+ | EC key generation (used by bash script) | + +**No external Go dependencies** — uses only standard library. + diff --git a/deploy/helm/openbao/injector/nginx.yaml b/deploy/helm/openbao/injector/nginx.yaml new file mode 100644 index 000000000..8d6f60512 --- /dev/null +++ b/deploy/helm/openbao/injector/nginx.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + app: nginx + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/agent-init-first: 'true' + vault.hashicorp.com/agent-configmap: 'nginx-vault-config' + vault.hashicorp.com/service: http://openbao-server.vault-system.svc.cluster.local:8200 +spec: + serviceAccountName: nvcf-api + volumes: + - name: token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + audience: http://openbao-server.vault-system.svc.cluster.local:8200 + containers: + - name: nginx + image: nginx:latest + volumeMounts: + - name: token + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-configmap +data: + config-init.hcl: | + "auto_auth" = { + "method" = { + "mount_path" = "auth/jwt" + "config" = { + "role" = "nginx" + "path" = "/run/secrets/kubernetes.io/serviceaccount/token" + "remove_jwt_after_reading" = false + } + "type" = "jwt" + } + "sink" = { + "config" = { + "path" = "/home/vault/.vault-token" + } + "type" = "file" + } + } + + "exit_after_auth" = true + + "template" = { + "source" = "/vault/configs/config.ctmpl" + "destination" = "/vault/secrets/test" + "exec" = { + "command" = "cat /vault/secrets/test" + } + } + + "pid_file" = "/home/vault/.pid" + + config.hcl: | + "auto_auth" = { + "method" = { + "mount_path" = "auth/jwt" + "config" = { + "role" = "nginx" + "path" = "/run/secrets/kubernetes.io/serviceaccount/token" + "remove_jwt_after_reading" = false + } + "type" = "jwt" + } + + "sink" = { + "config" = { + "path" = "/home/vault/.vault-token" + } + "type" = "file" + } + } + + "template" = { + "source" = "/vault/configs/config.ctmpl" + "destination" = "/vault/secrets/test" + "exec" = { + "command" = "cat /vault/secrets/test" + } + } + + "pid_file" = "/home/vault/.pid" + + config.ctmpl: | + {{ with secret "services/nginx/kv/test" }} + { "foo": "{{ .Data.data.foo }}" } + {{ end }} diff --git a/deploy/helm/openbao/test.md b/deploy/helm/openbao/test.md new file mode 100644 index 000000000..18f267c58 --- /dev/null +++ b/deploy/helm/openbao/test.md @@ -0,0 +1,125 @@ +# OpenBao Testing Guide + +## Quick Reference + +| Task | Command | +|------|---------| +| **Fresh Install (baseline)** | `make install additional_values=values.local.yaml` | +| **Fresh Install (new version)** | `make install additional_values=values.local.yaml,upgrade/values-upgrades.yaml` | +| **Uninstall** | `make uninstall` | +| **Setup LoadBalancer** | `./utils/lb.sh` | +| **Cleanup LoadBalancer** | `./utils/lb.sh --cleanup` | +| **HA Test: Seal Status** | `./tests/openbao/loadbalancer_ha_seal_status_check.sh ` | +| **HA Test: KV Read/Write** | `./tests/openbao/loadbalancer_ha_kv_rw_check.sh ` | +| **HA Test: Basic KV** | `./tests/openbao/kv_rw_test.sh ` | +| **Upgrade** | See [upgrade runbook](./upgrade/runbook.md) | + +> **Important**: Set your kubectl context before running install/uninstall commands. +> **Version config**: Baseline in `values.local.yaml`, new version in `upgrade/values-upgrades.yaml` + +--- + +## Prerequisites + +- **Image Pull Secrets**: Configure NGC credentials — see [Image Pull Secrets](README.md#image-pull-secrets---ngcnvcrio) +- **Kubernetes Cluster**: Running cluster (e.g., Colima, k3d, minikube) + ```bash + # Example with Colima (isolated setup for testing) + colima start --profile amd64 --kubernetes + ``` +- **Set kubectl context**: Ensure you're targeting the correct cluster + ```bash + # Check current context + kubectl config current-context + + # List context + kubectl config get-contexts + + # Switch context if needed (example for Colima) + kubectl config use-context colima-amd64 + ``` + +--- + +## 1. Deploy a Fresh Cluster + +### Option A: Deploy with baseline version +```bash +make uninstall # Clean previous deployment +make install additional_values=values.local.yaml # Deploy baseline version +``` + +### Option B: Deploy with new version +```bash +# 1. Build the image locally (from repo root) +unset DOCKER_HOST +docker context use + +# if image not available in nvcr.io, then build it locally +VERSION=2.4.4 +docker build -f upgrade/Dockerfile.upgrade \ + --build-arg OPENBAO_VERSION=${VERSION} \ + -t //nvcf-openbao:${VERSION} . + +# 3. Deploy (ensure correct kubectl context first) +kubectl config use-context colima-amd64 +make uninstall +make install additional_values=values.local.yaml,upgrade/values-upgrades.yaml +``` + +### Verify Deployment +```bash +kubectl get pods -n vault-system +./tools/raft-list-peers.sh +``` + +#### HA Tests + +> **Setup**: Run `./utils/lb.sh` once before running HA tests. LoadBalancer persists across tests. + +### Test 1: Seal Status Check +Continuously verifies seal status through LoadBalancer during pod failures. + +```bash +# Terminal 1: Run test +./tests/openbao/loadbalancer_ha_seal_status_check.sh amd64 + +# Terminal 2: Simulate failures +kubectl delete pod openbao-server-0 -n vault-system --force --grace-period=0 +``` + +**Expected**: Seal status remains `false`, LoadBalancer routes to healthy pods. + +--- + +### Test 2: KV Read/Write/Delete +Continuous KV operations through LoadBalancer to verify data plane HA. + +```bash +# Terminal 1: Run test +./tests/openbao/loadbalancer_ha_kv_rw_check.sh amd64 + +# Terminal 2: Simulate failures +./tools/operator-stepdown.sh +``` + +**Expected**: All operations succeed continuously, no interruption during failures. + +--- + +### Test 3: Basic KV Test +Direct KV operations against individual pods. + +```bash +# Terminal 1: Run test +./tests/openbao/kv_rw_test.sh amd64 + +# Terminal 2: Kill a pod +kubectl delete pod openbao-server-1 -n vault-system --force --grace-period=0 +``` + +**Expected**: Operations continue unaffected. + + +## 2. Upgrade an Existing Cluster +For detailed walkthrough, see **[upgrade runbook](./upgrade/runbook.md)**. diff --git a/deploy/helm/openbao/tests/openbao/kv_rw_test.sh b/deploy/helm/openbao/tests/openbao/kv_rw_test.sh new file mode 100755 index 000000000..b730c9cb7 --- /dev/null +++ b/deploy/helm/openbao/tests/openbao/kv_rw_test.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../../utils" + exit 1 +fi +source "$SCRIPT_DIR/../../utils/utils.sh" + +test_kv() { + local namespace=$1 + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + local secret_path="services/all/kv/test" + log_section "Starting continuous KV test (Ctrl+C to stop)..." + while true; do + log_step "Writing test value - openbao-server-2 (bao kv put $secret_path foo=bar)" + if output=$(kubectl exec $statefulset-2 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao kv put $secret_path foo=bar 2>&1); then + log_success "Successfully wrote to KV2 store: $output" + else + log_error "Failed to write to KV2 store (exit code: $?): $output" + fi + + log_step "Reading test value - openbao-server-2 (bao kv get $secret_path)" + if output=$(kubectl exec $statefulset-2 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao kv get $secret_path 2>&1); then + log_success "Successfully read the KV2 secret: $output" + else + log_error "Failed to read the KV2 secret (exit code: $?): $output" + fi + + log_step "Deleting test value - openbao-server-2 (bao kv delete $secret_path)" + if output=$(kubectl exec $statefulset-2 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao kv delete $secret_path 2>&1); then + log_success "Successfully deleted the KV2 secret: $output" + else + log_error "Failed to delete the KV2 secret (exit code: $?): $output" + fi + sleep 1 + done +} + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_kubernetes; then + exit 1 +fi + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +if ! test_kv "$namespace" "$statefulset"; then + log_error "Failed to test KV store" + exit 1 +fi \ No newline at end of file diff --git a/deploy/helm/openbao/tests/openbao/loadbalancer_ha_kv_rw_check.sh b/deploy/helm/openbao/tests/openbao/loadbalancer_ha_kv_rw_check.sh new file mode 100755 index 000000000..e94eefe5f --- /dev/null +++ b/deploy/helm/openbao/tests/openbao/loadbalancer_ha_kv_rw_check.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../../utils" + exit 1 +fi +source "$SCRIPT_DIR/../../utils/utils.sh" + +# Usage +show_usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " COLIMA_PROFILE Colima profile name (required)" + echo "" + echo "Examples:" + echo " $0 amd64" + echo " $0 default" + exit 1 +} + +# Get Colima profile from argument (required) +if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then + show_usage +elif [[ -n "${1-}" ]]; then + COLIMA_PROFILE="$1" +else + echo "Error: COLIMA_PROFILE argument is required" + echo "" + show_usage +fi + +namespace="vault-system" +statefulset="openbao-server" + +if ! check_kubernetes; then + exit 1 +fi + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +# Check if MetalLB is deployed +if ! kubectl get pods -n metallb-system >/dev/null 2>&1; then + log_error "MetalLB is not deployed. Please deploy MetalLB using 'utils/lb.sh'." + exit 1 +fi + +# Get loadbalancer IP +curr_ip=$(kubectl get svc openbao-server-lb -n vault-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null) +if [[ -z "$curr_ip" ]]; then + log_error "LoadBalancer service 'openbao-server-lb' not found." + echo "" + echo "To set up the LoadBalancer, run:" + echo " ./utils/lb.sh" + exit 1 +fi +log_info "Using LoadBalancer IP: $curr_ip" + +OPENBAO_ADDR="http://${curr_ip}:8200" +echo "Using OpenBao at: $OPENBAO_ADDR" + +TOKEN=$(get_root_token $namespace $statefulset) + +# Verify Colima profile is running +if ! colima -p "$COLIMA_PROFILE" status >/dev/null 2>&1; then + log_error "Colima profile '$COLIMA_PROFILE' is not running. Start it with: colima start -p $COLIMA_PROFILE" + exit 1 +fi +log_info "Using Colima profile: $COLIMA_PROFILE" + +while true; do + TIMESTAMP=$(date +%s) + # KVv2 mount path and secret name + KV_MOUNT="services/all/kv" + SECRET_NAME="test" + RANDOM_VALUE="value-$RANDOM-$TIMESTAMP" + + log_section "$(date) - Starting test cycle #$TIMESTAMP" + + # Write with dynamic value (KVv2 API: /v1/{mount}/data/{path}) + log_info "WRITE: Creating secret at $KV_MOUNT/data/$SECRET_NAME with value: $RANDOM_VALUE" + WRITE_RESULT=$(colima -p "$COLIMA_PROFILE" ssh -- curl -s -w "%{http_code}" \ + -H "X-Vault-Token: $TOKEN" \ + -X POST \ + -d "{\"data\":{\"timestamp\":\"$TIMESTAMP\",\"value\":\"$RANDOM_VALUE\"}}" \ + $OPENBAO_ADDR/v1/$KV_MOUNT/data/$SECRET_NAME) + + # Last 3 chars are the HTTP status code; the rest is body + if [[ -z "$WRITE_RESULT" || ${#WRITE_RESULT} -lt 3 ]]; then + log_error "Write failed: empty or invalid response" + sleep 5 + continue + fi + WRITE_CODE="${WRITE_RESULT: -3}" + WRITE_BODY="${WRITE_RESULT:0:${#WRITE_RESULT}-3}" + if [[ "$WRITE_CODE" == "200" || "$WRITE_CODE" == "204" ]]; then + log_success "Write successful" + else + log_error "Write failed: HTTP $WRITE_CODE, body: $WRITE_BODY" + sleep 5 + continue + fi + + # Read and verify value + log_info "READ: Fetching secret from $KV_MOUNT/data/$SECRET_NAME" + READ_RESULT=$(colima -p "$COLIMA_PROFILE" ssh -- curl -s \ + -H "X-Vault-Token: $TOKEN" \ + $OPENBAO_ADDR/v1/$KV_MOUNT/data/$SECRET_NAME) + + # Extract the value from the JSON response + READ_VALUE=$(echo "$READ_RESULT" | grep -o "\"value\":\"$RANDOM_VALUE\"" || echo "") + + if [[ "$READ_VALUE" == "\"value\":\"$RANDOM_VALUE\"" ]]; then + log_success "Read successful and value verified: $RANDOM_VALUE" + else + log_error "Read failed or value mismatch, Expected: $RANDOM_VALUE, Response: $READ_RESULT" + fi + + # Delete step (KVv2 API: DELETE /v1/{mount}/data/{path}) + log_info "DELETE: Removing secret at $KV_MOUNT/data/$SECRET_NAME" + DELETE_RESULT=$(colima -p "$COLIMA_PROFILE" ssh -- curl -s -w "%{http_code}" \ + -H "X-Vault-Token: $TOKEN" \ + -X DELETE \ + $OPENBAO_ADDR/v1/$KV_MOUNT/data/$SECRET_NAME) + + # Last 3 chars are the HTTP status code; the rest is body + if [[ -z "$DELETE_RESULT" || ${#DELETE_RESULT} -lt 3 ]]; then + log_error "Delete failed: empty or invalid response" + sleep 5 + continue + fi + DELETE_CODE="${DELETE_RESULT: -3}" + DELETE_BODY="${DELETE_RESULT:0:${#DELETE_RESULT}-3}" + if [[ "$DELETE_CODE" == "204" ]]; then + log_success "Delete successful" + else + log_error "Delete failed: HTTP $DELETE_CODE, body: $DELETE_BODY" + fi + + # Verify deletion (KV v2 returns 200 with deletion_time, not 404) + VERIFY_RESULT=$(colima -p "$COLIMA_PROFILE" ssh -- curl -s -w "%{http_code}" \ + -H "X-Vault-Token: $TOKEN" \ + $OPENBAO_ADDR/v1/$KV_MOUNT/data/$SECRET_NAME) + + # Last 3 chars are the HTTP status code; the rest is body + if [[ -z "$VERIFY_RESULT" || ${#VERIFY_RESULT} -lt 3 ]]; then + log_error "Verify failed: empty or invalid response" + sleep 5 + continue + fi + VERIFY_CODE="${VERIFY_RESULT: -3}" + VERIFY_BODY="${VERIFY_RESULT:0:${#VERIFY_RESULT}-3}" + # Check if either 404 (destroyed) or has deletion_time set (soft-deleted in KV v2) + if [[ "$VERIFY_CODE" == "404" ]] || echo "$VERIFY_BODY" | grep -q '"deletion_time"'; then + log_success "Deletion verified" + else + log_error "Deletion verification failed: HTTP $VERIFY_CODE, body: $VERIFY_BODY" + fi + + sleep 5 +done \ No newline at end of file diff --git a/deploy/helm/openbao/tests/openbao/loadbalancer_ha_seal_status_check.sh b/deploy/helm/openbao/tests/openbao/loadbalancer_ha_seal_status_check.sh new file mode 100755 index 000000000..5ee823bd2 --- /dev/null +++ b/deploy/helm/openbao/tests/openbao/loadbalancer_ha_seal_status_check.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +TEST_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${TEST_SCRIPT_DIR}/../../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${TEST_SCRIPT_DIR}/../../utils" + exit 1 +fi +source "${TEST_SCRIPT_DIR}/../../utils/utils.sh" + +# Usage +show_usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " COLIMA_PROFILE Colima profile name (required)" + echo "" + echo "Examples:" + echo " $0 amd64" + echo " $0 default" + exit 1 +} + +# Get Colima profile from argument (required) +if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then + show_usage +elif [[ -n "${1-}" ]]; then + COLIMA_PROFILE="$1" +else + echo "Error: COLIMA_PROFILE argument is required" + echo "" + show_usage +fi + +# Check Colima is installed +if ! check_colima; then + echo "Please install Colima before proceeding." + exit 1 +fi + +# Verify Colima profile is running +if ! colima -p "$COLIMA_PROFILE" status >/dev/null 2>&1; then + echo "Error: Colima profile '$COLIMA_PROFILE' is not running" + echo "Start it with: colima start -p $COLIMA_PROFILE" + exit 1 +fi +echo "Using Colima profile: $COLIMA_PROFILE" + +test_seal_ha_status_via_lb() { + log_section "Starting continuous seal status check via LoadBalancer (Ctrl+C to stop)..." + while true; do + colima -p "$COLIMA_PROFILE" ssh -- curl -s http://${curr_ip}:8200/v1/sys/health | jq .sealed + sleep 1 + done +} + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_kubernetes; then + exit 1 +fi + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +# get loadbalancer IP (after k8s checks) +curr_ip=$(kubectl get svc openbao-server-lb -n vault-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null) +if [[ -z "$curr_ip" ]]; then + echo "Error: LoadBalancer service 'openbao-server-lb' not found" + echo "" + echo "To set up the LoadBalancer, run:" + echo " ./utils/lb.sh" + exit 1 +fi +echo "Using LoadBalancer IP: $curr_ip" + +# Run infinite loop (Ctrl+C to stop) +test_seal_ha_status_via_lb \ No newline at end of file diff --git a/deploy/helm/openbao/tests/services/INJECTOR.md b/deploy/helm/openbao/tests/services/INJECTOR.md new file mode 100644 index 000000000..c835a7604 --- /dev/null +++ b/deploy/helm/openbao/tests/services/INJECTOR.md @@ -0,0 +1,66 @@ +# OpenBao K8s Injector Test + +This guide provides instructions for testing the OpenBao agent injector on a local +Kubernetes cluster using Colima. + +## Overview + +The `nginx.yaml` manifest deploys: +- ServiceAccount `nvcf-api` in namespace `nvcf` +- Pod `nvcf-api` with OpenBao agent injection annotations +- ConfigMap with Vault Agent configuration for JWT auth and secret templates + +## Prerequisites + +- OpenBao cluster running (`make install additional_values=values.local.yaml`) +- Colima profile running (e.g., `amd64`) +- Migrations completed (creates `nvcf-api` JWT role and secrets) + +## Test Injector + +Deploy a test pod with OpenBao annotations to verify secret injection: + +```bash +./tests/services/test.sh +# Example: ./tests/services/test.sh amd64 +``` + +**What it does**: +1. Creates `nvcf` namespace +2. Deploys `nvcf-api` pod with OpenBao annotations +3. Waits for pod to be ready +4. Verifies injector injected the agent sidecar +5. Shows injected agent image version +6. Checks if secrets were rendered + +**Expected**: Pod shows `2/2` containers (nginx + openbao-agent) + +## Re-deploy Pod + +```bash +kubectl delete pod nvcf-api -n nvcf +./tests/services/test.sh +``` + +## Cleanup + +```bash +./tests/services/test.sh --cleanup +``` + +## Troubleshooting + +**Agent logs**: +```bash +kubectl logs nvcf-api -n nvcf -c openbao-agent +``` + +**Pod events**: +```bash +kubectl describe pod nvcf-api -n nvcf +``` + +**Check injected agent version**: +```bash +kubectl get pod nvcf-api -n nvcf -o jsonpath='{.spec.containers[?(@.name=="openbao-agent")].image}' +``` diff --git a/deploy/helm/openbao/tests/services/nginx.yaml b/deploy/helm/openbao/tests/services/nginx.yaml new file mode 100644 index 000000000..cf5f6a1aa --- /dev/null +++ b/deploy/helm/openbao/tests/services/nginx.yaml @@ -0,0 +1,162 @@ +### +# This is a test of known service nvcf-api that uses OpenBao to sign JWT tokens. +# It is used to test the migration of secrets from Vault to OpenBao. +# There is a template example and injection of the vault-agent +# +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nvcf-api + namespace: nvcf + labels: + app: nvcf-api + +--- + +apiVersion: v1 +kind: Pod +metadata: + name: nvcf-api + labels: + app: nvcf-api + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/agent-init-first: 'true' + vault.hashicorp.com/agent-inject-status: 'update' + vault.hashicorp.com/agent-configmap: 'vault-agent-configmap' + vault.hashicorp.com/secret-volume-path: '/vault/secrets' + vault.hashicorp.com/copy-volume-mounts: 'nginx' + vault.hashicorp.com/service: http://openbao-server.vault-system.svc.cluster.local:8200 +spec: + serviceAccountName: nvcf-api + volumes: + - name: token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + audience: http://openbao-server.vault-system.svc.cluster.local:8200 + - name: vault-agent-configmap + configMap: + name: vault-agent-configmap + + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + resources: + limits: + cpu: "0.5" + memory: "512Mi" + requests: + cpu: "250m" + memory: "256Mi" + volumeMounts: + - name: token + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + - name: vault-agent-configmap + mountPath: /vault/configs + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-agent-configmap +data: + secrets.tmpl: | + { + "tokens": { + [[- with secret `services/nvcf-api/jwt/sign/nvcf-api` ]] + "notary": "[[ .Data.token ]]", + [[- end ]] + [[- with secret `services/ess-api/jwt/sign/nvcf-api` ]] + "jwtToken": "[[ .Data.token ]]", + [[- end ]] + } + + "kv": { + [[- with secret `services/nvcf-api/kv/data/cassandra/creds` ]] + "cassandra": { + "username": "[[ .Data.data.username ]]", + "password": "[[ .Data.data.password ]]" + }, + [[- end ]] + [[- with secret `services/all/kv/data/encryption/keys/audit` ]] + "audit": { + "kid": "[[ .Data.data.current_kid ]]", + "keys": "[[ .Data.data.keys ]]" + }, + [[- end ]] + [[- with secret `services/all/kv/data/encryption/keys/stored_data` ]] + [[- $keysObj := .Data.data.keys | base64Decode | parseJSON ]] + "jwks": { + "kid": "[[ .Data.data.current_kid ]]", + "currentKey": "[[ index $keysObj.keys 0 | toJSON | base64Encode ]]", + "allKeys": "[[ $keysObj.keys | toJSON | base64Encode ]]" + } + [[- end ]] + } + } + config.hcl: | + # Comment this out if running as sidecar instead of initContainer + exit_after_auth = false + + pid_file = "/home/vault/pidfile" + + auto_auth { + method "jwt" { + mount_path = "auth/jwt" + config = { + role = "nvcf-api" + path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + remove_jwt_after_reading = false + } + } + + sink "file" { + config = { + path = "/home/vault/.vault-token" + } + } + } + + template { + destination = "/vault/secrets/secrets.json" + source = "/vault/configs/secrets.tmpl" + left_delimiter = "[[" + right_delimiter = "]]" + } + + config-init.hcl: | + # Comment this out if running as sidecar instead of initContainer + exit_after_auth = true + + pid_file = "/home/vault/pidfile" + + auto_auth { + method "jwt" { + mount_path = "auth/jwt" + config = { + role = "nvcf-api" + path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + remove_jwt_after_reading = false + } + } + + sink "file" { + config = { + path = "/home/vault/.vault-token" + } + } + } + + template { + destination = "/vault/secrets/secrets.json" + source = "/vault/configs/secrets.tmpl" + left_delimiter = "[[" + right_delimiter = "]]" + } diff --git a/deploy/helm/openbao/tests/services/test.sh b/deploy/helm/openbao/tests/services/test.sh new file mode 100755 index 000000000..479109f66 --- /dev/null +++ b/deploy/helm/openbao/tests/services/test.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +# Exit on Ctrl+C +trap 'echo ""; exit 130' INT + +# Save paths before sourcing utils.sh (which overwrites SCRIPT_DIR) +TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_ROOT="$(cd "$TEST_DIR/../.." && pwd)" + +if [ ! -f "${REPO_ROOT}/utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${REPO_ROOT}/utils" + exit 1 +fi + +source "$REPO_ROOT/utils/utils.sh" + +# Usage +usage() { + echo "Usage: $0 [--cleanup]" + echo "" + echo "Tests the OpenBao agent injector by deploying the nvcf-api pod with secret injection." + echo "" + echo "Arguments:" + echo " colima_profile Name of the Colima profile (e.g., amd64, arm64)" + echo " --cleanup Remove the test namespace and pod" + echo "" + echo "Examples:" + echo " $0 amd64 # Deploy test pod" + echo " $0 amd64 --cleanup # Clean up test resources" + exit 1 +} + +# Parse arguments +if [[ -z "$1" ]]; then + usage +fi + +COLIMA_PROFILE="$1" +CLEANUP=false + +if [[ "$2" == "--cleanup" ]] || [[ "$1" == "--cleanup" ]]; then + CLEANUP=true + if [[ "$1" == "--cleanup" ]]; then + echo "Error: colima_profile is required even for cleanup" + usage + fi +fi + +namespace="nvcf" +manifest_path="$TEST_DIR/nginx.yaml" + +# Switch to correct kubectl context +log_step "Switching kubectl context to colima-${COLIMA_PROFILE}..." +if ! kubectl config use-context "colima-${COLIMA_PROFILE}" > /dev/null 2>&1; then + log_error "Failed to switch to context colima-${COLIMA_PROFILE}" + log_info "Available contexts:" + kubectl config get-contexts -o name + exit 1 +fi +log_success "Using context: colima-${COLIMA_PROFILE}" + +if ! check_kubernetes; then + exit 1 +fi + +if [[ "$CLEANUP" == "true" ]]; then + log_step "Cleaning up namespace '$namespace'..." + kubectl delete namespace "$namespace" --ignore-not-found --wait > /dev/null 2>&1 + log_success "Namespace '$namespace' cleaned up successfully." + exit 0 +fi + +# Step 1: Create the namespace +log_step "Creating namespace '$namespace'..." +if ! kubectl create namespace "$namespace" > /dev/null 2>&1; then + if kubectl get namespace "$namespace" > /dev/null 2>&1; then + log_success "Namespace '$namespace' already exists." + else + log_error "Failed to create namespace '$namespace'." + exit 1 + fi +else + log_success "Namespace '$namespace' created successfully." +fi + +# Step 2: Apply the manifest to the namespace +log_step "Applying manifest '$manifest_path' to namespace '$namespace'..." +if ! kubectl apply -f "$manifest_path" -n "$namespace"; then + log_error "Failed to apply manifest '$manifest_path' to namespace '$namespace'." + exit 1 +fi +log_success "Manifest applied successfully." + +# Step 3: Wait for pod to be ready +log_step "Waiting for pod to be ready..." +if ! kubectl wait --for=condition=ready pod -l app=nvcf-api -n "$namespace" --timeout=120s; then + log_warn "Pod not ready within timeout. Checking status..." +fi + +# Step 4: Show pod status and injector results +log_step "Pod status:" +kubectl get pod -n "$namespace" -o wide + +log_step "Checking injected agent..." +POD_NAME=$(kubectl get pod -n "$namespace" -l app=nvcf-api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [[ -n "$POD_NAME" ]]; then + echo "" + log_info "Pod containers:" + kubectl get pod "$POD_NAME" -n "$namespace" -o jsonpath='{.spec.containers[*].name}' | tr ' ' '\n' + echo "" + + # Show injected agent image version + log_info "Injected OpenBao Agent image:" + AGENT_IMAGE=$(kubectl get pod "$POD_NAME" -n "$namespace" -o jsonpath='{.spec.containers[?(@.name=="openbao-agent")].image}') + if [[ -n "$AGENT_IMAGE" ]]; then + echo " $AGENT_IMAGE" + AGENT_VERSION=$(echo "$AGENT_IMAGE" | sed 's/.*://') + log_success "Agent version: $AGENT_VERSION" + else + log_warn "openbao-agent container not found - injection may have failed" + fi + echo "" + + log_info "Checking if secrets were rendered..." + if kubectl exec "$POD_NAME" -n "$namespace" -c nginx -- ls /vault/secrets/ 2>/dev/null; then + log_success "Secrets directory exists!" + echo "" + log_info "Secret contents:" + kubectl exec "$POD_NAME" -n "$namespace" -c nginx -- cat /vault/secrets/secrets.json 2>/dev/null || true + else + log_warn "Secrets not yet rendered. Check agent logs:" + echo " kubectl logs $POD_NAME -n $namespace -c openbao-agent" + fi +else + log_error "No pod found with label app=nvcf-api" +fi + +echo "" +log_success "Injector test complete!" +echo "" +echo "Next steps:" +echo " - View agent logs: kubectl logs $POD_NAME -n $namespace -c openbao-agent" +echo " - View init logs: kubectl logs $POD_NAME -n $namespace -c openbao-agent-init" +echo " - Cleanup: $0 $COLIMA_PROFILE --cleanup" diff --git a/deploy/helm/openbao/tests/signing-key-harness/go.mod b/deploy/helm/openbao/tests/signing-key-harness/go.mod new file mode 100644 index 000000000..82c211ee4 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/go.mod @@ -0,0 +1,4 @@ +module signing-key-harness + +go 1.21 + diff --git a/deploy/helm/openbao/tests/signing-key-harness/jwt/jwt.go b/deploy/helm/openbao/tests/signing-key-harness/jwt/jwt.go new file mode 100644 index 000000000..10d2b8961 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/jwt/jwt.go @@ -0,0 +1,155 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jwt + +import ( + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "strings" + "time" +) + +// Header represents a JWT header +type Header struct { + Alg string `json:"alg"` + Typ string `json:"typ"` + Kid string `json:"kid,omitempty"` +} + +// Payload represents a JWT payload +type Payload struct { + Iss string `json:"iss"` + Sub string `json:"sub"` + Aud string `json:"aud"` + Iat int64 `json:"iat"` + Exp int64 `json:"exp"` + Jti string `json:"jti"` +} + +// Sign creates a signed JWT using the given private key +func Sign(privateKey *ecdsa.PrivateKey, kid string) (string, error) { + // Create header + header := Header{ + Alg: "ES256", + Typ: "JWT", + Kid: kid, + } + + // Create payload + now := time.Now().Unix() + payload := Payload{ + Iss: "test-harness", + Sub: "signing-key-test", + Aud: "verification", + Iat: now, + Exp: now + 3600, + Jti: fmt.Sprintf("test-%d", now), + } + + // Encode header and payload + headerJSON, _ := json.Marshal(header) + payloadJSON, _ := json.Marshal(payload) + + headerB64 := base64URLEncode(headerJSON) + payloadB64 := base64URLEncode(payloadJSON) + + // Create signing input + signingInput := headerB64 + "." + payloadB64 + + // Sign with ECDSA + hash := sha256.Sum256([]byte(signingInput)) + r, s, err := ecdsa.Sign(rand.Reader, privateKey, hash[:]) + if err != nil { + return "", fmt.Errorf("signing failed: %w", err) + } + + // Convert r, s to fixed-size byte arrays (32 bytes each for P-256) + rBytes := padToLength(r.Bytes(), 32) + sBytes := padToLength(s.Bytes(), 32) + + // Concatenate r || s + signature := append(rBytes, sBytes...) + signatureB64 := base64URLEncode(signature) + + return signingInput + "." + signatureB64, nil +} + +// Verify validates a JWT signature using the given public key +func Verify(token string, publicKey *ecdsa.PublicKey) (bool, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return false, fmt.Errorf("invalid JWT format: expected 3 parts, got %d", len(parts)) + } + + signingInput := parts[0] + "." + parts[1] + signatureB64 := parts[2] + + // Decode signature + signature, err := base64URLDecode(signatureB64) + if err != nil { + return false, fmt.Errorf("failed to decode signature: %w", err) + } + + // For ES256, signature should be 64 bytes (32 + 32) + if len(signature) != 64 { + return false, fmt.Errorf("invalid signature length: expected 64, got %d", len(signature)) + } + + // Extract r and s + r := new(big.Int).SetBytes(signature[:32]) + s := new(big.Int).SetBytes(signature[32:]) + + // Verify + hash := sha256.Sum256([]byte(signingInput)) + valid := ecdsa.Verify(publicKey, hash[:], r, s) + + return valid, nil +} + +// base64URLEncode encodes bytes to base64url without padding +func base64URLEncode(data []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(data), "=") +} + +// base64URLDecode decodes base64url string +func base64URLDecode(s string) ([]byte, error) { + // Add padding if needed + switch len(s) % 4 { + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} + +// padToLength pads a byte slice to the specified length with leading zeros +func padToLength(b []byte, length int) []byte { + if len(b) >= length { + return b[len(b)-length:] + } + padded := make([]byte, length) + copy(padded[length-len(b):], b) + return padded +} + diff --git a/deploy/helm/openbao/tests/signing-key-harness/keygen/keygen.go b/deploy/helm/openbao/tests/signing-key-harness/keygen/keygen.go new file mode 100644 index 000000000..cefed2275 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/keygen/keygen.go @@ -0,0 +1,160 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package keygen + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "os" + "os/exec" +) + +// JWK represents an EC JSON Web Key +type JWK struct { + Kty string `json:"kty"` + Crv string `json:"crv"` + D string `json:"d"` + X string `json:"x"` + Y string `json:"y"` + Kid string `json:"kid"` + Alg string `json:"alg"` + Use string `json:"use"` + Iat int64 `json:"iat"` +} + +// KeyGenResult contains the generated key and metadata +type KeyGenResult struct { + JWK JWK + RawOutput string + DLength int + XLength int + YLength int + ParseError error +} + +// Generator handles key generation via bash subprocess +type Generator struct { + ScriptPath string +} + +// NewGenerator creates a generator with the specified script +func NewGenerator(scriptPath string) *Generator { + return &Generator{ScriptPath: scriptPath} +} + +// Generate calls bash to generate a signing key and returns the result +func (g *Generator) Generate() (*KeyGenResult, error) { + result := &KeyGenResult{} + + // Generate a UUID for the kid using Go + kid := generateUUID() + + // Build bash command that sources the script and calls the function + bashScript := fmt.Sprintf(` + source "%s" + generate_asymmetric_signing_key "%s" + `, g.ScriptPath, kid) + + cmd := exec.Command("bash", "-c", bashScript) + cmd.Env = os.Environ() + + output, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("bash execution failed: %w\nstderr: %s", err, string(exitErr.Stderr)) + } + return nil, fmt.Errorf("bash execution failed: %w", err) + } + + result.RawOutput = string(output) + + // Parse the JWK JSON + if err := json.Unmarshal(output, &result.JWK); err != nil { + result.ParseError = err + return result, nil + } + + // Record lengths + result.DLength = len(result.JWK.D) + result.XLength = len(result.JWK.X) + result.YLength = len(result.JWK.Y) + + return result, nil +} + +// ToECDSAPrivateKey converts JWK to *ecdsa.PrivateKey +func (j *JWK) ToECDSAPrivateKey() (*ecdsa.PrivateKey, error) { + // Decode base64url values + dBytes, err := base64URLDecode(j.D) + if err != nil { + return nil, fmt.Errorf("failed to decode d: %w", err) + } + + xBytes, err := base64URLDecode(j.X) + if err != nil { + return nil, fmt.Errorf("failed to decode x: %w", err) + } + + yBytes, err := base64URLDecode(j.Y) + if err != nil { + return nil, fmt.Errorf("failed to decode y: %w", err) + } + + // Construct the private key + privateKey := &ecdsa.PrivateKey{ + PublicKey: ecdsa.PublicKey{ + Curve: elliptic.P256(), + X: new(big.Int).SetBytes(xBytes), + Y: new(big.Int).SetBytes(yBytes), + }, + D: new(big.Int).SetBytes(dBytes), + } + + return privateKey, nil +} + +// base64URLDecode decodes a base64url string (no padding) +func base64URLDecode(s string) ([]byte, error) { + // Add padding if needed + switch len(s) % 4 { + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} + +// generateUUID creates a UUID v4 +func generateUUID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + + // Set version (4) and variant bits + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + diff --git a/deploy/helm/openbao/tests/signing-key-harness/main.go b/deploy/helm/openbao/tests/signing-key-harness/main.go new file mode 100644 index 000000000..aa2adf371 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/main.go @@ -0,0 +1,208 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "time" + + "signing-key-harness/jwt" + "signing-key-harness/keygen" + "signing-key-harness/stats" +) + +func main() { + // CLI flags + iterations := flag.Int("n", 1000, "Number of iterations to run") + scriptPath := flag.String("script", "", "Path to encryption_setup.sh (required)") + codeVersion := flag.String("version", "unknown", "Code version label (e.g., 'buggy' or 'fixed')") + workers := flag.Int("workers", runtime.NumCPU(), "Number of parallel workers") + flag.Parse() + + if *iterations <= 0 { + fmt.Fprintln(os.Stderr, "Error: --n must be a positive integer") + os.Exit(1) + } + + if *scriptPath == "" { + fmt.Println("Error: --script flag is required") + fmt.Println("Usage: signing-key-harness --script /path/to/encryption_setup.sh") + os.Exit(1) + } + + // Resolve absolute path + absPath, err := filepath.Abs(*scriptPath) + if err != nil { + fmt.Printf("Error resolving script path: %v\n", err) + os.Exit(1) + } + + // Verify script exists + if _, err := os.Stat(absPath); os.IsNotExist(err) { + fmt.Printf("Error: script not found: %s\n", absPath) + os.Exit(1) + } + + // Print header + fmt.Println() + fmt.Println("╔══════════════════════════════════════════════════════════════╗") + fmt.Println("║ JWT Signing Key Test Harness (Go) ║") + fmt.Println("╠══════════════════════════════════════════════════════════════╣") + fmt.Printf("║ Script: %-47s ║\n", truncateString(filepath.Base(absPath), 47)) + fmt.Printf("║ Version: %-47s ║\n", truncateString(*codeVersion, 47)) + fmt.Printf("║ Iterations: %-47d ║\n", *iterations) + fmt.Printf("║ Workers: %-47d ║\n", *workers) + fmt.Println("╚══════════════════════════════════════════════════════════════╝") + fmt.Println() + + // Create generator and stats + gen := keygen.NewGenerator(absPath) + s := stats.New() + + // Progress tracking + var completed int64 + total := int64(*iterations) + + // Progress display goroutine + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-done: + return + case <-ticker.C: + current := atomic.LoadInt64(&completed) + pct := float64(current) / float64(total) * 100 + bar := progressBar(int(pct), 40) + fmt.Printf("\rRunning... [%s] %d/%d (%.1f%%)", bar, current, total, pct) + } + } + }() + + // Run tests + startTime := time.Now() + + // Use worker pool for parallel execution + var wg sync.WaitGroup + workChan := make(chan int, *iterations) + + // Start workers + for w := 0; w < *workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for iteration := range workChan { + runSingleTest(gen, s, iteration) + atomic.AddInt64(&completed, 1) + } + }() + } + + // Send work + for i := 0; i < *iterations; i++ { + workChan <- i + } + close(workChan) + + // Wait for completion + wg.Wait() + close(done) + + elapsed := time.Since(startTime) + + // Clear progress line + fmt.Printf("\rRunning... [%s] %d/%d (100.0%%) \n", progressBar(100, 40), total, total) + fmt.Printf("\nCompleted in %v (%.1f keys/sec)\n", elapsed.Round(time.Millisecond), float64(*iterations)/elapsed.Seconds()) + + // Print results + s.PrintSummary(*codeVersion) + + // Exit with error code if failures detected + if s.Failed > 0 || s.ParseError > 0 { + os.Exit(1) + } +} + +// runSingleTest executes one iteration of the test +func runSingleTest(gen *keygen.Generator, s *stats.Stats, iteration int) { + result, err := gen.Generate() + if err != nil { + s.Record(iteration, "unknown", 0, 0, 0, false, err.Error()) + return + } + + if result.ParseError != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, result.ParseError.Error()) + return + } + + // Convert to ECDSA key + privateKey, err := result.JWK.ToECDSAPrivateKey() + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + // Sign JWT + token, err := jwt.Sign(privateKey, result.JWK.Kid) + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + // Verify JWT + valid, err := jwt.Verify(token, &privateKey.PublicKey) + if err != nil { + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, false, err.Error()) + return + } + + s.Record(iteration, result.JWK.Kid, result.DLength, result.XLength, result.YLength, valid, "") +} + +// progressBar creates an ASCII progress bar +func progressBar(percent, width int) string { + filled := percent * width / 100 + bar := "" + for i := 0; i < width; i++ { + if i < filled { + bar += "█" + } else { + bar += "░" + } + } + return bar +} + +// truncateString truncates a string to maxLen +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} + diff --git a/deploy/helm/openbao/tests/signing-key-harness/stats/stats.go b/deploy/helm/openbao/tests/signing-key-harness/stats/stats.go new file mode 100644 index 000000000..bad267edb --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/stats/stats.go @@ -0,0 +1,190 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + "strings" + "sync" +) + +// FailedKey records details of a failed key +type FailedKey struct { + Iteration int + Kid string + DLength int + XLength int + YLength int + Error string +} + +// Stats collects test results +type Stats struct { + mu sync.Mutex + + Total int + Passed int + Failed int + ParseError int + + DLengthDist map[int]int + FailedKeys []FailedKey +} + +// New creates a new Stats collector +func New() *Stats { + return &Stats{ + DLengthDist: make(map[int]int), + FailedKeys: make([]FailedKey, 0), + } +} + +// Record records the result of a single test iteration +func (s *Stats) Record(iteration int, kid string, dLen, xLen, yLen int, valid bool, errMsg string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.Total++ + s.DLengthDist[dLen]++ + + if errMsg != "" { + s.ParseError++ + s.FailedKeys = append(s.FailedKeys, FailedKey{ + Iteration: iteration, + Kid: kid, + DLength: dLen, + XLength: xLen, + YLength: yLen, + Error: errMsg, + }) + } else if valid { + s.Passed++ + } else { + s.Failed++ + s.FailedKeys = append(s.FailedKeys, FailedKey{ + Iteration: iteration, + Kid: kid, + DLength: dLen, + XLength: xLen, + YLength: yLen, + Error: "signature verification failed", + }) + } +} + +// PrintSummary outputs the test results +func (s *Stats) PrintSummary(codeVersion string) { + s.mu.Lock() + defer s.mu.Unlock() + + width := 65 + + fmt.Println() + fmt.Println(strings.Repeat("═", width)) + fmt.Println(" JWT SIGNING KEY TEST HARNESS - RESULTS") + fmt.Println(strings.Repeat("═", width)) + fmt.Println() + + fmt.Printf(" Code Version: %s\n", codeVersion) + fmt.Printf(" Iterations: %d\n", s.Total) + fmt.Println() + + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" RESULTS") + fmt.Println(strings.Repeat("─", width)) + + passRate := float64(s.Passed) / float64(s.Total) * 100 + failRate := float64(s.Failed) / float64(s.Total) * 100 + parseRate := float64(s.ParseError) / float64(s.Total) * 100 + + fmt.Printf(" Passed: %5d (%6.2f%%)\n", s.Passed, passRate) + fmt.Printf(" Failed: %5d (%6.2f%%) ← Signature verification failed\n", s.Failed, failRate) + fmt.Printf(" Parse Errors: %5d (%6.2f%%)\n", s.ParseError, parseRate) + fmt.Println() + + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" 'd' LENGTH DISTRIBUTION") + fmt.Println(strings.Repeat("─", width)) + + // Show lengths from 40 to 44 + for l := 40; l <= 44; l++ { + count := s.DLengthDist[l] + pct := float64(count) / float64(s.Total) * 100 + + indicator := "" + if l < 43 && count > 0 { + indicator = " ← CORRUPTED" + } else if l == 43 { + indicator = " ← VALID" + } + + fmt.Printf(" %d chars: %5d (%6.2f%%)%s\n", l, count, pct, indicator) + } + fmt.Println() + + // Show failed samples (up to 10) + if len(s.FailedKeys) > 0 { + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" FAILED SAMPLES") + fmt.Println(strings.Repeat("─", width)) + + limit := 10 + if len(s.FailedKeys) < limit { + limit = len(s.FailedKeys) + } + + for i := 0; i < limit; i++ { + f := s.FailedKeys[i] + kidDisplay := f.Kid + if len(kidDisplay) > 18 { + kidDisplay = kidDisplay[:18] + "..." + } + fmt.Printf(" #%-4d d=%d chars kid=%s\n", f.Iteration, f.DLength, kidDisplay) + } + + if len(s.FailedKeys) > limit { + fmt.Printf(" ... and %d more\n", len(s.FailedKeys)-limit) + } + fmt.Println() + } + + // Conclusion + fmt.Println(strings.Repeat("─", width)) + fmt.Println(" CONCLUSION") + fmt.Println(strings.Repeat("─", width)) + + if s.Failed > 0 || s.ParseError > 0 { + expectedRate := 0.39 + observedRate := float64(s.Failed+s.ParseError) / float64(s.Total) * 100 + + fmt.Println(" Bug Confirmed: YES") + fmt.Printf(" Expected Rate: ~%.2f%%\n", expectedRate) + fmt.Printf(" Observed Rate: %.2f%%\n", observedRate) + + if s.Total >= 100 && (s.Failed+s.ParseError) >= 1 { + fmt.Println(" Statistically Significant: ✓") + } + } else { + fmt.Println(" Bug Confirmed: NO") + fmt.Println(" All keys valid: ✓") + } + + fmt.Println() + fmt.Println(strings.Repeat("═", width)) +} + diff --git a/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_buggy.sh b/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_buggy.sh new file mode 100644 index 000000000..acfa43004 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_buggy.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# +# BUGGY VERSION - For testing purposes only +# This version has the space-padding bug on line 147 +# + +# Include guard: Ensures this script's contents are only processed once. +if [[ -n "${_ENCRYPTION_SETUP_SH_SOURCED:-}" ]]; then + return 0 +fi +readonly _ENCRYPTION_SETUP_SH_SOURCED=1 + +# Function to check if OpenSSL is available +check_openssl() { + if ! command -v openssl &> /dev/null; then + echo "Error: OpenSSL is required but not installed." + exit 1 + fi +} + +# Function to generate data domain key +generate_rand_key() { + local length=${1:-256} + local type=${2:-hex} + + case $type in + hex) + local raw_key=$(openssl rand -hex $length) + echo -n "$raw_key" + ;; + base64) + # Don't use the -base64 flag; instead stream raw bytes directly to base64 to avoid null-byte truncation + local base64_key=$(openssl rand $length | base64 -w 0 | tr -d '\n=' | tr '/+' '_-') + echo -n "$base64_key" + ;; + *) + echo "Error: Invalid type '$type'. Supported types: hex, base64" >&2 + return 1 + ;; + esac +} + +# Function to generate a random kid (Key ID) +generate_kid() { + kid=$(uuidgen -t | tr '[:upper:]' '[:lower:]') + echo "$kid" +} + +# Main function to generate JWK +generate_jwk() { + local kid=$1 + # Generate 256-bit (32 byte) random key + local raw_key=$(openssl rand -base64 32) + + # Convert to base64url encoding (replace / with _, + with -, remove =) + local base64url_key=$(echo -n "$raw_key" | tr '/+' '_-' | tr -d '=\n') + + # Create JWK JSON + echo -n "{\"kty\":\"oct\",\"use\":\"enc\",\"kid\":\"$kid\",\"k\":\"$base64url_key\",\"alg\":\"A256GCM\"}" +} + +generate_hmac_json() { + local kid=$1 + local key_length=$2 + local key=$(generate_rand_key $key_length) + echo -n "{\"keys\":[{\"kid\":\"$kid\",\"key\":\"$key\"}]}" | base64 -w 0 +} + +# Function to generate JWKS +generate_jwks_escaped_json() { + local key=$1 + echo -n "{\"keys\":[$key]}" +} + +# Function to generate JWKS +generate_base64_encoded_json() { + local key=$1 + echo -n "{\"keys\":[$key]}" | base64 -w 0 +} + +base64_encode() { + echo -n "$1" | base64 -w 0 +} + +# Base64URL encoding function (RFC 4648 Section 5) +base64url_encode() { + base64 -w 0 | tr '+/' '-_' | tr -d '=' +} + +# Generate a new signing key +generate_asymmetric_signing_key() { + + if [ -z "$1" ]; then + echo "Usage: generate_asymmetric_signing_key " >&2 + return 1 + fi + + # set KID + KID=$1 + + # Get current timestamp + TIMESTAMP=$(date +%s) + + # Generate EC private key (P-256 curve) + PRIVATE_KEY=$(openssl ecparam -genkey -name prime256v1 -noout) + + # Create temporary files for key processing + TEMP_KEY=$(mktemp) + TEMP_PUB=$(mktemp) + trap "rm -f $TEMP_KEY $TEMP_PUB" EXIT + + # Save private key to temp file + echo "$PRIVATE_KEY" > "$TEMP_KEY" + + # Extract the private key value (d parameter) - improved method + D_VALUE=$(openssl ec -in "$TEMP_KEY" -text -noout 2>/dev/null | \ + awk '/priv:/{flag=1; next} /pub:/{flag=0} flag{print}' | \ + tr -d ' \n:' | \ + sed 's/^00*//') + + # Generate public key from private key + openssl ec -in "$TEMP_KEY" -pubout -out "$TEMP_PUB" 2>/dev/null + + # Extract x and y coordinates from public key - improved method + PUBLIC_HEX=$(openssl ec -pubin -in "$TEMP_PUB" -text -noout 2>/dev/null | \ + awk '/pub:/{flag=1; next} /ASN1 OID:/{flag=0} flag{print}' | \ + tr -d ' \n:') + + # Remove the 04 prefix (uncompressed point indicator) and extract x,y coordinates + if [[ ${#PUBLIC_HEX} -ge 130 ]]; then + COORDS=${PUBLIC_HEX:2} + X_HEX=${COORDS:0:64} + Y_HEX=${COORDS:64:64} + else + echo "Error: Could not extract public key coordinates" >&2 + exit 1 + fi + + # Convert hex to base64url (without padding) + hex_to_base64url() { + local hex_input="$1" + # Ensure even length by padding with leading zero if needed + if [[ $((${#hex_input} % 2)) -eq 1 ]]; then + hex_input="0$hex_input" + fi + echo -n "$hex_input" | xxd -r -p | base64 | tr '+/' '-_' | tr -d '=' + } + + # Pad private key to 64 hex characters (32 bytes) + # BUG: printf %s pads with SPACES, not zeros! + D_PADDED=$(printf "%064s" "$D_VALUE") + + # Convert private key hex to base64url + D_B64=$(hex_to_base64url "$D_PADDED") + + # Convert coordinates to base64url + X_B64=$(hex_to_base64url "$X_HEX") + Y_B64=$(hex_to_base64url "$Y_HEX") + + # Verify we got values + if [[ -z "$D_B64" || -z "$X_B64" || -z "$Y_B64" ]]; then + echo "Error: Failed to convert key values to base64url" >&2 + echo "D_VALUE: $D_VALUE" >&2 + echo "X_HEX: $X_HEX" >&2 + echo "Y_HEX: $Y_HEX" >&2 + exit 1 + fi + + # Output the JWK + echo -n "{\"kty\":\"EC\",\"d\":\"$D_B64\",\"use\":\"sig\",\"crv\":\"P-256\",\"kid\":\"$KID\",\"x\":\"$X_B64\",\"y\":\"$Y_B64\",\"alg\":\"ES256\",\"iat\":$TIMESTAMP}" +} + + +# always check for openssl +check_openssl + diff --git a/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_fixed.sh b/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_fixed.sh new file mode 100644 index 000000000..ed4d53476 --- /dev/null +++ b/deploy/helm/openbao/tests/signing-key-harness/testdata/encryption_setup_fixed.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# +# FIXED VERSION - For testing purposes +# This version has the corrected zero-padding on line 147 +# + +# Include guard: Ensures this script's contents are only processed once. +if [[ -n "${_ENCRYPTION_SETUP_SH_SOURCED:-}" ]]; then + return 0 +fi +readonly _ENCRYPTION_SETUP_SH_SOURCED=1 + +# Function to check if OpenSSL is available +check_openssl() { + if ! command -v openssl &> /dev/null; then + echo "Error: OpenSSL is required but not installed." + exit 1 + fi +} + +# Function to generate data domain key +generate_rand_key() { + local length=${1:-256} + local type=${2:-hex} + + case $type in + hex) + local raw_key=$(openssl rand -hex $length) + echo -n "$raw_key" + ;; + base64) + # Don't use the -base64 flag; instead stream raw bytes directly to base64 to avoid null-byte truncation + local base64_key=$(openssl rand $length | base64 -w 0 | tr -d '\n=' | tr '/+' '_-') + echo -n "$base64_key" + ;; + *) + echo "Error: Invalid type '$type'. Supported types: hex, base64" >&2 + return 1 + ;; + esac +} + +# Function to generate a random kid (Key ID) +generate_kid() { + kid=$(uuidgen -t | tr '[:upper:]' '[:lower:]') + echo "$kid" +} + +# Main function to generate JWK +generate_jwk() { + local kid=$1 + # Generate 256-bit (32 byte) random key + local raw_key=$(openssl rand -base64 32) + + # Convert to base64url encoding (replace / with _, + with -, remove =) + local base64url_key=$(echo -n "$raw_key" | tr '/+' '_-' | tr -d '=\n') + + # Create JWK JSON + echo -n "{\"kty\":\"oct\",\"use\":\"enc\",\"kid\":\"$kid\",\"k\":\"$base64url_key\",\"alg\":\"A256GCM\"}" +} + +generate_hmac_json() { + local kid=$1 + local key_length=$2 + local key=$(generate_rand_key $key_length) + echo -n "{\"keys\":[{\"kid\":\"$kid\",\"key\":\"$key\"}]}" | base64 -w 0 +} + +# Function to generate JWKS +generate_jwks_escaped_json() { + local key=$1 + echo -n "{\"keys\":[$key]}" +} + +# Function to generate JWKS +generate_base64_encoded_json() { + local key=$1 + echo -n "{\"keys\":[$key]}" | base64 -w 0 +} + +base64_encode() { + echo -n "$1" | base64 -w 0 +} + +# Base64URL encoding function (RFC 4648 Section 5) +base64url_encode() { + base64 -w 0 | tr '+/' '-_' | tr -d '=' +} + +# Generate a new signing key +generate_asymmetric_signing_key() { + + if [ -z "$1" ]; then + echo "Usage: generate_asymmetric_signing_key " >&2 + return 1 + fi + + # set KID + KID=$1 + + # Get current timestamp + TIMESTAMP=$(date +%s) + + # Generate EC private key (P-256 curve) + PRIVATE_KEY=$(openssl ecparam -genkey -name prime256v1 -noout) + + # Create temporary files for key processing + TEMP_KEY=$(mktemp) + TEMP_PUB=$(mktemp) + trap "rm -f $TEMP_KEY $TEMP_PUB" EXIT + + # Save private key to temp file + echo "$PRIVATE_KEY" > "$TEMP_KEY" + + # Extract the private key value (d parameter) - improved method + D_VALUE=$(openssl ec -in "$TEMP_KEY" -text -noout 2>/dev/null | \ + awk '/priv:/{flag=1; next} /pub:/{flag=0} flag{print}' | \ + tr -d ' \n:' | \ + sed 's/^00*//') + + # Generate public key from private key + openssl ec -in "$TEMP_KEY" -pubout -out "$TEMP_PUB" 2>/dev/null + + # Extract x and y coordinates from public key - improved method + PUBLIC_HEX=$(openssl ec -pubin -in "$TEMP_PUB" -text -noout 2>/dev/null | \ + awk '/pub:/{flag=1; next} /ASN1 OID:/{flag=0} flag{print}' | \ + tr -d ' \n:') + + # Remove the 04 prefix (uncompressed point indicator) and extract x,y coordinates + if [[ ${#PUBLIC_HEX} -ge 130 ]]; then + COORDS=${PUBLIC_HEX:2} + X_HEX=${COORDS:0:64} + Y_HEX=${COORDS:64:64} + else + echo "Error: Could not extract public key coordinates" >&2 + exit 1 + fi + + # Convert hex to base64url (without padding) + hex_to_base64url() { + local hex_input="$1" + # Ensure even length by padding with leading zero if needed + if [[ $((${#hex_input} % 2)) -eq 1 ]]; then + hex_input="0$hex_input" + fi + echo -n "$hex_input" | xxd -r -p | base64 | tr '+/' '-_' | tr -d '=' + } + + # Pad private key to 64 hex characters (32 bytes) + # FIX: printf %s pads with spaces, so we must replace them with zeros + D_PADDED=$(printf "%064s" "$D_VALUE" | tr ' ' '0') + + # Convert private key hex to base64url + D_B64=$(hex_to_base64url "$D_PADDED") + + # Convert coordinates to base64url + X_B64=$(hex_to_base64url "$X_HEX") + Y_B64=$(hex_to_base64url "$Y_HEX") + + # Verify we got values + if [[ -z "$D_B64" || -z "$X_B64" || -z "$Y_B64" ]]; then + echo "Error: Failed to convert key values to base64url" >&2 + echo "D_VALUE: $D_VALUE" >&2 + echo "X_HEX: $X_HEX" >&2 + echo "Y_HEX: $Y_HEX" >&2 + exit 1 + fi + + # Output the JWK + echo -n "{\"kty\":\"EC\",\"d\":\"$D_B64\",\"use\":\"sig\",\"crv\":\"P-256\",\"kid\":\"$KID\",\"x\":\"$X_B64\",\"y\":\"$Y_B64\",\"alg\":\"ES256\",\"iat\":$TIMESTAMP}" +} + + +# always check for openssl +check_openssl + diff --git a/deploy/helm/openbao/tools/bao.sh b/deploy/helm/openbao/tools/bao.sh new file mode 100755 index 000000000..77d7ba9b9 --- /dev/null +++ b/deploy/helm/openbao/tools/bao.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../utils" + exit 1 +fi +source "$SCRIPT_DIR/../utils/utils.sh" + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +# Build the command to execute inside the pod +COMMAND="env BAO_TOKEN=$(get_root_token "$namespace" $statefulset) bao $@" +ECHO_COMMAND="env BAO_TOKEN= bao $@" +# Execute the command inside the OpenBao pod using kubectl +echo "kubectl exec -n $namespace $statefulset-0 -c openbao -- $ECHO_COMMAND" +kubectl exec -n $namespace $statefulset-2 -c openbao -- $COMMAND diff --git a/deploy/helm/openbao/tools/generate-admin-token.sh b/deploy/helm/openbao/tools/generate-admin-token.sh new file mode 100755 index 000000000..62d6addfb --- /dev/null +++ b/deploy/helm/openbao/tools/generate-admin-token.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../utils" + exit 1 +fi +source "$SCRIPT_DIR/../utils/utils.sh" + +generate_nvcf_api_admin_token() { + local namespace=$1 + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + + log_step "Generating admin JWT token for nvcf-api" + + local jwt_output + if jwt_output=$(kubectl exec -it $statefulset-0 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao write -f services/nvcf-api/jwt/sign/nvcf-api-admin 2>&1); then + + log_success "Admin JWT token generated successfully for nvcf-api" + echo "JWT Token Output:" + echo "$jwt_output" + + # Try to extract just the token value if it's in the expected format + local token=$(echo "$jwt_output" | grep -E "^token\s+" | awk '{print $2}' | tr -d '\r') + if [ -n "$token" ]; then + echo "" + echo "Extracted Token:" + echo "$token" + + # Optionally decode the JWT to show claims (requires base64 and jq) + if command -v jq >/dev/null 2>&1; then + echo "" + echo "JWT Claims (decoded):" + # Extract payload (second part of JWT) + local payload=$(echo "$token" | cut -d'.' -f2) + # Add padding if needed for base64 decoding + local padded_payload + case $((${#payload} % 4)) in + 2) padded_payload="${payload}==" ;; + 3) padded_payload="${payload}=" ;; + *) padded_payload="$payload" ;; + esac + echo "$padded_payload" | base64 -d 2>/dev/null | jq . 2>/dev/null || echo "Could not decode JWT payload" + fi + fi + else + log_error "Failed to generate admin JWT token: $jwt_output" + return 1 + fi +} + +# Default OpenBao configuration +namespace="vault-system" +statefulset="openbao-server" + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +if ! generate_nvcf_api_admin_token "$namespace" "$statefulset"; then + log_error "Failed to generate nvcf-api admin JWT token" + exit 1 +fi \ No newline at end of file diff --git a/deploy/helm/openbao/tools/operator-stepdown.sh b/deploy/helm/openbao/tools/operator-stepdown.sh new file mode 100755 index 000000000..c9f90b066 --- /dev/null +++ b/deploy/helm/openbao/tools/operator-stepdown.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../utils" + exit 1 +fi +source "$SCRIPT_DIR/../utils/utils.sh" + +operator_stepdown() { + local namespace=$1 + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + + kubectl exec -it $statefulset-0 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao operator step-down +} + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +if ! operator_stepdown "$namespace" "$statefulset"; then + log_error "Failed to issue operator stepdown" + exit 1 +fi diff --git a/deploy/helm/openbao/tools/raft-list-peers.sh b/deploy/helm/openbao/tools/raft-list-peers.sh new file mode 100755 index 000000000..bc849e89d --- /dev/null +++ b/deploy/helm/openbao/tools/raft-list-peers.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../utils" + exit 1 +fi +source "$SCRIPT_DIR/../utils/utils.sh" + +check_raft_status() { + local namespace=$1 + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + + kubectl exec $statefulset-0 -c openbao -n $namespace -- \ + env BAO_TOKEN=$root_token \ + bao operator raft list-peers +} + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +if ! check_raft_status "$namespace" "$statefulset"; then + log_error "Failed to get Raft status" + exit 1 +fi diff --git a/deploy/helm/openbao/tools/register_plugin.sh b/deploy/helm/openbao/tools/register_plugin.sh new file mode 100755 index 000000000..2ba18fe7b --- /dev/null +++ b/deploy/helm/openbao/tools/register_plugin.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/../utils/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}/../utils" + exit 1 +fi +source "$SCRIPT_DIR/../utils/utils.sh" + +register_and_enable_plugin() { + local namespace="$1" + local statefulset=$2 + local root_token=$(get_root_token "$namespace" "$statefulset") + local plugin_dir="/openbao/plugins" + local plugin_name="vault-plugin-secrets-jwt" + local enable_path="jwt" + local pod_name=$statefulset-0 + + # Step 1: Verify Plugin Binary Exists + log_step "Verifying if plugin binary exists at $plugin_dir/$plugin_name..." + if ! kubectl exec -n "$namespace" "$pod_name" -c openbao -- test -f "$plugin_dir/$plugin_name"; then + log_error "Plugin binary not found at $plugin_dir/$plugin_name. Ensure it has been copied to this location." + return 1 + fi + log_success "Plugin binary found" + + # Step 2: Calculate SHA256 Checksum of the Plugin Binary + log_step "Calculating SHA256 checksum for $plugin_name..." + local plugin_sha256=$(kubectl exec -n "$namespace" "$pod_name" -c openbao -- sha256sum "$plugin_dir/$plugin_name" | awk '{print $1}') + if [[ -z "$plugin_sha256" ]]; then + log_error "Failed to calculate SHA256 checksum for $plugin_name." + return 1 + fi + log_success "SHA256 checksum: $plugin_sha256" + + # Step 3: Register the Plugin + log_step "Registering plugin '$plugin_name'..." + kubectl exec -n "$namespace" "$pod_name" -c openbao -- \ + env BAO_TOKEN="$root_token" \ + bao plugin register \ + -sha256="$plugin_sha256" \ + -command="$plugin_name" \ + secret "$plugin_name" + if [[ $? -ne 0 ]]; then + log_error "Failed to register plugin '$plugin_name'." + return 1 + fi + log_success "Plugin '$plugin_name' registered successfully." + + # Step 4: Enable the Plugin as a Secrets Engine + log_step "Enabling plugin '$plugin_name' as a secrets engine at path '$enable_path'..." + kubectl exec -n "$namespace" "$pod_name" -c openbao -- \ + env BAO_TOKEN="$root_token" \ + bao secrets enable -path="$enable_path" "$plugin_name" + if [[ $? -ne 0 ]]; then + log_error "Failed to enable plugin '$plugin_name' as a secrets engine." + return 1 + fi + + log_success "Plugin '$plugin_name' enabled as a secrets engine at path '$enable_path'." +} + +# Usage +namespace="vault-system" +statefulset="openbao-server" + +if ! check_resources "$namespace" "$statefulset"; then + exit 1 +fi + +if ! register_and_enable_plugin "$namespace" "$statefulset"; then + log_error "Failed to register the sample plugin (outfoxx/vault-plugin-secrets-jwt)" + exit 1 +fi diff --git a/deploy/helm/openbao/upgrade/runbook.md b/deploy/helm/openbao/upgrade/runbook.md new file mode 100644 index 000000000..43b79ae56 --- /dev/null +++ b/deploy/helm/openbao/upgrade/runbook.md @@ -0,0 +1,525 @@ +# OpenBao Upgrade Runbook + +**Upgrade Path**: `` → `` +**Estimated Time**: ~15-20 minutes +**Difficulty**: Intermediate + +> **Note**: Replace `` (e.g., `2.2.2-nv-1`) and `` (e.g., `2.4.4`) with your actual versions throughout this runbook. + +--- + +## Prerequisites + +- **Image Pull Secrets**: Configure NGC credentials as described in [README.md - Image Pull Secrets](../README.md#image-pull-secrets---ngcnvcrio) +- **Colima**: Running Kubernetes cluster (e.g., `colima start --profile amd64 --kubernetes`) +- **kubectl context**: Set to your target cluster before running commands + ```bash + kubectl config use-context colima-amd64 # or your cluster context + ``` +- **Baseline Deployment**: OpenBao v2.2.2 deployed via `make install additional_values=values.local.yaml` + +--- + +## What You'll Learn + +This runbook walks you through a **near zero-downtime upgrade** of OpenBao in Kubernetes. By the end, you'll understand: + +- How **RollingUpdate** restarts pods automatically (no manual `kubectl delete` needed) +- How **PreStop hooks** gracefully hand off leadership before pod termination +- How to **measure HA availability** during rolling upgrades +- How to **validate** the upgrade with continuous testing + +--- + +## Architecture Overview + +We deploy a 3-node OpenBao cluster locally using Colima (Kubernetes on macOS): + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ OpenBao HA Cluster │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ openbao-0 │ │ openbao-1 │ │ openbao-2 │ │ +│ │ (Leader) │◄──│ (Follower) │◄──│ (Follower) │ │ +│ │ v2.2.2 │ │ v2.2.2 │ │ v2.2.2 │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ │ │ │ +│ └─────────────────┴─────────────────┘ │ +│ Raft Consensus │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Testing Strategy: LoadBalancer + +To accurately measure the impact of a rolling upgrade with HA enabled, we stand up a **LoadBalancer** in front of the cluster: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Test Traffic → LoadBalancer (MetalLB IP) → Healthy Pods │ +└─────────────────────────────────────────────────────────────────┘ +``` + +> The LoadBalancer IP is assigned by MetalLB from the pool configured in `utils/lb.sh` +> (default: `192.168.5.240-250`). The first available IP is typically `192.168.5.240`. + +**Why LoadBalancer for testing?** +- Routes requests to healthy pods automatically +- Shows real availability during pod restarts +- Gives accurate picture of client-side impact during rollout + +> 💡 **Note**: Production environments may use different ingress methods (Ingress Controller, Service Mesh, etc.). The LoadBalancer approach gives us the best estimate of HA behavior during upgrades, regardless of your production setup. + +**During upgrade**, pods restart in reverse order (2 → 1 → 0). The LoadBalancer routes traffic to healthy pods, so we can measure actual downtime/errors. + +--- + +## Prerequisites + +### Tools Required + +```bash +# Check these are installed +brew install yq # YAML processor +brew install jq # JSON processor +brew install kustomize # Kubernetes manifests +``` + +### Environment + +| Component | Value | +|-----------|-------| +| Colima Profile | `` (e.g., `amd64`) | +| Kubernetes | v1.32.1+k3s1 | +| Namespace | `vault-system` | + +--- + +## Part 1: Reset to Baseline + +> **Goal**: Start with a clean cluster running OpenBao 2.2.2-nv-1 + +### Step 1.1: Clean Up Existing Deployment + +**What you're doing**: Removing any existing OpenBao installation to start fresh. + +```bash +# Stop any running test scripts first (Ctrl+C in those terminals) + +# Uninstall OpenBao +make uninstall +``` + +**What this does**: +- Runs `helm uninstall openbao-server` +- Deletes the `vault-system` namespace +- Removes all PVCs (Raft data is deleted!) + +> ⚠️ **Warning**: This deletes all secrets stored in OpenBao. Only do this in test environments! + +**If cluster-scoped resources remain** (webhooks, clusterroles): +```bash +kubectl delete mutatingwebhookconfiguration,clusterrole,clusterrolebinding \ + -l app.kubernetes.io/instance=openbao-server +``` + +### Step 1.2: Verify Cleanup + +**What you're doing**: Confirming everything is gone before reinstalling. + +```bash +# This should return "not found" +kubectl get ns vault-system +``` + +✅ **Expected**: `Error from server (NotFound): namespaces "vault-system" not found` + +### Step 1.3: Install Baseline (2.2.2-nv-1) + +**What you're doing**: Installing the "old" version that you'll upgrade from. + +```bash +make install additional_values=values.local.yaml +``` + +⏱️ **Wait time**: ~2-3 minutes for pods to be ready + +**Watch the installation**: +```bash +kubectl get pods -n vault-system -w +``` + +### Step 1.4: Verify Baseline Installation + +**What you're doing**: Confirming all 3 pods are running the baseline version. + +```bash +kubectl get pods -n vault-system +``` + +✅ **Expected output**: +``` +NAME READY STATUS RESTARTS AGE +openbao-server-0 2/2 Running 0 2m +openbao-server-1 2/2 Running 0 2m +openbao-server-2 2/2 Running 0 2m +openbao-server-agent-injector-xxx 1/1 Running 0 2m +openbao-server-initialize-cluster-xxx 0/1 Completed 0 2m +openbao-server-migrations-xxx 0/1 Completed 0 1m +``` + +> 💡 **Tip**: `2/2` means both containers are running: the main `openbao` container and the `auto-unseal-sidecar`. + +**Verify versions**: +```bash +for i in 0 1 2; do + echo -n "openbao-server-$i: " + kubectl exec openbao-server-$i -n vault-system -c openbao -- bao version | head -1 +done +``` + +✅ **Expected**: All pods show `OpenBao v2.2.2` + +### Step 1.5: Test Injector with Baseline Version (Optional) + +**What you're doing**: Verifying the agent injector works before upgrading. + +```bash +./tests/services/test.sh +``` + +**What this does**: +1. Creates a `nvcf` namespace +2. Deploys an nginx pod with OpenBao annotations +3. Verifies the injector injects the agent sidecar +4. Shows if secrets are being rendered + +✅ **Expected**: Pod shows `2/2` containers (nginx + openbao-agent) + +**Cleanup** (or keep running to test post-upgrade): +```bash +./tests/services/test.sh --cleanup +``` + +> 💡 You can leave this pod running during the upgrade to verify that existing injected pods +> continue working after the server upgrade. + +--- + +## Part 2: Setup LoadBalancer for Testing + +> **Goal**: Stand up a LoadBalancer to measure real HA availability during the upgrade + +### Why LoadBalancer for Testing? + +Without a LoadBalancer, you'd connect directly to a specific pod. When that pod restarts during upgrade, your connection fails—but that doesn't tell you if the *cluster* is still available. + +The LoadBalancer routes to healthy pods, giving you an accurate picture of client-side impact: + +``` +Direct pod: Client → Pod-0 (restarting) → ❌ Connection Failed (pod down, but cluster OK) +Via LB: Client → LoadBalancer → Pod-1 (healthy) → ✅ Request Succeeds (cluster available) +``` + +> 💡 This simulates what real clients experience in production, regardless of your actual ingress method. + +### Step 2.1: Create LoadBalancer + +**What you're doing**: Setting up MetalLB and creating a LoadBalancer service. + +```bash +# Check if LoadBalancer already exists +kubectl get svc openbao-server-lb -n vault-system + +# If not found, create it +./utils/lb.sh +``` + +### Step 2.2: Verify LoadBalancer + +```bash +# Get the LoadBalancer IP +LB_IP=$(kubectl get svc openbao-server-lb -n vault-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +echo "LoadBalancer IP: $LB_IP" +``` + +✅ **Expected**: `192.168.5.240` (first IP from MetalLB pool configured in `utils/lb.sh`) + +**Quick health check**: +```bash +./tests/openbao/loadbalancer_ha_seal_status_check.sh +# Let it run for a few cycles, then Ctrl+C +``` + +✅ **Expected**: `Seal status = false` (unsealed and healthy) + +--- + +## Part 3: Start Continuous Tests + +> **Goal**: Run tests that will show cluster health during the upgrade + +### Why Continuous Tests? + +These tests run in a loop, constantly reading/writing to OpenBao. They'll show you: +- **Green**: Cluster is healthy, requests succeeding +- **Brief blips**: Normal during pod restarts (1-2 seconds) +- **Extended errors**: Something is wrong, investigate + +### Step 3.1: Open Terminal Windows + +You'll need: +| Terminal | Purpose | +|----------|---------| +| Terminal 1 | KV read/write test (keep running during upgrade) | +| Terminal 2 | Seal status test (keep running during upgrade) | + +> 💡 The upgrade script can run in any terminal. It pauses to let you start the test +> scripts, then handles the helm upgrade automatically when you press Enter. + +### Step 3.2: Start KV Test (Terminal 1) + +**What you're doing**: Continuously writing, reading, and deleting secrets. + +```bash +# Terminal 1 - Keep this running during upgrade! +./tests/openbao/loadbalancer_ha_kv_rw_check.sh +``` + +**What to watch for**: +- ✅ `SUCCESS` messages = cluster is healthy +- ⚠️ Occasional failures during upgrade = normal (LoadBalancer re-routing) +- ❌ Continuous failures = something is wrong + +### Step 3.3: Start Seal Status Test (Terminal 2) + +**What you're doing**: Continuously checking that pods stay unsealed. + +```bash +# Terminal 2 - Keep this running during upgrade! +./tests/openbao/loadbalancer_ha_seal_status_check.sh +``` + +**What to watch for**: +- ✅ `sealed: false` = pod is healthy and accepting requests +- ❌ `sealed: true` = pod needs manual unsealing (shouldn't happen with auto-unseal) + +> 💡 **Let tests run for 1-2 minutes** to establish a baseline before starting the upgrade. + +--- + +## Part 4: Execute the Upgrade + +> **Goal**: Upgrade all pods from 2.2.2 to 2.4.4 with minimal downtime + +### Step 4.1: (Optional) Watch Pods in Terminal 3 + +**What you're doing**: Observing the rolling restart in real-time. + +```bash +# Terminal 3 (optional but recommended) +kubectl get pods -n vault-system -w +``` + +### Step 4.2: Run the Upgrade Script + +**What you're doing**: Building the new image and triggering a rolling upgrade. + +> ⚠️ **Important**: Keep Terminals 1 & 2 (the test scripts) running while you do this! + +```bash +# In a new terminal (or the one you used for baseline verification) +./upgrade/upgrade-test.sh 2.4.4 +``` + +**The script will**: +1. Switch Docker context to your Colima profile +2. Build the 2.4.4 image (or skip if already built) +3. Show current pod versions +4. Prompt you to confirm before starting +5. **Automatically run** `helm upgrade` with RollingUpdate strategy +6. Verify all pods upgraded + +> 💡 The script pauses and shows you what monitoring terminals to open. Once you confirm +> Terminals 1 & 2 are running, press Enter and the script handles the rest. + +### Step 4.3: What You'll See (Terminal 3 - Pod Watch) + +```bash +kubectl get pods -n vault-system -w +``` + +**What you'll see**: + +``` +NAME READY STATUS RESTARTS AGE +openbao-server-2 2/2 Terminating 0 10m ← Pod 2 shutting down +openbao-server-2 0/2 Pending 0 0s ← New pod creating +openbao-server-2 0/2 Init:0/1 0 1s ← Initializing +openbao-server-2 2/2 Running 0 30s ← Back online with v2.4.4 +openbao-server-1 2/2 Terminating 0 10m ← Pod 1 next +... (repeats for pod 1, then pod 0) +``` + +**Restart order**: `2 → 1 → 0` (reverse ordinal order) + +### What's Happening Behind the Scenes + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ Pod-2 Terminating │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ 1. PreStop Hook Runs: │ │ +│ │ - Checks if this pod is the Raft leader │ │ +│ │ - If leader: runs "bao operator step-down" │ │ +│ │ - Waits 5s for new leader election │ │ +│ │ 2. Container Terminates │ │ +│ │ 3. New Pod Created with v2.4.4 Image │ │ +│ │ 4. Auto-Unseal Sidecar Unseals the Pod │ │ +│ │ 5. Pod Joins Raft Cluster as Follower │ │ +│ └─────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Part 5: Verify the Upgrade + +> **Goal**: Confirm all pods are upgraded and the cluster is healthy + +### Step 5.1: Check Pod Versions + +```bash +for i in 0 1 2; do + echo -n "openbao-server-$i: " + kubectl exec openbao-server-$i -n vault-system -c openbao -- bao version | head -1 +done +``` + +✅ **Expected**: All pods show `OpenBao v2.4.4` + +### Step 5.2: Check Raft Cluster Health + +```bash +./tools/raft-list-peers.sh +``` + +✅ **Expected**: 3 nodes (1 leader, 2 followers), all `voter: true` + +### Step 5.3: Check Seal Status + +```bash +for i in 0 1 2; do + echo -n "openbao-server-$i: " + kubectl exec openbao-server-$i -n vault-system -c openbao -- bao status 2>/dev/null | grep "Sealed" +done +``` + +✅ **Expected**: All pods show `Sealed: false` + +### Step 5.4: Review Test Results + +Go back to Terminals 1 and 2. Stop the tests with `Ctrl+C`. + +**Analyze the output**: +- **Success rate >99%**: Upgrade successful ✅ +- **Success rate 95-99%**: Acceptable, brief leader election delays +- **Success rate <95%**: Investigate what went wrong + +--- + +## Part 6: Update Injector Agent Image + +> **Goal**: Ensure newly injected pods use the matching OpenBao agent version + +### Why This Matters + +The agent injector injects a sidecar container into your application pods. If the agent version doesn't match the server, you'll see warnings: + +``` +OpenBao Agent version does not match OpenBao server version. +Agent: 2.2.2, Server: 2.4.4 +``` + +### Step 6.1: Verify Agent Image Updated + +The `upgrade-test.sh` script includes the agent image in `/tmp/upgrade-values.yaml`, so this should already be done. Verify with: + +```bash +kubectl get deploy openbao-server-agent-injector -n vault-system \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -i image +``` + +✅ **Expected**: Shows `2.4.4` tag + +### Step 6.2: Re-test Injector with New Version + +**What you're doing**: Delete the test pod and re-deploy to verify the upgraded injector +injects the new agent version. + +```bash +# Delete existing test pod (if running from Step 1.5) +./tests/services/test.sh --cleanup + +# Re-deploy test pod +./tests/services/test.sh +``` + +**Verify the injected agent uses the new version**: +```bash +kubectl get pod -n nvcf -o jsonpath='{.items[0].spec.containers[?(@.name=="openbao-agent")].image}' +``` + +✅ **Expected**: Image tag shows `2.4.4` + +**Cleanup when done**: +```bash +./tests/services/test.sh --cleanup +``` + +> 💡 This confirms the injector is using the upgraded agent image for newly created pods. + +--- + +## Quick Reference + +### Commands Cheat Sheet + +```bash +# Set context first +kubectl config use-context colima-amd64 # or your cluster context + +# Full reset and reinstall +make uninstall +make install additional_values=values.local.yaml + +# Test injector (optional) +./tests/services/test.sh +./tests/services/test.sh --cleanup + +# Setup LoadBalancer +./utils/lb.sh + +# Verify versions +for i in 0 1 2; do + kubectl exec openbao-server-$i -n vault-system -c openbao -- bao version | head -1 +done + +# Run upgrade +./upgrade/upgrade-test.sh 2.4.4 + +# Verify versions +for i in 0 1 2; do + kubectl exec openbao-server-$i -n vault-system -c openbao -- bao version | head -1 +done + +# Check cluster health +./tools/raft-list-peers.sh + +# Run HA tests +./tests/openbao/loadbalancer_ha_kv_rw_check.sh +./tests/openbao/loadbalancer_ha_seal_status_check.sh + +# Cleanup LoadBalancer +./utils/lb.sh --cleanup +``` diff --git a/deploy/helm/openbao/upgrade/upgrade-test.sh b/deploy/helm/openbao/upgrade/upgrade-test.sh new file mode 100755 index 000000000..f197a0139 --- /dev/null +++ b/deploy/helm/openbao/upgrade/upgrade-test.sh @@ -0,0 +1,327 @@ +#!/bin/bash +# Usage: ./upgrade-test.sh [COLIMA_PROFILE] [OPENBAO_VERSION] [IMAGE_TAG] +# Examples: +# ./upgrade-test.sh amd64 2.5.1 +# ./upgrade-test.sh amd64 2.5.1 2.5.1-nv-1 +# ./upgrade-test.sh kubernetes 2.5.1 +# Defaults: COLIMA_PROFILE=amd64, OPENBAO_VERSION=2.5.1, IMAGE_TAG=-nv-1 + +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" +source "$REPO_ROOT/utils/utils.sh" + +COLIMA_PROFILE="${1:-amd64}" +UPGRADE_VERSION="${2:-2.5.1}" +LOCAL_TEST_TAG="${3:-${UPGRADE_VERSION}-nv-1}" # Default to the published production tag +CURRENT_VERSION="2.2.2-nv-1" +IMAGE_NAME="//nvcf-openbao" + +log_section "OpenBao Upgrade Test: ${CURRENT_VERSION} -> ${UPGRADE_VERSION} (Colima profile: ${COLIMA_PROFILE})" + +# Step 1: Switch Docker context to Colima profile +log_step "Switching Docker context to colima-${COLIMA_PROFILE}..." +ORIGINAL_CONTEXT=$(docker context show) +ORIGINAL_DOCKER_HOST="${DOCKER_HOST:-}" + +# Unset DOCKER_HOST so context switch works +unset DOCKER_HOST + +# Trap to restore context and DOCKER_HOST on exit +trap "docker context use ${ORIGINAL_CONTEXT} 2>/dev/null || true; export DOCKER_HOST='${ORIGINAL_DOCKER_HOST}'" EXIT + +docker context use colima-${COLIMA_PROFILE} + +if [ $? -ne 0 ]; then + log_error "Failed to switch Docker context to colima-${COLIMA_PROFILE}" + exit 1 +fi +log_success "Docker context switched to colima-${COLIMA_PROFILE}" + +# Step 2: Pull upgrade image into Colima's containerd +IMAGE_TAG="${IMAGE_NAME}:${LOCAL_TEST_TAG}" + +log_info "Pulling upgrade image: ${IMAGE_TAG}" + +if docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^${IMAGE_TAG}$"; then + log_info "Image ${IMAGE_TAG} already present locally, skipping pull" + log_info "To force re-pull: docker rmi ${IMAGE_TAG}" +else + log_step "Pulling OpenBao v${UPGRADE_VERSION} image from registry..." + docker pull "${IMAGE_TAG}" + + if [ $? -ne 0 ]; then + log_error "Failed to pull image ${IMAGE_TAG} — ensure the image has been built and pushed by the nvcf-openbao pipeline" + trap - EXIT + docker context use ${ORIGINAL_CONTEXT} > /dev/null 2>&1 || true + exit 1 + fi + log_success "Image pulled successfully (available in Colima ${COLIMA_PROFILE})" +fi + +# Step 3: Verify current deployment +log_step "Verifying current deployment..." +kubectl get pods -n vault-system + +# Step 4: Show current versions and check if upgrade is needed +log_info "Current OpenBao versions:" +CURRENT_RUNNING_VERSION="" +for i in 0 1 2; do + version=$(kubectl exec openbao-server-${i} -n vault-system -- bao version 2>/dev/null | head -1) + echo " openbao-server-${i}: ${version}" + + # Extract version number (e.g., "v2.2.2" from "OpenBao v2.2.2 (...)") + if [ -z "$CURRENT_RUNNING_VERSION" ]; then + CURRENT_RUNNING_VERSION=$(echo "$version" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+') + fi +done + +# Fail fast if version cannot be determined +if [ -z "$CURRENT_RUNNING_VERSION" ]; then + log_error "Unable to determine current OpenBao version from server pods; aborting upgrade." + exit 1 +fi + +# Check if already running target version +TARGET_VERSION="v${UPGRADE_VERSION}" +if [ "$CURRENT_RUNNING_VERSION" == "$TARGET_VERSION" ]; then + log_info "" + log_info "Cluster is already running OpenBao ${TARGET_VERSION}" + log_info "Nothing to upgrade. Exiting..." + trap - EXIT # Clear trap to avoid double restore + docker context use ${ORIGINAL_CONTEXT} > /dev/null 2>&1 || true + export DOCKER_HOST="${ORIGINAL_DOCKER_HOST}" + exit 0 +fi + +# Check if running unexpected version +EXPECTED_VERSION="v2.2.2" +if [ "$CURRENT_RUNNING_VERSION" != "$EXPECTED_VERSION" ] && [ "$CURRENT_RUNNING_VERSION" != "$TARGET_VERSION" ]; then + log_warn "" + log_warn "Warning: Cluster is running ${CURRENT_RUNNING_VERSION}, but expected ${EXPECTED_VERSION}" + log_warn "This script is designed to upgrade from ${CURRENT_VERSION} -> ${UPGRADE_VERSION}" + read -p "Do you want to continue anyway? (yes/no): " CONTINUE + if [ "$CONTINUE" != "yes" ]; then + log_info "Upgrade cancelled" + trap - EXIT # Clear trap to avoid double restore + docker context use ${ORIGINAL_CONTEXT} > /dev/null 2>&1 || true + export DOCKER_HOST="${ORIGINAL_DOCKER_HOST}" + exit 0 + fi +fi + +log_info "" +log_info "Proceeding with upgrade: ${CURRENT_RUNNING_VERSION} -> ${TARGET_VERSION}" + +# Step 5: Prepare upgrade values +log_step "Creating upgrade values file..." +log_info "Using local test tag for Helm upgrade: ${LOCAL_TEST_TAG}" +cat > /tmp/upgrade-values.yaml </nvcf-openbao + tag: ${LOCAL_TEST_TAG} + pullPolicy: IfNotPresent + + # Graceful leader stepdown before deletion (for server pods) + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - | + # Get root token from secret + ROOT_TOKEN=\$(cat /home/openbao/unseal/root_token 2>/dev/null || echo "") + if [ -z "\$ROOT_TOKEN" ]; then + echo "Warning: Root token not found, cannot step down" + exit 0 + fi + + # Check if this pod is the active leader + export BAO_ADDR="http://127.0.0.1:8200" + export BAO_TOKEN="\$ROOT_TOKEN" + + STATUS=\$(bao status -format=json 2>/dev/null || echo "{}") + HA_MODE=\$(echo "\$STATUS" | jq -r '.ha_mode // "unknown"') + + if [ "\$HA_MODE" = "active" ]; then + echo "Pod is active leader, stepping down gracefully..." + bao operator step-down || true + sleep 5 + echo "Leader stepdown complete" + else + echo "Pod is not leader (mode: \$HA_MODE), no stepdown needed" + fi + + # Update auto-unseal sidecar image (for server pods) + extraContainers: + - name: auto-unseal-sidecar + image: ${IMAGE_NAME}:${LOCAL_TEST_TAG} + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: openbao-server-unseal + mountPath: /vault/userconfig/unseal + readOnly: true + command: ["/bin/sh", "-c"] + args: + - | + echo "Starting auto-unseal monitor..." + echo http://\$HOSTNAME:8200 + export BAO_ADDR=http://\$HOSTNAME:8200 + while true; do + if [ -f /vault/userconfig/unseal/unseal_key ]; then + UNSEAL_KEY=\$(cat /vault/userconfig/unseal/unseal_key) + if [ ! -z "\$UNSEAL_KEY" ]; then + bao operator unseal \$UNSEAL_KEY + sleep 60 + continue + else + echo "Unseal key is empty, waiting..." + fi + else + echo "Unseal key file not found, waiting..." + fi + sleep 10 + done + + # Update injector agent image (image used when injecting sidecars into app pods) + injector: + # Disable anti-affinity for single-node clusters (allows rolling update to work) + affinity: "" + agentImage: + registry: nvcr.io + repository: /nvcf-openbao + tag: ${LOCAL_TEST_TAG} + pullPolicy: IfNotPresent +EOF + +log_success "Upgrade values created at /tmp/upgrade-values.yaml" + +# Step 6: Instructions before upgrade +log_section "Ready to upgrade!" +echo "" +echo "Before pressing Enter, open separate terminals for monitoring:" +echo "" +echo " Terminal 1: ./tests/openbao/loadbalancer_ha_kv_rw_check.sh ${COLIMA_PROFILE}" +echo " (Keep running - shows KV read/write health during upgrade)" +echo "" +echo " Terminal 2: ./tests/openbao/loadbalancer_ha_seal_status_check.sh ${COLIMA_PROFILE}" +echo " (Keep running - shows seal status during upgrade)" +echo "" +echo " Terminal 3: kubectl get pods -n vault-system -w" +echo " (Optional - watch pods restart in real-time)" +echo "" +echo "Expected behavior during upgrade:" +echo " - LoadBalancer tests show <1% errors (routes to healthy pods)" +echo " - Pods restart one at a time (rolling update: 2 → 1 → 0)" +echo " - Leader may change during upgrade" +echo " - Seal status stays false throughout" +echo "" + +read -p "Press Enter to start the upgrade, or Ctrl+C to cancel..." + +# Step 7: Perform upgrade +log_step "Performing Helm upgrade..." +cd "$REPO_ROOT" +helm upgrade openbao-server helm -n vault-system \ + --values helm/values.yaml \ + --values values.local.yaml \ + --values /tmp/upgrade-values.yaml \ + --force-conflicts \ + --wait --wait-for-jobs --timeout 20m + +if [ $? -ne 0 ]; then + log_error "Upgrade failed!" + exit 1 +fi + +log_success "Upgrade completed!" + +# Step 8: Verify +log_step "Verifying upgraded versions..." +for i in 0 1 2; do + version=$(kubectl exec openbao-server-${i} -n vault-system -- bao version 2>/dev/null | head -1) + log_info "openbao-server-${i}: ${version}" +done + +# Step 9: Restore original Docker context and DOCKER_HOST +log_step "Restoring Docker context to ${ORIGINAL_CONTEXT}..." +trap - EXIT # Clear trap to avoid double restore +docker context use ${ORIGINAL_CONTEXT} > /dev/null 2>&1 || true +export DOCKER_HOST="${ORIGINAL_DOCKER_HOST}" + +# Step 10: Show files that need manual update +log_step "Files to update for version ${UPGRADE_VERSION}" +echo "" +echo " The following files contain version references that should be updated:" +echo "" + +NEW_VERSION="${UPGRADE_VERSION}" +OLD_VERSION="${CURRENT_VERSION}" + +cd "$REPO_ROOT" +FILES_NEEDING_UPDATE="" + +# Check each file and show what needs updating +check_file() { + local file=$1 + local desc=$2 + if [ -f "$file" ]; then + if grep -q "$OLD_VERSION" "$file"; then + echo " - $file" + echo " Purpose: $desc" + echo " Update: ${OLD_VERSION} → ${NEW_VERSION}" + echo "" + FILES_NEEDING_UPDATE="${FILES_NEEDING_UPDATE} ${file}" + fi + fi +} + +check_file "helm/Chart.yaml" "Helm chart appVersion (shown in 'helm list')" +check_file "helm/values.yaml" "Default image tags for production deployments" +check_file "values.local.yaml" "Local development image tags" +check_file "service-migrations/Dockerfile" "Base image for migration jobs" + +if [ -z "$FILES_NEEDING_UPDATE" ]; then + log_success "All files already at version ${NEW_VERSION}" +else + echo "" + log_warn "ACTION REQUIRED: Manually update the files above and create an MR" +fi + +log_success "Upgrade test completed successfully!" +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " SUMMARY" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo " ✅ Local upgrade test passed: ${CURRENT_VERSION} → ${LOCAL_TEST_TAG}" +echo "" +echo " Next steps to complete the upgrade:" +echo "" +echo " 1. Merge the nvcf-openbao MR to publish the ${UPGRADE_VERSION}-nv-1 image" +echo "" +echo " 2. Update version references in repository files (see above)" +echo "" +echo " 3. Create MR with changes in k8s-openbao:" +echo " git checkout -b upgrade/openbao-${UPGRADE_VERSION}" +echo " git add -A" +echo " git commit -m 'feat: upgrade OpenBao from ${CURRENT_VERSION} to ${UPGRADE_VERSION}-nv-1'" +echo " git push -u origin HEAD" +echo "" +echo " 4. After MR merge, deploy to environments" +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/deploy/helm/openbao/upgrade/values-upgrades.yaml b/deploy/helm/openbao/upgrade/values-upgrades.yaml new file mode 100644 index 000000000..ef31fa116 --- /dev/null +++ b/deploy/helm/openbao/upgrade/values-upgrades.yaml @@ -0,0 +1,48 @@ +# OpenBao upgrade values +# Usage: make install additional_values=values.local.yaml,upgrade/values-upgrades.yaml + +openbao: + server: + image: + tag: 2.5.5-nv-1.3.1 + + # Update auto-unseal sidecar to match server version + extraContainers: + - name: auto-unseal-sidecar + image: '{{ .Values.server.image.registry }}/{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default (trimPrefix "v" .Chart.AppVersion) }}' + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: openbao-server-unseal + mountPath: /vault/userconfig/unseal + readOnly: true + command: ["/bin/sh", "-c"] + args: + - | + echo "Starting auto-unseal monitor..." + echo http://$HOSTNAME:8200 + export BAO_ADDR=http://$HOSTNAME:8200 + while true; do + if [ -f /vault/userconfig/unseal/unseal_key ]; then + UNSEAL_KEY=$(cat /vault/userconfig/unseal/unseal_key) + if [ ! -z "$UNSEAL_KEY" ]; then + bao operator unseal $UNSEAL_KEY + sleep 60 + continue + else + echo "Unseal key is empty, waiting..." + fi + else + echo "Unseal key file not found, waiting..." + fi + sleep 10 + done + + injector: + agentImage: + tag: 2.5.5-nv-1.3.1 diff --git a/deploy/helm/openbao/utils/lb.sh b/deploy/helm/openbao/utils/lb.sh new file mode 100755 index 000000000..ec5213917 --- /dev/null +++ b/deploy/helm/openbao/utils/lb.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ ! -f "${SCRIPT_DIR}/utils.sh" ]; then + echo "Error: utils.sh not found in ${SCRIPT_DIR}" + exit 1 +fi +source "$SCRIPT_DIR/utils.sh" + +if ! check_kubernetes; then + log_error "kubernetes check failed" + exit 1 +fi + +configure_kube_proxy() { + log_step "Configuring kube-proxy..." + + # Check if ConfigMap exists + if ! kubectl get configmap kube-proxy -n kube-system >/dev/null 2>&1; then + log_info "Creating kube-proxy ConfigMap..." + cat < /tmp/kube-proxy.yaml; then + log_error "Failed to get existing kube-proxy ConfigMap" + return 1 + fi + + if ! sed -i.bak -e "s/strictARP: false/strictARP: true/" /tmp/kube-proxy.yaml; then + log_error "Failed to modify kube-proxy configuration" + return 1 + fi + + if ! kubectl apply -f /tmp/kube-proxy.yaml; then + log_error "Failed to apply updated kube-proxy ConfigMap" + return 1 + fi + + rm -f /tmp/kube-proxy.yaml /tmp/kube-proxy.yaml.bak + fi + + log_success "kube-proxy configuration completed" + return 0 +} + +setup_metallb() { + local statefulset=$1 + local metallb_version="v0.14.9" + local pool_start="192.168.5.240" + local pool_end="192.168.5.250" + local pool_name="openbao-server-pool" + local l2_name="${pool_name}-l2" + + log_section "Setting up MetalLB ..." + + # Configure kube-proxy before MetalLB installation + if ! configure_kube_proxy; then + log_error "Failed to configure kube-proxy" + return 1 + fi + + log_step "Installing MetalLB version ${metallb_version}..." + if ! kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/${metallb_version}/config/manifests/metallb-native.yaml; then + log_error "Failed to install MetalLB" + return 1 + fi + + log_step "Waiting for MetalLB components to be ready..." + if ! kubectl wait --namespace metallb-system \ + --for=condition=ready pod \ + --selector app=metallb \ + --timeout=90s; then + log_error "MetalLB pods failed to become ready" + return 1 + fi + + log_step "Configuring kube-proxy for strict ARP..." + if ! kubectl get configmap kube-proxy -n kube-system -o yaml | \ + sed -e "s/strictARP: false/strictARP: true/" | \ + kubectl apply -f - -n kube-system; then + log_error "Failed to configure kube-proxy" + return 1 + fi + + log_step "Creating MetalLB IP address pool..." + kubectl apply -f - </dev/null 2>&1; then + log_error "IP address pool not configured properly" + return 1 + fi + + if ! kubectl get l2advertisements -n metallb-system ${l2_name} >/dev/null 2>&1; then + log_error "L2 advertisement not configured properly" + return 1 + fi + + log_step "Creating LoadBalancer service for ${statefulset}..." + kubectl apply -f - </dev/null; then + log_info "LoadBalancer service not found or already removed" + fi + + # Remove L2 advertisements + log_info "Removing L2 advertisements..." + if ! kubectl delete l2advertisements.metallb.io -n metallb-system --all; then + log_error "Failed to remove L2 advertisements" + return 1 + fi + + # Remove IP address pools + log_info "Removing IP address pools..." + if ! kubectl delete ipaddresspools.metallb.io -n metallb-system --all; then + log_error "Failed to remove IP address pools" + return 1 + fi + + # Remove MetalLB installation + log_info "Removing MetalLB installation..." + if ! kubectl delete -f https://raw.githubusercontent.com/metallb/metallb/v0.14.9/config/manifests/metallb-native.yaml; then + log_error "Failed to remove MetalLB installation" + return 1 + fi + + log_success "MetalLB cleanup completed successfully" + return 0 +} + +# Usage +statefulset="openbao-server" + +if [[ "$1" == "--cleanup" ]]; then + if ! cleanup_metallb $statefulset; then + log_error "MetalLB cleanup failed" + exit 1 + fi +else + if ! setup_metallb $statefulset; then + log_error "MetalLB setup failed" + exit 1 + fi +fi diff --git a/deploy/helm/openbao/utils/log.sh b/deploy/helm/openbao/utils/log.sh new file mode 100755 index 000000000..6a2ccab6e --- /dev/null +++ b/deploy/helm/openbao/utils/log.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Timestamp format +timestamp() { + date '+%Y-%m-%d %H:%M:%S' +} + +# Log levels with emojis +log_info() { + echo -e "${GREEN}[INFO]${NC} ℹ️ $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} ⚠️ $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} ❌ $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} 🚀 $1" +} + +log_success() { + echo -e "${GREEN}[DONE]${NC} ✅ $1" +} + +# Section separator (no timestamp) +log_section() { + echo -e "\n${BLUE}=================================${NC}" + echo -e "${BLUE} $1${NC}" + echo -e "${BLUE}=================================${NC}\n" +} \ No newline at end of file diff --git a/deploy/helm/openbao/utils/utils.sh b/deploy/helm/openbao/utils/utils.sh new file mode 100755 index 000000000..c2cb791b5 --- /dev/null +++ b/deploy/helm/openbao/utils/utils.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +UTILS_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ ! -f "${UTILS_SCRIPT_DIR}/log.sh" ]; then + echo "Error: log.sh not found in ${UTILS_SCRIPT_DIR}" + exit 1 +fi +source "${UTILS_SCRIPT_DIR}/log.sh" + +get_root_token() { + local namespace=$1 + local statefulset=$2 + kubectl get secret $statefulset-root-token -n $namespace -o jsonpath='{.data.root_token}' | base64 -d +} + +check_docker() { + if ! command -v docker &>/dev/null; then + log_error "docker is not installed" + return 1 + fi + return 0 +} + +check_jq() { + if ! command -v jq &>/dev/null; then + log_error "jq is not installed" + return 1 + fi + return 0 +} + +check_helm() { + if ! command -v helm &>/dev/null; then + log_error "Failed to retrieve Helm version. Please ensure Helm is installed." + return 1 + fi + + helm_version=$(helm version --template="{{.Version}}" | tr -d "v") + if ! version_gt "$helm_version" "3.12.0"; then + log_error "Helm version 3.12+ is required, found version: $helm_version" + return 1 + fi + return 0 +} + +version_gt() { + local version1=$1 + local version2=$2 + if [[ "$(printf '%s\n' "$version2" "$version1" | sort -V | head -n1)" != "$version1" ]]; then + return 0 + fi + return 1 +} + +check_kubernetes() { + if ! command -v kubectl &>/dev/null; then + log_error "kubectl is not installed" + return 1 + fi + + k8s_version=$(kubectl version --output=json | jq -r '.serverVersion.gitVersion' | tr -d "v") + if ! version_gt "$k8s_version" "1.29.0"; then + log_error "Kubernetes version 1.29+ is required, found version: $k8s_version" + return 1 + fi + return 0 +} + +check_colima() { + if ! command -v colima &>/dev/null; then + log_error "Colima is not installed" + return 1 + fi + return 0 +} + +check_resources() { + local namespace=$1 + local statefulset=$2 + + # Check if kubernetes version is >= 1.29+ + if ! check_kubernetes; then + log_error "Kubernetes check failed" + exit 1 + fi + + # Check if namespace exists + if ! kubectl get namespace "$namespace" &>/dev/null; then + log_error "Namespace '$namespace' not found" + return 1 + fi + + # Check if pod exists and is running + if ! kubectl get pod "$statefulset-0" -n "$namespace" &>/dev/null; then + log_error "Pod '$statefulset-0' not found in namespace '$namespace'" + return 1 + fi + + pod_status=$(kubectl get pod "$statefulset-0" -n "$namespace" -o jsonpath='{.status.phase}') + if [ "$pod_status" != "Running" ]; then + log_error "Pod '$statefulset-0' is not running (status: $pod_status)" + return 1 + fi + + # Check if root token secret exists + if ! kubectl get secret "$statefulset-root-token" -n "$namespace" &>/dev/null; then + log_error "Secret '$statefulset-root-token' not found in namespace '$namespace'" + return 1 + fi + + return 0 +} diff --git a/deploy/helm/openbao/values.local.yaml b/deploy/helm/openbao/values.local.yaml new file mode 100644 index 000000000..a9a48c6a8 --- /dev/null +++ b/deploy/helm/openbao/values.local.yaml @@ -0,0 +1,55 @@ +openbao: + + # Enable if needed + global: + imagePullSecrets: + - name: nvcr-secret + + migrations: + image: + registry: "" + repository: "" + pullPolicy: IfNotPresent + env: + - name: DEFAULT_CASSANDRA_PASSWORD + value: "ch@ng3m3" + - name: NVCF_API_SIDECARS_IMAGE_PULL_SECRET + value: "your-actual-image-pull-secret-value" # Replace with real value + + server: + image: + registry: "" + repository: "" + pullPolicy: IfNotPresent + + # Keep HA and 3 replicas + ha: + enabled: true + replicas: 3 + + # Override anti-affinity for single-node testing (allows all 3 pods on 1 node) + # Use soft affinity instead of hard requirement + affinity: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "openbao.name" . }} + app.kubernetes.io/instance: "{{ .Release.Name }}" + component: server + topologyKey: kubernetes.io/hostname + + injector: + # Disable anti-affinity for single-node clusters (allows rolling updates to work) + affinity: "" + image: + registry: "" + repository: "" + tag: 1.7.4 + pullPolicy: IfNotPresent + agentImage: + registry: "" + repository: "" + pullPolicy: IfNotPresent diff --git a/tools/ci/helm-validate-values/admin-token-issuer-proxy.yaml b/tools/ci/helm-validate-values/admin-token-issuer-proxy.yaml new file mode 100644 index 000000000..e1a05a8ca --- /dev/null +++ b/tools/ci/helm-validate-values/admin-token-issuer-proxy.yaml @@ -0,0 +1,5 @@ +# CI-only values for helm lint/template validation. +adminIssuerProxy: + image: + registry: example.com + repository: foo/bar/baz diff --git a/tools/ci/helm-validate-values/api-keys-colocated.yaml b/tools/ci/helm-validate-values/api-keys-colocated.yaml new file mode 100644 index 000000000..ecdba4069 --- /dev/null +++ b/tools/ci/helm-validate-values/api-keys-colocated.yaml @@ -0,0 +1,5 @@ +# CI-only values for helm lint/template validation. +apikeys: + image: + registry: example.com + repository: foo/bar/baz diff --git a/tools/ci/helm-validate-values/cassandra.yaml b/tools/ci/helm-validate-values/cassandra.yaml new file mode 100644 index 000000000..c2cd99e71 --- /dev/null +++ b/tools/ci/helm-validate-values/cassandra.yaml @@ -0,0 +1,4 @@ +# CI-only values for helm lint/template validation. +global: + security: + allowInsecureImages: true diff --git a/tools/ci/helm-validate-values/gateway-routes-vanity.yaml b/tools/ci/helm-validate-values/gateway-routes-vanity.yaml new file mode 100644 index 000000000..86da33f10 --- /dev/null +++ b/tools/ci/helm-validate-values/gateway-routes-vanity.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nvcfGatewayRoutes: + routes: + vanityGateway: + enabled: true + name: ci-vanity-gateway + hostnames: + - ci-vanity.localhost + backend: + name: ci-vanity-backend + namespace: ci-vanity + port: 9090 + routeAnnotations: + ci.nvidia.com/validated: enabled diff --git a/tools/ci/helm-validate-values/grpc-proxy.yaml b/tools/ci/helm-validate-values/grpc-proxy.yaml new file mode 100644 index 000000000..71404646c --- /dev/null +++ b/tools/ci/helm-validate-values/grpc-proxy.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +grpcproxy: + image: + registry: example.com + repository: foo/bar/baz + deploymentType: deployment diff --git a/tools/ci/helm-validate-values/llm-api-gateway.yaml b/tools/ci/helm-validate-values/llm-api-gateway.yaml new file mode 100644 index 000000000..64b6027fe --- /dev/null +++ b/tools/ci/helm-validate-values/llm-api-gateway.yaml @@ -0,0 +1,7 @@ +# CI-only values for helm lint/template validation. +llmApiGateway: + image: + registry: example.com + repository: foo/bar/lpu-nvcf-gateway + config: + requestRouterUrl: http://llm-request-router-headless.nvcf.svc.cluster.local:8000 diff --git a/tools/ci/helm-validate-values/llm-request-router.yaml b/tools/ci/helm-validate-values/llm-request-router.yaml new file mode 100644 index 000000000..bd295c345 --- /dev/null +++ b/tools/ci/helm-validate-values/llm-request-router.yaml @@ -0,0 +1,7 @@ +# CI-only values for helm lint/template validation. +llmRequestRouter: + image: + registry: example.com + repository: foo/bar/stargate + discovery: + dnsName: llm-request-router-headless.nvcf.svc.cluster.local diff --git a/tools/ci/helm-validate-values/nvca-operator.yaml b/tools/ci/helm-validate-values/nvca-operator.yaml new file mode 100644 index 000000000..e4f1cf10a --- /dev/null +++ b/tools/ci/helm-validate-values/nvca-operator.yaml @@ -0,0 +1,27 @@ +# CI-only values for helm lint/template validation. +image: + repository: example.com/nvcf/nvca-operator + tag: "3.0.4" + +nvcaImage: + repositoryOverride: example.com/nvcf/nvca + +ngcConfig: + clusterSource: self-managed + serviceKey: not-used + +selfManaged: + icmsServiceURL: http://icms.example.invalid:8080 + revalServiceURL: http://reval.example.invalid:8080 + natsURL: nats://nats.example.invalid:4222 + nvcaVersion: "3.0.4" + featureGateValues: ["DynamicGPUDiscovery", "SelfHosted"] + imageCredHelper: + imageRepository: example.com/nvcf/nvcf-image-credential-helper + imageTag: "0.5.0" + sharedStorage: + imageRepository: example.com/nvcf/samba + imageTag: "1.0.5" + +generateImagePullSecret: false +imagePullSecrets: [] diff --git a/tools/ci/helm-validate-values/openbao.yaml b/tools/ci/helm-validate-values/openbao.yaml new file mode 100644 index 000000000..0c0d58e72 --- /dev/null +++ b/tools/ci/helm-validate-values/openbao.yaml @@ -0,0 +1,11 @@ +# CI-only values for helm lint/template validation. +global: + security: + allowInsecureImages: true +openbao: + global: + imagePullSecrets: [] + migrations: + image: + registry: example.com + repository: nvcf/nvcf-openbao-migrations diff --git a/tools/ci/helm-validate-values/ratelimiter.yaml b/tools/ci/helm-validate-values/ratelimiter.yaml new file mode 100644 index 000000000..13c976a2a --- /dev/null +++ b/tools/ci/helm-validate-values/ratelimiter.yaml @@ -0,0 +1,8 @@ +# CI-only values for helm lint/template validation. +rateLimiter: + image: + registry: example.com + repository: foo/bar/nvcf-ratelimiter + env: + OAUTH2_ISSUER: https://issuer.example.com + AUDIENCE: ratelimit:check_invocation