From cdda734dcb4c3b530c4d6cd765272135bc82ce95 Mon Sep 17 00:00:00 2001 From: Ankit Kr Chowdhury Date: Sat, 2 Aug 2025 15:23:56 +0530 Subject: [PATCH] fix(topology): support unique helm values per worker cluster Signed-off-by: Ankit Kr Chowdhury --- go.mod | 3 - go.sum | 3 - pkg/internal/bootstrap-configuration.go | 11 +- pkg/internal/worker.go | 30 ++- pkg/internal/worker_integration_test.go | 338 ++++++++++++++++++++++++ pkg/internal/worker_test.go | 287 ++++++++++++++++++++ samples/per-worker-helm-values.yaml | 79 ++++++ samples/template.yaml | 10 + 8 files changed, 749 insertions(+), 12 deletions(-) create mode 100644 pkg/internal/worker_integration_test.go create mode 100644 pkg/internal/worker_test.go create mode 100644 samples/per-worker-helm-values.yaml diff --git a/go.mod b/go.mod index 005627b..81667c2 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,9 @@ require ( ) require ( - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tidwall/gjson v1.14.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index c8e91cf..07afaf7 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -6,7 +5,6 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= @@ -25,7 +23,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/internal/bootstrap-configuration.go b/pkg/internal/bootstrap-configuration.go index 3cb5300..f8db62b 100644 --- a/pkg/internal/bootstrap-configuration.go +++ b/pkg/internal/bootstrap-configuration.go @@ -45,11 +45,12 @@ type ClusterConfiguration struct { } type Cluster struct { - Name string `yaml:"name"` - ContextName string `yaml:"context_name"` - KubeConfigPath string `yaml:"kube_config_path"` - ControlPlaneAddress string `yaml:"control_plane_address"` - NodeIP string `yaml:"node_ip"` + Name string `yaml:"name"` + ContextName string `yaml:"context_name"` + KubeConfigPath string `yaml:"kube_config_path"` + ControlPlaneAddress string `yaml:"control_plane_address"` + NodeIP string `yaml:"node_ip"` + HelmValues map[string]interface{} `yaml:"helm_values,omitempty"` } type ImagePullSecrets struct { diff --git a/pkg/internal/worker.go b/pkg/internal/worker.go index ba0fe45..ad6cae0 100644 --- a/pkg/internal/worker.go +++ b/pkg/internal/worker.go @@ -84,6 +84,30 @@ func Retry(backoffLimit int, sleep time.Duration, f func() error) (err error) { return fmt.Errorf("retry failed after %d attempts (took %d seconds), last error: %s", backoffLimit, int(elapsed.Seconds()), err) } +// CreateWorkerSpecificHelmChart merges global worker chart values with cluster-specific values +func CreateWorkerSpecificHelmChart(globalChart HelmChart, cluster Cluster) HelmChart { + // Start with a copy of the global chart + workerChart := HelmChart{ + ChartName: globalChart.ChartName, + Version: globalChart.Version, + Values: make(map[string]interface{}), + } + + // Copy global values first + for k, v := range globalChart.Values { + workerChart.Values[k] = v + } + + // Override with cluster-specific values if they exist + if cluster.HelmValues != nil { + for k, v := range cluster.HelmValues { + workerChart.Values[k] = v + } + } + + return workerChart +} + func generateWorkerValuesFile(cluster Cluster, valuesFile string, config Configuration, insecureMetrics bool) { var secrets map[string]string err := Retry(3, 1*time.Second, func() (err error) { @@ -96,7 +120,11 @@ func generateWorkerValuesFile(cluster Cluster, valuesFile string, config Configu if err != nil { log.Fatalf("Unable to fetch secrets\n%s", err) } - err = generateValuesFile(kubesliceDirectory+"/"+valuesFile, &config.HelmChartConfiguration.WorkerChart, fmt.Sprintf(workerValuesTemplate+generateImagePullSecretsValue(config.HelmChartConfiguration.ImagePullSecret), secrets["namespace"], secrets["controllerEndpoint"], secrets["ca.crt"], secrets["token"], insecureMetrics, cluster.Name, cluster.ControlPlaneAddress)) + + // Create a worker-specific helm chart configuration + workerChart := CreateWorkerSpecificHelmChart(config.HelmChartConfiguration.WorkerChart, cluster) + + err = generateValuesFile(kubesliceDirectory+"/"+valuesFile, &workerChart, fmt.Sprintf(workerValuesTemplate+generateImagePullSecretsValue(config.HelmChartConfiguration.ImagePullSecret), secrets["namespace"], secrets["controllerEndpoint"], secrets["ca.crt"], secrets["token"], insecureMetrics, cluster.Name, cluster.ControlPlaneAddress)) if err != nil { log.Fatalf("%s %s", util.Cross, err) } diff --git a/pkg/internal/worker_integration_test.go b/pkg/internal/worker_integration_test.go new file mode 100644 index 0000000..360cbb6 --- /dev/null +++ b/pkg/internal/worker_integration_test.go @@ -0,0 +1,338 @@ +package internal + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func assertValuesEqual(t *testing.T, got, want map[string]interface{}) { + t.Helper() + + if !reflect.DeepEqual(got, want) { + t.Errorf("Values mismatch:\nGot: %v\nWant: %v", got, want) + } +} + +func assertValueExistsIntegration(t *testing.T, values map[string]interface{}, key string, expectedValue interface{}) { + t.Helper() + + if actualValue, exists := values[key]; !exists { + t.Errorf("Expected key %s not found in values", key) + } else if actualValue != expectedValue { + t.Errorf("Value for %s = %v, want %v", key, actualValue, expectedValue) + } +} + +func TestWorkerChartCreation_Integration(t *testing.T) { + globalValues := map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + } + + tests := []struct { + name string + cluster Cluster + expectedValues map[string]interface{} + }{ + { + name: "Standard worker with global values only", + cluster: Cluster{ + Name: "standard-worker", + ContextName: "standard-context", + HelmValues: nil, + }, + expectedValues: globalValues, + }, + { + name: "OpenShift worker with overrides", + cluster: Cluster{ + Name: "openshift-worker", + ContextName: "openshift-context", + HelmValues: map[string]interface{}{ + "global.profile.openshift": true, + "nodeSelector.zone": "us-west-1a", + }, + }, + expectedValues: map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": true, // Overridden + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "nodeSelector.zone": "us-west-1a", // New + }, + }, + { + name: "High-performance worker with resource overrides", + cluster: Cluster{ + Name: "high-perf-worker", + ContextName: "high-perf-context", + HelmValues: map[string]interface{}{ + "resources.limits.cpu": "2000m", + "resources.limits.memory": "4Gi", + "nodeSelector.performance": "high", + }, + }, + expectedValues: map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "2000m", // Overridden + "resources.limits.memory": "4Gi", // Overridden + "nodeSelector.performance": "high", // New + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + globalChart := HelmChart{ + ChartName: "kubeslice-worker", + Version: "1.0.0", + Values: globalValues, + } + + workerChart := CreateWorkerSpecificHelmChart(globalChart, tt.cluster) + assertValuesEqual(t, workerChart.Values, tt.expectedValues) + }) + } +} + +func TestConfigurationParsing_WithPerWorkerValues(t *testing.T) { + yamlContent := ` +configuration: + cluster_configuration: + controller: + name: controller + context_name: controller-context + workers: + - name: standard-worker + context_name: standard-worker-context + - name: openshift-worker + context_name: openshift-worker-context + helm_values: + global.profile.openshift: true + nodeSelector.zone: us-west-1a + - name: high-perf-worker + context_name: high-perf-worker-context + helm_values: + resources.limits.cpu: "2000m" + resources.limits.memory: "4Gi" + nodeSelector.performance: "high" + kubeslice_configuration: + project_name: test-project + helm_chart_configuration: + repo_alias: kubeslice + worker_chart: + chart_name: kubeslice-worker + values: + kubesliceNetworking.enabled: true + global.profile.openshift: false + resources.limits.cpu: "500m" + resources.limits.memory: "512Mi" +` + + var config ConfigurationSpecs + err := yaml.Unmarshal([]byte(yamlContent), &config) + if err != nil { + t.Fatalf("Failed to parse YAML: %v", err) + } + + t.Run("Configuration parsing", func(t *testing.T) { + if len(config.Configuration.ClusterConfiguration.WorkerClusters) != 3 { + t.Errorf("Expected 3 worker clusters, got %d", len(config.Configuration.ClusterConfiguration.WorkerClusters)) + } + }) + + t.Run("Standard worker has no helm values", func(t *testing.T) { + worker := config.Configuration.ClusterConfiguration.WorkerClusters[0] + if worker.Name != "standard-worker" { + t.Errorf("Expected worker name 'standard-worker', got '%s'", worker.Name) + } + if worker.HelmValues != nil { + t.Errorf("Expected standard worker HelmValues to be nil, got %v", worker.HelmValues) + } + }) + + t.Run("OpenShift worker has correct helm values", func(t *testing.T) { + worker := config.Configuration.ClusterConfiguration.WorkerClusters[1] + if worker.Name != "openshift-worker" { + t.Errorf("Expected worker name 'openshift-worker', got '%s'", worker.Name) + } + + if worker.HelmValues == nil { + t.Fatal("Expected openshift worker to have helm_values, but it was nil") + } + + assertValueExistsIntegration(t, worker.HelmValues, "global.profile.openshift", true) + assertValueExistsIntegration(t, worker.HelmValues, "nodeSelector.zone", "us-west-1a") + }) + + t.Run("High-performance worker has correct helm values", func(t *testing.T) { + worker := config.Configuration.ClusterConfiguration.WorkerClusters[2] + if worker.Name != "high-perf-worker" { + t.Errorf("Expected worker name 'high-perf-worker', got '%s'", worker.Name) + } + + if worker.HelmValues == nil { + t.Fatal("Expected high-perf worker to have helm_values, but it was nil") + } + + assertValueExistsIntegration(t, worker.HelmValues, "resources.limits.cpu", "2000m") + assertValueExistsIntegration(t, worker.HelmValues, "resources.limits.memory", "4Gi") + assertValueExistsIntegration(t, worker.HelmValues, "nodeSelector.performance", "high") + }) +} + +func TestBackwardCompatibility_ExistingConfigurations(t *testing.T) { + yamlContent := ` +configuration: + cluster_configuration: + controller: + name: controller + context_name: controller-context + workers: + - name: legacy-worker + context_name: legacy-context + kubeslice_configuration: + project_name: test-project + helm_chart_configuration: + repo_alias: kubeslice + worker_chart: + chart_name: kubeslice-worker + values: + kubesliceNetworking.enabled: true + global.profile.openshift: false +` + + var config ConfigurationSpecs + err := yaml.Unmarshal([]byte(yamlContent), &config) + if err != nil { + t.Fatalf("Failed to parse YAML: %v", err) + } + + t.Run("Legacy configuration parsing", func(t *testing.T) { + if len(config.Configuration.ClusterConfiguration.WorkerClusters) != 1 { + t.Errorf("Expected 1 worker cluster, got %d", len(config.Configuration.ClusterConfiguration.WorkerClusters)) + } + + worker := config.Configuration.ClusterConfiguration.WorkerClusters[0] + if worker.Name != "legacy-worker" { + t.Errorf("Expected worker name 'legacy-worker', got '%s'", worker.Name) + } + + if worker.HelmValues != nil { + t.Errorf("Expected legacy worker HelmValues to be nil, got %v", worker.HelmValues) + } + }) + + t.Run("Legacy worker chart creation", func(t *testing.T) { + globalChart := config.Configuration.HelmChartConfiguration.WorkerChart + worker := config.Configuration.ClusterConfiguration.WorkerClusters[0] + + workerChart := CreateWorkerSpecificHelmChart(globalChart, worker) + + expectedValues := map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + } + + assertValuesEqual(t, workerChart.Values, expectedValues) + }) +} + +func TestMultiWorkerScenario_Integration(t *testing.T) { + t.Run("Multiple workers with different configurations", func(t *testing.T) { + globalValues := map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "replicaCount": 2, + } + + globalChart := HelmChart{ + ChartName: "kubeslice-worker", + Version: "1.0.0", + Values: globalValues, + } + + workers := []struct { + cluster Cluster + expectedValues map[string]interface{} + }{ + { + cluster: Cluster{ + Name: "standard-worker", + ContextName: "standard-context", + HelmValues: nil, + }, + expectedValues: globalValues, + }, + { + cluster: Cluster{ + Name: "openshift-worker", + ContextName: "openshift-context", + HelmValues: map[string]interface{}{ + "global.profile.openshift": true, + "nodeSelector.zone": "us-east-1a", + }, + }, + expectedValues: map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": true, // Overridden + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "replicaCount": 2, + "nodeSelector.zone": "us-east-1a", // New + }, + }, + { + cluster: Cluster{ + Name: "gpu-worker", + ContextName: "gpu-context", + HelmValues: map[string]interface{}{ + "nodeSelector.accelerator": "nvidia-tesla-k80", + "resources.limits.nvidia.com/gpu": 1, + }, + }, + expectedValues: map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "replicaCount": 2, + "nodeSelector.accelerator": "nvidia-tesla-k80", // New + "resources.limits.nvidia.com/gpu": 1, // New + }, + }, + { + cluster: Cluster{ + Name: "edge-worker", + ContextName: "edge-context", + HelmValues: map[string]interface{}{ + "resources.limits.cpu": "200m", + "resources.limits.memory": "256Mi", + "replicaCount": 1, + }, + }, + expectedValues: map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "200m", // Overridden + "resources.limits.memory": "256Mi", // Overridden + "replicaCount": 1, // Overridden + }, + }, + } + + for _, w := range workers { + t.Run("Worker_"+w.cluster.Name, func(t *testing.T) { + workerChart := CreateWorkerSpecificHelmChart(globalChart, w.cluster) + assertValuesEqual(t, workerChart.Values, w.expectedValues) + }) + } + }) +} \ No newline at end of file diff --git a/pkg/internal/worker_test.go b/pkg/internal/worker_test.go new file mode 100644 index 0000000..b86a124 --- /dev/null +++ b/pkg/internal/worker_test.go @@ -0,0 +1,287 @@ +package internal + +import ( + "reflect" + "testing" +) + +// Test data helpers +func testHelmChart(chartName, version string, values map[string]interface{}) HelmChart { + if values == nil { + values = make(map[string]interface{}) + } + return HelmChart{ + ChartName: chartName, + Version: version, + Values: values, + } +} + +func testCluster(name, contextName string, helmValues map[string]interface{}) Cluster { + return Cluster{ + Name: name, + ContextName: contextName, + KubeConfigPath: "/test/kubeconfig", + ControlPlaneAddress: "https://api." + name + ".local", + NodeIP: "10.0.0.1", + HelmValues: helmValues, + } +} + +func globalValues() map[string]interface{} { + return map[string]interface{}{ + "kubesliceNetworking.enabled": true, + "global.profile.openshift": false, + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "resources.requests.cpu": "100m", + "resources.requests.memory": "128Mi", + "replicaCount": 2, + "image.pullPolicy": "IfNotPresent", + } +} + +func openShiftWorkerValues() map[string]interface{} { + return map[string]interface{}{ + "global.profile.openshift": true, + "nodeSelector.node-role.kubernetes.io/worker": "", + "tolerations[0].key": "node-role.kubernetes.io/master", + "tolerations[0].effect": "NoSchedule", + } +} + +func highPerformanceWorkerValues() map[string]interface{} { + return map[string]interface{}{ + "resources.limits.cpu": "2000m", + "resources.limits.memory": "4Gi", + "resources.requests.cpu": "1000m", + "resources.requests.memory": "2Gi", + "nodeSelector.performance": "high", + "nodeSelector.zone": "us-west-1a", + "replicaCount": 3, + } +} + +func gpuWorkerValues() map[string]interface{} { + return map[string]interface{}{ + "nodeSelector.accelerator": "nvidia-tesla-k80", + "tolerations[0].key": "nvidia.com/gpu", + "tolerations[0].operator": "Exists", + "tolerations[0].effect": "NoSchedule", + "resources.limits.nvidia.com/gpu": 1, + } +} + +func edgeWorkerValues() map[string]interface{} { + return map[string]interface{}{ + "resources.limits.cpu": "200m", + "resources.limits.memory": "256Mi", + "resources.requests.cpu": "100m", + "resources.requests.memory": "128Mi", + "nodeSelector.node-type": "edge", + "replicaCount": 1, + } +} + +func mergeValues(base, override map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + // Copy base values + for k, v := range base { + result[k] = v + } + + // Override with new values + for k, v := range override { + result[k] = v + } + + return result +} + +func assertHelmChartEqual(t *testing.T, got, want HelmChart) { + t.Helper() + + if got.ChartName != want.ChartName { + t.Errorf("ChartName = %v, want %v", got.ChartName, want.ChartName) + } + + if got.Version != want.Version { + t.Errorf("Version = %v, want %v", got.Version, want.Version) + } + + if !reflect.DeepEqual(got.Values, want.Values) { + t.Errorf("Values mismatch:\nGot: %v\nWant: %v", got.Values, want.Values) + } +} + +func assertValueExists(t *testing.T, values map[string]interface{}, key string, expectedValue interface{}) { + t.Helper() + + if actualValue, exists := values[key]; !exists { + t.Errorf("Expected key %s not found in values", key) + } else if actualValue != expectedValue { + t.Errorf("Value for %s = %v, want %v", key, actualValue, expectedValue) + } +} + +func TestCreateWorkerSpecificHelmChart(t *testing.T) { + tests := []struct { + name string + globalChart HelmChart + cluster Cluster + expectedChart HelmChart + }{ + { + name: "Global values only - backward compatibility", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + cluster: testCluster("standard-worker", "standard-context", nil), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + }, + { + name: "OpenShift worker with overrides", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + cluster: testCluster("openshift-worker", "openshift-context", openShiftWorkerValues()), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", + mergeValues(globalValues(), openShiftWorkerValues())), + }, + { + name: "High-performance worker with resource overrides", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + cluster: testCluster("high-perf-worker", "high-perf-context", highPerformanceWorkerValues()), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", + mergeValues(globalValues(), highPerformanceWorkerValues())), + }, + { + name: "GPU worker with specialized configuration", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + cluster: testCluster("gpu-worker", "gpu-context", gpuWorkerValues()), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", + mergeValues(globalValues(), gpuWorkerValues())), + }, + { + name: "Edge worker with minimal resources", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", globalValues()), + cluster: testCluster("edge-worker", "edge-context", edgeWorkerValues()), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", + mergeValues(globalValues(), edgeWorkerValues())), + }, + { + name: "Empty global values with cluster-specific values", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", map[string]interface{}{}), + cluster: testCluster("custom-worker", "custom-context", openShiftWorkerValues()), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", openShiftWorkerValues()), + }, + { + name: "No values at all", + globalChart: testHelmChart("kubeslice-worker", "1.0.0", map[string]interface{}{}), + cluster: testCluster("minimal-worker", "minimal-context", nil), + expectedChart: testHelmChart("kubeslice-worker", "1.0.0", map[string]interface{}{}), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := CreateWorkerSpecificHelmChart(tt.globalChart, tt.cluster) + assertHelmChartEqual(t, result, tt.expectedChart) + }) + } +} + +func TestCreateWorkerSpecificHelmChart_BackwardCompatibility(t *testing.T) { + t.Run("Legacy configuration without HelmValues", func(t *testing.T) { + globalChart := testHelmChart("kubeslice-worker", "1.0.0", globalValues()) + + // Simulate old cluster configuration without HelmValues field + cluster := Cluster{ + Name: "legacy-worker", + ContextName: "legacy-context", + KubeConfigPath: "/path/to/kubeconfig", + ControlPlaneAddress: "https://api.cluster.local", + NodeIP: "10.0.0.1", + // HelmValues is nil (not set) + } + + result := CreateWorkerSpecificHelmChart(globalChart, cluster) + + // Should return exactly the global chart values + expectedChart := testHelmChart("kubeslice-worker", "1.0.0", globalValues()) + assertHelmChartEqual(t, result, expectedChart) + }) +} + +func TestCreateWorkerSpecificHelmChart_ValueMerging(t *testing.T) { + t.Run("Values are properly merged and overridden", func(t *testing.T) { + globalValues := map[string]interface{}{ + "resources.limits.cpu": "500m", + "resources.limits.memory": "512Mi", + "nodeSelector.environment": "production", + "replicaCount": 2, + "image.tag": "v1.0.0", + } + + clusterValues := map[string]interface{}{ + "resources.limits.cpu": "2000m", // Override + "resources.limits.memory": "2Gi", // Override + "nodeSelector.performance": "high", // New + "replicaCount": 5, // Override + } + + globalChart := testHelmChart("kubeslice-worker", "1.0.0", globalValues) + cluster := testCluster("test-worker", "test-context", clusterValues) + + result := CreateWorkerSpecificHelmChart(globalChart, cluster) + + // Check that global values are preserved when not overridden + assertValueExists(t, result.Values, "nodeSelector.environment", "production") + assertValueExists(t, result.Values, "image.tag", "v1.0.0") + + // Check that cluster values override global values + assertValueExists(t, result.Values, "resources.limits.cpu", "2000m") + assertValueExists(t, result.Values, "resources.limits.memory", "2Gi") + assertValueExists(t, result.Values, "replicaCount", 5) + + // Check that new cluster values are added + assertValueExists(t, result.Values, "nodeSelector.performance", "high") + }) +} + +func TestCreateWorkerSpecificHelmChart_EdgeCases(t *testing.T) { + t.Run("Nil global values with cluster values", func(t *testing.T) { + globalChart := HelmChart{ + ChartName: "kubeslice-worker", + Version: "1.0.0", + Values: nil, // Nil values + } + + clusterValues := map[string]interface{}{ + "test.key": "test.value", + } + + cluster := testCluster("test-worker", "test-context", clusterValues) + result := CreateWorkerSpecificHelmChart(globalChart, cluster) + + assertValueExists(t, result.Values, "test.key", "test.value") + }) + + t.Run("Empty string values are preserved", func(t *testing.T) { + globalValues := map[string]interface{}{ + "emptyString": "", + "normalValue": "test", + } + + clusterValues := map[string]interface{}{ + "emptyString": "overridden", + "newEmpty": "", + } + + globalChart := testHelmChart("kubeslice-worker", "1.0.0", globalValues) + cluster := testCluster("test-worker", "test-context", clusterValues) + + result := CreateWorkerSpecificHelmChart(globalChart, cluster) + + assertValueExists(t, result.Values, "emptyString", "overridden") + assertValueExists(t, result.Values, "normalValue", "test") + assertValueExists(t, result.Values, "newEmpty", "") + }) +} \ No newline at end of file diff --git a/samples/per-worker-helm-values.yaml b/samples/per-worker-helm-values.yaml new file mode 100644 index 0000000..26659d4 --- /dev/null +++ b/samples/per-worker-helm-values.yaml @@ -0,0 +1,79 @@ +# Per-Worker Helm Values Example +# This example demonstrates how to use per-worker helm values to customize +# each worker cluster with specific configurations while maintaining global defaults. + +configuration: + cluster_configuration: + kube_config_path: ~/.kube/config + controller: + name: controller + context_name: controller-context + workers: + # Standard worker - uses global values only (backward compatibility) + - name: standard-worker + context_name: standard-worker-context + # No helm_values specified - inherits all global values + + # OpenShift worker - platform-specific overrides + - name: openshift-worker + context_name: openshift-worker-context + helm_values: + global.profile.openshift: true # Override global setting + nodeSelector.node-role.kubernetes.io/worker: "" # OpenShift node selector + tolerations[0].key: "node-role.kubernetes.io/master" # Master node toleration + tolerations[0].effect: "NoSchedule" + + # High-performance worker - resource and performance overrides + - name: high-perf-worker + context_name: high-perf-worker-context + helm_values: + resources.limits.cpu: "2000m" # 4x more CPU than global + resources.limits.memory: "4Gi" # 8x more memory than global + resources.requests.cpu: "1000m" # Higher CPU requests + nodeSelector.performance: "high" # Performance node selector + nodeSelector.zone: "us-west-1a" # Specific availability zone + replicaCount: 3 # More replicas for HA + + # GPU worker - specialized hardware configuration + - name: gpu-worker + context_name: gpu-worker-context + helm_values: + nodeSelector.accelerator: "nvidia-tesla-k80" # GPU node selector + tolerations[0].key: "nvidia.com/gpu" # GPU toleration + tolerations[0].operator: "Exists" + tolerations[0].effect: "NoSchedule" + resources.limits.nvidia.com/gpu: 1 # GPU resource limit + + # Edge worker - minimal resource configuration + - name: edge-worker + context_name: edge-worker-context + helm_values: + resources.limits.cpu: "200m" # Minimal CPU for edge + resources.limits.memory: "256Mi" # Minimal memory for edge + resources.requests.cpu: "100m" # Low CPU requests + resources.requests.memory: "128Mi" # Low memory requests + nodeSelector.node-type: "edge" # Edge node selector + replicaCount: 1 # Single replica for edge + + kubeslice_configuration: + project_name: per-worker-demo + + helm_chart_configuration: + repo_alias: kubeslice + repo_url: https://kubeslice.github.io/kubeslice/ + cert_manager_chart: + chart_name: cert-manager + controller_chart: + chart_name: kubeslice-controller + worker_chart: + chart_name: kubeslice-worker + # Global values applied to all workers (unless overridden by helm_values) + values: + kubesliceNetworking.enabled: true # Enable networking by default + global.profile.openshift: false # Default to standard Kubernetes + resources.limits.cpu: "500m" # Default CPU limit + resources.limits.memory: "512Mi" # Default memory limit + resources.requests.cpu: "100m" # Default CPU request + resources.requests.memory: "128Mi" # Default memory request + replicaCount: 2 # Default replica count + image.pullPolicy: "IfNotPresent" # Default image pull policy diff --git a/samples/template.yaml b/samples/template.yaml index 07ea44a..c950a53 100644 --- a/samples/template.yaml +++ b/samples/template.yaml @@ -21,6 +21,11 @@ configuration: #{Override this flag if the address in kubeconfig is not reachable by other clusters in topology} node_ip: #{the IP address of one of the node in this cluster. kubeslice-cli determines this address from kubectl get nodes} #{Override this flag to an address which is discoverable by other clusters in the topology} + helm_values: #{optional: per-worker helm values that override global values} + #{These values are specific to this worker cluster} + #{Example: resources.limits.cpu: "2000m"} + #{Example: nodeSelector.zone: "us-west-1a"} + #{Leave empty or omit entirely for backward compatibility} - name: #{the user defined name of the worker cluster} context_name: #{the name of the context to use from the kubeconfig file; for topology only} kube_config_path: #{the path to kube config file to use for worker installation; for topology only.} @@ -29,6 +34,11 @@ configuration: #{Override this flag if the address in kubeconfig is not reachable by other clusters in topology} node_ip: #{the IP address of one of the node in this cluster. kubeslice-cli determines this address from kubectl get nodes} #{Override this flag to an address which is discoverable by other clusters in the topology} + helm_values: #{optional: per-worker helm values that override global values} + #{These values are specific to this worker cluster} + #{Example: resources.limits.cpu: "2000m"} + #{Example: nodeSelector.zone: "us-west-1a"} + #{Leave empty or omit entirely for backward compatibility} kubeslice_configuration: project_name: #{the name of the KubeSlice Project} project_users: #{optional: specify KubeSlice Project users with Readw-Write access. Default is admin}