feat(nvca): add worker-based health per-instance feature

src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel

@@ -60,6 +60,7 @@ go_library(
         "//vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config",
         "//vendor/github.com/go-logr/logr",
         "//vendor/github.com/google/go-cmp/cmp",
+        "//vendor/github.com/google/go-cmp/cmp/cmpopts",
         "//vendor/github.com/google/go-containerregistry/pkg/name",
         "//vendor/github.com/hashicorp/go-retryablehttp",
         "//vendor/github.com/imdario/mergo",

src/compute-plane-services/nvca/internal/miniservice/controller.go

@@ -136,6 +136,8 @@ type registrationInstanceTypeCache interface {
 }
 
 const (
+	controllerName = "miniservice-controller"
+
 	defaultCacheDir = "/var/run/nvca/reval-rendered-helmcharts"
 )
 
@@ -185,7 +187,7 @@ func BuildController(ctx context.Context,
 		tracer:            otel.NewTracer(),
 		Decoder:           newFlexibleDecoder(mgr.GetScheme(), extraGVKs...),
 		NFClient:          nflient,
-		eventRecorder:     mgr.GetEventRecorderFor("miniservice-controller"),
+		eventRecorder:     mgr.GetEventRecorderFor(controllerName),
 		chartCache:        chartcache.New(opts.cacheDir),
 		regITCache:        regITCache,
 		enabledAttrs:      enabledAttrs,

src/compute-plane-services/nvca/internal/miniservice/reconcile.go

@@ -39,6 +39,8 @@ import (
 	translateutil "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/util"
 	cmnotel "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/otel"
 	nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/google/go-containerregistry/pkg/name"
 	otelattr "go.opentelemetry.io/otel/attribute"
 	otelcodes "go.opentelemetry.io/otel/codes"
@@ -443,6 +445,41 @@ func (r *Reconciler) patchMiniService(ctx context.Context, oldObj, newObj *v1alp
 	return nil
 }
 
+// saveWorkloadConfig persists the workload config decoded from the rendered workload config
+// ConfigMap onto the MiniService spec, patching only that field inline. It is a no-op when the
+// config is unchanged. On success ms is updated in place so callers see the persisted value
+// and current ResourceVersion.
+func (r *Reconciler) saveWorkloadConfig(
+	ctx context.Context,
+	ms *v1alpha1.MiniService,
+	desired *v1alpha1.WorkloadConfig,
+) error {
+	if cmp.Equal(ms.Spec.WorkloadConfig, desired, cmpopts.EquateEmpty()) {
+		return nil
+	}
+
+	base := &v1alpha1.MiniService{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: v1alpha1.SchemeGroupVersion.String(),
+			Kind:       "MiniService",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            ms.Name,
+			ResourceVersion: ms.ResourceVersion,
+		},
+		Spec: v1alpha1.MiniServiceSpec{
+			WorkloadConfig: desired.DeepCopy(),
+		},
+	}
+	if err := r.Client.Patch(ctx, base, client.Apply, client.ForceOwnership, client.FieldOwner(managedByValue)); err != nil {
+		return fmt.Errorf("patch miniservice %s workload config: %w", ms.Name, err)
+	}
+
+	ms.Spec.WorkloadConfig = desired
+	ms.ResourceVersion = base.ResourceVersion
+	return nil
+}
+
 // prepareUpdateIfNeeded detects spec changes by comparing metadata.generation
 // to status.observedGeneration. Because generation increments on any spec field change,
 // this method additionally compares helm values against the latest revision ConfigMap.
@@ -560,10 +597,15 @@ func (r *Reconciler) doInstall(ctx context.Context,
 		}
 	}
 
-	workloadObjs, resources, err := decodeObjects(ctx, r.Decoder, objsData)
+	workloadObjs, resources, workloadConfig, err := decodeObjects(ctx, r.Decoder, objsData)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
+	// Persist workload config decoded from the rendered workload config ConfigMap so later
+	// reconciles (e.g. status) can source it from the spec.
+	if err := r.saveWorkloadConfig(ctx, ms, workloadConfig); err != nil {
+		return reconcile.Result{}, err
+	}
 	// Update the resources status in the MiniService status.
 	updateResourcesStatus(ms, resources)
 	// ReVal may render filtered workload pull secrets, which should be used if possible.
@@ -911,7 +953,7 @@ func (r *Reconciler) prepareUpdateWorkload(ctx context.Context,
 		}
 	}
 
-	workloadObjs, resources, err := decodeObjects(ctx, r.Decoder, objsData)
+	workloadObjs, resources, _, err := decodeObjects(ctx, r.Decoder, objsData)
 	if err != nil {
 		return nil, nil, "", "", err
 	}
@@ -1274,7 +1316,7 @@ func (r *Reconciler) doCleanup(ctx context.Context, //nolint:gocyclo
 	} else {
 		log.V(1).Info("Using cached rendered object data in cleanup")
 
-		if objs, _, err = decodeObjects(ctx, r.Decoder, objsData); err != nil {
+		if objs, _, _, err = decodeObjects(ctx, r.Decoder, objsData); err != nil {
 			return reconcile.Result{}, err
 		}
 		for i := 0; i < len(objs); i++ {
@@ -1511,8 +1553,6 @@ func (r *Reconciler) applyWorkload(ctx context.Context,
 	return r.create(ctx, ms, objectMutatorSet{}, genericMutator, c, objs...)
 }
 
-const fieldManagerName = "miniservice-controller"
-
 // applySSAWorkload applies workload objects using server-side apply for updates.
 // Like applyWorkload, it may use an impersonating client for RBAC enforcement.
 func (r *Reconciler) applySSAWorkload(ctx context.Context,
@@ -1587,7 +1627,7 @@ func (r *Reconciler) applySSA(ctx context.Context,
 		obj.SetResourceVersion("")
 		obj.GetObjectKind().SetGroupVersionKind(gvk)
 
-		if err := c.Patch(ctx, obj, client.Apply, client.FieldOwner(fieldManagerName), client.ForceOwnership); err != nil {
+		if err := c.Patch(ctx, obj, client.Apply, client.FieldOwner(managedByValue), client.ForceOwnership); err != nil {
 			if apierrors.IsInvalid(err) || apierrors.IsForbidden(err) {
 				err = reconcile.TerminalError(err)
 			}
@@ -1802,29 +1842,46 @@ func weighObject(obj client.Object) int8 {
 	}
 }
 
-func decodeObjects(ctx context.Context, decoder runtime.Decoder, objsData []byte) ([]client.Object, []v1alpha1.ResourceStatus, error) {
+// decodeObjects decodes the ReVal-rendered object data into typed client objects and a
+// per-GVK resource summary. The workload config ConfigMap
+// (featureflag.WorkloadConfigConfigMapName) is a control signal from the chart and is never
+// created on-cluster: it is extracted and returned as a decoded WorkloadConfig, and excluded
+// from the returned objects and resource summary so it is never applied or status-checked.
+func decodeObjects(ctx context.Context, decoder runtime.Decoder, objsData []byte) (
+	[]client.Object, []v1alpha1.ResourceStatus, *v1alpha1.WorkloadConfig, error,
+) {
 	log := logf.FromContext(ctx)
 
 	var rawObjs []json.RawMessage
 	if err := json.Unmarshal(objsData, &rawObjs); err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
-	objs := make([]client.Object, len(rawObjs))
+	objs := make([]client.Object, 0, len(rawObjs))
 	resourcesByGVK := map[string]v1alpha1.ResourceStatus{}
+	var workloadConfig *v1alpha1.WorkloadConfig
 	for i, rawObj := range rawObjs {
 		robj, _, err := decoder.Decode(rawObj, nil, nil)
 		if err != nil {
 			log.Error(err, "Error decoding object from ReVal", "index", i)
-			return nil, nil, reconcile.TerminalError(err)
+			return nil, nil, nil, reconcile.TerminalError(err)
 		}
 
 		cobj, ok := robj.(client.Object)
 		if !ok {
 			log.Error(nil, "Object does not implement client.Object", "index", i)
-			return nil, nil, reconcile.TerminalError(fmt.Errorf("bad object type"))
+			return nil, nil, nil, reconcile.TerminalError(fmt.Errorf("bad object type"))
+		}
+
+		// Extract the workload config ConfigMap. It must never be applied on-cluster,
+		// so drop it from the objects to create and status-check.
+		if cm, ok := cobj.(*corev1.ConfigMap); ok && cm.Name == featureflag.WorkloadConfigConfigMapName {
+			if workloadConfig, err = featureflag.DecodeWorkloadConfig(ctx, log, cm); err != nil {
+				return nil, nil, nil, reconcile.TerminalError(fmt.Errorf("decode workload config: %w", err))
+			}
+			continue
 		}
 
-		objs[i] = cobj
+		objs = append(objs, cobj)
 		gvk := cobj.GetObjectKind().GroupVersionKind().String()
 		if _, ok := resourcesByGVK[gvk]; ok {
 			resource := resourcesByGVK[gvk]
@@ -1846,7 +1903,7 @@ func decodeObjects(ctx context.Context, decoder runtime.Decoder, objsData []byte
 		log.V(1).Info("Decoded object", "gvk", resource.GVK, "names", resource.Names, "count", resource.Count)
 	}
 
-	return objs, resources, nil
+	return objs, resources, workloadConfig, nil
 }
 
 // getObjectGVK returns the GVK for an object using the gvkCache from context for better performance.

src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go

@@ -4340,7 +4340,7 @@ func TestDecodeObjects_Success(t *testing.T) {
 	data, err := json.Marshal(helmObjs)
 	require.NoError(t, err)
 
-	objs, resources, derr := decodeObjects(ctx, decoder, data)
+	objs, resources, _, derr := decodeObjects(ctx, decoder, data)
 	require.NoError(t, derr)
 	require.Len(t, objs, 2)
 	require.Len(t, resources, 2)
@@ -4366,6 +4366,57 @@ func TestDecodeObjects_Success(t *testing.T) {
 	})
 }
 
+func TestDecodeObjects_ExtractsWorkloadConfig(t *testing.T) {
+	ctx := newTestContext()
+	s := mgrScheme
+	decoder := serializer.NewCodecFactory(s).UniversalDeserializer()
+
+	helmObjs := []client.Object{
+		&corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: featureflag.WorkloadConfigConfigMapName,
+			},
+			Data: map[string]string{
+				featureflag.WorkloadConfigDataKey: "featureFlags:\n" +
+					"  " + featureflag.StatusByWorkerReadiness + ": true\n" +
+					"  SOME_UNKNOWN_FLAG: true\n",
+			},
+		},
+		&appsv1.Deployment{
+			TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "dep-1",
+			},
+		},
+	}
+
+	data, err := json.Marshal(helmObjs)
+	require.NoError(t, err)
+
+	objs, resources, workloadConfig, derr := decodeObjects(ctx, decoder, data)
+	require.NoError(t, derr)
+
+	// The workload config ConfigMap must be stripped from the applied/status-checked objects
+	// and from the resource summary.
+	require.Len(t, objs, 1)
+	if dep, ok := objs[0].(*appsv1.Deployment); assert.True(t, ok) {
+		assert.Equal(t, "dep-1", dep.Name)
+	}
+	require.Len(t, resources, 1)
+	assert.Contains(t, resources, v1alpha1.ResourceStatus{
+		GVK:   "apps/v1, Kind=Deployment",
+		Names: []string{"dep-1"},
+		Count: 1,
+	})
+
+	// Recognized flags are decoded; unknown keys are dropped.
+	assert.Equal(t, &v1alpha1.WorkloadConfig{
+		FeatureFlags: map[string]bool{featureflag.StatusByWorkerReadiness: true},
+	}, workloadConfig)
+	assert.True(t, workloadConfig.IsFeatureFlagEnabled(featureflag.StatusByWorkerReadiness))
+}
+
 func TestDecodeObjects_JSONUnmarshalError(t *testing.T) {
 	ctx := newTestContext()
 	s := mgrScheme
@@ -4374,7 +4425,7 @@ func TestDecodeObjects_JSONUnmarshalError(t *testing.T) {
 	// Invalid JSON
 	data := []byte("not-json")
 
-	objs, resources, err := decodeObjects(ctx, decoder, data)
+	objs, resources, _, err := decodeObjects(ctx, decoder, data)
 	assert.Error(t, err)
 	assert.Nil(t, objs)
 	assert.Nil(t, resources)
@@ -4394,7 +4445,7 @@ func TestDecodeObjects_NonClientObject(t *testing.T) {
 	data, err := json.Marshal(objs)
 	require.NoError(t, err)
 
-	gotObjs, gotResources, derr := decodeObjects(ctx, decoder, data)
+	gotObjs, gotResources, _, derr := decodeObjects(ctx, decoder, data)
 	assert.EqualError(t, derr, "terminal error: bad object type")
 	assert.Nil(t, gotObjs)
 	assert.Nil(t, gotResources)

src/compute-plane-services/nvca/internal/miniservice/revision.go

@@ -40,7 +40,7 @@ const (
 	revisionConfigMapPrefix = "miniservice-revision-v"
 	revisionLabel           = "nvca.nvcf.nvidia.io/revision"
 	managedByLabel          = "app.kubernetes.io/managed-by"
-	managedByValue          = "miniservice-controller"
+	managedByValue          = controllerName
 
 	revisionDataKeyValues     = "values"
 	revisionDataKeyRenderHash = "renderHash"