diff --git a/migration.mk b/migration.mk index 9d565b3..ea67d46 100644 --- a/migration.mk +++ b/migration.mk @@ -264,7 +264,7 @@ migration/test-e2e-cos-supersession: migration/build ## Verify migration COS is mkdir -p "$$(dirname "$$coverage_profile")"; \ KUBECONFIG="$(E2E_FIXTURE_KUBECONFIG)" ./hack/e2e/migration/delete-v1.sh "$$package"; \ KUBECONFIG="$(E2E_FIXTURE_KUBECONFIG)" ./hack/e2e/migration/install-fixture-v0.sh "$$package"; \ - KUBECONFIG="$(E2E_FIXTURE_KUBECONFIG)" E2E_ARTIFACTS="$(E2E_ARTIFACTS)/cos-supersession" E2E_SUITE=fixture E2E_COS_SUPERSESSION_TEST=true E2E_NAMESPACE=migration-e2e-ecr-secret E2E_SUBSCRIPTION="$$package" go test -count=1 -tags=e2e -covermode=count -coverpkg=./migration/... -coverprofile="$$coverage_profile" ./test/e2e/migration -run '^TestPrecreatedClusterObjectSetSupersession$$' -timeout "$(E2E_TIMEOUT)"; \ + KUBECONFIG="$(E2E_FIXTURE_KUBECONFIG)" E2E_ARTIFACTS="$(E2E_ARTIFACTS)/cos-supersession" E2E_SUITE=fixture E2E_COS_SUPERSESSION_TEST=true E2E_NAMESPACE=migration-e2e-ecr-secret E2E_SUBSCRIPTION="$$package" go test -count=1 -tags=e2e -covermode=count -coverpkg=./migration/... -coverprofile="$$coverage_profile" ./test/e2e/migration -run '^Test(PrecreatedClusterObjectSetSupersession|CODProtectedRolloutCollision)$$' -timeout "$(E2E_TIMEOUT)"; \ if [[ "$(E2E_KEEP_RESOURCES)" != true ]]; then \ KUBECONFIG="$(E2E_FIXTURE_KUBECONFIG)" ./hack/e2e/migration/delete-v1.sh "$$package"; \ else \ diff --git a/migration/examples/cmd/migrate-operators-v0-to-v1/convert.go b/migration/examples/cmd/migrate-operators-v0-to-v1/convert.go index c4c1b02..8631277 100644 --- a/migration/examples/cmd/migrate-operators-v0-to-v1/convert.go +++ b/migration/examples/cmd/migrate-operators-v0-to-v1/convert.go @@ -274,10 +274,12 @@ func printDryRunPreview(opts, resourceOpts migration.Options, info *migration.Mi success(fmt.Sprintf("Package: %s Version: %s Channel: %s", info.PackageName, info.Version, valueOrDefault(info.Channel, "(default)"))) fmt.Printf("\n Resources that would be created:\n") if info.UseClusterObjectDeployment { + detail("ClusterObjectSet:", fmt.Sprintf("%s-1 (CollisionProtection: None; adopted by the COD)", opts.ClusterExtensionName)) detail("ClusterObjectDeployment:", fmt.Sprintf("%s (wait for rollout completion and availability before creating the ClusterExtension)", opts.ClusterExtensionName)) + detail("Template CollisionProtection:", "Prevent") fmt.Printf("\n Resources that would be placed into its template and controller-created COS revisions:\n") } else { - detail("ClusterObjectSet:", fmt.Sprintf("%s-1 (wait for completion before creating the ClusterExtension)", opts.ClusterExtensionName)) + detail("ClusterObjectSet:", fmt.Sprintf("%s-1 (CollisionProtection: None; wait for completion before creating the ClusterExtension)", opts.ClusterExtensionName)) fmt.Printf("\n Resources that would be placed into ClusterObjectSet %s-1:\n", opts.ClusterExtensionName) } @@ -303,7 +305,6 @@ func printDryRunPreview(opts, resourceOpts migration.Options, info *migration.Mi detail("Version:", "(unset — automatic channel-based upgrades)") } detail("Channel:", valueOrDefault(info.Channel, "(none set)")) - detail("CollisionProtection:", "None") fmt.Printf("\n OLMv0 resources that would be deleted or changed:\n") for _, line := range dryRunCleanupPlan(opts, resourceOpts, info) { diff --git a/migration/pkg/migration/create_rejection_test.go b/migration/pkg/migration/create_rejection_test.go index f9850ad..a24237d 100644 --- a/migration/pkg/migration/create_rejection_test.go +++ b/migration/pkg/migration/create_rejection_test.go @@ -41,29 +41,31 @@ func TestMigrationCreateRejectionRecovery(t *testing.T) { attempted := false m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{ Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { - switch obj.(type) { - case *ocv1.ClusterObjectSet, *ocv1.ClusterObjectDeployment: + _, isCOS := obj.(*ocv1.ClusterObjectSet) + _, isCOD := obj.(*ocv1.ClusterObjectDeployment) + if mode == "COS" && isCOS || mode == "COD" && isCOD { attempted = true return tc.err - default: - return c.Create(ctx, obj, opts...) } + return c.Create(ctx, obj, opts...) }, }) + m.Client = failingMigrationClient{Client: m.Client, succeedCOS: true} m.Client = reconcilingSubscriptionClient{Client: m.Client} sub, _ := healthySubscriptionFixtures() result, err := m.CreateMigrationResources(ctx, Options{ClusterExtensionName: "example", SubscriptionName: sub.Name, SubscriptionNamespace: sub.Namespace}, &MigrationInfo{}, &Backup{Subscription: sub}) if !attempted || !errors.Is(err, tc.err) { t.Fatalf("expected create rejection %v, got %v (attempted: %v)", tc.err, err, attempted) } - if result.TargetMayBeActive != tc.ambiguous { - t.Fatalf("TargetMayBeActive = %v, want %v", result.TargetMayBeActive, tc.ambiguous) + wantActive := tc.ambiguous || mode == "COD" + if result.TargetMayBeActive != wantActive { + t.Fatalf("TargetMayBeActive = %v, want %v", result.TargetMayBeActive, wantActive) } var restored operatorsv1alpha1.Subscription getErr := m.Client.Get(ctx, client.ObjectKeyFromObject(sub), &restored) - if tc.ambiguous { + if wantActive { if !apierrors.IsNotFound(getErr) { - t.Fatalf("ambiguous create restored OLMv0: %v", getErr) + t.Fatalf("potentially active migration restored OLMv0: %v", getErr) } } else if getErr != nil || restored.Status.State != operatorsv1alpha1.SubscriptionStateAtLatest { t.Fatalf("definitive rejection did not restore OLMv0: %v, state %s", getErr, restored.Status.State) diff --git a/migration/pkg/migration/deployment.go b/migration/pkg/migration/deployment.go index c0098e3..371f72b 100644 --- a/migration/pkg/migration/deployment.go +++ b/migration/pkg/migration/deployment.go @@ -3,6 +3,7 @@ package migration import ( "context" "encoding/json" + "errors" "fmt" "regexp" "slices" @@ -98,7 +99,8 @@ func (m *Migrator) createClusterObjectDeployment(ctx context.Context, opts Optio // waitForDeployment waits for the created deployment to complete its current rollout and become available. func (m *Migrator) waitForDeployment(ctx context.Context, created *ocv1.ClusterObjectDeployment) error { - return wait.PollUntilContextTimeout(ctx, cosWaitPollInterval, cosWaitTimeout, true, func(ctx context.Context) (bool, error) { + var collision error + err := wait.PollUntilContextTimeout(ctx, cosWaitPollInterval, cosWaitTimeout, true, func(ctx context.Context) (bool, error) { var cod ocv1.ClusterObjectDeployment if err := m.Client.Get(ctx, client.ObjectKeyFromObject(created), &cod); err != nil { return false, err @@ -106,11 +108,18 @@ func (m *Migrator) waitForDeployment(ctx context.Context, created *ocv1.ClusterO if cod.UID != created.UID || cod.Generation != created.Generation || cod.DeletionTimestamp != nil { return false, fmt.Errorf("migration ClusterObjectDeployment %s was replaced, changed or deleted", cod.Name) } + collision = nil progress := apimeta.FindStatusCondition(cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeProgressing) available := apimeta.FindStatusCondition(cod.Status.Conditions, ocv1.ClusterObjectDeploymentTypeAvailable) if progress != nil && progress.ObservedGeneration == cod.Generation { if progress.Status == metav1.ConditionFalse && progress.Reason != ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed { - return false, fmt.Errorf("ClusterObjectDeployment %s rollout failed (%s): %s", cod.Name, progress.Reason, progress.Message) + failure := fmt.Errorf("ClusterObjectDeployment %s rollout failed (%s): %s", cod.Name, progress.Reason, progress.Message) + if progress.Reason != ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision { + return false, failure + } + // Ownership can briefly collide while sibling revisions hand off. + // Continue observing; only actual completion permits migration. + collision = failure } if progress.Status == metav1.ConditionFalse && progress.Reason == ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed && available != nil && available.ObservedGeneration == cod.Generation && available.Status == metav1.ConditionTrue { @@ -120,6 +129,10 @@ func (m *Migrator) waitForDeployment(ctx context.Context, created *ocv1.ClusterO m.progress(ProgressEvent{Step: ProgressStepCreate, Status: ProgressWaiting, Message: fmt.Sprintf("Waiting for ClusterObjectDeployment %s rollout completion and availability", cod.Name)}) return false, nil }) + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return errors.Join(err, collision) + } + return err } // rollbackDeployment permits the COD API to be absent, but never hides @@ -286,6 +299,7 @@ func buildClusterObjectDeployment(opts Options, info *MigrationInfo, phases []*o LabelBundleName: info.BundleName, LabelBundleVersion: info.Version, } + addStagingBackup(annotations, info) if info.BundleImage != "" { annotations[LabelBundleReference] = info.BundleImage } @@ -297,7 +311,7 @@ func buildClusterObjectDeployment(opts Options, info *MigrationInfo, phases []*o WithMetadata(ocv1ac.ClusterObjectDeploymentTemplateMetadata(). WithLabels(map[string]string{LabelOwnerKind: ocv1.ClusterExtensionKind, LabelOwnerName: opts.ClusterExtensionName}). WithAnnotations(annotations)).WithSpec( - ocv1ac.ClusterObjectDeploymentTemplateSpec().WithPhases(phases...).WithCollisionProtection(ocv1.CollisionProtectionNone)))) + ocv1ac.ClusterObjectDeploymentTemplateSpec().WithPhases(phases...).WithCollisionProtection(ocv1.CollisionProtectionPrevent)))) data, err := json.Marshal(configuration) if err != nil { return nil, fmt.Errorf("marshal ClusterObjectDeployment: %w", err) diff --git a/migration/pkg/migration/deployment_test.go b/migration/pkg/migration/deployment_test.go index 8272402..8651049 100644 --- a/migration/pkg/migration/deployment_test.go +++ b/migration/pkg/migration/deployment_test.go @@ -3,6 +3,7 @@ package migration import ( "context" "errors" + "fmt" "reflect" "strings" "testing" @@ -12,6 +13,8 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -126,7 +129,6 @@ func TestDeploymentInvalidSpecification(t *testing.T) { func TestDeploymentTerminalRevisionFailures(t *testing.T) { for _, reason := range []string{ - ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision, ocv1.ClusterObjectDeploymentReasonRevisionInvalidSpecification, ocv1.ClusterObjectDeploymentReasonUnmanagedRevision, "FutureTerminalReason", @@ -231,6 +233,10 @@ func TestRollbackDeploymentOwnership(t *testing.T) { type completingDeploymentClient struct{ client.Client } func (c completingDeploymentClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if cos, ok := obj.(*ocv1.ClusterObjectSet); ok { + cos.Status.CompletedAt = metav1.Now() + cos.Status.Conditions = []metav1.Condition{{Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionTrue}} + } if cod, ok := obj.(*ocv1.ClusterObjectDeployment); ok { ready := readyDeployment() cod.UID, cod.Generation, cod.Status = ready.UID, ready.Generation, ready.Status @@ -268,7 +274,7 @@ func TestCreateDeploymentTemplate(t *testing.T) { if err != nil { t.Fatal(err) } - if resources.cod == nil || resources.cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionNone { + if resources.cod == nil || resources.cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { t.Fatalf("unexpected deployment: %+v", resources.cod) } if resources.cod.Annotations[migrationInvocationAnnotation] != "invocation" { @@ -502,3 +508,209 @@ func establishedClusterObjectDeploymentCRD() *apiextensionsv1.CustomResourceDefi crd.Name = clusterObjectDeploymentCRDName return crd } + +// TestCreateCOSBeforeProtectedCOD verifies staging waits for adoption before creating the COD +// and that permissive phase/object overrides cannot leak into future rollouts. +func TestCreateCOSBeforeProtectedCOD(t *testing.T) { + for _, rejectCOD := range []bool{false, true} { + t.Run(fmt.Sprint("rejectCOD=", rejectCOD), func(t *testing.T) { + m := migrationTestClient(t, establishedClusterObjectDeploymentCRD()) + var order []string + denied := apierrors.NewForbidden(schema.GroupResource{Resource: "clusterobjectdeployments"}, "example", errors.New("denied")) + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + switch obj := obj.(type) { + case *ocv1.ClusterObjectSet: + order = append(order, "COS") + obj.UID = "initial" + obj.Status.CompletedAt = metav1.Now() + obj.Status.Conditions = []metav1.Condition{{Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionTrue}} + case *ocv1.ClusterObjectDeployment: + order = append(order, "COD") + if rejectCOD { + return denied + } + ready := readyDeployment() + obj.UID, obj.Generation, obj.Status = ready.UID, ready.Generation, ready.Status + } + return c.Create(ctx, obj, opts...) + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + obj := unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]interface{}{"name": "existing", "namespace": "ns"}}} + resources, err := m.createClusterObjectSet(ctx, Options{ClusterExtensionName: "example", SubscriptionNamespace: "ns", SubscriptionName: "sub"}, &MigrationInfo{CollectedObjects: []unstructured.Unstructured{obj}}) + if !reflect.DeepEqual(order, []string{"COS", "COD"}) { + t.Fatalf("create order = %v, error %v", order, err) + } + if rejectCOD { + if !errors.Is(err, denied) { + t.Fatalf("error = %v", err) + } + if err := m.cleanupCreatedClusterObjectSet(ctx, resources); err == nil { + t.Fatal("allowed unsafe cleanup after COS creation") + } + if err := m.recoverCreatedMigrationResources(ctx, Options{}, &Backup{}, resources); err == nil { + t.Fatal("allowed OLMv0 recovery after COS creation") + } + var preserved ocv1.ClusterObjectSet + if err := m.Client.Get(ctx, client.ObjectKey{Name: "example-1"}, &preserved); err != nil { + t.Fatal(err) + } + return + } + if err != nil { + t.Fatal(err) + } + if resources.cos.Spec.CollisionProtection != ocv1.CollisionProtectionNone || resources.cos.Spec.Group != "example" || resources.cos.Spec.Revision != 1 { + t.Fatalf("invalid initial COS: %+v", resources.cos.Spec) + } + if resources.cos.Labels[controllerManagedByLabel] != "object-controller" { + t.Fatal("COS cannot be adopted") + } + if resources.cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + t.Fatal("COD must prevent collisions") + } + for _, phase := range resources.cod.Spec.Template.Spec.Phases { + if phase.CollisionProtection != "" { + t.Fatal("phase overrides COD protection") + } + for _, entry := range phase.Objects { + if entry.CollisionProtection != "" { + t.Fatal("object overrides COD protection") + } + } + } + if resources.cos.Spec.Phases[0].CollisionProtection != ocv1.CollisionProtectionNone { + t.Fatal("mutated initial COS while building COD") + } + }) + } +} + +// TestCreateMigrationResourcesSharesDefaults verifies COD and CE creation use the +// same defaulted identity and namespace, while retaining explicit CE names. +func TestCreateMigrationResourcesSharesDefaults(t *testing.T) { + for _, name := range []string{"", "custom"} { + t.Run("name="+name, func(t *testing.T) { + m := migrationTestClient(t, establishedClusterObjectDeploymentCRD()) + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + switch resource := obj.(type) { + case *ocv1.ClusterObjectSet: + resource.Status.CompletedAt = metav1.Now() + resource.Status.Conditions = []metav1.Condition{{Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionTrue}} + case *ocv1.ClusterObjectDeployment: + ready := readyDeployment() + resource.UID, resource.Generation, resource.Status = ready.UID, ready.Generation, ready.Status + case *ocv1.ClusterExtension: + resource.UID = "extension" + resource.Status.Conditions = []metav1.Condition{{Type: ocv1.TypeInstalled, Status: metav1.ConditionTrue}} + } + return c.Create(ctx, obj, opts...) + }, + }) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + opts := Options{SubscriptionName: "subscription", SubscriptionNamespace: "source", ClusterExtensionName: name, SystemNamespace: "controller"} + if _, err := m.CreateMigrationResources(ctx, opts, &MigrationInfo{PackageName: "package", Channel: "stable"}, &Backup{}); err != nil { + t.Fatal(err) + } + expected := name + if expected == "" { + expected = "subscription" + } + var cod ocv1.ClusterObjectDeployment + if err := m.Client.Get(ctx, client.ObjectKey{Name: expected}, &cod); err != nil { + t.Fatal(err) + } + var ce ocv1.ClusterExtension + if err := m.Client.Get(ctx, client.ObjectKey{Name: expected}, &ce); err != nil { + t.Fatal(err) + } + if cod.Spec.Template.Metadata.Labels[LabelOwnerName] != expected || ce.Spec.Namespace != "source" { + t.Fatalf("inconsistent defaults: COD owner label=%q CE namespace=%q", cod.Spec.Template.Metadata.Labels[LabelOwnerName], ce.Spec.Namespace) + } + }) + } +} + +// TestIncompleteAdoptionPreventsProtectedCOD ensures no protected rollout or CE +// can overtake an incomplete initial adoption, and preserves recovery state. +func TestIncompleteAdoptionPreventsProtectedCOD(t *testing.T) { + for _, completed := range []bool{false, true} { + t.Run(fmt.Sprint("completed=", completed), func(t *testing.T) { + m := migrationTestClient(t, establishedClusterObjectDeploymentCRD()) + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + switch resource := obj.(type) { + case *ocv1.ClusterObjectSet: + if completed { + resource.Status.CompletedAt = metav1.Now() + } else { + resource.Status.Conditions = []metav1.Condition{{Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionTrue}} + } + case *ocv1.ClusterObjectDeployment, *ocv1.ClusterExtension: + t.Fatal("created protected target before initial COS completed and became ready") + } + return c.Create(ctx, obj, opts...) + }, + }) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + sub, _ := healthySubscriptionFixtures() + result, err := m.CreateMigrationResources(ctx, Options{SubscriptionName: sub.Name, SubscriptionNamespace: sub.Namespace, ClusterExtensionName: "staged"}, &MigrationInfo{}, &Backup{Subscription: sub}) + if !errors.Is(err, context.DeadlineExceeded) || !result.TargetMayBeActive { + t.Fatalf("expected retained target and timeout, got %+v, %v", result, err) + } + var cos ocv1.ClusterObjectSet + if err := m.Client.Get(t.Context(), client.ObjectKey{Name: "staged-1"}, &cos); err != nil { + t.Fatal(err) + } + if cos.Annotations[MigrationSubscriptionBackupAnnotation] == "" { + t.Fatal("retained COS has no recovery backup") + } + var restored operatorsv1alpha1.Subscription + if err := m.Client.Get(t.Context(), client.ObjectKeyFromObject(sub), &restored); !apierrors.IsNotFound(err) { + t.Fatalf("unexpected automatic recovery: %v", err) + } + }) + } +} + +func TestDeploymentCollisionWait(t *testing.T) { + for _, resolves := range []bool{false, true} { + t.Run(fmt.Sprint("resolves=", resolves), func(t *testing.T) { + created := readyDeployment() + current := created.DeepCopy() + current.Status.Conditions[0].Reason = ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision + current.Status.Conditions[0].Message = "existing object is still owned" + m := migrationTestClient(t, current) + reads := 0 + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + reads++ + if resolves && reads > 1 { + obj.(*ocv1.ClusterObjectDeployment).Status = created.Status + } + return nil + }}) + timeout := 50 * time.Millisecond + if resolves { + timeout = 2 * cosWaitPollInterval + } + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + err := m.waitForDeployment(ctx, created) + if resolves { + if err != nil || reads < 2 { + t.Fatalf("transient collision did not resolve: %v", err) + } + } else if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "existing object is still owned") { + t.Fatalf("persistent collision lost deadline or diagnostic: %v", err) + } + }) + } +} diff --git a/migration/pkg/migration/labels.go b/migration/pkg/migration/labels.go index 0b0d896..fa5bdde 100644 --- a/migration/pkg/migration/labels.go +++ b/migration/pkg/migration/labels.go @@ -34,10 +34,10 @@ const ( // Value is "/" of the source Subscription. MigratedFromSubscriptionAnnotation = "olm.operatorframework.io/migrated-from-subscription" - // MigrationSubscriptionBackupAnnotation holds JSON-encoded Subscription spec on the CE (R2.5). + // MigrationSubscriptionBackupAnnotation holds JSON-encoded Subscription spec on retained staging resources and the CE (R2.5). MigrationSubscriptionBackupAnnotation = "olm.operatorframework.io/migration-subscription-backup" - // MigrationOperatorGroupBackupAnnotation holds JSON-encoded OperatorGroup spec on the CE (R2.5). + // MigrationOperatorGroupBackupAnnotation holds JSON-encoded OperatorGroup spec on retained staging resources and the CE (R2.5). MigrationOperatorGroupBackupAnnotation = "olm.operatorframework.io/migration-operatorgroup-backup" // AnnotationAcknowledgedPrefix is the prefix for per-flag audit annotations on the CE (R2.5). diff --git a/migration/pkg/migration/migration.go b/migration/pkg/migration/migration.go index 0e41f4a..ccbfbf6 100644 --- a/migration/pkg/migration/migration.go +++ b/migration/pkg/migration/migration.go @@ -41,11 +41,12 @@ var annotationPrefixesToStrip = []string{ // Recovery must never infer ownership from predictable names or labels: another // migration may have created objects with the same values after this one started. type createdMigrationResources struct { - cos *ocv1.ClusterObjectSet - cod *ocv1.ClusterObjectDeployment - secrets []corev1.Secret - ce *ocv1.ClusterExtension - ownershipUnknown bool + cos *ocv1.ClusterObjectSet + cod *ocv1.ClusterObjectDeployment + secrets []corev1.Secret + ce *ocv1.ClusterExtension + ownershipUnknown bool + deploymentRequested bool } // MigrationResourcesResult records whether target resources may have started @@ -633,6 +634,12 @@ func (m *Migrator) CreateClusterObjectSet(ctx context.Context, opts Options, inf // as one recoverable operation. Its result reports whether a target COS may have // started reconciling, including when cleanup was requested after a failure. func (m *Migrator) CreateMigrationResources(ctx context.Context, opts Options, info *MigrationInfo, backup *Backup) (MigrationResourcesResult, error) { + var err error + info, err = withStagingBackup(info, backup) + if err != nil { + return MigrationResourcesResult{}, err + } + opts.ApplyDefaults() resources, err := m.createClusterObjectSet(ctx, opts, info) if err != nil { result := MigrationResourcesResult{TargetMayBeActive: resources.cos != nil || resources.cod != nil || resources.ownershipUnknown} @@ -665,6 +672,7 @@ func (m *Migrator) createClusterObjectSet(ctx context.Context, opts Options, inf return resources, err } opts.UseClusterObjectDeployment = useCOD + resources.deploymentRequested = useCOD if useCOD { if err := m.prepareClusterObjectDeployment(ctx, opts.ClusterExtensionName); err != nil { return resources, err @@ -721,14 +729,6 @@ func (m *Migrator) createClusterObjectSet(ctx context.Context, opts Options, inf resources.secrets = append(resources.secrets, *secret) } - if opts.UseClusterObjectDeployment { - err := m.createClusterObjectDeployment(ctx, opts, info, phases, invocationMarker, resources) - if err != nil && resources.cod == nil { - return resources, failWithSecretCleanup(err) - } - return resources, err - } - cosObj, err := buildClusterObjectSet(opts, info, phases, invocationMarker) if err != nil { return resources, failWithSecretCleanup(err) @@ -760,6 +760,19 @@ func (m *Migrator) createClusterObjectSet(ctx context.Context, opts Options, inf if err := m.WaitForClusterObjectSetAvailable(ctx, cosName); err != nil { return resources, err } + + if opts.UseClusterObjectDeployment { + // Complete initial adoption before a protected revision can supersede it. + // Only the initial COS may adopt collected OLMv0 objects blindly. + for _, phase := range phases { + phase.CollisionProtection = nil + for i := range phase.Objects { + phase.Objects[i].CollisionProtection = nil + } + } + return resources, m.createClusterObjectDeployment(ctx, opts, info, phases, invocationMarker, resources) + } + return resources, nil } @@ -768,8 +781,8 @@ func (m *Migrator) cleanupCreatedClusterObjectSet(ctx context.Context, resources if resources == nil { return nil } - if resources.cod != nil { - return fmt.Errorf("ClusterObjectDeployment may have reconciled; preserving its revisions and Secrets for explicit recovery") + if resources.cod != nil || resources.deploymentRequested && resources.cos != nil { + return fmt.Errorf("ClusterObjectSet or ClusterObjectDeployment may have reconciled; preserving its revisions and Secrets for explicit recovery") } if resources.ownershipUnknown { return fmt.Errorf("ClusterObjectSet creation outcome is unknown; refusing automatic cleanup") diff --git a/migration/pkg/migration/revision_schema.go b/migration/pkg/migration/revision_schema.go index fee0ced..462e8f5 100644 --- a/migration/pkg/migration/revision_schema.go +++ b/migration/pkg/migration/revision_schema.go @@ -27,18 +27,29 @@ func (m *Migrator) validateMigrationRevision(ctx context.Context, opts Options, return err } marker := string(uuid.NewUUID()) - var candidate client.Object - if opts.UseClusterObjectDeployment { - candidate, err = buildClusterObjectDeployment(opts, info, phases, marker) - } else { - candidate, err = buildClusterObjectSet(opts, info, phases, marker) - } + cos, err := buildClusterObjectSet(opts, info, phases, marker) if err != nil { return err } - if err := m.Client.Create(ctx, candidate, client.DryRunAll, client.FieldValidation(metav1.FieldValidationStrict)); err != nil { - return fmt.Errorf("validate migration revision with server-side dry run: %w", err) + if err := m.Client.Create(ctx, cos, client.DryRunAll, client.FieldValidation(metav1.FieldValidationStrict)); err != nil { + return fmt.Errorf("validate migration ClusterObjectSet with server-side dry run: %w", err) + } + if opts.UseClusterObjectDeployment { + for _, phase := range phases { + phase.CollisionProtection = nil + for i := range phase.Objects { + phase.Objects[i].CollisionProtection = nil + } + } + cod, err := buildClusterObjectDeployment(opts, info, phases, marker) + if err != nil { + return err + } + if err := m.Client.Create(ctx, cod, client.DryRunAll, client.FieldValidation(metav1.FieldValidationStrict)); err != nil { + return fmt.Errorf("validate migration ClusterObjectDeployment with server-side dry run: %w", err) + } } + return nil } @@ -100,6 +111,7 @@ func buildClusterObjectSet(opts Options, info *MigrationInfo, phases []*ocv1ac.C LabelBundleVersion: info.Version, migrationInvocationAnnotation: marker, } + addStagingBackup(cosAnnotations, info) if info.BundleImage != "" { cosAnnotations[LabelBundleReference] = info.BundleImage } diff --git a/migration/pkg/migration/revision_schema_test.go b/migration/pkg/migration/revision_schema_test.go index 4fd5634..41acd3d 100644 --- a/migration/pkg/migration/revision_schema_test.go +++ b/migration/pkg/migration/revision_schema_test.go @@ -20,13 +20,19 @@ import ( func TestRevisionServerDryRun(t *testing.T) { rejection := errors.New("API server rejected candidate") for _, deployment := range []bool{false, true} { - for _, serverErr := range []error{nil, rejection} { + for _, rejectAt := range []int{0, 1, 2} { + if !deployment && rejectAt == 2 { + continue + } name := "COS" if deployment { name = "COD" } - if serverErr != nil { - name += "/rejected" + if rejectAt == 1 { + name += "/cos-rejected" + } + if rejectAt == 2 { + name += "/cod-rejected" } t.Run(name, func(t *testing.T) { m := migrationTestClient(t) @@ -41,12 +47,12 @@ func TestRevisionServerDryRun(t *testing.T) { var phases []ocv1.ClusterObjectSetPhase switch candidate := obj.(type) { case *ocv1.ClusterObjectSet: - if deployment || candidate.Name != "example-1" { + if calls != 1 || candidate.Name != "example-1" { t.Fatalf("unexpected COS: %+v", candidate) } phases = candidate.Spec.Phases case *ocv1.ClusterObjectDeployment: - if !deployment || candidate.Name != "example" { + if !deployment || calls != 2 || candidate.Name != "example" || candidate.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { t.Fatalf("unexpected COD: %+v", candidate) } phases = candidate.Spec.Template.Spec.Phases @@ -60,7 +66,10 @@ func TestRevisionServerDryRun(t *testing.T) { if object.Type != ocv1.ObjectTypeSecretReference || object.SecretRef.Namespace != "controller" || object.SecretRef.Key == "" { t.Fatalf("unpacked candidate: %+v", object) } - return serverErr + if calls == rejectAt { + return rejection + } + return nil }, Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { t.Fatal("dry run deleted an object") @@ -70,7 +79,11 @@ func TestRevisionServerDryRun(t *testing.T) { opts := Options{ClusterExtensionName: "example", SystemNamespace: "controller", UseClusterObjectDeployment: deployment} info := &MigrationInfo{CollectedObjects: []unstructured.Unstructured{{Object: map[string]interface{}{"apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]interface{}{"name": "config", "namespace": "operators"}, "data": map[string]interface{}{"key": "value"}}}}} err := m.validateMigrationRevision(t.Context(), opts, info) - if !errors.Is(err, serverErr) || calls != 1 { + wantCalls := 1 + if deployment && rejectAt != 1 { + wantCalls = 2 + } + if (rejectAt == 0 && err != nil) || (rejectAt != 0 && !errors.Is(err, rejection)) || calls != wantCalls { t.Fatalf("calls=%d, error=%v", calls, err) } var secrets corev1.SecretList diff --git a/migration/pkg/migration/staging_backup.go b/migration/pkg/migration/staging_backup.go new file mode 100644 index 0000000..5b7c197 --- /dev/null +++ b/migration/pkg/migration/staging_backup.go @@ -0,0 +1,44 @@ +package migration + +import ( + "encoding/json" + "fmt" +) + +// withStagingBackup copies recovery specs into the migration plan before any +// target is created. The caller's plan and backup remain unchanged. +func withStagingBackup(info *MigrationInfo, backup *Backup) (*MigrationInfo, error) { + if info == nil { + return nil, fmt.Errorf("migration information is required") + } + copy := *info + if backup == nil { + return ©, nil + } + if backup.Subscription != nil { + data, err := json.Marshal(backup.Subscription.Spec) + if err != nil { + return nil, fmt.Errorf("encode Subscription recovery backup: %w", err) + } + copy.SubscriptionBackupJSON = string(data) + } + if backup.OperatorGroup != nil { + data, err := json.Marshal(backup.OperatorGroup.Spec) + if err != nil { + return nil, fmt.Errorf("encode OperatorGroup recovery backup: %w", err) + } + copy.OperatorGroupBackupJSON = string(data) + } + return ©, nil +} + +// addStagingBackup persists the same recovery specs used by CE rollback on the +// retained staging resource, so a failure before CE creation cannot lose them. +func addStagingBackup(annotations map[string]string, info *MigrationInfo) { + if info.SubscriptionBackupJSON != "" { + annotations[MigrationSubscriptionBackupAnnotation] = info.SubscriptionBackupJSON + } + if info.OperatorGroupBackupJSON != "" { + annotations[MigrationOperatorGroupBackupAnnotation] = info.OperatorGroupBackupJSON + } +} diff --git a/migration/pkg/migration/staging_backup_test.go b/migration/pkg/migration/staging_backup_test.go new file mode 100644 index 0000000..34d92d7 --- /dev/null +++ b/migration/pkg/migration/staging_backup_test.go @@ -0,0 +1,128 @@ +package migration + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + ocv1 "github.com/operator-framework/operator-controller/api/v1" +) + +// TestStagingBackupSurvivesFailedHandoff proves that recovery data remains on +// the server after the original caller and its in-memory backup are discarded. +func TestStagingBackupSurvivesFailedHandoff(t *testing.T) { + sub, _ := healthySubscriptionFixtures() + sub.Spec.Channel = "original-channel" + sub.Spec.InstallPlanApproval = operatorsv1alpha1.ApprovalManual + og := &operatorsv1.OperatorGroup{Spec: operatorsv1.OperatorGroupSpec{TargetNamespaces: []string{"watched"}}} + m := migrationTestClient(t, establishedClusterObjectDeploymentCRD()) + m.Client = failingMigrationClient{Client: m.Client, succeedCOS: true} + info := &MigrationInfo{} + opts := Options{SubscriptionName: sub.Name, SubscriptionNamespace: sub.Namespace, ClusterExtensionName: "staged"} + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + result, err := m.CreateMigrationResources(ctx, opts, info, &Backup{Subscription: sub, OperatorGroup: og}) + if err == nil || !result.TargetMayBeActive { + t.Fatalf("expected retained staging resources: %+v %v", result, err) + } + if info.SubscriptionBackupJSON != "" || info.OperatorGroupBackupJSON != "" { + t.Fatal("modified caller's plan") + } + var cod ocv1.ClusterObjectDeployment + if err := m.Client.Get(t.Context(), client.ObjectKey{Name: "staged"}, &cod); err != nil { + t.Fatal(err) + } + var spec operatorsv1alpha1.SubscriptionSpec + if err := json.Unmarshal([]byte(cod.Annotations[MigrationSubscriptionBackupAnnotation]), &spec); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&spec, sub.Spec) { + t.Fatalf("lost original Subscription spec: %+v", spec) + } + var ogSpec operatorsv1.OperatorGroupSpec + if err := json.Unmarshal([]byte(cod.Annotations[MigrationOperatorGroupBackupAnnotation]), &ogSpec); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ogSpec, og.Spec) { + t.Fatal("lost OperatorGroup spec") + } + var extensions ocv1.ClusterExtensionList + if err := m.Client.List(t.Context(), &extensions); err != nil { + t.Fatal(err) + } + if len(extensions.Items) != 0 { + t.Fatal("created CE after failed rollout") + } + // This fake has no object controller or workloads. Remove the target before + // exercising explicit recovery using only the persisted Subscription spec. + if err := m.Client.Delete(t.Context(), &cod); err != nil { + t.Fatal(err) + } + ns, name, err := splitSubRef(cod.Annotations[MigratedFromSubscriptionAnnotation]) + if err != nil { + t.Fatal(err) + } + fresh := &Migrator{Client: reconcilingSubscriptionClient{Client: m.Client}} + recovered := &operatorsv1alpha1.Subscription{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: &spec} + if err := fresh.RecoverFromBackup(t.Context(), opts, &Backup{Subscription: recovered}); err != nil { + t.Fatal(err) + } + var actual operatorsv1alpha1.Subscription + if err := m.Client.Get(t.Context(), client.ObjectKeyFromObject(sub), &actual); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual.Spec, sub.Spec) { + t.Fatal("persisted backup did not restore original spec") + } +} + +// TestInitialCOSBackupSurvivesCODRejection covers the failure window unique to +// COS+COD: the initial COS is retained even when no COD or CE can be created. +func TestInitialCOSBackupSurvivesCODRejection(t *testing.T) { + sub, _ := healthySubscriptionFixtures() + m := migrationTestClient(t, establishedClusterObjectDeploymentCRD()) + denied := apierrors.NewForbidden(schema.GroupResource{Resource: "clusterobjectdeployments"}, "staged", errors.New("denied")) + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*ocv1.ClusterObjectDeployment); ok { + return denied + } + if _, ok := obj.(*ocv1.ClusterExtension); ok { + t.Fatal("created CE after rejected COD") + } + if cos, ok := obj.(*ocv1.ClusterObjectSet); ok { + cos.Status.CompletedAt = metav1.Now() + cos.Status.Conditions = []metav1.Condition{{Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionTrue}} + } + return c.Create(ctx, obj, opts...) + }}) + result, err := m.CreateMigrationResources(t.Context(), Options{SubscriptionName: sub.Name, SubscriptionNamespace: sub.Namespace, ClusterExtensionName: "staged"}, &MigrationInfo{}, &Backup{Subscription: sub}) + if !errors.Is(err, denied) || !result.TargetMayBeActive { + t.Fatalf("result=%+v error=%v", result, err) + } + var cos ocv1.ClusterObjectSet + if err := m.Client.Get(t.Context(), client.ObjectKey{Name: "staged-1"}, &cos); err != nil { + t.Fatal(err) + } + var spec operatorsv1alpha1.SubscriptionSpec + if err := json.Unmarshal([]byte(cos.Annotations[MigrationSubscriptionBackupAnnotation]), &spec); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&spec, sub.Spec) { + t.Fatal("initial COS lost Subscription backup") + } + var restored operatorsv1alpha1.Subscription + if err := m.Client.Get(t.Context(), client.ObjectKeyFromObject(sub), &restored); !apierrors.IsNotFound(err) { + t.Fatalf("unexpected automatic recovery: %v", err) + } +} diff --git a/specs/20260821-migration-v0-to-v1/e2e.md b/specs/20260821-migration-v0-to-v1/e2e.md index 3fdb630..1ae71dd 100644 --- a/specs/20260821-migration-v0-to-v1/e2e.md +++ b/specs/20260821-migration-v0-to-v1/e2e.md @@ -25,18 +25,18 @@ the COD controller stamps its own generated revisions. Unit regressions cover definitive create rejection recovery, terminal rollout diagnostics, and rollback retries with dangling references to verified COD UIDs. -Migration automatically creates a COD named after the CE when the COD CRD exists; +Migration creates an initial COS with `None`, waits for completed adoption and +readiness, then creates a COD named after the CE with `Prevent` when the COD CRD exists; otherwise it directly creates a COS using the current API definitions. A present but unready COD CRD or a discovery error blocks migration rather than falling back. COD migration requires the COD API/controller and the COD-backed CE applier merged in operator-controller PR #3058. The library's pinned API dependency alone does not install that controller. Unit tests exercise creation, preflight, readiness and rollback ownership and invalid-template diagnostics. The existing handoff scenario -also checks COD identity preservation, CE ownership of the COD, and COD ownership of both -revisions when a COD-capable controller is installed. On 2026-10-29 those assertions -passed against operator-controller main `255a4026` in kind. A separate full CLI fixture -conversion and acknowledged rollback preserved the operator Deployment UID and its one -available replica, restored the Subscription, and removed the COD and both COS revisions. +also checks COD identity preservation, CE ownership of the COD, and COD ownership of +all revisions. The initial migration revision uses `None`; protected staging and +catalog revisions use `Prevent`. A negative rollout test verifies that a newly +introduced, unrelated existing object is not adopted or modified. The `migration/test-e2e-cod-rollback` target now automates those rollback assertions and collects CLI coverage. A separate CI job builds and installs operator-controller from pinned commit `dae9b613163a8fc5df36e3b9ad5a770a64c0048c`, then runs the handoff and @@ -211,9 +211,10 @@ live, COS-supersession, and COD tests publish coverage inputs; a dependent cover their combined report. The Job intentionally does not upload diagnostics or coverage artifacts. The supersession check is deliberately separate from the conversion matrix. With the pinned -experimental controller, migration creates revision 1 with collision protection `None`; -creating the ClusterExtension results in a controller-owned, catalog-derived revision 2 with -collision protection `Prevent`. The check requires both revisions to succeed. +experimental controller, migration creates revision 1 with collision protection `None` +and waits for it to complete adoption. The COD creates protected staging revision 2; +creating the ClusterExtension then produces a catalog-derived protected revision. +The check requires initial adoption to complete and the catalog revision to succeed. ## CI rollout @@ -387,3 +388,48 @@ Setup refuses existing OLM namespaces or APIs, including partial installations. The live namespace-deletion target uses Redis. The ecr-secret-operator bundle includes a custom-resource sample with `spec.aws_iam_secret`, which is absent from its CRD schema and can block OLMv1 preflight in a new namespace. Ecr-secret-operator remains in the live migration matrix; the focused Redis scenario preserves all source-namespace deletion and label-transfer assertions. Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. + +### Alternative COS + COD handoff + +`make migration/test-e2e-cos-supersession` also runs +`TestCODProtectedRolloutCollision`. With the pinned controller, migration creates +revision 1 with `None`, waits for completed adoption and readiness, then creates a COD with `Prevent`; the COD +adopts revision 1 and creates protected revision 2. CE adoption retains the COD +UID and creates the catalog revision. The collision test adds an unrelated +existing ConfigMap to a later COD template and requires a collision while that +ConfigMap remains unchanged. Both tests collect package coverage in the existing +handoff profile and run in the handoff CI jobs. + +### Recovering a handoff that stopped before CE creation + +Migration stores the original Subscription and OperatorGroup specs on its retained +staging resource using the same backup annotations as the CE. These annotations +are written with resource creation, before any rollout wait or handoff can fail, +and are propagated to COD-generated revisions. This does not depend on `--backup`. + +Export the retained COD before removing any staging resources. If COD creation +failed after a directly staged COS, export that COS instead: + +```bash +kubectl get clusterobjectdeployment "$ce_name" -o json > staging.json +# Alternative when only the initial COS exists: +# kubectl get clusterobjectset "${ce_name}-1" -o json > staging.json +jq -e ' + .metadata.annotations as $a | + ($a["olm.operatorframework.io/migrated-from-subscription"] | split("/")) as $source | + {apiVersion: "operators.coreos.com/v1alpha1", kind: "Subscription", + metadata: {namespace: $source[0], name: $source[1]}, + spec: ($a["olm.operatorframework.io/migration-subscription-backup"] | fromjson)} +' staging.json > subscription-recovery.json +jq -e '.metadata.annotations["olm.operatorframework.io/migration-operatorgroup-backup"] | fromjson' staging.json > operatorgroup-spec-recovery.json +``` + +Exporting does not restore OLMv0. Before restoring the Subscription, stop the target +controllers, release their workload ownership, and verify the source namespace and +OperatorGroup are usable; otherwise OLMv0 and OLMv1 could reconcile concurrently. +The ordinary rollback CLI requires a CE and does not automate this pre-CE recovery. +Library callers can decode the saved spec into `Backup.Subscription` and use +`RecoverFromBackup` after safely releasing the target. Keep the exported files until +recovery has succeeded. + +Revision collision observations during COD handoff are retried within the existing rollout timeout. Only current-generation completion and availability allow migration to continue; persistent collisions fail with the controller diagnostic and preserve staged recovery resources. diff --git a/specs/20260821-migration-v0-to-v1/plan.md b/specs/20260821-migration-v0-to-v1/plan.md index f6df942..f9240ce 100644 --- a/specs/20260821-migration-v0-to-v1/plan.md +++ b/specs/20260821-migration-v0-to-v1/plan.md @@ -310,3 +310,28 @@ Sources: [shared converter](https://github.com/operator-framework/library-olm/pu [typed construction](https://github.com/operator-framework/library-olm/pull/93#discussion_r4556794570). Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. + + +### Alternative COS + COD handoff + +This draft evaluates COS + COD instead of COD-only staging. Create an adoptable +initial COS with `None`, waits for it to complete adoption and become ready, then +creates an adoptable COD with `Prevent`. The +COD adopts the initial revision and creates a protected staging revision; the CE +then adopts the same COD and rolls out catalog content. No permissive phase or +object override may remain in the COD template. Failure after initial COS creation +preserves staging resources rather than restarting OLMv0 automatically. + +The focused handoff suite covers the additional staging revision and a negative +rollout collision. It remains part of the existing coverage-enabled handoff CI +jobs. The direct-COS fallback inherited from the base remains unchanged; choosing +the handoff protocol and removing the fallback are separate decisions. + +Recovery before CE creation must not rely only on an in-memory backup. Persist the +original Subscription and OperatorGroup specs on retained staging resources at +creation, before rollout/handoff failures can strand the operator. Preserve them +on failed or uncertain handoffs and refuse automatic OLMv0 restart. Regression +tests must decode the server-side backup after discarding the original plan and +verify it can restore the original Subscription spec after safe target release. + +Revision collision observations during COD handoff are retried within the existing rollout timeout. Only current-generation completion and availability allow migration to continue; persistent collisions fail with the controller diagnostic and preserve staged recovery resources. diff --git a/specs/20260821-migration-v0-to-v1/requirements.md b/specs/20260821-migration-v0-to-v1/requirements.md index 66d4f38..64e5cfd 100644 --- a/specs/20260821-migration-v0-to-v1/requirements.md +++ b/specs/20260821-migration-v0-to-v1/requirements.md @@ -155,8 +155,14 @@ ClusterExtension, with controller-managed COS revisions. The initial implementat selected automatically when the COD CRD exists; when absent, the direct COS workflow uses the current COS definitions. Discovery errors must fail rather than silently fall back. COD migration requires an established COD CRD and operator-controller's COD-backed ClusterExtension applier (upstream PR #3058). -Create the deployment with collision protection `None`, wait for current-generation rollout -completion and availability, then create the CE to hand management to operator-controller. +Create the initial COS (revision 1 in the COD's group) with collision protection `None`, +wait for its completion timestamp and readiness, then create the COD with `Prevent`. The COD +adopts the initial COS and creates a protected revision from the migration template. +Clear permissive phase/object overrides from that template. Wait for the COD's +current-generation rollout completion and availability before creating the CE. +Only the initial COS may blindly adopt collected OLMv0 objects; subsequent templates +must reject unrelated existing objects. If COD creation fails after COS creation, +preserve the initial COS and its Secrets and refuse automatic OLMv0 recovery. Set `apps.kubernetes.io/managed-by: operator-controller` on the COD itself so operator-controller retains its rollout data and can adopt it while ownerless. Preserve the same COD UID through adoption; owner-kind and @@ -179,7 +185,7 @@ Reject unverified controller identities. Definitively rejected COS/COD creates ambiguous creation outcomes continue to require explicit recovery. Carry bundle and migration annotations and extension identity labels through `spec.template.metadata` onto generated COS revisions. Report current-generation -terminal `Progressing=False` conditions other than successful rollout completion immediately, preserving the +terminal `Progressing=False` conditions other than successful rollout completion or retryable revision collisions immediately, preserving the controller's diagnostic message; ignore stale-generation failure conditions. --- @@ -382,3 +388,12 @@ unreferenced source would be deleted without changing it. Readiness reporting includes a lightweight COS API availability check; this is diagnostic, not a guarantee that a later write will succeed. Preparation does not repeat that COS CRD check. COD discovery remains necessary to select the revision manager. Gather (including CLI dry-run) and migration validate the prepared revision with a strict server-side dry-run create before removing OLMv0 management. Validation and creation share the same revision and Secret-reference builders; dry-run does not persist revisions or packed Secrets. The API server validates schema, admission, and permissions, but does not run controller-level validation of referenced workloads. Rollback permits verified newer sibling handoffs while retaining resource-UID, owner, and management-label checks. Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. + +Recovery before CE creation must not rely only on an in-memory backup. Persist the +original Subscription and OperatorGroup specs on retained staging resources at +creation, before rollout/handoff failures can strand the operator. Preserve them +on failed or uncertain handoffs and refuse automatic OLMv0 restart. Regression +tests must decode the server-side backup after discarding the original plan and +verify it can restore the original Subscription spec after safe target release. + +Revision collision observations during COD handoff are retried within the existing rollout timeout. Only current-generation completion and availability allow migration to continue; persistent collisions fail with the controller diagnostic and preserve staged recovery resources. diff --git a/specs/20260821-migration-v0-to-v1/validation.md b/specs/20260821-migration-v0-to-v1/validation.md index 43a328b..7c42229 100644 --- a/specs/20260821-migration-v0-to-v1/validation.md +++ b/specs/20260821-migration-v0-to-v1/validation.md @@ -12,7 +12,7 @@ kind cluster with both OLMv0 and OLMv1 installed. ## V1. Per-command behavior - **V1.1** `check -n ` on a healthy AllNamespaces operator with an available catalog reports all checks green and exits 0; makes no cluster changes. -- **V1.2** `convert -n --dry-run` lists every resource that would be migrated (grouped by kind), including CRDs, and reports the detected COS or COD manager with `CollisionProtection: None` on the COS or COD template; makes no cluster changes. +- **V1.2** `convert -n --dry-run` lists every resource that would be migrated (grouped by kind), including CRDs, and reports the detected COS or COD manager with `CollisionProtection: None` on the initial COS and `Prevent` on the COD template; makes no cluster changes. - **V1.3** `convert -n ` results in a `ClusterExtension` reaching `Installed=True`; the `Subscription` and `CSV` are deleted; CRDs remain and are adopted by OLMv1. In direct COS mode, the COS sets `status.completedAt` before the CE is created. In COD mode, the COD's current-generation rollout completes and becomes available before the CE is created; the CE then owns the same COD and its catalog revision uses `CollisionProtection: Prevent`. - **V1.4** `rollback --acknowledge-installed` starts CE orphan deletion with a rollback finalizer to stop new revisions, records verified ownership from in-flight replacement revisions before deleting them, waits for orphaned resources to lose COS controller references, removes their OLMv1 revision metadata, restores the `Subscription` from the backup annotation, then removes the CE finalizer. A matching Subscription from a prior attempt permits retry; a conflicting Subscription or failed restoration retains the CE backup. The operator returns to OLMv0 management (`AtLatestKnown`/`UpgradePending`), and a subsequent migration can adopt preserved resources. Without `--acknowledge-installed` on an `Installed=True` CE, rollback refuses and exits non-zero. In COD mode, verify CE-to-COD and COD-to-COS ownership and stop the COD with orphan deletion before removing its COS revisions, preventing replacement revisions during rollback. Before deletion and during subsequent revision discovery, reject controlled COS revisions unless the controller's API version, kind, name, and UID match the CE or its verified COD. Require the current `spec.group` identity even for unowned staging or orphaned revisions; a name or owner label alone cannot authorize deletion. Preserve workloads and retain the CE backup if cleanup or restoration fails. @@ -21,20 +21,20 @@ kind cluster with both OLMv0 and OLMv1 installed. - **V1.7** `convert --all` stops on the first failure by default; with `--continue-on-error` it logs the failure and continues, exiting non-zero if any operator failed. - **V1.8** No command ever blocks on interactive input. An operator whose name collides with a verb (e.g. `check`) is migratable as a positional argument (`convert check -n `). - **V1.9** Manager detection selects direct COS when the COD CRD is absent and COD when it is present and established. Discovery errors and unmet prerequisites fail before deleting OLMv0 resources. Validate COD name/group constraints and reject an existing COD rather than overwriting it. CRD presence is the supported capability signal; controller-version detection is not required. -- **V1.10** The COD template preserves migration labels, annotations, packed Secret references, and `CollisionProtection: None`. Readiness requires completion and availability for the current generation; stale status cannot satisfy the wait. Invalid-specification and deadline failures report actionable errors. Once a COD may have reconciled, failure preserves migration resources and refuses automatic OLMv0 recovery. +- **V1.10** The COD template preserves migration labels, annotations, packed Secret references, and `CollisionProtection: Prevent` without permissive phase/object overrides. Readiness requires completion and availability for the current generation; stale status cannot satisfy the wait. Invalid-specification and deadline failures report actionable errors. Once a COD may have reconciled, failure preserves migration resources and refuses automatic OLMv0 recovery. The COD itself carries `apps.kubernetes.io/managed-by: operator-controller` before CE creation and after adoption. It has no controller owner during staging, then gains the CE controller reference while retaining its UID. The template continues to carry extension identity metadata without copying this COD-specific adoption label onto operator workloads. Directly created migration COS revisions carry `apps.kubernetes.io/managed-by: - object-controller` and remain ownerless during staging. COD-generated migration + object-controller` and start ownerless; in COD mode the COD adopts them. COD-generated staging and catalog revisions carry the same COS label, while the COD retains its distinct `operator-controller` value. Assert inline and Secret-backed object encodings using the current COS/COD definitions, including workload preservation during rollback. Older experimental schemas are outside the compatibility requirements. - Assert prompt failures for all current-generation terminal `Progressing=False` reasons except successful completion, including revision collisions and invalid revisions; preserve the controller's reason and message. Definitive COS/COD create rejections restore OLMv0 and report that the target cannot be active; timeouts and transport failures refuse automatic recovery. + Assert prompt failures for all current-generation terminal `Progressing=False` reasons except successful completion and retryable revision collisions, including invalid revisions; preserve the controller's reason and message. Definitive initial COS create rejections restore OLMv0 and report that the target cannot be active; a COD rejection after COS creation preserves staging resources and refuses automatic recovery; timeouts and transport failures refuse automatic recovery. Rollback persists verified COD UIDs before deletion and accepts matching dangling references even after restarting rollback. Unknown UIDs remain rejected, and failure to persist verified identity prevents COD deletion. ## V2. Eligibility matrix (one case per check, R3) @@ -108,7 +108,7 @@ flag flips it to `Eligible`: - **V5.8** `rollback --acknowledge-installed` → Subscription restored, CE and COS revisions deleted, preserved Deployments released from OLMv1 ownership; a second `convert` creates new CE/COS UIDs, installs successfully, and adopts the same Deployment UIDs. In COD mode, also verify the COD is deleted and no COS remains in its group, while the Deployment UID and availability are preserved. Fixture tests assert restored Subscription state; OLMv0 reconciliation requires the live environment. - **V5.9** Fresh cluster with operators in all four states → `check --all` → correct four-section output; `convert --all` warns on Conflicts and migrates only Eligible. -- **V5.10** On a COD-capable controller, fixture migration creates one COD named for the CE. Catalog handoff retains that COD's UID, establishes CE ownership of the COD and COD ownership of successive COS revisions in the same group, and retains the migration revision (`None`) alongside a new catalog revision (`Prevent`). Verify the migration COS retains its UID and both revisions retain their respective collision-protection settings. Run handoff and fixture conversion/rollback (V5.8) in the dedicated COD CI job, collecting coverage separately from the other fixture tests. +- **V5.10** On a COD-capable controller, fixture migration creates one COD named for the CE. Catalog handoff retains that COD's UID, establishes CE ownership of the COD and COD ownership of successive COS revisions in the same group, and retains the migration revision (`None`) alongside a new catalog revision (`Prevent`). Verify the migration COS retains its UID and all revisions retain their respective collision-protection settings. Before COD creation, require initial COS completion and readiness; a timeout must retain the COS and backups without creating a COD or CE. Before CE creation, assert revision 1 completed adoption and uses `None` and the COD-created staging revision uses `Prevent`; catalog handoff produces another protected revision. Add an unrelated existing object to a later COD template and assert `RevisionObjectCollision` without changing that object’s UID, data, labels, or ownership. Run handoff and fixture conversion/rollback (V5.8) in the dedicated COD CI job, collecting coverage separately from the other fixture tests. ## V6. Non-functional (Jira deployment considerations) @@ -158,3 +158,12 @@ flag flips it to `Eligible`: Readiness reporting includes a lightweight COS API availability check; this is diagnostic, not a guarantee that a later write will succeed. Preparation does not repeat that COS CRD check. COD discovery remains necessary to select the revision manager. Gather (including CLI dry-run) and migration validate the prepared revision with a strict server-side dry-run create before removing OLMv0 management. Validation and creation share the same revision and Secret-reference builders; dry-run does not persist revisions or packed Secrets. The API server validates schema, admission, and permissions, but does not run controller-level validation of referenced workloads. Rollback permits verified newer sibling handoffs while retaining resource-UID, owner, and management-label checks. Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. + +Recovery before CE creation must not rely only on an in-memory backup. Persist the +original Subscription and OperatorGroup specs on retained staging resources at +creation, before rollout/handoff failures can strand the operator. Preserve them +on failed or uncertain handoffs and refuse automatic OLMv0 restart. Regression +tests must decode the server-side backup after discarding the original plan and +verify it can restore the original Subscription spec after safe target release. + +Revision collision observations during COD handoff are retried within the existing rollout timeout. Only current-generation completion and availability allow migration to continue; persistent collisions fail with the controller diagnostic and preserve staged recovery resources. diff --git a/test/e2e/migration/cod_collision_test.go b/test/e2e/migration/cod_collision_test.go new file mode 100644 index 0000000..2732957 --- /dev/null +++ b/test/e2e/migration/cod_collision_test.go @@ -0,0 +1,111 @@ +//go:build e2e + +package e2e + +import ( + "context" + "os" + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + + "github.com/operator-framework/library-olm/migration/pkg/migration" +) + +// TestCODProtectedRolloutCollision proves only the initial migration objects can +// be adopted blindly; an additional object in a later template must collide. +func TestCODProtectedRolloutCollision(t *testing.T) { + if os.Getenv("E2E_COS_SUPERSESSION_TEST") != "true" { + t.Skip("requires the focused handoff suite") + } + m, c, _ := newMigrator(t) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "migration-collision-"}} + if err := c.Create(ctx, ns); err != nil { + t.Fatal(err) + } + name := ns.Name + systemNamespace := "" + t.Cleanup(func() { + collectArtifacts(t, ns.Name) + run(t, "kubectl", "delete", "clusterobjectdeployment/"+name, "--ignore-not-found", "--wait=true", "--timeout=60s") + run(t, "kubectl", "delete", "clusterobjectset", "-l", migration.LabelOwnerName+"="+name, "--ignore-not-found", "--wait=true", "--timeout=60s") + if systemNamespace != "" { + run(t, "kubectl", "delete", "secret", "-n", systemNamespace, "-l", migration.LabelOwnerName+"="+name, "--ignore-not-found", "--wait=true", "--timeout=60s") + } + run(t, "kubectl", "delete", "namespace/"+ns.Name, "--wait=false") + }) + original := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "migrated", Namespace: ns.Name}, Data: map[string]string{"value": "original"}} + foreign := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foreign", Namespace: ns.Name}, Data: map[string]string{"value": "do-not-change"}} + for _, obj := range []client.Object{original, foreign} { + if err := c.Create(ctx, obj); err != nil { + t.Fatal(err) + } + } + manifest := func(name, value string) unstructured.Unstructured { + return unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]interface{}{"name": name, "namespace": ns.Name}, "data": map[string]interface{}{"value": value}}} + } + opts, err := m.PrepareClusterObjectSet(ctx, migration.Options{SubscriptionName: name, SubscriptionNamespace: ns.Name}) + if err != nil { + t.Fatal(err) + } + systemNamespace = opts.SystemNamespace + if !opts.UseClusterObjectDeployment { + t.Fatal("test requires COD support") + } + if err := m.CreateClusterObjectSet(ctx, opts, &migration.MigrationInfo{CollectedObjects: []unstructured.Unstructured{manifest(original.Name, "original")}}); err != nil { + t.Fatal(err) + } + var cod ocv1.ClusterObjectDeployment + if err := c.Get(ctx, client.ObjectKey{Name: name}, &cod); err != nil { + t.Fatal(err) + } + if cod.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + t.Fatal("COD is not protected") + } + var adopted corev1.ConfigMap + if err := c.Get(ctx, client.ObjectKeyFromObject(original), &adopted); err != nil { + t.Fatal(err) + } + if adopted.UID != original.UID || metav1.GetControllerOf(&adopted) == nil { + t.Fatal("initial object was not adopted in place") + } + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := c.Get(ctx, client.ObjectKey{Name: name}, &cod); err != nil { + return err + } + before := cod.DeepCopy() + cod.Spec.Template.Spec.Phases[0].Objects = append(cod.Spec.Template.Spec.Phases[0].Objects, ocv1.ClusterObjectSetObject{Type: ocv1.ObjectTypeInlineManifest, InlineManifest: manifest(foreign.Name, "must-not-be-applied")}) + return c.Patch(ctx, &cod, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) + }); err != nil { + t.Fatal(err) + } + if err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + var current ocv1.ClusterObjectDeployment + if err := c.Get(ctx, client.ObjectKey{Name: name}, ¤t); err != nil { + return false, err + } + condition := apimeta.FindStatusCondition(current.Status.Conditions, ocv1.ClusterObjectDeploymentTypeProgressing) + return condition != nil && condition.ObservedGeneration == current.Generation && condition.Status == metav1.ConditionFalse && condition.Reason == ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision, nil + }); err != nil { + t.Fatalf("expected collision on newly introduced object: %v", err) + } + var preserved corev1.ConfigMap + if err := c.Get(ctx, client.ObjectKeyFromObject(foreign), &preserved); err != nil { + t.Fatal(err) + } + if preserved.UID != foreign.UID || !reflect.DeepEqual(preserved.Data, foreign.Data) || !reflect.DeepEqual(preserved.OwnerReferences, foreign.OwnerReferences) || !reflect.DeepEqual(preserved.Labels, foreign.Labels) { + t.Fatalf("foreign object was modified: %+v", preserved) + } +} diff --git a/test/e2e/migration/e2e_test.go b/test/e2e/migration/e2e_test.go index be5cd4f..98690e5 100644 --- a/test/e2e/migration/e2e_test.go +++ b/test/e2e/migration/e2e_test.go @@ -4,12 +4,14 @@ package e2e import ( "bytes" + "cmp" "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "slices" "strings" "testing" @@ -329,9 +331,14 @@ func TestPrecreatedClusterObjectSetSupersession(t *testing.T) { }); err != nil { t.Fatalf("list pre-existing COS: %v", err) } - if len(before.Items) != 1 { - t.Fatalf("pre-existing COS count = %d, want 1", len(before.Items)) + expectedBefore := 1 + if opts.UseClusterObjectDeployment { + expectedBefore = 2 + } + if len(before.Items) != expectedBefore { + t.Fatalf("pre-existing COS count = %d, want %d", len(before.Items), expectedBefore) } + slices.SortFunc(before.Items, func(a, b ocv1.ClusterObjectSet) int { return cmp.Compare(a.Spec.Revision, b.Spec.Revision) }) precreatedUID := before.Items[0].UID var originalCOD ocv1.ClusterObjectDeployment if opts.UseClusterObjectDeployment { @@ -341,6 +348,16 @@ func TestPrecreatedClusterObjectSetSupersession(t *testing.T) { if !metav1.IsControlledBy(&before.Items[0], &originalCOD) || before.Items[0].Spec.Group != subscription { t.Fatal("migration revision is not controlled by its COD") } + protected := &before.Items[1] + if protected.Spec.Revision != 2 || protected.Spec.Group != subscription || !metav1.IsControlledBy(protected, &originalCOD) { + t.Fatal("protected staging revision is not revision 2 controlled by its COD") + } + if protected.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + t.Fatal("staging revision must enforce Prevent before CE adoption") + } + if originalCOD.Spec.Template.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + t.Fatal("staging COD must enforce Prevent before CE adoption") + } if originalCOD.Labels["apps.kubernetes.io/managed-by"] != "operator-controller" || metav1.GetControllerOf(&originalCOD) != nil { t.Fatal("staging COD must be ownerless and carry the operator-controller adoption label") } @@ -369,8 +386,8 @@ func TestPrecreatedClusterObjectSetSupersession(t *testing.T) { }); err != nil { t.Fatalf("list COS after CE creation: %v", err) } - if len(after.Items) != 2 { - t.Fatalf("COS count after CE creation = %d, want 2 (migration and catalog revisions)", len(after.Items)) + if len(after.Items) != expectedBefore+1 { + t.Fatalf("COS count after CE creation = %d, want %d", len(after.Items), expectedBefore+1) } var migrationCOS, catalogCOS *ocv1.ClusterObjectSet for i := range after.Items { @@ -378,12 +395,12 @@ func TestPrecreatedClusterObjectSetSupersession(t *testing.T) { switch cos.Spec.Revision { case 1: migrationCOS = cos - case 2: + case int64(expectedBefore + 1): catalogCOS = cos } } if migrationCOS == nil || catalogCOS == nil { - t.Fatalf("COS revisions = %#v, want migration revision 1 and catalog revision 2", after.Items) + t.Fatalf("COS revisions = %#v, want migration revision 1 and catalog revision %d", after.Items, expectedBefore+1) } if migrationCOS.UID != precreatedUID { t.Fatalf("migration COS UID = %s, want pre-created UID %s", migrationCOS.UID, precreatedUID) @@ -816,16 +833,30 @@ func TestSystemManagedNamespaceMigration(t *testing.T) { if err := json.Unmarshal([]byte(namespaceJSON), &managedNamespace); err != nil { t.Fatalf("decode system-managed namespace: %v", err) } - catalogCOS := subscription + "-2" - ownedByCatalogCOS := false - for _, owner := range managedNamespace.OwnerReferences { - if owner.Kind == ocv1.ClusterObjectSetKind && owner.Name == catalogCOS { - ownedByCatalogCOS = true - break - } + owner := metav1.GetControllerOf(&managedNamespace) + if owner == nil || owner.Kind != ocv1.ClusterObjectSetKind || owner.APIVersion != ocv1.GroupVersion.String() { + t.Fatalf("system-managed namespace has no COS controller: %#v", managedNamespace.OwnerReferences) } - if !ownedByCatalogCOS { - t.Fatalf("system-managed namespace %q ownerReferences = %#v, want catalog ClusterObjectSet %q", targetNamespace, managedNamespace.OwnerReferences, catalogCOS) + _, kubeClient, _ := newMigrator(t) + var revision ocv1.ClusterObjectSet + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: owner.Name}, &revision); err != nil { + t.Fatal(err) + } + if revision.UID != owner.UID || revision.Spec.Group != subscription || revision.Spec.CollisionProtection != ocv1.CollisionProtectionPrevent { + t.Fatalf("namespace owner is not a protected revision of %s: %+v", subscription, revision) + } + var extension ocv1.ClusterExtension + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: subscription}, &extension); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(&revision, &extension) { + var deployment ocv1.ClusterObjectDeployment + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: subscription}, &deployment); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(&revision, &deployment) || !metav1.IsControlledBy(&deployment, &extension) { + t.Fatal("namespace revision is not managed by the migrated ClusterExtension") + } } targetDeployments, err := output("kubectl", "get", "deployment", "-n", targetNamespace, "-o", "name") if err != nil || strings.TrimSpace(targetDeployments) == "" {