From 1df22dd4c25721e3e92183873874932449aedff8 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Fri, 13 Nov 2026 10:07:40 -0500 Subject: [PATCH] fix: preserve rollback resource identities for remigration Record verified resource identities on the restored Subscription before removing the ClusterExtension backup. Retry conflicting updates and refuse to overwrite a changed Subscription. During remigration, include retained unowned resources missing from the OLMv0 inventory, skipping missing and duplicate objects while rejecting replacements and newly controlled objects. Add regression tests and update the migration specifications. Validation: make migration/test-unit; make lint; go test -tags=e2e ./test/e2e/migration -run ^$. Signed-off-by: Todd Short --- migration/pkg/migration/collector.go | 7 + migration/pkg/migration/remigration.go | 52 ++++++++ migration/pkg/migration/remigration_test.go | 121 ++++++++++++++++++ migration/pkg/migration/scan.go | 28 ++++ specs/20260821-migration-v0-to-v1/e2e.md | 2 + specs/20260821-migration-v0-to-v1/plan.md | 2 + .../requirements.md | 2 + .../20260821-migration-v0-to-v1/validation.md | 2 + 8 files changed, 216 insertions(+) create mode 100644 migration/pkg/migration/remigration.go create mode 100644 migration/pkg/migration/remigration_test.go diff --git a/migration/pkg/migration/collector.go b/migration/pkg/migration/collector.go index d473976..3d9f0d4 100644 --- a/migration/pkg/migration/collector.go +++ b/migration/pkg/migration/collector.go @@ -276,6 +276,13 @@ func (m *Migrator) CollectResources(ctx context.Context, opts Options, csv *oper } } + retained, err := m.collectRollbackResources(ctx, opts, seen) + if err != nil { + return nil, err + } + for _, obj := range retained { + addIfNew(obj) + } return collected, nil } diff --git a/migration/pkg/migration/remigration.go b/migration/pkg/migration/remigration.go new file mode 100644 index 0000000..0b205fc --- /dev/null +++ b/migration/pkg/migration/remigration.go @@ -0,0 +1,52 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" +) + +// collectRollbackResources recovers identities recorded before rollback orphaned +// catalog-generated resources, which need not appear in OLMv0's InstallPlan. +func (m *Migrator) collectRollbackResources(ctx context.Context, opts Options, seen map[string]bool) ([]unstructured.Unstructured, error) { + var sub operatorsv1alpha1.Subscription + if err := m.Client.Get(ctx, client.ObjectKey{Namespace: opts.SubscriptionNamespace, Name: opts.SubscriptionName}, &sub); err != nil { + return nil, client.IgnoreNotFound(err) + } + encoded := sub.Annotations[rollbackReleaseSnapshotAnnotation] + if encoded == "" { + return nil, nil + } + var snapshot []rollbackObject + if err := json.Unmarshal([]byte(encoded), &snapshot); err != nil { + return nil, fmt.Errorf("decode rollback resource identities: %w", err) + } + var result []unstructured.Unstructured + for _, saved := range snapshot { + obj := saved.Identity.DeepCopy() + if seen[resourceKey(*obj)] { + continue + } + if saved.UID == "" || obj.GetName() == "" || obj.GetAPIVersion() == "" || obj.GetKind() == "" || olmv0OnlyKinds[obj.GetKind()] { + return nil, fmt.Errorf("invalid rollback resource identity: %s", resourceKey(*obj)) + } + if err := m.Client.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, fmt.Errorf("read rollback resource: %w", err) + } + if obj.GetUID() != saved.UID || obj.GetDeletionTimestamp() != nil || metav1.GetControllerOf(obj) != nil { + return nil, fmt.Errorf("rollback resource %s was replaced, claimed or deleted", resourceKey(*obj)) + } + result = append(result, *obj) + } + return result, nil +} diff --git a/migration/pkg/migration/remigration_test.go b/migration/pkg/migration/remigration_test.go new file mode 100644 index 0000000..62b96a0 --- /dev/null +++ b/migration/pkg/migration/remigration_test.go @@ -0,0 +1,121 @@ +package migration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + ocv1 "github.com/operator-framework/operator-controller/api/v1" +) + +func TestCollectRollbackResources(t *testing.T) { + for _, mode := range []string{"retained", "missing", "replaced", "claimed", "already collected", "invalid snapshot"} { + t.Run(mode, func(t *testing.T) { + role := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "catalog-role", UID: "original"}} + identity := unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", "metadata": map[string]interface{}{"name": role.Name}}} + data, err := json.Marshal([]rollbackObject{{Identity: identity, UID: role.UID}}) + if err != nil { + t.Fatal(err) + } + sub := &operatorsv1alpha1.Subscription{ObjectMeta: metav1.ObjectMeta{Name: "operator", Namespace: "source", Annotations: map[string]string{rollbackReleaseSnapshotAnnotation: string(data)}}} + if mode == "replaced" { + role.UID = "replacement" + } + if mode == "claimed" { + yes := true + role.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "other", UID: "other", Controller: &yes}} + } + if mode == "invalid snapshot" { + sub.Annotations[rollbackReleaseSnapshotAnnotation] = "{" + } + objects := []runtime.Object{sub} + if mode != "missing" { + objects = append(objects, role) + } + m := migrationTestClient(t, objects...) + seen := map[string]bool{} + if mode == "already collected" { + seen[resourceKey(identity)] = true + } + got, err := m.collectRollbackResources(t.Context(), Options{SubscriptionName: sub.Name, SubscriptionNamespace: sub.Namespace}, seen) + wantError := mode == "replaced" || mode == "claimed" || mode == "invalid snapshot" + if (err != nil) != wantError { + t.Fatalf("error = %v", err) + } + if mode == "retained" && (len(got) != 1 || got[0].GetUID() != role.UID) { + t.Fatalf("lost retained catalog RBAC: %+v", got) + } + if (mode == "missing" || mode == "already collected") && len(got) != 0 { + t.Fatal("collected absent or duplicate resource") + } + }) + } +} + +func TestRollbackInventoryRetriesStatusConflict(t *testing.T) { + for _, changedSpec := range []bool{false, true} { + t.Run(fmt.Sprint("changedSpec=", changedSpec), func(t *testing.T) { + sub := &operatorsv1alpha1.Subscription{ObjectMeta: metav1.ObjectMeta{Name: "sub", Namespace: "source"}, Spec: &operatorsv1alpha1.SubscriptionSpec{Package: "widgets", CatalogSource: "catalog", CatalogSourceNamespace: "olm"}} + encoded, err := json.Marshal(sub.Spec) + if err != nil { + t.Fatal(err) + } + ce := &ocv1.ClusterExtension{ObjectMeta: metav1.ObjectMeta{Name: "widgets", Annotations: map[string]string{MigratedFromSubscriptionAnnotation: "source/sub", MigrationSubscriptionBackupAnnotation: string(encoded)}}} + m := migrationTestClient(t, sub, ce, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source"}}) + attempts := 0 + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*operatorsv1alpha1.Subscription); !ok { + return c.Patch(ctx, obj, patch, opts...) + } + attempts++ + if attempts != 1 { + return c.Patch(ctx, obj, patch, opts...) + } + if changedSpec { + var current operatorsv1alpha1.Subscription + if err := c.Get(ctx, client.ObjectKeyFromObject(sub), ¤t); err != nil { + return err + } + current.Spec.Channel = "changed" + if err := c.Update(ctx, ¤t); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Resource: "subscriptions"}, sub.Name, errors.New("concurrent update")) + }}) + err = m.Rollback(t.Context(), Options{ClusterExtensionName: ce.Name, AcknowledgeInstalled: true}) + if changedSpec { + if err == nil { + t.Fatal("accepted changed Subscription") + } + if err := m.Client.Get(t.Context(), client.ObjectKeyFromObject(ce), &ocv1.ClusterExtension{}); err != nil { + t.Fatalf("lost recovery backup: %v", err) + } + return + } + if err != nil || attempts != 2 { + t.Fatalf("rollback did not retry conflict: attempts=%d error=%v", attempts, err) + } + var restored operatorsv1alpha1.Subscription + if err := m.Client.Get(t.Context(), client.ObjectKeyFromObject(sub), &restored); err != nil { + t.Fatal(err) + } + if restored.Annotations[rollbackReleaseSnapshotAnnotation] == "" { + t.Fatal("lost rollback inventory") + } + }) + } +} diff --git a/migration/pkg/migration/scan.go b/migration/pkg/migration/scan.go index 2de36db..76c1c95 100644 --- a/migration/pkg/migration/scan.go +++ b/migration/pkg/migration/scan.go @@ -13,6 +13,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" @@ -501,6 +502,11 @@ func (m *Migrator) RollbackClusterExtension(ctx context.Context, ceName string, restoredSub.Name = name restoredSub.Namespace = ns restoredSub.Spec = &subSpec + snapshotJSON, err := json.Marshal(ownedObjects) + if err != nil { + return fmt.Errorf("encode remigration resource identities: %w", err) + } + restoredSub.Annotations = map[string]string{rollbackReleaseSnapshotAnnotation: string(snapshotJSON)} if err := m.Client.Create(ctx, restoredSub); err != nil { if !apierrors.IsAlreadyExists(err) { return fmt.Errorf("failed to restore Subscription %s/%s: %w", ns, name, err) @@ -514,6 +520,28 @@ func (m *Migrator) RollbackClusterExtension(ctx context.Context, ceName string, } } + // Persist verified identities before removing the CE backup, including on retry. + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var restored operatorsv1alpha1.Subscription + if err := m.Client.Get(ctx, subKey, &restored); err != nil { + return err + } + if restored.DeletionTimestamp != nil || restored.Spec == nil || !apiequality.Semantic.DeepEqual(*restored.Spec, subSpec) { + return fmt.Errorf("restored Subscription %s/%s changed before saving remigration identities", ns, name) + } + if restored.Annotations[rollbackReleaseSnapshotAnnotation] == string(snapshotJSON) { + return nil + } + before := restored.DeepCopy() + if restored.Annotations == nil { + restored.Annotations = map[string]string{} + } + restored.Annotations[rollbackReleaseSnapshotAnnotation] = string(snapshotJSON) + return m.Client.Patch(ctx, &restored, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) + }); err != nil { + return fmt.Errorf("save remigration resource identities: %w", err) + } + if err := m.finishRollbackDeletion(ctx, ceName); err != nil { return err } diff --git a/specs/20260821-migration-v0-to-v1/e2e.md b/specs/20260821-migration-v0-to-v1/e2e.md index a8910c7..b3a4633 100644 --- a/specs/20260821-migration-v0-to-v1/e2e.md +++ b/specs/20260821-migration-v0-to-v1/e2e.md @@ -385,3 +385,5 @@ Readiness reporting includes a lightweight COS API availability check; this is d Setup refuses existing OLM namespaces or APIs, including partial installations. Run `E2E_CLUSTER_NAME= make migration/e2e-teardown` before repeating setup. It installs the pinned controller from scratch and waits for the revision APIs, object-controller, operator-controller, and catalogd. Test targets reuse the prepared cluster without invoking setup. The live namespace-deletion target uses Redis. The ecr-secret-operator bundle includes a custom-resource sample with `spec.aws_iam_secret`, which is absent from its CRD schema and can block OLMv1 preflight in a new namespace. Ecr-secret-operator remains in the live migration matrix; the focused Redis scenario preserves all source-namespace deletion and label-transfer assertions. + +Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. diff --git a/specs/20260821-migration-v0-to-v1/plan.md b/specs/20260821-migration-v0-to-v1/plan.md index 901d843..004bdba 100644 --- a/specs/20260821-migration-v0-to-v1/plan.md +++ b/specs/20260821-migration-v0-to-v1/plan.md @@ -308,3 +308,5 @@ Sources: [COD labels](https://github.com/operator-framework/library-olm/pull/93# Sources: [shared converter](https://github.com/operator-framework/library-olm/pull/93#discussion_r4554296554), [typed construction](https://github.com/operator-framework/library-olm/pull/93#discussion_r4556794570). + +Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. diff --git a/specs/20260821-migration-v0-to-v1/requirements.md b/specs/20260821-migration-v0-to-v1/requirements.md index fa74913..66d4f38 100644 --- a/specs/20260821-migration-v0-to-v1/requirements.md +++ b/specs/20260821-migration-v0-to-v1/requirements.md @@ -380,3 +380,5 @@ unreferenced source would be deleted without changing it. - Console UI. Readiness reporting includes a lightweight COS API availability check; this is diagnostic, not a guarantee that a later write will succeed. Preparation does not repeat that COS CRD check. COD discovery remains necessary to select the revision manager. Gather (including CLI dry-run) and migration validate the prepared revision with a strict server-side dry-run create before removing OLMv0 management. Validation and creation share the same revision and Secret-reference builders; dry-run does not persist revisions or packed Secrets. The API server validates schema, admission, and permissions, but does not run controller-level validation of referenced workloads. Rollback permits verified newer sibling handoffs while retaining resource-UID, owner, and management-label checks. + +Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated. diff --git a/specs/20260821-migration-v0-to-v1/validation.md b/specs/20260821-migration-v0-to-v1/validation.md index 71cba89..43a328b 100644 --- a/specs/20260821-migration-v0-to-v1/validation.md +++ b/specs/20260821-migration-v0-to-v1/validation.md @@ -156,3 +156,5 @@ flag flips it to `Eligible`: | R10 Non-goals | V6.4, V6.5 | Readiness reporting includes a lightweight COS API availability check; this is diagnostic, not a guarantee that a later write will succeed. Preparation does not repeat that COS CRD check. COD discovery remains necessary to select the revision manager. Gather (including CLI dry-run) and migration validate the prepared revision with a strict server-side dry-run create before removing OLMv0 management. Validation and creation share the same revision and Secret-reference builders; dry-run does not persist revisions or packed Secrets. The API server validates schema, admission, and permissions, but does not run controller-level validation of referenced workloads. Rollback permits verified newer sibling handoffs while retaining resource-UID, owner, and management-label checks. + +Rollback persists verified resource identities and UIDs on the restored Subscription before deleting the CE backup. Remigration includes still-unowned retained resources (including catalog-generated RBAC) omitted from the OLMv0 inventory; missing resources are skipped, replacements and newly controlled resources are rejected, and normal collection is deduplicated.