diff --git a/Makefile b/Makefile index 832376c87..35562a86d 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,8 @@ GOLANGCI_LINT_VERSION := 2.1.6 GORELEASER_VERSION := 2.13.0 GOTESTSUM_VERSION := 1.8.1 HELM_VERSION := 3.18.6 -KCP_VERSION := 0.29.0 +# unreleased kcp version with vw code for schemas +KCP_VERSION := 301a8f749e7b99a0c81f43b37aa5b5e5ff0fc0b4 KUBE_APPLYCONFIGURATION_GEN_VERSION := v0.32.0 KUBE_CLIENT_GEN_VERSION := v0.32.0 KUBE_INFORMER_GEN_VERSION := v0.32.0 @@ -156,7 +157,11 @@ install-boilerplate: .PHONY: install-kcp install-kcp: - @hack/uget.sh https://github.com/kcp-dev/kcp/releases/download/v{VERSION}/kcp_{VERSION}_{GOOS}_{GOARCH}.tar.gz kcp $(KCP_VERSION) + @if echo "$(KCP_VERSION)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \ + hack/uget.sh https://github.com/kcp-dev/kcp/releases/download/v{VERSION}/kcp_{VERSION}_{GOOS}_{GOARCH}.tar.gz kcp $(KCP_VERSION); \ + else \ + GOBIN=$(abspath $(UGET_DIRECTORY)) hack/go-install.sh github.com/kcp-dev/kcp/cmd/kcp kcp $(KCP_VERSION); \ + fi GORELEASER = $(UGET_DIRECTORY)/goreleaser-$(GORELEASER_VERSION) diff --git a/backend/provider/kcp/controllers/apibindingtemplate/controller.go b/backend/provider/kcp/controllers/apibindingtemplate/controller.go new file mode 100644 index 000000000..4df27e3bf --- /dev/null +++ b/backend/provider/kcp/controllers/apibindingtemplate/controller.go @@ -0,0 +1,180 @@ +/* +Copyright 2026 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apibindingtemplate contains a kcp-specific controller that watches +// APIBindings in provider/backend workspaces and automatically creates or +// updates APIServiceExportTemplates based on the APIResourceSchemas exposed by +// each bound APIExport. +package apibindingtemplate + +import ( + "context" + "fmt" + "strings" + + apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + "github.com/kube-bind/kube-bind/backend/provider/kcp/controllers/shared" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +const controllerName = "kube-bind-kcp-apibinding-template" + +// APIBindingTemplateReconciler watches APIBindings and ensures an +// APIServiceExportTemplate exists for every bound APIExport. +type APIBindingTemplateReconciler struct { + manager mcmanager.Manager + opts controller.TypedOptions[mcreconcile.Request] + ignorePrefixes []string + scheme *runtime.Scheme + vwCache *shared.VWClientCache +} + +// New returns a new APIBindingTemplateReconciler. +func New( + ctx context.Context, + mgr mcmanager.Manager, + opts controller.TypedOptions[mcreconcile.Request], + ignorePrefixes []string, + baseConfig *rest.Config, + scheme *runtime.Scheme, +) (*APIBindingTemplateReconciler, error) { + r := &APIBindingTemplateReconciler{ + manager: mgr, + opts: opts, + ignorePrefixes: ignorePrefixes, + scheme: scheme, + vwCache: shared.NewVWClientCache(baseConfig, scheme), + } + + return r, nil +} + +// shouldIgnore returns true if the APIBinding name matches any of the +// configured ignore prefixes. +func (r *APIBindingTemplateReconciler) shouldIgnore(name string) bool { + for _, prefix := range r.ignorePrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// Reconcile implements reconcile.Reconciler for multicluster-runtime. +func (r *APIBindingTemplateReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + if r.shouldIgnore(req.Name) { + logger.V(4).Info("Ignoring APIBinding matching ignore prefix", "name", req.Name) + return ctrl.Result{}, nil + } + + logger.Info("Reconciling APIBinding", "request", req) + + cl, err := r.manager.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get client for cluster %q: %w", req.ClusterName, err) + } + + c := cl.GetClient() + clusterConfig := cl.GetConfig() + + binding := &apisv1alpha2.APIBinding{} + if err := c.Get(ctx, req.NamespacedName, binding); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to get APIBinding %q: %w", req.Name, err) + } + + // Build the schema getter with VW fallback using the shared cache. + getSchema := shared.SchemaGetterWithFallback(c, clusterConfig, r.vwCache) + + rec := reconciler{ + client: c, + scheme: r.scheme, + getAPIResourceSchema: getSchema, + } + + if err := rec.reconcile(ctx, binding); err != nil { + logger.Error(err, "Failed to reconcile APIBinding", "name", req.Name) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// getTemplateMapper returns a mapper that enqueues the owning APIBinding when +// an APIServiceExportTemplate changes. +// +// This function has the signature func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler +// because multicluster-runtime's mcbuilder.Watches accepts a "per-cluster event handler factory" +// rather than a plain handler — it calls this factory for each cluster that is engaged. +func getTemplateMapper(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + annotations := obj.GetAnnotations() + if annotations == nil { + return nil + } + ownerName, ok := annotations[shared.AnnotationOwnerBinding] + if !ok { + return nil + } + + c := cl.GetClient() + var binding apisv1alpha2.APIBinding + if err := c.Get(ctx, client.ObjectKey{Name: ownerName}, &binding); err != nil { + return nil + } + + return []mcreconcile.Request{ + { + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{Name: ownerName}, + }, + ClusterName: clusterName, + }, + } + }) +} + +// SetupWithManager registers the controller with the multicluster-runtime Manager. +func (r *APIBindingTemplateReconciler) SetupWithManager(mgr mcmanager.Manager) error { + return mcbuilder.ControllerManagedBy(mgr). + For(&apisv1alpha2.APIBinding{}). + Watches( + &kubebindv1alpha2.APIServiceExportTemplate{}, + getTemplateMapper, + ). + WithOptions(r.opts). + Named(controllerName). + Complete(r) +} diff --git a/backend/provider/kcp/controllers/apibindingtemplate/reconcile.go b/backend/provider/kcp/controllers/apibindingtemplate/reconcile.go new file mode 100644 index 000000000..67456f366 --- /dev/null +++ b/backend/provider/kcp/controllers/apibindingtemplate/reconcile.go @@ -0,0 +1,236 @@ +/* +Copyright 2026 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apibindingtemplate + +import ( + "context" + "fmt" + + apisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1" + apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kube-bind/kube-bind/backend/provider/kcp/controllers/shared" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +// templateNameForBinding derives a deterministic APIServiceExportTemplate name +// from the APIBinding name and scope. The naming convention is: +// +// -cluster for ClusterScope +// -namespaced for NamespacedScope +// +// Note: if a binding name already ends in "-cluster" or "-namespaced", this +// could theoretically collide with another binding's template name. In practice +// kcp APIBinding names are generated and unlikely to hit this edge case. +func templateNameForBinding(bindingName string, scope kubebindv1alpha2.InformerScope) string { + switch scope { + case kubebindv1alpha2.ClusterScope: + return bindingName + "-cluster" + default: + return bindingName + "-namespaced" + } +} + +type reconciler struct { + client client.Client + scheme *runtime.Scheme + getAPIResourceSchema func(ctx context.Context, name string) (*apisv1alpha1.APIResourceSchema, error) +} + +func (r *reconciler) reconcile(ctx context.Context, binding *apisv1alpha2.APIBinding) error { + logger := klog.FromContext(ctx) + + if binding.Status.Phase != apisv1alpha2.APIBindingPhaseBound { + return nil + } + if len(binding.Status.BoundResources) == 0 { + logger.V(4).Info("APIBinding has no bound resources yet, skipping", "binding", binding.Name) + return nil + } + + // Build separate templates per scope. + templates, err := r.buildTemplates(ctx, binding) + if err != nil { + return err + } + + desiredNames := make(map[string]bool, len(templates)) + for _, desired := range templates { + desiredNames[desired.Name] = true + + // Set owner reference: APIBinding owns the template. + if err := controllerutil.SetOwnerReference(binding, desired, r.scheme); err != nil { + return fmt.Errorf("failed to set owner reference on APIServiceExportTemplate %q: %w", desired.Name, err) + } + + if err := r.ensureTemplate(ctx, binding, desired); err != nil { + return err + } + } + + // Prune orphaned templates: if a binding used to have resources of a + // given scope but no longer does, the old template must be deleted. + return r.pruneOrphanedTemplates(ctx, binding, desiredNames) +} + +// pruneOrphanedTemplates lists templates owned by this binding and deletes any +// that are no longer in the desired set. +func (r *reconciler) pruneOrphanedTemplates(ctx context.Context, binding *apisv1alpha2.APIBinding, desiredNames map[string]bool) error { + logger := klog.FromContext(ctx) + + var allTemplates kubebindv1alpha2.APIServiceExportTemplateList + if err := r.client.List(ctx, &allTemplates); err != nil { + return fmt.Errorf("failed to list APIServiceExportTemplates: %w", err) + } + + for i := range allTemplates.Items { + tmpl := &allTemplates.Items[i] + ann := tmpl.Annotations + if ann == nil || ann[shared.AnnotationOwnerBinding] != binding.Name { + continue + } + if desiredNames[tmpl.Name] { + continue + } + logger.Info("Deleting orphaned APIServiceExportTemplate", "name", tmpl.Name, "binding", binding.Name) + if err := r.client.Delete(ctx, tmpl); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete orphaned APIServiceExportTemplate %q: %w", tmpl.Name, err) + } + } + + return nil +} + +func (r *reconciler) ensureTemplate(ctx context.Context, binding *apisv1alpha2.APIBinding, desired *kubebindv1alpha2.APIServiceExportTemplate) error { + logger := klog.FromContext(ctx) + + existing := &kubebindv1alpha2.APIServiceExportTemplate{} + err := r.client.Get(ctx, types.NamespacedName{Name: desired.Name}, existing) + if errors.IsNotFound(err) { + logger.Info("Creating APIServiceExportTemplate", "name", desired.Name, "binding", binding.Name) + if err := r.client.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create APIServiceExportTemplate %q: %w", desired.Name, err) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get APIServiceExportTemplate %q: %w", desired.Name, err) + } + + needsUpdate := false + + if !equality.Semantic.DeepEqual(existing.Spec, desired.Spec) { + existing.Spec = desired.Spec + needsUpdate = true + } + + // Ensure owner reference is set. + if !metav1.IsControlledBy(existing, binding) { + if err := controllerutil.SetOwnerReference(binding, existing, r.scheme); err != nil { + return fmt.Errorf("failed to set owner reference on APIServiceExportTemplate %q: %w", desired.Name, err) + } + needsUpdate = true + } + + if !needsUpdate { + return nil + } + + logger.Info("Updating APIServiceExportTemplate", "name", desired.Name, "binding", binding.Name) + if err := r.client.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update APIServiceExportTemplate %q: %w", desired.Name, err) + } + + return nil +} + +// buildTemplates constructs one APIServiceExportTemplate per scope from the +// APIBinding's bound resources. Resources are grouped by scope so that each +// template contains only resources of the same scope (Namespaced or Cluster). +func (r *reconciler) buildTemplates(ctx context.Context, binding *apisv1alpha2.APIBinding) ([]*kubebindv1alpha2.APIServiceExportTemplate, error) { + type scopedResource struct { + scope kubebindv1alpha2.InformerScope + resource kubebindv1alpha2.APIServiceExportResource + } + + grouped := make([]scopedResource, 0, len(binding.Status.BoundResources)) + for _, boundRes := range binding.Status.BoundResources { + schema, err := r.getAPIResourceSchema(ctx, boundRes.Schema.Name) + if err != nil { + return nil, fmt.Errorf("failed to get APIResourceSchema %q for binding %q: %w", + boundRes.Schema.Name, binding.Name, err) + } + + var versions []string + for _, v := range schema.Spec.Versions { + if v.Served { + versions = append(versions, v.Name) + } + } + + scope := kubebindv1alpha2.NamespacedScope + if schema.Spec.Scope == apiextensionsv1.ClusterScoped { + scope = kubebindv1alpha2.ClusterScope + } + + grouped = append(grouped, scopedResource{ + scope: scope, + resource: kubebindv1alpha2.APIServiceExportResource{ + GroupResource: kubebindv1alpha2.GroupResource{ + Group: boundRes.Group, + Resource: boundRes.Resource, + }, + Versions: versions, + }, + }) + } + + // Group resources by scope. + byScope := map[kubebindv1alpha2.InformerScope][]kubebindv1alpha2.APIServiceExportResource{} + for _, sr := range grouped { + byScope[sr.scope] = append(byScope[sr.scope], sr.resource) + } + + templates := make([]*kubebindv1alpha2.APIServiceExportTemplate, 0, len(byScope)) + for scope, resources := range byScope { + templateName := templateNameForBinding(binding.Name, scope) + tmpl := &kubebindv1alpha2.APIServiceExportTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: templateName, + Annotations: map[string]string{ + shared.AnnotationOwnerBinding: binding.Name, + }, + }, + Spec: kubebindv1alpha2.APIServiceExportTemplateSpec{ + Scope: scope, + Resources: resources, + }, + } + templates = append(templates, tmpl) + } + + return templates, nil +} diff --git a/backend/provider/kcp/controllers/apiresourceschema/controller.go b/backend/provider/kcp/controllers/apiresourceschema/controller.go new file mode 100644 index 000000000..6055a94df --- /dev/null +++ b/backend/provider/kcp/controllers/apiresourceschema/controller.go @@ -0,0 +1,136 @@ +/* +Copyright 2026 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apiresourceschema contains a kcp-specific controller that watches +// APIServiceExportTemplates and copies the corresponding APIResourceSchemas +// into the workspace so that the serviceexportrequest controller can find them. +package apiresourceschema + +import ( + "context" + "fmt" + + apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/log" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + "github.com/kube-bind/kube-bind/backend/provider/kcp/controllers/shared" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +const controllerName = "kube-bind-kcp-apiresourceschema" + +// APIResourceSchemaReconciler watches APIServiceExportTemplates and ensures +// that the APIResourceSchemas referenced by the owning APIBinding are copied +// into the workspace with the kube-bind.io/exported=true label. +type APIResourceSchemaReconciler struct { + manager mcmanager.Manager + opts controller.TypedOptions[mcreconcile.Request] + scheme *runtime.Scheme + vwCache *shared.VWClientCache +} + +// New returns a new APIResourceSchemaReconciler. +func New( + ctx context.Context, + mgr mcmanager.Manager, + opts controller.TypedOptions[mcreconcile.Request], + baseConfig *rest.Config, + scheme *runtime.Scheme, +) (*APIResourceSchemaReconciler, error) { + return &APIResourceSchemaReconciler{ + manager: mgr, + opts: opts, + scheme: scheme, + vwCache: shared.NewVWClientCache(baseConfig, scheme), + }, nil +} + +// Reconcile implements reconcile.Reconciler for multicluster-runtime. +func (r *APIResourceSchemaReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.Info("Reconciling APIServiceExportTemplate for schema copy", "request", req) + + cl, err := r.manager.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get cluster %q: %w", req.ClusterName, err) + } + + c := cl.GetClient() + clusterConfig := cl.GetConfig() + + // Get the template. + tmpl := &kubebindv1alpha2.APIServiceExportTemplate{} + if err := c.Get(ctx, req.NamespacedName, tmpl); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to get APIServiceExportTemplate %q: %w", req.Name, err) + } + + // Find the owning APIBinding via annotation. + bindingName, ok := tmpl.Annotations[shared.AnnotationOwnerBinding] + if !ok { + logger.V(4).Info("Template has no owner-binding annotation, skipping", "name", tmpl.Name) + return ctrl.Result{}, nil + } + + binding := &apisv1alpha2.APIBinding{} + if err := c.Get(ctx, types.NamespacedName{Name: bindingName}, binding); err != nil { + if errors.IsNotFound(err) { + logger.Info("Owning APIBinding not found, skipping", "binding", bindingName) + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to get APIBinding %q: %w", bindingName, err) + } + + if binding.Status.Phase != apisv1alpha2.APIBindingPhaseBound { + return ctrl.Result{}, nil + } + + // Build schema getter with VW fallback using the shared cache. + getSchema := shared.SchemaGetterWithFallback(c, clusterConfig, r.vwCache) + + rec := reconciler{ + client: c, + scheme: r.scheme, + getAPIResourceSchema: getSchema, + } + + if err := rec.reconcile(ctx, tmpl, binding); err != nil { + logger.Error(err, "Failed to reconcile schemas for template", "name", tmpl.Name) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager registers the controller with the multicluster-runtime Manager. +func (r *APIResourceSchemaReconciler) SetupWithManager(mgr mcmanager.Manager) error { + return mcbuilder.ControllerManagedBy(mgr). + For(&kubebindv1alpha2.APIServiceExportTemplate{}). + WithOptions(r.opts). + Named(controllerName). + Complete(r) +} diff --git a/backend/provider/kcp/controllers/apiresourceschema/reconcile.go b/backend/provider/kcp/controllers/apiresourceschema/reconcile.go new file mode 100644 index 000000000..b617654ba --- /dev/null +++ b/backend/provider/kcp/controllers/apiresourceschema/reconcile.go @@ -0,0 +1,139 @@ +/* +Copyright 2026 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiresourceschema + +import ( + "context" + "fmt" + + apisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1" + apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kube-bind/kube-bind/backend/kubernetes/resources" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" +) + +type reconciler struct { + client client.Client + scheme *runtime.Scheme + getAPIResourceSchema func(ctx context.Context, name string) (*apisv1alpha1.APIResourceSchema, error) +} + +func (r *reconciler) reconcile(ctx context.Context, tmpl *kubebindv1alpha2.APIServiceExportTemplate, binding *apisv1alpha2.APIBinding) error { + logger := klog.FromContext(ctx) + + // Build a set of resources listed in this template so we only copy + // schemas that match this template's scope. + templateResources := make(map[string]bool, len(tmpl.Spec.Resources)) + for _, res := range tmpl.Spec.Resources { + templateResources[res.Group+"/"+res.Resource] = true + } + + for _, boundRes := range binding.Status.BoundResources { + if !templateResources[boundRes.Group+"/"+boundRes.Resource] { + continue + } + + schema, err := r.getAPIResourceSchema(ctx, boundRes.Schema.Name) + if err != nil { + return fmt.Errorf("failed to get APIResourceSchema %q for binding %q: %w", + boundRes.Schema.Name, binding.Name, err) + } + + if err := r.ensureSchema(ctx, tmpl, schema); err != nil { + logger.Error(err, "Failed to ensure APIResourceSchema copy", + "schema", schema.Name, "template", tmpl.Name) + return err + } + } + + return nil +} + +// ensureSchema creates or updates a local copy of the APIResourceSchema, +// owned by the template and labeled for discovery by serviceexportrequest. +func (r *reconciler) ensureSchema(ctx context.Context, tmpl *kubebindv1alpha2.APIServiceExportTemplate, source *apisv1alpha1.APIResourceSchema) error { + logger := klog.FromContext(ctx) + + desired := &apisv1alpha1.APIResourceSchema{ + ObjectMeta: metav1.ObjectMeta{ + Name: source.Name, + Labels: map[string]string{ + resources.ExportedCRDsLabel: "true", + }, + }, + Spec: *source.Spec.DeepCopy(), + } + + // Set owner reference: template owns the schema copy. + if err := controllerutil.SetOwnerReference(tmpl, desired, r.scheme); err != nil { + return fmt.Errorf("failed to set owner reference on APIResourceSchema %q: %w", desired.Name, err) + } + + existing := &apisv1alpha1.APIResourceSchema{} + err := r.client.Get(ctx, client.ObjectKey{Name: desired.Name}, existing) + if errors.IsNotFound(err) { + logger.Info("Creating APIResourceSchema copy", "name", desired.Name, "template", tmpl.Name) + if err := r.client.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create APIResourceSchema %q: %w", desired.Name, err) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get APIResourceSchema %q: %w", desired.Name, err) + } + + // Check if spec or labels need updating. + needsUpdate := false + + if !equality.Semantic.DeepEqual(existing.Spec, desired.Spec) { + existing.Spec = desired.Spec + needsUpdate = true + } + + if existing.Labels == nil || existing.Labels[resources.ExportedCRDsLabel] != "true" { + if existing.Labels == nil { + existing.Labels = make(map[string]string) + } + existing.Labels[resources.ExportedCRDsLabel] = "true" + needsUpdate = true + } + + // Ensure owner reference is set. + if !metav1.IsControlledBy(existing, tmpl) { + if err := controllerutil.SetOwnerReference(tmpl, existing, r.scheme); err != nil { + return fmt.Errorf("failed to set owner reference on APIResourceSchema %q: %w", existing.Name, err) + } + needsUpdate = true + } + + if needsUpdate { + logger.Info("Updating APIResourceSchema copy", "name", existing.Name, "template", tmpl.Name) + if err := r.client.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update APIResourceSchema %q: %w", existing.Name, err) + } + } + + return nil +} diff --git a/backend/provider/kcp/controllers/shared/vw.go b/backend/provider/kcp/controllers/shared/vw.go new file mode 100644 index 000000000..4286c543c --- /dev/null +++ b/backend/provider/kcp/controllers/shared/vw.go @@ -0,0 +1,161 @@ +/* +Copyright 2026 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package shared provides utilities shared across kcp controllers. +package shared + +import ( + "context" + "fmt" + "net/url" + "strings" + "sync" + + apisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// AnnotationOwnerBinding is the annotation key used to link an +// APIServiceExportTemplate back to the APIBinding that owns it. +const AnnotationOwnerBinding = "apibindingtemplate.kube-bind.io/owner-binding" + +// ExtractClusterID extracts the cluster ID from an apiexport virtual workspace +// URL. The expected URL format (as of kcp v0.26+) is: +// +// https://host:port/services/apiexport///clusters/ +// +// where path segments 0-3 are /services/apiexport//, +// segment 4 is "clusters", and segment 5 is the cluster ID. +func ExtractClusterID(clusterConfig *rest.Config) (string, error) { + u, err := url.Parse(clusterConfig.Host) + if err != nil { + return "", fmt.Errorf("failed to parse cluster host URL: %w", err) + } + + pathParts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(pathParts) < 6 || pathParts[4] != "clusters" { + return "", fmt.Errorf("unexpected apiexport URL format: %s", u.Path) + } + + return pathParts[5], nil +} + +// VWClientCache caches controller-runtime clients keyed by cluster ID to avoid +// creating a new client on every schema lookup. +type VWClientCache struct { + mu sync.RWMutex + clients map[string]client.Client + baseConfig *rest.Config + scheme *runtime.Scheme +} + +// NewVWClientCache creates a new VWClientCache. +func NewVWClientCache(baseConfig *rest.Config, scheme *runtime.Scheme) *VWClientCache { + return &VWClientCache{ + clients: make(map[string]client.Client), + baseConfig: baseConfig, + scheme: scheme, + } +} + +// GetClient returns a cached client for the given cluster ID, creating one if +// necessary. +func (c *VWClientCache) GetClient(clusterID string) (client.Client, error) { + c.mu.RLock() + if cl, ok := c.clients[clusterID]; ok { + c.mu.RUnlock() + return cl, nil + } + c.mu.RUnlock() + + c.mu.Lock() + defer c.mu.Unlock() + + // Double-check after acquiring write lock. + if cl, ok := c.clients[clusterID]; ok { + return cl, nil + } + + cl, err := newVWClient(c.baseConfig, c.scheme, clusterID) + if err != nil { + return nil, err + } + c.clients[clusterID] = cl + return cl, nil +} + +// newVWClient creates a client pointing at the apiresourceschema virtual workspace +// for the given cluster ID: +// https://host:port/services/apiresourceschema/{clusterID}/clusters/*/ +func newVWClient(baseConfig *rest.Config, scheme *runtime.Scheme, clusterID string) (client.Client, error) { + cfg := rest.CopyConfig(baseConfig) + u, err := url.Parse(cfg.Host) + if err != nil { + return nil, fmt.Errorf("failed to parse base config host: %w", err) + } + + u.Path = fmt.Sprintf("/services/apiresourceschema/%s/clusters/*", clusterID) + cfg.Host = u.String() + + return client.New(cfg, client.Options{Scheme: scheme}) +} + +// SchemaGetterWithFallback returns a function that first tries to get an +// APIResourceSchema from the workspace client, and if not found, falls back to +// the apiresourceschema virtual workspace via the cached VW client. +func SchemaGetterWithFallback( + workspaceClient client.Client, + clusterConfig *rest.Config, + vwCache *VWClientCache, +) func(ctx context.Context, name string) (*apisv1alpha1.APIResourceSchema, error) { + return func(ctx context.Context, name string) (*apisv1alpha1.APIResourceSchema, error) { + logger := log.FromContext(ctx) + + // 1. Try the current workspace first. + var schema apisv1alpha1.APIResourceSchema + err := workspaceClient.Get(ctx, client.ObjectKey{Name: name}, &schema) + if err == nil { + return &schema, nil + } + if !errors.IsNotFound(err) { + return nil, err + } + + // 2. Fallback: try the apiresourceschema virtual workspace. + logger.V(2).Info("APIResourceSchema not found in workspace, trying VW fallback", "schema", name) + + clusterID, err := ExtractClusterID(clusterConfig) + if err != nil { + return nil, fmt.Errorf("cannot build VW fallback client: %w", err) + } + + vwClient, err := vwCache.GetClient(clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get VW client for cluster %q: %w", clusterID, err) + } + + var vwSchema apisv1alpha1.APIResourceSchema + if err := vwClient.Get(ctx, client.ObjectKey{Name: name}, &vwSchema); err != nil { + return nil, fmt.Errorf("APIResourceSchema %q not found in workspace or VW: %w", name, err) + } + + return &vwSchema, nil + } +} diff --git a/backend/provider/kcp/options/options.go b/backend/provider/kcp/options/options.go index 3349c0fc6..1f1f1a4cf 100644 --- a/backend/provider/kcp/options/options.go +++ b/backend/provider/kcp/options/options.go @@ -26,6 +26,12 @@ type Options struct { type ExtraOptions struct { APIExportEndpointSliceName string + + // APIBindingIgnorePrefixes is a list of name prefixes for APIBindings + // that should be ignored by the APIBindingTemplate controller. System + // bindings like "kube-bind.io", "tenancy.kcp.io-*", "topology.kcp.io-*" + // should typically be excluded. + APIBindingIgnorePrefixes []string } type completedOptions struct { @@ -40,12 +46,18 @@ func NewOptions() *Options { return &Options{ ExtraOptions: ExtraOptions{ APIExportEndpointSliceName: "kube-bind.io", + APIBindingIgnorePrefixes: []string{ + "kube-bind.io", + "tenancy.kcp.io", + "topology.kcp.io", + }, }, } } func (options *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&options.ExtraOptions.APIExportEndpointSliceName, "apiexport-endpoint-slice-name", options.ExtraOptions.APIExportEndpointSliceName, "name of the APIExport EndpointSlice to watch") + fs.StringSliceVar(&options.ExtraOptions.APIBindingIgnorePrefixes, "apibinding-ignore-prefixes", options.ExtraOptions.APIBindingIgnorePrefixes, "name prefixes of APIBindings to ignore when generating APIServiceExportTemplates") } func (options *Options) Complete() (*CompletedOptions, error) { diff --git a/backend/server.go b/backend/server.go index f8faecb33..176906293 100644 --- a/backend/server.go +++ b/backend/server.go @@ -38,6 +38,8 @@ import ( "github.com/kube-bind/kube-bind/backend/controllers/servicenamespace" http "github.com/kube-bind/kube-bind/backend/http" kube "github.com/kube-bind/kube-bind/backend/kubernetes" + "github.com/kube-bind/kube-bind/backend/provider/kcp/controllers/apibindingtemplate" + "github.com/kube-bind/kube-bind/backend/provider/kcp/controllers/apiresourceschema" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" ) @@ -61,6 +63,10 @@ type Controllers struct { ServiceNamespace *servicenamespace.APIServiceNamespaceReconciler Cluster *cluster.ClusterReconciler BindableResourcesRequest *bindableresourcesrequest.BindableResourcesRequestReconciler + + // Provider specific controllers - only set when provider is "kcp" + APIBindingTemplate *apibindingtemplate.APIBindingTemplateReconciler + APIResourceSchema *apiresourceschema.APIResourceSchemaReconciler } func NewServer(ctx context.Context, c *Config) (*Server, error) { @@ -264,6 +270,28 @@ func NewServer(ctx context.Context, c *Config) (*Server, error) { logger.Info("BindableResourcesRequest controller enabled for CRD-based binding") } + // Setup kcp-specific controllers. Only active when provider is kcp + // (APIBindings and APIResourceSchemas only exist in kcp workspaces). + if c.Options.Provider == "kcp" { + s.APIBindingTemplate, err = apibindingtemplate.New(ctx, s.Config.Manager, opts, c.Options.ProviderKcp.APIBindingIgnorePrefixes, c.ClientConfig, c.Scheme) + if err != nil { + return nil, fmt.Errorf("error setting up APIBindingTemplate Controller: %w", err) + } + if err := s.APIBindingTemplate.SetupWithManager(s.Config.Manager); err != nil { + return nil, fmt.Errorf("error setting up APIBindingTemplate controller with manager: %w", err) + } + logger.Info("APIBindingTemplate controller enabled for kcp provider") + + s.APIResourceSchema, err = apiresourceschema.New(ctx, s.Config.Manager, opts, c.ClientConfig, c.Scheme) + if err != nil { + return nil, fmt.Errorf("error setting up APIResourceSchema Controller: %w", err) + } + if err := s.APIResourceSchema.SetupWithManager(s.Config.Manager); err != nil { + return nil, fmt.Errorf("error setting up APIResourceSchema controller with manager: %w", err) + } + logger.Info("APIResourceSchema controller enabled for kcp provider") + } + return s, nil } diff --git a/contrib/kcp/README.md b/contrib/kcp/README.md index ef9c2c282..828e7aa21 100644 --- a/contrib/kcp/README.md +++ b/contrib/kcp/README.md @@ -1,47 +1,97 @@ # kcp -kcp folder contains isolated set of tooling to bootstrap the kube-bind to allow it to work with kcp instance. -It is split into separate package to avoid vendoring pollution. +kcp folder contains isolated set of tooling to bootstrap kube-bind to work with a kcp instance. +It is split into a separate package to avoid vendoring pollution. -kcp requires initial setup to be run before it can be used. -This includes setting up workspace/provider and setting up all the APIResourceSchemas and APIExports. +kcp requires initial setup before it can be used. +This includes setting up workspaces, `APIResourceSchema`s, and `APIExport`s. -It was its own GO module to avoid kcp dependencies in the main kube-bind module. +It was its own Go module to avoid kcp dependencies in the main kube-bind module. -This is not required if you are doing deeper integration, and controlling the setup with your own scripts. +This is not required if you are doing deeper integration and controlling the setup with your own scripts. + +## Architecture + +This example demonstrates a **three-tier** kube-bind setup with kcp: -It will do the following: -1. Create a provider workspace: -``` -:root:kube-bind -``` -2. Create apiexport inside the workspace: ``` -:root:kube-bind/apiexport/kube-bind.io +:root:provider — owns the Cowboys APIResourceSchema + APIExport (source of truth) + | + | (APIBinding: :root:backend binds cowboys from :root:provider) + v +:root:backend — binds cowboys from provider, re-exposes them to consumers via kube-bind + | + | (kube-bind runs in :root:kube-bind, serves the binding API for consumers) + v + consumer — any kcp workspace or external cluster that wants cowboys ``` +Three kubeconfigs are used throughout this guide: -# How to run +| File | Points to | Used for | +|------|-----------|----------| +| `provider.kubeconfig` | `:root:provider` | creating cowboys `APIResourceSchema` + `APIExport` | +| `backend.kubeconfig` | `:root:backend` | binding cowboys from provider, creating templates | +| `kube-bind.kubeconfig` | `:root:kube-bind` | running the kube-bind backend process | -## Preparation +- **Provider** (`:root:provider`): defines the `cowboys` API (`APIResourceSchema` + `APIExport`). The upstream API owner. +- **Backend** (`:root:backend`): binds provider's `cowboys` APIExport, creates `APIServiceExport` templates. The broker layer. +- **kube-bind** (`:root:kube-bind`): where the kube-bind backend process runs, serving the binding HTTP API to consumers. +- **Consumer**: binds cowboys via kube-bind, with no direct dependency on the provider. -1. Start kcp +## How to run + +### Preparation + +1. Start kcp: ```bash make run-kcp ``` -## Backend +--- + +### Provider + +The provider workspace owns the `cowboys` `APIResourceSchema` and `APIExport`. + +2. Set up the provider workspace: + -2. Bootstrap kcp: ```bash -cp .kcp/admin.kubeconfig .kcp/backend.kubeconfig -export KUBECONFIG=.kcp/backend.kubeconfig -./bin/kcp-init --kcp-kubeconfig $KUBECONFIG +cp .kcp/admin.kubeconfig .kcp/provider.kubeconfig +export KUBECONFIG=.kcp/provider.kubeconfig +kubectl ws use :root +kubectl ws create provider --enter ``` -3. Run the backend: + +3. Create the `APIResourceSchema` and `APIExport` for cowboys: + +```bash +kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml +kubectl apply -f contrib/kcp/deploy/examples/apiexport-cowboys.yaml ``` -k ws use :root:kube-bind + +The `APIExport` named `cowboys` makes the cowboys API available for other workspaces to bind. + +--- + +### kube-bind + +The `:root:kube-bind` workspace is where the kube-bind backend process runs. `kcp-init` creates this workspace and installs the kube-bind `APIExport` into it. + +4. Bootstrap the kube-bind workspace: + +```bash +cp .kcp/admin.kubeconfig .kcp/kube-bind.kubeconfig +export KUBECONFIG=.kcp/kube-bind.kubeconfig +./bin/kcp-init --kcp-kubeconfig $KUBECONFIG +``` + +5. Run the kube-bind backend: + +```bash +kubectl ws use :root:kube-bind go run ./cmd/backend \ --multicluster-runtime-provider kcp \ @@ -55,20 +105,22 @@ go run ./cmd/backend \ --consumer-scope=cluster ``` -This process will keep running, so open a new terminal. +This process keeps running — open a new terminal. -## Provider +--- -4. Copy the kubeconfig to the provider and create provider workspace: -```bash -cp .kcp/admin.kubeconfig .kcp/provider.kubeconfig -export KUBECONFIG=.kcp/provider.kubeconfig -k ws use :root -kubectl create-workspace provider --enter -``` +### Backend + +The backend workspace (`:root:backend`) binds the provider's cowboys `APIExport` and sets up `APIServiceExport` templates so kube-bind knows what to offer consumers. + +6. Set up the backend workspace and bind the kube-bind `APIExport`: -5. Bind the APIExport to the provider workspace ```bash +cp .kcp/admin.kubeconfig .kcp/backend.kubeconfig +export KUBECONFIG=.kcp/backend.kubeconfig +kubectl ws use :root +kubectl ws create backend --enter + kubectl kcp bind apiexport root:kube-bind:kube-bind.io \ --accept-permission-claim clusterrolebindings.rbac.authorization.k8s.io \ --accept-permission-claim clusterroles.rbac.authorization.k8s.io \ @@ -80,22 +132,17 @@ kubectl kcp bind apiexport root:kube-bind:kube-bind.io \ --accept-permission-claim roles.rbac.authorization.k8s.io \ --accept-permission-claim subjectaccessreviews.authorization.k8s.io \ --accept-permission-claim rolebindings.rbac.authorization.k8s.io \ - --accept-permission-claim apiresourceschemas.apis.kcp.io + --accept-permission-claim apiresourceschemas.apis.kcp.io \ + --accept-permission-claim apibindings.apis.kcp.io ``` -6. Create CRD in provider: -```bash -kubectl apply -f contrib/kcp/deploy/examples/apiexport.yaml -kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml -kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-sheriffs.yaml -kubectl kcp bind apiexport root:provider:cowboys-stable +7. Bind the provider's cowboys `APIExport` into `:root:backend`: -kubectl apply -f deploy/examples/template-cowboys.yaml -kubectl apply -f deploy/examples/template-sheriffs.yaml -kubectl apply -f deploy/examples/collection.yaml +```bash +kubectl kcp bind apiexport root:provider:cowboys ``` -7. Get LogicalCluster: +8. Get the LogicalCluster identity (needed for consumer binding): ```bash kubectl get logicalcluster @@ -103,9 +150,14 @@ kubectl get logicalcluster # cluster Ready https://192.168.2.166:6443/clusters/2ocmmccjkme8bof4 ``` -## Consumer +--- + +### Consumer + +The consumer binds cowboys from kube-bind (backed by `:root:backend`), with no direct dependency on the provider. + +9. Set up the consumer workspace: -8. Now we gonna initiate consumer: ```bash cp .kcp/admin.kubeconfig .kcp/consumer.kubeconfig export KUBECONFIG=.kcp/consumer.kubeconfig @@ -113,62 +165,97 @@ kubectl ws use :root kubectl ws create consumer --enter ``` -9. Bind the thing: +// TODO: Backend in kcp should dynamically lifecycle schemas. + +10. Log in and bind cowboys: ```bash -./bin/kubectl-bind login http://127.0.0.1:8080 --cluster 2ocmmccjkme8bof4 +./bin/kubectl-bind login http://127.0.0.1:8080 --cluster 216gnjm1cqct4x1j + ./bin/kubectl-bind --dry-run -o yaml > apiserviceexport.yaml -# Extract secret for binding process. Note that secret name is not the same as output from command above. Check secret -# name by running `kubectl get secret -n kube-bind` +# Extract the secret for the binding process. +# Note: the secret name may differ — check with `kubectl get secret -n kube-bind` kubectl get secrets -n kube-bind -o jsonpath='{.items[0].data.kubeconfig}' | base64 -d > remote.kubeconfig namespace=$(yq '.contexts[0].context.namespace' remote.kubeconfig) -./bin/kubectl-bind apiservice -v 6 --remote-kubeconfig remote.kubeconfig -f apiserviceexport.yaml --skip-konnector --remote-namespace "$namespace" +./bin/kubectl-bind apiservice -v 6 \ + --remote-kubeconfig remote.kubeconfig \ + -f apiserviceexport.yaml \ + --skip-konnector \ + --remote-namespace "$namespace" ``` -This will keep running, so switch to a new terminal. - -### Consumer Konnector +This process keeps running — switch to a new terminal. -Start konnector: +11. Start the konnector: ```bash ./bin/konnector --lease-namespace default --kubeconfig .kcp/consumer.kubeconfig ``` -# Create an instance +--- + +### Create an instance + +Once binding is complete, create a cowboy on the consumer: ```bash kubectl apply -f deploy/examples/cr-cowboy.yaml -kubectl apply -f deploy/examples/cr-sheriff.yaml ``` +The cowboy is synced to `:root:backend`, which in turn stores it via the provider's APIExport in `:root:provider`. -# Backend only mode +--- -Sometimes when integrating with existing systems, you might want to run kube-bind backend without any frontend, API and OIDC. -This is useful when running in multi-tenant environments where each tenant has its own identity provider and frontend (or no frontend at all). +## Backend-only mode +Sometimes you want to run kube-bind without any frontend, HTTP API, or OIDC — useful in multi-tenant environments, GitOps pipelines, or when each tenant has their own identity provider. -1. Start kcp +The same three kubeconfigs apply: + +| File | Points to | Used for | +|------|-----------|----------| +| `provider.kubeconfig` | `:root:provider` | creating cowboys `APIResourceSchema` + `APIExport` | +| `backend.kubeconfig` | `:root:backend` | binding cowboys from provider, creating templates | +| `kube-bind.kubeconfig` | `:root:kube-bind` | running the kube-bind backend process | + +### Preparation + +1. Start kcp: ```bash make run-kcp ``` -## Backend +### Provider + +2. Set up the provider workspace and cowboys API: -2. Bootstrap kcp: ```bash -cp .kcp/admin.kubeconfig .kcp/backend.kubeconfig -export KUBECONFIG=.kcp/backend.kubeconfig -./bin/kcp-init --kcp-kubeconfig $KUBECONFIG +cp .kcp/admin.kubeconfig .kcp/provider.kubeconfig +export KUBECONFIG=.kcp/provider.kubeconfig +kubectl ws use :root +kubectl ws create provider --enter +kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml +kubectl apply -f contrib/kcp/deploy/examples/apiexport-cowboys.yaml ``` -3. Run the backend with `--isolation-mode=None` for no isolation mode. + +### kube-bind + +3. Bootstrap the kube-bind workspace: + +```bash +cp .kcp/admin.kubeconfig .kcp/kube-bind.kubeconfig +export KUBECONFIG=.kcp/kube-bind.kubeconfig +./bin/kcp-init --kcp-kubeconfig $KUBECONFIG ``` -k ws use :root:kube-bind + +4. Run the backend in backend-only mode (no OIDC, no HTTP frontend): + +```bash +kubectl ws use :root:kube-bind go run ./cmd/backend \ --multicluster-runtime-provider kcp \ @@ -181,20 +268,18 @@ go run ./cmd/backend \ --isolation=None ``` -This process will keep running, so open a new terminal. +This process keeps running — open a new terminal. -## Provider +### Backend -4. Copy the kubeconfig to the provider and create provider workspace: -```bash -cp .kcp/admin.kubeconfig .kcp/provider.kubeconfig -export KUBECONFIG=.kcp/provider.kubeconfig -k ws use :root -kubectl create-workspace provider --enter -``` +5. Set up the backend workspace: -5. Bind the APIExport to the provider workspace ```bash +cp .kcp/admin.kubeconfig .kcp/backend.kubeconfig +export KUBECONFIG=.kcp/backend.kubeconfig +kubectl ws use :root +kubectl ws create backend --enter + kubectl kcp bind apiexport root:kube-bind:kube-bind.io \ --accept-permission-claim clusterrolebindings.rbac.authorization.k8s.io \ --accept-permission-claim clusterroles.rbac.authorization.k8s.io \ @@ -206,32 +291,35 @@ kubectl kcp bind apiexport root:kube-bind:kube-bind.io \ --accept-permission-claim roles.rbac.authorization.k8s.io \ --accept-permission-claim subjectaccessreviews.authorization.k8s.io \ --accept-permission-claim rolebindings.rbac.authorization.k8s.io \ - --accept-permission-claim apiresourceschemas.apis.kcp.io + --accept-permission-claim apiresourceschemas.apis.kcp.io \ + --accept-permission-claim apibindings.apis.kcp.io ``` -6. Create CRD in provider: +6. Bind the provider's cowboys `APIExport` into `:root:backend`: + ```bash -kubectl apply -f contrib/kcp/deploy/examples/apiexport.yaml -kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml -kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-sheriffs.yaml -kubectl kcp bind apiexport root:provider:cowboys-stable +kubectl kcp bind apiexport root:provider:cowboys +``` + +7. Create the `APIResourceSchema` and `APIServiceExport` template for cowboys: +```bash +kubectl apply -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml kubectl apply -f deploy/examples/template-cowboys.yaml -kubectl apply -f deploy/examples/template-sheriffs.yaml -kubectl apply -f deploy/examples/collection.yaml ``` -7. Get LogicalCluster: +8. Get the LogicalCluster identity: ```bash kubectl get logicalcluster # NAME PHASE URL AGE -# cluster Ready https://192.168.2.166:6443/clusters/20nuv280snhqd5j4 +# cluster Ready https://192.168.2.166:6443/clusters/20nuv280snhqd5j4 ``` -## Consumer +### Consumer (backend-only) + +9. Set up the consumer workspace: -8. Now we gonna initiate consumer: ```bash cp .kcp/admin.kubeconfig .kcp/consumer.kubeconfig export KUBECONFIG=.kcp/consumer.kubeconfig @@ -239,27 +327,34 @@ kubectl ws use :root kubectl ws create consumer --enter ``` -This is where it starts to differ from normal setup. CLI login and binding will not work without OIDC and API. -So we need manually create binding request. +In backend-only mode there is no HTTP API or OIDC, so the CLI login flow does not apply. +Instead, binding is initiated via a `BindableResourcesRequest` CRD. -**Backend-Only Binding Process:** +**How backend-only binding works:** -In traditional mode, `BindableResourcesRequest` is sent as a REST API payload to the backend HTTP server, which then: +In traditional mode, `BindableResourcesRequest` is sent as a REST API payload to the backend HTTP server, which: 1. Creates a dedicated namespace for the consumer 2. Generates a kubeconfig for that namespace 3. Returns it as an HTTP response -In backend-only mode, `BindableResourcesRequest` is a **CRD** that triggers the same binding process: -1. A controller watches for `BindableResourcesRequest` resources -2. Creates the namespace and kubeconfig automatically -3. Stores the response in a Secret (specified by `kubeconfigSecretRef`) +In backend-only mode, `BindableResourcesRequest` is a **CRD** watched by a controller, which: +1. Creates the namespace and kubeconfig automatically +2. Stores the result in a Secret specified by `kubeconfigSecretRef` This enables GitOps and automation without requiring HTTP API access. -We need first to get identity for the cluster we gonna use. +10. Get the cluster identity: + +```bash +./bin/kubectl-bind cluster-identity ``` -kubectl bind cluster-identity + +11. Create the `BindableResourcesRequest` (applied against `:root:kube-bind` where the backend is running): + ```bash +export KUBECONFIG=.kcp/kube-bind.kubeconfig +kubectl ws use :root:kube-bind + kubectl apply -f - < remote.data +```bash +kubectl get secret 0ac6800e-bc4f-4c70-814b-45b44e04aa02-response \ + -o jsonpath='{.data.response}' | base64 -d > remote.data ``` -9. Bind the thing on the consumer cluster: +13. Deploy the binding on the consumer: ```bash +export KUBECONFIG=.kcp/consumer.kubeconfig ./bin/kubectl-bind deploy --file remote.data --skip-konnector ``` -### Consumer Konnector - -Start konnector: +14. Start the konnector: ```bash ./bin/konnector --lease-namespace default --kubeconfig .kcp/consumer.kubeconfig ``` -We will be doing wild card pull, meaning pull every contract we have access to. +15. Create an `APIServiceBindingBundle` to pull all available contracts: ```bash kubectl apply -f - <= 1, fmt.Sprintf("Error listing %s instances: %v", resource, err) + }, wait.ForeverTestTimeout, time.Millisecond*100) + + require.GreaterOrEqual(t, len(instances.Items), 1, "Expected at least one %s instance on backend side", resource) + }) + + t.Run("instance spec updated downstream syncs upstream", func(t *testing.T) { + t.Logf("Updating %s spec on consumer side", resource) + + require.Eventually(t, func() bool { + obj, err := consumerClient.Get(t.Context(), "test-3tier-"+resource, metav1.GetOptions{}) + if err != nil { + return false + } + + unstructured.SetNestedField(obj.Object, "holster", "spec", "intent") //nolint:errcheck + _, err = consumerClient.Update(t.Context(), obj, metav1.UpdateOptions{}) + return err == nil + }, wait.ForeverTestTimeout, 5*time.Second, "waiting for %s spec to be updated on consumer side", resource) + + t.Logf("Waiting for %s spec update to sync to backend side", resource) + require.Eventually(t, func() bool { + instances, err := backendClient.List(t.Context(), metav1.ListOptions{}) + if err != nil || len(instances.Items) == 0 { + return false + } + + for _, item := range instances.Items { + intent, found, err := unstructured.NestedString(item.Object, "spec", "intent") + if err == nil && found && intent == "holster" { + return true + } + } + return false + }, wait.ForeverTestTimeout, 5*time.Second, "waiting for %s spec update to sync to backend side", resource) + }) + + t.Run("instance deleted downstream is deleted upstream", func(t *testing.T) { + t.Logf("Deleting %s instance on consumer side", resource) + + err := consumerClient.Delete(t.Context(), "test-3tier-"+resource, metav1.DeleteOptions{}) + require.NoError(t, err, "Failed to delete %s on consumer side", resource) + + t.Logf("Waiting for %s instance to be deleted on backend side", resource) + require.Eventually(t, func() bool { + instances, err := backendClient.List(t.Context(), metav1.ListOptions{}) + if err != nil { + return false + } + for _, item := range instances.Items { + if item.GetName() == "test-3tier-"+resource { + return false + } + } + return true + }, wait.ForeverTestTimeout, 5*time.Second, "waiting for %s instance to be deleted on backend side", resource) + }) +} diff --git a/hack/go-install.sh b/hack/go-install.sh new file mode 100755 index 000000000..f93f2a473 --- /dev/null +++ b/hack/go-install.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kube Bind Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Originally copied from +# https://github.com/kubernetes-sigs/cluster-api-provider-gcp/blob/c26a68b23e9317323d5d37660fe9d29b3d2ff40c/scripts/go_install.sh +# +# Installs a Go binary either via "go install" or by cloning and building +# from source (for modules with replace directives that block go install). + +set -o errexit +set -o nounset +set -o pipefail + +if [[ -z "${1:-}" ]]; then + echo "must provide module as first parameter" + exit 1 +fi + +if [[ -z "${2:-}" ]]; then + echo "must provide binary name as second parameter" + exit 1 +fi + +if [[ -z "${3:-}" ]]; then + echo "must provide version as third parameter" + exit 1 +fi + +if [[ -z "${GOBIN:-}" ]]; then + echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory." + exit 1 +fi + +MODULE="${1}" +BINARY="${2}" +VERSION="${3}" + +# Extract the repo URL from the module path (everything before /cmd/...) +# e.g. github.com/kcp-dev/kcp/cmd/kcp -> github.com/kcp-dev/kcp +REPO_URL="${MODULE}" +if [[ "${REPO_URL}" == *"/cmd/"* ]]; then + REPO_URL="${REPO_URL%%/cmd/*}" +fi +# Relative path within the repo to the binary (e.g. ./cmd/kcp) +CMD_PATH="./${MODULE#"${REPO_URL}"/}" +if [[ "${CMD_PATH}" == "./${MODULE}" ]]; then + CMD_PATH="." +fi + +mkdir -p "${GOBIN}" + +# If the versioned binary already exists, just ensure the symlink and exit. +if [[ -f "${GOBIN}/${BINARY}-${VERSION}" ]]; then + ln -sf "${GOBIN}/${BINARY}-${VERSION}" "${GOBIN}/${BINARY}" + exit 0 +fi + +tmp_dir=$(mktemp -d -t goinstall_XXXXXXXXXX) +function clean { + rm -rf "${tmp_dir}" +} +trap clean EXIT + +rm "${GOBIN}/${BINARY}"* > /dev/null 2>&1 || true + +cd "${tmp_dir}" + +# Try go install first (works when the module has no replace directives). +# The || true prevents errexit from killing the script on failure. +go mod init fake/mod +if go install -tags tools "${MODULE}@${VERSION}" 2>/dev/null; then + mv "${GOBIN}/${BINARY}" "${GOBIN}/${BINARY}-${VERSION}" + ln -sf "${GOBIN}/${BINARY}-${VERSION}" "${GOBIN}/${BINARY}" + exit 0 +fi + +# Fallback: clone the repo and build from source. +# This handles modules with replace directives in go.mod. +echo "go install failed (module likely has replace directives), cloning and building from source..." +git clone --quiet "https://${REPO_URL}.git" repo +cd repo +git checkout --quiet "${VERSION}" +go build -trimpath -o "${GOBIN}/${BINARY}-${VERSION}" "${CMD_PATH}" +ln -sf "${GOBIN}/${BINARY}-${VERSION}" "${GOBIN}/${BINARY}"