From 7ac165020fe0994449eeb4a9c892fa77e5841d77 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Tue, 18 Aug 2026 16:06:37 +0200 Subject: [PATCH 1/5] feat: router support --- .github/workflows/e2e-weekly.yml | 2 +- .github/workflows/test-e2e.yml | 1 + .gitignore | 1 + AGENTS.md | 4 +- Makefile | 18 +- api/v1beta2/cloudscalecluster_types.go | 161 ++- api/v1beta2/condition_types.go | 19 + api/v1beta2/zz_generated.deepcopy.go | 91 ++ cmd/main.go | 2 +- ...e.cluster.x-k8s.io_cloudscaleclusters.yaml | 159 ++- ...r.x-k8s.io_cloudscaleclustertemplates.yaml | 98 +- docs/getting-started.md | 107 +- go.mod | 8 +- go.sum | 20 +- internal/cloudscale/client.go | 4 +- internal/cloudscale/client_test.go | 2 +- internal/cloudscale/flavors.go | 2 +- internal/cloudscale/flavors_test.go | 2 +- internal/cloudscale/regions.go | 2 +- internal/cloudscale/regions_test.go | 2 +- internal/cloudscale/services.go | 12 +- internal/controller/cloudscale_services.go | 2 +- .../controller/cloudscale_services_test.go | 2 +- internal/controller/cloudscale_tags.go | 2 +- .../cloudscalecluster_controller.go | 46 +- .../cloudscalecluster_controller_test.go | 59 + .../cloudscalecluster_floatingip.go | 2 +- .../cloudscalecluster_floatingip_test.go | 2 +- .../cloudscalecluster_loadbalancer.go | 7 +- .../cloudscalecluster_loadbalancer_test.go | 2 +- .../controller/cloudscalecluster_network.go | 27 +- .../cloudscalecluster_network_test.go | 30 +- .../cloudscalecluster_reconcile_test.go | 2 +- .../controller/cloudscalecluster_router.go | 615 +++++++++ .../cloudscalecluster_router_test.go | 1099 +++++++++++++++++ .../cloudscalecluster_servergroup.go | 2 +- .../cloudscalecluster_servergroup_test.go | 2 +- .../cloudscalemachine_reconcile_test.go | 6 +- .../controller/cloudscalemachine_server.go | 12 +- .../cloudscalemachine_server_test.go | 12 +- .../cloudscalemachine_servergroup.go | 2 +- .../cloudscalemachine_servergroup_test.go | 2 +- internal/testutils/cluster_scope.go | 61 +- internal/testutils/fixtures.go | 2 +- internal/testutils/mocks.go | 63 +- .../v1beta2/cloudscalecluster_webhook.go | 420 ++++++- .../v1beta2/cloudscalecluster_webhook_test.go | 590 ++++++++- .../cloudscaleclustertemplate_webhook.go | 2 +- .../cloudscaleclustertemplate_webhook_test.go | 117 +- templates/cluster-class-router-nat.yaml | 377 ++++++ ...ster-template-public-lb-private-nodes.yaml | 6 + templates/cluster-template-router-nat.yaml | 165 +++ .../cluster-template-topology-router-nat.yaml | 68 + test/e2e/config/cloudscale.yaml | 3 + .../kustomization.yaml | 9 + .../router-nat.yaml | 37 + .../cluster-topology.yaml | 69 ++ .../kustomization.yaml | 6 + .../cluster-class.yaml | 369 ++++++ .../kustomization.yaml | 4 + .../main/clusterclass-quick-start.yaml | 311 ----- test/e2e/e2e_test.go | 52 + test/e2e/helpers.go | 67 + 63 files changed, 4994 insertions(+), 456 deletions(-) create mode 100644 internal/controller/cloudscalecluster_router.go create mode 100644 internal/controller/cloudscalecluster_router_test.go create mode 100644 templates/cluster-class-router-nat.yaml create mode 100644 templates/cluster-template-router-nat.yaml create mode 100644 templates/cluster-template-topology-router-nat.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml create mode 100644 test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml delete mode 100644 test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml diff --git a/.github/workflows/e2e-weekly.yml b/.github/workflows/e2e-weekly.yml index b7673a7..0cf7456 100644 --- a/.github/workflows/e2e-weekly.yml +++ b/.github/workflows/e2e-weekly.yml @@ -42,7 +42,7 @@ jobs: TAG: e2e-weekly-${{ github.sha }} run: | make test-e2e \ - GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || topology || conformance || pre-existing-networking" \ + GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || topology || conformance || pre-existing-networking || router-nat" \ KUBETEST_CONFIGURATION=./data/kubetest/conformance-fast.yaml - name: Install regctl diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 389ef01..5cc2489 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -17,6 +17,7 @@ on: - test-e2e-self-hosted - test-e2e-md-remediation - test-e2e-pre-existing-networking + - test-e2e-router-nat - test-e2e-topology - test-e2e-upgrade - test-e2e-conformance diff --git a/.gitignore b/.gitignore index 4afe9fd..f93a469 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ go.work _artifacts/ test/e2e/config/*.generated.yaml test/e2e/data/infrastructure-cloudscale/main/cluster-template*.yaml +test/e2e/data/infrastructure-cloudscale/main/clusterclass*.yaml # release dist/ diff --git a/AGENTS.md b/AGENTS.md index 43672eb..fc8e791 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,10 +109,12 @@ For the prose architecture sketch see [`docs/development.md`](docs/development.m - The e2e overlay `test/infrastructure/cloudscale/clusterclass-quick-start/cluster-class.yaml` intentionally duplicates `templates/cluster-class.yaml`. Keep them in sync, but do not attempt to merge or deduplicate them — both copies are required. + The same applies to `clusterclass-router-nat/cluster-class.yaml` and + `templates/cluster-class-router-nat.yaml`; those two are byte-identical. ## Cloudscale SDK usage -- Do not `import "github.com/cloudscale-ch/cloudscale-go-sdk/v9"` outside +- Do not `import "github.com/cloudscale-ch/cloudscale-go-sdk/v10"` outside `internal/cloudscale/`. Controllers and webhooks talk to the SDK through the service interfaces on `cloudscale.Client` (`internal/cloudscale/client.go:32`). diff --git a/Makefile b/Makefile index 5eca562..5617378 100644 --- a/Makefile +++ b/Makefile @@ -162,9 +162,12 @@ E2E_CLUSTER_TEMPLATES := cluster-template \ cluster-template-md-remediation \ cluster-template-pre-existing-network \ cluster-template-public-lb-private-nodes \ + cluster-template-router-nat \ cluster-template-topology \ + cluster-template-topology-router-nat \ cluster-template-upgrades \ - clusterclass-quick-start + clusterclass-quick-start \ + clusterclass-router-nat .PHONY: generate-e2e-templates generate-e2e-templates: $(KUSTOMIZE) generate-e2e-cni generate-e2e-ccm ## Generate e2e cluster templates using kustomize overlays @@ -285,6 +288,19 @@ test-e2e-topology: $(GINKGO) generate-e2e-templates generate-e2e-config docker-b -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) +.PHONY: test-e2e-router-nat +test-e2e-router-nat: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run managed-router NAT e2e tests + $(GINKGO) -v --trace --tags=e2e \ + --nodes=$(GINKGO_NODES) \ + --label-filter="router-nat" \ + --timeout=90m \ + --output-dir="$(E2E_ARTIFACTS_FOLDER)" --junit-report="junit.e2e_router_nat.xml" \ + ./test/e2e -- \ + -e2e.config=$(E2E_CONF_FILE) \ + -e2e.artifacts-folder=$(E2E_ARTIFACTS_FOLDER) \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + .PHONY: test-e2e-conformance test-e2e-conformance: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run K8s conformance e2e tests $(GINKGO) -v --trace --tags=e2e \ diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index f55a010..f967d3e 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -72,6 +72,14 @@ type CloudscaleClusterSpec struct { // +optional Networks []NetworkSpec `json:"networks,omitempty"` + // Routers define routers for this cluster. + // Each router can be attached to one or more networks via its interfaces. + // If empty, no router is provisioned. + // +listType=map + // +listMapKey=name + // +optional + Routers []RouterSpec `json:"routers,omitempty"` + // ControlPlaneLoadBalancer configures the load balancer for the control plane. // +optional ControlPlaneLoadBalancer LoadBalancerSpec `json:"controlPlaneLoadBalancer,omitzero"` @@ -127,16 +135,87 @@ type NetworkSpec struct { // +optional CIDR string `json:"cidr,omitempty"` + // MTU defines the MTU size for this network. + // Immutable after creation + // +kubebuilder:validation:Minimum=1400 + // +kubebuilder:validation:Maximum=9000 + // +optional + MTU int `json:"mtu,omitempty"` + // GatewayAddress is the gateway IP address for the subnet. - // Only applicable when CIDR is set (managed network). - // By default, no gateway is configured on the subnet. This ensures - // that outbound internet traffic uses the public network interface. - // Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - // a NAT gateway or similar infrastructure on the private network. + // Only applicable when CIDR is set (managed network). Immutable after creation. // +optional GatewayAddress string `json:"gatewayAddress,omitempty"` } +// RouterSpec defines a router managed or adopted for this cluster. +// Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. +type RouterSpec struct { + // Name identifies this router within the cluster. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // UUID references a pre-existing router by UUID. + // CAPCS attaches and detaches the interfaces it created but does not create or delete + // the router, and leaves interfaces it did not create alone. Interfaces the router + // already carries must be adopted explicitly via interfaces[].uuid. + // Mutually exclusive with InternetGateway. Immutable after creation. + // +optional + UUID string `json:"uuid,omitempty"` + + // InternetGateway enables SNAT for outbound internet access on a managed router. + // Only valid when UUID is not set. Immutable after creation. + // +optional + InternetGateway bool `json:"internetGateway,omitempty"` + + // Interfaces lists the networks this router is attached to. + // A router attaches to any given network at most once. + // +listType=map + // +listMapKey=network + // +optional + Interfaces []RouterInterfaceSpec `json:"interfaces,omitempty"` +} + +// RouterInterfaceSpec defines a single network attachment for a router. +type RouterInterfaceSpec struct { + // Network references spec.networks[].name. + // +kubebuilder:validation:Required + Network string `json:"network"` + + // UUID adopts a pre-existing interface of the router instead of attaching a new one. + // The interface must already be attached to the referenced network. CAPCS reads its + // address but never creates or detaches it. + // Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS + // creates has no pre-existing interfaces. Mutually exclusive with Address. + // Immutable after creation. + // +optional + UUID string `json:"uuid,omitempty"` + + // Address is the IP requested for this router interface within the referenced + // network's subnet. The API requires an explicit address per interface, so the + // admission webhook defaults it: the interface with configureSubnetGateway=true gets + // the network's gatewayAddress, or the first free address in the network's CIDR if + // that is unset. Must be set explicitly when the network is referenced by uuid — its + // subnet is not known at admission time. Must be left empty when this interface is + // adopted by uuid: its address is whatever the pre-existing interface holds. + // If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once + // the subnet gateway is configured. Immutable after creation. + // +optional + Address string `json:"address,omitempty"` + + // ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress + // to this router interface's assigned IP, making the router the default route for + // servers on that subnet. + // Set to false for transit/backbone networks where a different router (e.g. the + // internet-gateway router) owns the subnet gateway. + // Immutable after creation. + // +kubebuilder:default=true + // +optional + ConfigureSubnetGateway *bool `json:"configureSubnetGateway,omitempty"` +} + // LoadBalancerSpec defines the load balancer configuration for the control plane. type LoadBalancerSpec struct { // Enabled controls whether a load balancer is created for the control plane. @@ -248,6 +327,12 @@ type CloudscaleClusterStatus struct { // +optional Networks []NetworkStatus `json:"networks,omitempty"` + // Routers track the provisioned state of each router defined in spec.routers. + // +listType=map + // +listMapKey=name + // +optional + Routers []RouterStatus `json:"routers,omitempty"` + // FloatingIP is the cloudscale.ch floating IP. // +optional FloatingIP string `json:"floatingIP,omitempty"` @@ -297,9 +382,65 @@ type NetworkStatus struct { // +optional CIDR string `json:"cidr,omitempty"` + // MTU is the MTU of this network. + // +optional + MTU int `json:"mtu,omitempty"` + // Managed indicates whether CAPCS manages this network's lifecycle. // false for pre-existing networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). Managed bool `json:"managed"` + + // GatewayAddress is the subnet gateway IP that CAPCS has configured on the router interface. + // Empty until the router interface is attached and the subnet gateway is updated. + // +optional + GatewayAddress string `json:"gatewayAddress,omitempty"` +} + +// RouterStatus tracks the provisioned state of a single router. +type RouterStatus struct { + // Name matches spec.routers[].name. + Name string `json:"name"` + + // RouterID is the router UUID. + // +optional + RouterID string `json:"routerID,omitempty"` + + // Interfaces tracks this router's interfaces, at most one per network. + // +listType=map + // +listMapKey=network + // +optional + Interfaces []RouterInterfaceStatus `json:"interfaces,omitempty"` + + // Managed indicates whether CAPCS manages this router's lifecycle. + // false for pre-existing routers (referenced by UUID), true for CAPCS-created routers. + Managed bool `json:"managed"` +} + +// RouterInterfaceStatus tracks one router interface. +type RouterInterfaceStatus struct { + // Network matches spec.networks[].name. + Network string `json:"network"` + + // InterfaceID is the router interface UUID. + // Empty only while an attach is in flight: the entry is written before the create call, + // so an attach whose response was lost to a timeout is still recognized as CAPCS-created. + // +optional + InterfaceID string `json:"interfaceID,omitempty"` + + // Managed indicates whether CAPCS created this interface, i.e. whether the spec entry + // left uuid empty. Interfaces adopted by uuid are never detached on teardown. + Managed bool `json:"managed"` +} + +// GetInterfaceStatus returns the RouterInterfaceStatus for the given network name, or +// nil if the router has no recorded interface on that network. +func (rs *RouterStatus) GetInterfaceStatus(network string) *RouterInterfaceStatus { + for i := range rs.Interfaces { + if rs.Interfaces[i].Network == network { + return &rs.Interfaces[i] + } + } + return nil } // ClusterInitializationStatus contains v1beta2 initialization tracking for CloudscaleCluster. @@ -320,6 +461,16 @@ func (s *CloudscaleClusterStatus) GetNetworkStatus(name string) *NetworkStatus { return nil } +// GetRouterStatus returns the RouterStatus for the given router name, or nil if not found. +func (s *CloudscaleClusterStatus) GetRouterStatus(name string) *RouterStatus { + for i := range s.Routers { + if s.Routers[i].Name == name { + return &s.Routers[i] + } + } + return nil +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=cloudscaleclusters,scope=Namespaced,categories=cluster-api diff --git a/api/v1beta2/condition_types.go b/api/v1beta2/condition_types.go index d82e359..2c83d13 100644 --- a/api/v1beta2/condition_types.go +++ b/api/v1beta2/condition_types.go @@ -30,6 +30,10 @@ const ( // True when the pause annotation is present on the resource or parent cluster. PausedCondition = "Paused" + // RouterReadyCondition indicates all spec.routers are provisioned, active, and have their + // interfaces attached. Also set to True when no routers are defined. + RouterReadyCondition = "RouterReady" + // FloatingIPReadyCondition indicates whether the floating IP has been provisioned and assigned. // Only applicable to CloudscaleCluster when spec.floatingIP is set. FloatingIPReadyCondition = "FloatingIPReady" @@ -65,6 +69,21 @@ const ( // LoadBalancerDeletingReason indicates the load balancer is being deleted. LoadBalancerDeletingReason = "LoadBalancerDeleting" + // RouterProvisionedReason indicates all routers and their interfaces are provisioned. + RouterProvisionedReason = "RouterProvisioned" + + // RouterNotReadyReason indicates a router exists but its status is not yet active. + RouterNotReadyReason = "RouterNotReady" + + // RouterErrorReason indicates an error occurred during router operations. + RouterErrorReason = "RouterError" + + // RouterDisabledReason indicates spec.routers is empty. + RouterDisabledReason = "RouterDisabled" + + // RouterDeletingReason indicates routers are being deleted. + RouterDeletingReason = "RouterDeleting" + // FloatingIPProvisionedReason indicates the floating IP has been successfully provisioned and assigned. FloatingIPProvisionedReason = "FloatingIPProvisioned" diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index ef98f12..24e26b2 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -96,6 +96,13 @@ func (in *CloudscaleClusterSpec) DeepCopyInto(out *CloudscaleClusterSpec) { *out = make([]NetworkSpec, len(*in)) copy(*out, *in) } + if in.Routers != nil { + in, out := &in.Routers, &out.Routers + *out = make([]RouterSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.ControlPlaneLoadBalancer.DeepCopyInto(&out.ControlPlaneLoadBalancer) if in.FloatingIP != nil { in, out := &in.FloatingIP, &out.FloatingIP @@ -127,6 +134,13 @@ func (in *CloudscaleClusterStatus) DeepCopyInto(out *CloudscaleClusterStatus) { *out = make([]NetworkStatus, len(*in)) copy(*out, *in) } + if in.Routers != nil { + in, out := &in.Routers, &out.Routers + *out = make([]RouterStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.LoadBalancerMemberIDs != nil { in, out := &in.LoadBalancerMemberIDs, &out.LoadBalancerMemberIDs *out = make([]string, len(*in)) @@ -668,6 +682,83 @@ func (in *NodeInfo) DeepCopy() *NodeInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterInterfaceSpec) DeepCopyInto(out *RouterInterfaceSpec) { + *out = *in + if in.ConfigureSubnetGateway != nil { + in, out := &in.ConfigureSubnetGateway, &out.ConfigureSubnetGateway + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterInterfaceSpec. +func (in *RouterInterfaceSpec) DeepCopy() *RouterInterfaceSpec { + if in == nil { + return nil + } + out := new(RouterInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterInterfaceStatus) DeepCopyInto(out *RouterInterfaceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterInterfaceStatus. +func (in *RouterInterfaceStatus) DeepCopy() *RouterInterfaceStatus { + if in == nil { + return nil + } + out := new(RouterInterfaceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { + *out = *in + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]RouterInterfaceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterSpec. +func (in *RouterSpec) DeepCopy() *RouterSpec { + if in == nil { + return nil + } + out := new(RouterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterStatus) DeepCopyInto(out *RouterStatus) { + *out = *in + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]RouterInterfaceStatus, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterStatus. +func (in *RouterStatus) DeepCopy() *RouterStatus { + if in == nil { + return nil + } + out := new(RouterStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 55938f1..e051d06 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,7 +29,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" - "github.com/cloudscale-ch/cloudscale-go-sdk/v9/instrumentation" + "github.com/cloudscale-ch/cloudscale-go-sdk/v10/instrumentation" "go.opentelemetry.io/otel" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/runtime" diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index ea6d5f6..b2c4c3e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -223,12 +223,15 @@ spec: gatewayAddress: description: |- GatewayAddress is the gateway IP address for the subnet. - Only applicable when CIDR is set (managed network). - By default, no gateway is configured on the subnet. This ensures - that outbound internet traffic uses the public network interface. - Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - a NAT gateway or similar infrastructure on the private network. + Only applicable when CIDR is set (managed network). Immutable after creation. type: string + mtu: + description: |- + MTU defines the MTU size for this network. + Immutable after creation + maximum: 9000 + minimum: 1400 + type: integer name: description: |- Name identifies this network within the cluster. @@ -258,6 +261,90 @@ spec: - rma - lpg type: string + routers: + description: |- + Routers define routers for this cluster. + Each router can be attached to one or more networks via its interfaces. + If empty, no router is provisioned. + items: + description: |- + RouterSpec defines a router managed or adopted for this cluster. + Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. + properties: + interfaces: + description: |- + Interfaces lists the networks this router is attached to. + A router attaches to any given network at most once. + items: + description: RouterInterfaceSpec defines a single network + attachment for a router. + properties: + address: + description: |- + Address is the IP requested for this router interface within the referenced + network's subnet. The API requires an explicit address per interface, so the + admission webhook defaults it: the interface with configureSubnetGateway=true gets + the network's gatewayAddress, or the first free address in the network's CIDR if + that is unset. Must be set explicitly when the network is referenced by uuid — its + subnet is not known at admission time. Must be left empty when this interface is + adopted by uuid: its address is whatever the pre-existing interface holds. + If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once + the subnet gateway is configured. Immutable after creation. + type: string + configureSubnetGateway: + default: true + description: |- + ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress + to this router interface's assigned IP, making the router the default route for + servers on that subnet. + Set to false for transit/backbone networks where a different router (e.g. the + internet-gateway router) owns the subnet gateway. + Immutable after creation. + type: boolean + network: + description: Network references spec.networks[].name. + type: string + uuid: + description: |- + UUID adopts a pre-existing interface of the router instead of attaching a new one. + The interface must already be attached to the referenced network. CAPCS reads its + address but never creates or detaches it. + Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS + creates has no pre-existing interfaces. Mutually exclusive with Address. + Immutable after creation. + type: string + required: + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + internetGateway: + description: |- + InternetGateway enables SNAT for outbound internet access on a managed router. + Only valid when UUID is not set. Immutable after creation. + type: boolean + name: + description: Name identifies this router within the cluster. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + uuid: + description: |- + UUID references a pre-existing router by UUID. + CAPCS attaches and detaches the interfaces it created but does not create or delete + the router, and leaves interfaces it did not create alone. Interfaces the router + already carries must be adopted explicitly via interfaces[].uuid. + Mutually exclusive with InternetGateway. Immutable after creation. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map zone: description: |- Zone is the cloudscale.ch zone within Region. @@ -378,11 +465,19 @@ spec: CIDR is the subnet CIDR block. Set from spec for managed networks or discovered from the API for pre-existing networks. type: string + gatewayAddress: + description: |- + GatewayAddress is the subnet gateway IP that CAPCS has configured on the router interface. + Empty until the router interface is attached and the subnet gateway is updated. + type: string managed: description: |- Managed indicates whether CAPCS manages this network's lifecycle. false for pre-existing networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). type: boolean + mtu: + description: MTU is the MTU of this network. + type: integer name: description: Name matches the logical name from spec.networks[].name. type: string @@ -406,6 +501,60 @@ spec: format: int64 minimum: 1 type: integer + routers: + description: Routers track the provisioned state of each router defined + in spec.routers. + items: + description: RouterStatus tracks the provisioned state of a single + router. + properties: + interfaces: + description: Interfaces tracks this router's interfaces, at + most one per network. + items: + description: RouterInterfaceStatus tracks one router interface. + properties: + interfaceID: + description: |- + InterfaceID is the router interface UUID. + Empty only while an attach is in flight: the entry is written before the create call, + so an attach whose response was lost to a timeout is still recognized as CAPCS-created. + type: string + managed: + description: |- + Managed indicates whether CAPCS created this interface, i.e. whether the spec entry + left uuid empty. Interfaces adopted by uuid are never detached on teardown. + type: boolean + network: + description: Network matches spec.networks[].name. + type: string + required: + - managed + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + managed: + description: |- + Managed indicates whether CAPCS manages this router's lifecycle. + false for pre-existing routers (referenced by UUID), true for CAPCS-created routers. + type: boolean + name: + description: Name matches spec.routers[].name. + type: string + routerID: + description: RouterID is the router UUID. + type: string + required: + - managed + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object required: - spec diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml index 3e003da..9ede64f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml @@ -253,12 +253,15 @@ spec: gatewayAddress: description: |- GatewayAddress is the gateway IP address for the subnet. - Only applicable when CIDR is set (managed network). - By default, no gateway is configured on the subnet. This ensures - that outbound internet traffic uses the public network interface. - Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - a NAT gateway or similar infrastructure on the private network. + Only applicable when CIDR is set (managed network). Immutable after creation. type: string + mtu: + description: |- + MTU defines the MTU size for this network. + Immutable after creation + maximum: 9000 + minimum: 1400 + type: integer name: description: |- Name identifies this network within the cluster. @@ -288,6 +291,91 @@ spec: - rma - lpg type: string + routers: + description: |- + Routers define routers for this cluster. + Each router can be attached to one or more networks via its interfaces. + If empty, no router is provisioned. + items: + description: |- + RouterSpec defines a router managed or adopted for this cluster. + Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. + properties: + interfaces: + description: |- + Interfaces lists the networks this router is attached to. + A router attaches to any given network at most once. + items: + description: RouterInterfaceSpec defines a single + network attachment for a router. + properties: + address: + description: |- + Address is the IP requested for this router interface within the referenced + network's subnet. The API requires an explicit address per interface, so the + admission webhook defaults it: the interface with configureSubnetGateway=true gets + the network's gatewayAddress, or the first free address in the network's CIDR if + that is unset. Must be set explicitly when the network is referenced by uuid — its + subnet is not known at admission time. Must be left empty when this interface is + adopted by uuid: its address is whatever the pre-existing interface holds. + If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once + the subnet gateway is configured. Immutable after creation. + type: string + configureSubnetGateway: + default: true + description: |- + ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress + to this router interface's assigned IP, making the router the default route for + servers on that subnet. + Set to false for transit/backbone networks where a different router (e.g. the + internet-gateway router) owns the subnet gateway. + Immutable after creation. + type: boolean + network: + description: Network references spec.networks[].name. + type: string + uuid: + description: |- + UUID adopts a pre-existing interface of the router instead of attaching a new one. + The interface must already be attached to the referenced network. CAPCS reads its + address but never creates or detaches it. + Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS + creates has no pre-existing interfaces. Mutually exclusive with Address. + Immutable after creation. + type: string + required: + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + internetGateway: + description: |- + InternetGateway enables SNAT for outbound internet access on a managed router. + Only valid when UUID is not set. Immutable after creation. + type: boolean + name: + description: Name identifies this router within the + cluster. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + uuid: + description: |- + UUID references a pre-existing router by UUID. + CAPCS attaches and detaches the interfaces it created but does not create or delete + the router, and leaves interfaces it did not create alone. Interfaces the router + already carries must be adopted explicitly via interfaces[].uuid. + Mutually exclusive with InternetGateway. Immutable after creation. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map zone: description: |- Zone is the cloudscale.ch zone within Region. diff --git a/docs/getting-started.md b/docs/getting-started.md index 7828e14..3e6df50 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -113,44 +113,63 @@ Set them in your shell, or keep them in `clusterctl.yaml` alongside the token. ## 6. Pick a cluster template flavor -| Flavor | Network | Control plane endpoint | Node connectivity | Requires | -|---------------------------|--------------------------|------------------------|-------------------|------------------------------------------------------| -| *(default)* | Managed, `172.18.0.0/24` | Public LB, DualStack | Public + cluster | — | -| `fip` | Pre-existing | Floating IP, IPv4 | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | -| `pre-existing-network` | Pre-existing | Public LB, DualStack | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | -| `public-lb-private-nodes` | Pre-existing + NAT | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID`, with a NAT gateway set up | -| `topology` | Managed, `172.18.0.0/24` | Public LB, DualStack | Public + cluster | `CLUSTER_TOPOLOGY=true` feature gate | +| Flavor | Network | Router/sNAT | Control plane endpoint | Node connectivity | Requires | +|---------------------------|--------------|--------------|------------------------|-------------------|------------------------------------------------------| +| *(default)* | Managed | not required | Public LB, DualStack | Public + cluster | — | +| `fip` | Pre-existing | not required | Floating IP, IPv4 | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | +| `pre-existing-network` | Pre-existing | not required | Public LB, DualStack | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | +| `public-lb-private-nodes` | Pre-existing | pre-existing | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID`, with a NAT gateway set up | +| `router-nat` | Managed | managed | Public LB | Private only | — | +| `topology` | Managed | not required | Public LB, DualStack | Public + cluster | `CLUSTER_TOPOLOGY=true` feature gate | +| `topology-router-nat` | Managed | managed | Public LB | Private only | `CLUSTER_TOPOLOGY=true` feature gate | These flavors just show possible configurations. You're encouraged to copy and adjust them to your needs. -If you do need a NAT gateway set up for your cluster, set up a private network and contact cloudscale support with the -following details: +## Using Templates with Topology/ClusterClass -- Desired name of the gateway -- UUID of the network (in control panel details) -- Optional: desired IP of the gateway in the network +The flavors with a `topology` prefix generate a `Cluster` that references a +[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) in `spec.topology.classRef`. -### Using the topology flavor - -The `topology` flavor generates a `Cluster` that references the `quick-start` -[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) -shipped in `templates/cluster-class.yaml`. Apply it once per namespace before -generating any topology cluster: +To use a topology flavor, you first need to apply its referenced ClusterClass: ```bash +# adjust CLUSTER_CLASS_FLAVOR to the respective cluster class yaml file. clusterctl generate yaml \ - --from https://raw.githubusercontent.com/cloudscale-ch/cluster-api-provider-cloudscale/main/templates/cluster-class.yaml \ + --from https://raw.githubusercontent.com/cloudscale-ch/cluster-api-provider-cloudscale/main/templates/{CLUSTER_CLASS_FLAVOR}.yaml \ | kubectl apply -f - ``` -Per-cluster overrides go under `spec.topology.variables`. See `templates/cluster-class.yaml` for the full -variable list and defaults. +Per-cluster overrides go under `spec.topology.variables`. The available variables depend on which ClusterClass flavor you use. + +**Common variables** (both `topology` and `topology-router-nat`): + +| Variable | Type | Required | Default | Description | +|-----------------------------------------------|------------------|----------|-----------------------------------|--------------------------------------------------------------------------------------| +| `region` | string | yes | `lpg` | cloudscale region (`lpg` or `rma`) | +| `credentialsRefName` | string | yes | — | Name of the Secret holding API credentials (must match the Secret created in step 7) | +| `cloudscaleControlPlaneMachineFlavor` | string | yes | `flex-4-2` | Flavor for control plane nodes | +| `cloudscaleControlPlaneMachineImage` | string | yes | `custom:ubuntu-2404-kube-v1.36.0` | Image for control plane nodes | +| `cloudscaleControlPlaneMachineRootVolumeSize` | integer | yes | `20` | Root volume size in GB for control plane | +| `cloudscaleWorkerMachineFlavor` | string | yes | `flex-4-2` | Flavor for worker nodes | +| `cloudscaleWorkerMachineImage` | string | yes | `custom:ubuntu-2404-kube-v1.36.0` | Image for worker nodes | +| `cloudscaleWorkerMachineRootVolumeSize` | integer | yes | `20` | Root volume size in GB for workers | +| `cloudscaleSSHKeys` | array of strings | yes | — | SSH public keys for all nodes | +| `cloudscaleControlPlaneServerGroupName` | string | yes | — | Server group name for control plane | +| `cloudscaleWorkerServerGroupName` | string | yes | — | Server group name for workers | + +*(Source: [`templates/cluster-class.yaml`](../templates/cluster-class.yaml))* + +The `topology-router-nat` flavor adds these variables: -Caveat: the `quick-start` ClusterClass exposes a narrower configurability than the traditional templates — no floating -IP, no pre-existing network, no private load balancer, and pod/service CIDRs are pinned to -`192.168.0.0/16` / `10.96.0.0/12`. Adjust the template for your use-case. +| Variable | Type | Required | Default | Description | +|---------------|--------|----------|----------------|------------------------------------------------------------------------------| +| `networkName` | string | yes | `node-net` | Logical name for the private network | +| `networkCIDR` | string | yes | `10.10.0.0/24` | CIDR for the private network (router interface address is derived from this) | +| `routerName` | string | yes | `router` | Name of the managed router | -Use `--flavor topology` in the next step if you did use ClusterClass. +*(Source: [`templates/cluster-class-router-nat.yaml`](../templates/cluster-class-router-nat.yaml))* + +Use `--flavor topology` or `--flavor topology-router-nat` in the next step if you did use ClusterClass. ## 7. Generate and apply the workload cluster @@ -178,16 +197,26 @@ Watch progress: clusterctl describe cluster my-cluster ``` +Wait for the control plane to come up enough to hand out a kubeconfig. This +doesn't need a CCM or CNI yet: + +```bash +kubectl wait --for=condition=ControlPlaneInitialized cluster/my-cluster --timeout=15m +``` + ## 8. Install the cloudscale CCM CAPCS only provisions infrastructure. For a working cluster the [cloudscale CCM](https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager) must be installed on the **workload** cluster. -CAPCS provides a [`ClusterResourceSet` you apply on the **management** cluster](https://cluster-api.sigs.k8s.io/tasks/cluster-resource-set). -CAPI's CRS controller then deploys it into any workload cluster labelled `ccm: cloudscale` — every CAPCS template sets this label. +CAPCS provides a [`ClusterResourceSet` you apply on the +**management** cluster](https://cluster-api.sigs.k8s.io/tasks/cluster-resource-set). +CAPI's CRS controller then deploys it into any workload cluster labelled `ccm: cloudscale` — every CAPCS template sets +this label. -**N.B.:** the CCM also needs a CLOUDSCALE_API_TOKEN in the environment set. It's recommended to use a different token for the CCM and CAPCS. +**N.B.:** the CCM also needs a CLOUDSCALE_API_TOKEN in the environment set. It's recommended to use a different token +for the CCM and CAPCS. ```bash # the namespace must match the namespace of the workload cluster definition @@ -202,17 +231,16 @@ the CCM starts and removes the taint — that is expected. ## 9. Install a CNI (Cilium) -Fetch the workload cluster's kubeconfig and point `KUBECONFIG` at it: +Fetch the workload cluster's kubeconfig: ```bash clusterctl get kubeconfig my-cluster > ~/.kube/my-cluster.yaml -export KUBECONFIG=~/.kube/my-cluster.yaml ``` Install Cilium with defaults: ```bash -cilium install +KUBECONFIG=~/.kube/my-cluster.yaml cilium install ``` Nodes become `Ready` once Cilium is up. Any standard CNI works — Cilium is just @@ -222,26 +250,33 @@ the example. ```bash # management cluster -clusterctl describe cluster +clusterctl describe cluster my-cluster # workload cluster -kubectl get nodes -kubectl -n kube-system get pods +KUBECONFIG=~/.kube/my-cluster.yaml kubectl get nodes +KUBECONFIG=~/.kube/my-cluster.yaml kubectl -n kube-system get pods ``` All nodes should report `Ready`. In `kube-system` you should see `cloudscale-cloud-controller-manager-*` on each control plane node and `cilium-*` on every node. +Now wait for the cluster to report fully available. + +```bash +kubectl wait --for=condition=Available cluster/my-cluster --timeout=15m +``` + If something is stuck, see [troubleshooting](troubleshooting.md). ## Optional: persistent storage (cloudscale CSI) For `PersistentVolumeClaim` support, install the -[cloudscale CSI driver](https://github.com/cloudscale-ch/csi-cloudscale) on the -**workload** cluster. With `KUBECONFIG` still pointing at the workload cluster: +[cloudscale CSI driver](https://github.com/cloudscale-ch/csi-cloudscale) on the **workload** cluster: ```bash +export KUBECONFIG=~/.kube/my-cluster.yaml + kubectl -n kube-system create secret generic cloudscale \ --from-literal=access-token="$CLOUDSCALE_API_TOKEN" diff --git a/go.mod b/go.mod index 909a9fa..9feaa61 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cloudscale-ch/cluster-api-provider-cloudscale go 1.27.0 require ( - github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0 + github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0 github.com/go-logr/logr v1.4.4 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 @@ -105,10 +105,10 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.68.0 // indirect - github.com/prometheus/procfs v0.20.1 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/go.sum b/go.sum index a5658bd..ce38838 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0 h1:i9xlttrksSDd8HHyj8ezYRUuVxiBPUAeJ2rZS9DFAnM= -github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0/go.mod h1:cTPjnvp+5J6XVSgHl8UdBxokNye8lKv/MJxx5/rYk40= +github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0 h1:Ir0/Ei9mU1OWBPDb95THR5H/Cu6WiXL1W/O67KS5YMM= +github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0/go.mod h1:PbmyqFmr70walcN/nbphTlznb5XadWR4fRRmbd5tV+s= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -166,8 +166,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -227,14 +227,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.68.0 h1:8rQJvQmYltsR2L7h8Zw0Iyj8WYNNmpwikoQTZXwfVeA= -github.com/prometheus/common v0.68.0/go.mod h1:4soH+U8yJSROk7OJ//hmTiWKsxapv6zRGgTt3keN8gQ= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= diff --git a/internal/cloudscale/client.go b/internal/cloudscale/client.go index f1f4e05..5e8b658 100644 --- a/internal/cloudscale/client.go +++ b/internal/cloudscale/client.go @@ -25,13 +25,14 @@ import ( "strings" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "golang.org/x/oauth2" ) type Client struct { Networks NetworkService Subnets SubnetService + Routers RouterService Regions RegionService Servers ServerService ServerGroups ServerGroupService @@ -109,6 +110,7 @@ func NewClient(token, version string, transport http.RoundTripper) *Client { return &Client{ Networks: sdkClient.Networks, Subnets: sdkClient.Subnets, + Routers: sdkClient.Routers, Regions: sdkClient.Regions, Servers: sdkClient.Servers, ServerGroups: sdkClient.ServerGroups, diff --git a/internal/cloudscale/client_test.go b/internal/cloudscale/client_test.go index 5dc9e16..88b1db4 100644 --- a/internal/cloudscale/client_test.go +++ b/internal/cloudscale/client_test.go @@ -22,7 +22,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" ) diff --git a/internal/cloudscale/flavors.go b/internal/cloudscale/flavors.go index e9b54c9..54b0ca7 100644 --- a/internal/cloudscale/flavors.go +++ b/internal/cloudscale/flavors.go @@ -3,7 +3,7 @@ package cloudscale import ( "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) diff --git a/internal/cloudscale/flavors_test.go b/internal/cloudscale/flavors_test.go index d8b557a..d3c614d 100644 --- a/internal/cloudscale/flavors_test.go +++ b/internal/cloudscale/flavors_test.go @@ -3,7 +3,7 @@ package cloudscale import ( "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/internal/cloudscale/regions.go b/internal/cloudscale/regions.go index e0e891b..10627eb 100644 --- a/internal/cloudscale/regions.go +++ b/internal/cloudscale/regions.go @@ -17,7 +17,7 @@ limitations under the License. package cloudscale import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) // RegionInfo holds cloudscale.ch region and zone information for validation. diff --git a/internal/cloudscale/regions_test.go b/internal/cloudscale/regions_test.go index bbeeeed..159d919 100644 --- a/internal/cloudscale/regions_test.go +++ b/internal/cloudscale/regions_test.go @@ -19,7 +19,7 @@ package cloudscale import ( "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" ) diff --git a/internal/cloudscale/services.go b/internal/cloudscale/services.go index 464d0f5..7c2c28a 100644 --- a/internal/cloudscale/services.go +++ b/internal/cloudscale/services.go @@ -19,7 +19,7 @@ package cloudscale import ( "context" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) type NetworkService interface { @@ -34,6 +34,16 @@ type SubnetService interface { Get(ctx context.Context, id string) (*cloudscalesdk.Subnet, error) List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) Delete(ctx context.Context, id string) error + Update(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error +} + +type RouterService interface { + Create(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) + Get(ctx context.Context, id string) (*cloudscalesdk.Router, error) + List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) + Delete(ctx context.Context, id string) error + CreateInterface(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) + DeleteInterface(ctx context.Context, routerUUID, interfaceUUID string) error } type RegionService interface { diff --git a/internal/controller/cloudscale_services.go b/internal/controller/cloudscale_services.go index e7b6062..c3d21cc 100644 --- a/internal/controller/cloudscale_services.go +++ b/internal/controller/cloudscale_services.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" diff --git a/internal/controller/cloudscale_services_test.go b/internal/controller/cloudscale_services_test.go index 860fe8a..e6cc896 100644 --- a/internal/controller/cloudscale_services_test.go +++ b/internal/controller/cloudscale_services_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "github.com/go-logr/logr" . "github.com/onsi/gomega" diff --git a/internal/controller/cloudscale_tags.go b/internal/controller/cloudscale_tags.go index 3edffa3..8dd5801 100644 --- a/internal/controller/cloudscale_tags.go +++ b/internal/controller/cloudscale_tags.go @@ -1,7 +1,7 @@ package controller import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" ) diff --git a/internal/controller/cloudscalecluster_controller.go b/internal/controller/cloudscalecluster_controller.go index e1ac6d8..ab31e8c 100644 --- a/internal/controller/cloudscalecluster_controller.go +++ b/internal/controller/cloudscalecluster_controller.go @@ -176,6 +176,14 @@ func (r *CloudscaleClusterReconciler) reconcileNormal(ctx context.Context, clust return result, nil } + result, err = r.reconcileRouters(ctx, clusterScope) + if err != nil { + return ctrl.Result{}, fmt.Errorf("reconciling routers: %w", err) + } + if !result.IsZero() { + return result, nil + } + lb, result, err := r.reconcileLoadBalancer(ctx, clusterScope) if err != nil { return ctrl.Result{}, fmt.Errorf("reconciling load balancer: %w", err) @@ -241,6 +249,10 @@ func (r *CloudscaleClusterReconciler) reconcileDelete(ctx context.Context, clust return ctrl.Result{}, fmt.Errorf("deleting server groups: %w", err) } + if err := r.deleteRouters(ctx, clusterScope); err != nil { + return ctrl.Result{}, fmt.Errorf("deleting routers: %w", err) + } + if err := r.deleteNetwork(ctx, clusterScope); err != nil { if errors.Is(err, errNetworkNotReady) { return ctrl.Result{RequeueAfter: 10 * time.Second}, nil @@ -296,8 +308,9 @@ func (r *CloudscaleClusterReconciler) reconcileControlPlaneEndpoint(clusterScope } // isInfrastructureProvisioned returns true if all cluster infrastructure is ready. -// This includes all Networks+Subnets, Load Balancer (with pool and listener) if enabled, -// Floating IP if configured, and Control Plane Endpoint. +// This includes all Networks+Subnets, Routers (with all interfaces attached), Load +// Balancer (with pool and listener) if enabled, Floating IP if configured, and +// Control Plane Endpoint. func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope *scope.ClusterScope) bool { // All networks must have both network and subnet IDs if len(clusterScope.CloudscaleCluster.Status.Networks) == 0 { @@ -309,6 +322,34 @@ func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope * } } + // All routers must be resolved and all their spec interfaces attached. + for _, routerSpec := range clusterScope.CloudscaleCluster.Spec.Routers { + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + if rs == nil || rs.RouterID == "" { + return false + } + for _, ifaceSpec := range routerSpec.Interfaces { + iface := rs.GetInterfaceStatus(ifaceSpec.Network) + // interfaces are first created with an empty interfaceID in the status (see createRouterInterface) + // to avoid potential issues with timeouts during interface creation. An interface in status is only fully set up + // if there's an interfaceID set. + if iface == nil || iface.InterfaceID == "" { + return false + } + // When ConfigureSubnetGateway is true, the subnet gateway must be configured + if ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) + if ns == nil || ns.GatewayAddress == "" { + return false + } + // If the spec has an address, verify the gateway matches it + if ifaceSpec.Address != "" && ns.GatewayAddress != ifaceSpec.Address { + return false + } + } + } + } + // Load balancer, pool, and listener must exist (if LB is enabled) if ptr.Deref(clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { if clusterScope.CloudscaleCluster.Status.LoadBalancerID == "" || @@ -338,6 +379,7 @@ func (r *CloudscaleClusterReconciler) setReadyCondition(clusterScope *scope.Clus // Check all sub-conditions subConditions := []string{ infrastructurev1beta2.NetworkReadyCondition, + infrastructurev1beta2.RouterReadyCondition, infrastructurev1beta2.LoadBalancerReadyCondition, infrastructurev1beta2.FloatingIPReadyCondition, } diff --git a/internal/controller/cloudscalecluster_controller_test.go b/internal/controller/cloudscalecluster_controller_test.go index 117e4e6..cd68ea8 100644 --- a/internal/controller/cloudscalecluster_controller_test.go +++ b/internal/controller/cloudscalecluster_controller_test.go @@ -148,10 +148,20 @@ func TestReconcileControlPlaneEndpoint_ExternalControlPlaneNoSource(t *testing.T // TestIsInfrastructureProvisioned exercises the readiness predicate used by // reconcileNormal to decide when to flip Status.Initialization.Provisioned. func TestIsInfrastructureProvisioned(t *testing.T) { + // routerSpec is a single router "r" attached to the given network. + routerSpec := func(network string) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{ + Name: "r", + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{{Network: network}}, + }} + } + cases := []struct { name string lbEnabled bool networks []infrastructurev1beta2.NetworkStatus + routerSpecs []infrastructurev1beta2.RouterSpec + routerStatuses []infrastructurev1beta2.RouterStatus endpointHost string endpointPort int32 lbID string @@ -192,6 +202,41 @@ func TestIsInfrastructureProvisioned(t *testing.T) { lbEnabled: false, networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, }, + { + // A router with no status at all: it has not even been resolved yet. + name: "router not yet provisioned", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + }, + { + // The router exists, but the interface the spec asks for is not attached, so + // nodes on that subnet still have no route out. + name: "router provisioned but interface not attached", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + routerStatuses: []infrastructurev1beta2.RouterStatus{{Name: "r", RouterID: "router-123", Managed: true}}, + }, + { + name: "router provisioned with its interface attached", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, GatewayAddress: "10.0.0.1"}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + routerStatuses: []infrastructurev1beta2.RouterStatus{{ + Name: "r", RouterID: "router-123", Managed: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: "iface-123", Managed: true}, + }, + }}, + wantProvisioned: true, + }, { name: "Floating IP requested but status floating IP empty", lbEnabled: false, @@ -225,9 +270,11 @@ func TestIsInfrastructureProvisioned(t *testing.T) { Enabled: new(tc.lbEnabled), }, FloatingIP: tc.floatingIP, + Routers: tc.routerSpecs, }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ Networks: tc.networks, + Routers: tc.routerStatuses, LoadBalancerID: tc.lbID, LoadBalancerPoolID: tc.lbPoolID, LoadBalancerListenerID: tc.lbListenerID, @@ -255,6 +302,7 @@ func TestSetReadyCondition(t *testing.T) { name: "all sub-conditions True yields Ready=True", subConditions: []metav1.Condition{ {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.LoadBalancerReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.FloatingIPReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, }, @@ -275,12 +323,23 @@ func TestSetReadyCondition(t *testing.T) { name: "LoadBalancer=False propagates reason and message", subConditions: []metav1.Condition{ {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.LoadBalancerReadyCondition, Status: metav1.ConditionFalse, Reason: infrastructurev1beta2.LoadBalancerNotReadyReason, Message: "Load balancer is not running", ObservedGeneration: 1}, }, expectStatus: metav1.ConditionFalse, expectReason: infrastructurev1beta2.LoadBalancerNotReadyReason, expectMsgContains: "Load balancer is not running", }, + { + name: "Router=False propagates reason and message", + subConditions: []metav1.Condition{ + {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionFalse, Reason: infrastructurev1beta2.RouterNotReadyReason, Message: "Waiting for routers to become available", ObservedGeneration: 1}, + }, + expectStatus: metav1.ConditionFalse, + expectReason: infrastructurev1beta2.RouterNotReadyReason, + expectMsgContains: "Waiting for routers to become available", + }, { name: "no sub-conditions yields Ready=False/NotReady", subConditions: nil, diff --git a/internal/controller/cloudscalecluster_floatingip.go b/internal/controller/cloudscalecluster_floatingip.go index 43199a3..68c8aad 100644 --- a/internal/controller/cloudscalecluster_floatingip.go +++ b/internal/controller/cloudscalecluster_floatingip.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" diff --git a/internal/controller/cloudscalecluster_floatingip_test.go b/internal/controller/cloudscalecluster_floatingip_test.go index 3a80d4e..a5db7e8 100644 --- a/internal/controller/cloudscalecluster_floatingip_test.go +++ b/internal/controller/cloudscalecluster_floatingip_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalecluster_loadbalancer.go b/internal/controller/cloudscalecluster_loadbalancer.go index 8ff11ae..41d57b9 100644 --- a/internal/controller/cloudscalecluster_loadbalancer.go +++ b/internal/controller/cloudscalecluster_loadbalancer.go @@ -23,7 +23,7 @@ import ( "slices" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -513,8 +513,9 @@ func (r *CloudscaleClusterReconciler) getMemberSubnetCIDR(clusterScope *scope.Cl } // getPoolMemberSubnetID determines the subnet UUID for LB pool members. -// If the LB is on a private network, use that network's subnet. -// Otherwise (public LB), use the first network's subnet. +// 1. .PoolMemberNetwork's subnet if specified +// 2. .Network's subnet if specified +// 3. first network's subnet func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope.ClusterScope) (string, error) { if clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network != "" { return lbPrivateNetworkSubnetID(clusterScope) diff --git a/internal/controller/cloudscalecluster_loadbalancer_test.go b/internal/controller/cloudscalecluster_loadbalancer_test.go index e71315c..b2e44fe 100644 --- a/internal/controller/cloudscalecluster_loadbalancer_test.go +++ b/internal/controller/cloudscalecluster_loadbalancer_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalecluster_network.go b/internal/controller/cloudscalecluster_network.go index b4fcd91..207954d 100644 --- a/internal/controller/cloudscalecluster_network.go +++ b/internal/controller/cloudscalecluster_network.go @@ -23,7 +23,7 @@ import ( "strings" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -102,7 +102,7 @@ func (r *CloudscaleClusterReconciler) reconcilePreExistingNetwork(ctx context.Co return fmt.Errorf("pre-existing network %s has no subnets", netSpec.UUID) } - r.setNetworkStatus(clusterScope, netSpec.Name, network.UUID, network.Subnets[0].UUID, network.Subnets[0].CIDR, false) + r.setNetworkStatus(clusterScope, netSpec.Name, network.UUID, network.Subnets[0].UUID, network.Subnets[0].CIDR, false, network.MTU) clusterScope.Info("Discovered pre-existing network", "name", netSpec.Name, "networkID", network.UUID, "subnetID", network.Subnets[0].UUID, "cidr", network.Subnets[0].CIDR) return nil @@ -120,7 +120,7 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex tags := r.networkTags(clusterScope, netSpec.Name) - _, resolvedNetworkID, err := ensureResource(ctx, clusterScope, + network, resolvedNetworkID, err := ensureResource(ctx, clusterScope, networkID, fmt.Sprintf("network/%s", netSpec.Name), clusterScope.CloudscaleClient.Networks, @@ -136,12 +136,16 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex clusterScope.Info("Creating network", "name", netSpec.Name) createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) defer cancel() - network, err := clusterScope.CloudscaleClient.Networks.Create(createCtx, &cloudscalesdk.NetworkCreateRequest{ + req := &cloudscalesdk.NetworkCreateRequest{ Name: netSpec.Name, AutoCreateIPV4Subnet: new(false), Zone: clusterScope.CloudscaleCluster.Spec.Zone, Tags: new(tags), - }) + } + if netSpec.MTU > 0 { + req.MTU = netSpec.MTU + } + network, err = clusterScope.CloudscaleClient.Networks.Create(createCtx, req) if err != nil { if cloudscale.IsTimeoutError(err) { clusterScope.Info("Network creation timed out, waiting before retry", "requeueAfter", createNetworkTimeoutRequeueAfter) @@ -196,7 +200,7 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex "Created subnet %s (%s) with CIDR %s", netSpec.Name, subnet.UUID, netSpec.CIDR) } - r.setNetworkStatus(clusterScope, netSpec.Name, resolvedNetworkID, resolvedSubnetID, netSpec.CIDR, true) + r.setNetworkStatus(clusterScope, netSpec.Name, resolvedNetworkID, resolvedSubnetID, netSpec.CIDR, true, network.MTU) return ctrl.Result{}, nil } @@ -276,13 +280,14 @@ func isLBPoolMembersError(err error) bool { } // setNetworkStatus updates or appends the network status entry for the given name. -func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.ClusterScope, name, networkID, subnetID, cidr string, managed bool) { +func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.ClusterScope, name, networkID, subnetID, cidr string, managed bool, mtu int) { for i, ns := range clusterScope.CloudscaleCluster.Status.Networks { if ns.Name == name { clusterScope.CloudscaleCluster.Status.Networks[i].NetworkID = networkID clusterScope.CloudscaleCluster.Status.Networks[i].SubnetID = subnetID clusterScope.CloudscaleCluster.Status.Networks[i].CIDR = cidr clusterScope.CloudscaleCluster.Status.Networks[i].Managed = managed + clusterScope.CloudscaleCluster.Status.Networks[i].MTU = mtu return } } @@ -292,9 +297,17 @@ func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.Clust SubnetID: subnetID, CIDR: cidr, Managed: managed, + MTU: mtu, }) } +// setNetworkGatewayAddress records the configured subnet gateway IP in the network status entry. +func (r *CloudscaleClusterReconciler) setNetworkGatewayAddress(clusterScope *scope.ClusterScope, networkName, gatewayAddress string) { + if ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(networkName); ns != nil { + ns.GatewayAddress = gatewayAddress + } +} + // networkTags returns the tags for a specific named network, combining cluster ownership with network name. func (r *CloudscaleClusterReconciler) networkTags(clusterScope *scope.ClusterScope, networkName string) cloudscalesdk.TagMap { tags := cloudscalesdk.TagMap{ diff --git a/internal/controller/cloudscalecluster_network_test.go b/internal/controller/cloudscalecluster_network_test.go index 37e6154..4459525 100644 --- a/internal/controller/cloudscalecluster_network_test.go +++ b/internal/controller/cloudscalecluster_network_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" @@ -134,6 +134,34 @@ func TestReconcileNetwork_CreatesResources(t *testing.T) { g.Expect(capturedSubReq.GatewayAddress).To(Equal("10.0.0.254")) }, }, + { + name: "explicit mtu", + spec: func() (*testutils.MockNetworkService, *testutils.MockSubnetService, *scope.ClusterScope) { + networkService := &testutils.MockNetworkService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Network, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.NetworkCreateRequest) (*cloudscalesdk.Network, error) { + return &cloudscalesdk.Network{UUID: netUUID, Name: req.Name}, nil + }, + } + subnetService := &testutils.MockSubnetService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.SubnetCreateRequest) (*cloudscalesdk.Subnet, error) { + return &cloudscalesdk.Subnet{UUID: "subnet-uuid-123", CIDR: req.CIDR}, nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts(testutils.WithNetworkService(networkService), testutils.WithSubnetService(subnetService)) + clusterScope.CloudscaleCluster.Spec.Networks[0].MTU = 1500 + return networkService, subnetService, clusterScope + }, + want: func(g *WithT, capturedNetReq *cloudscalesdk.NetworkCreateRequest, capturedSubReq *cloudscalesdk.SubnetCreateRequest, clusterScope *scope.ClusterScope) { + g.Expect(capturedNetReq.MTU).To(Equal(1500)) + }, + }, } for _, tc := range tests { diff --git a/internal/controller/cloudscalecluster_reconcile_test.go b/internal/controller/cloudscalecluster_reconcile_test.go index 9781db0..563cdf3 100644 --- a/internal/controller/cloudscalecluster_reconcile_test.go +++ b/internal/controller/cloudscalecluster_reconcile_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalecluster_router.go b/internal/controller/cloudscalecluster_router.go new file mode 100644 index 0000000..f4a1e14 --- /dev/null +++ b/internal/controller/cloudscalecluster_router.go @@ -0,0 +1,615 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "time" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/observability" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" +) + +// routerRequeueAfter is how long to wait before looking again at a router that is still +// settling: a create or attach that timed out, or a router that is not active yet. +const routerRequeueAfter = 5 * time.Second + +// reconcileRouters orchestrates router provisioning for all routers +// defined in spec.routers. A single RouterReadyCondition covers all routers. +func (r *CloudscaleClusterReconciler) reconcileRouters(ctx context.Context, clusterScope *scope.ClusterScope) (result ctrl.Result, reterr error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouters") + defer done() + + defer func() { + switch { + case reterr != nil: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterErrorReason, reterr.Error()) + case !result.IsZero(): + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterNotReadyReason, "Waiting for routers to become available") + case len(clusterScope.CloudscaleCluster.Spec.Routers) == 0: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.RouterDisabledReason, "") + default: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.RouterProvisionedReason, "") + } + }() + + logger.Info("Reconciling routers") + + // Process all routers in one reconcile call by returning the lowest requeueAfter time (or just a non-zero result). + // This ensures routers get created in parallel and avoids longer waiting times if there is more than one router + // specified. + for _, routerSpec := range clusterScope.CloudscaleCluster.Spec.Routers { + res, err := r.reconcileRouter(ctx, clusterScope, routerSpec) + if err != nil { + return ctrl.Result{}, fmt.Errorf("reconciling router %q: %w", routerSpec.Name, err) + } + noResultYet := result.IsZero() + lowerRequeueAfter := res.RequeueAfter < result.RequeueAfter + if !res.IsZero() && (noResultYet || lowerRequeueAfter) { + result = res + } + } + return result, nil +} + +// reconcileRouter brings one router to the state its spec asks for: resolve or create it, +// record it in status, and attach its interfaces once it is active. +func (r *CloudscaleClusterReconciler) reconcileRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouter") + defer done() + + var ( + router *cloudscalesdk.Router + result ctrl.Result + err error + ) + if routerSpec.UUID != "" { + router, err = r.adoptRouter(ctx, clusterScope, routerSpec) + } else { + router, result, err = r.ensureManagedRouter(ctx, clusterScope, routerSpec) + } + if err != nil || !result.IsZero() { + return result, err + } + + r.setRouterStatus(clusterScope, routerSpec, router.UUID) + + if router.Status != cloudscalesdk.RouterActive { + logger.Info("Router is not active yet, waiting before retry", + "name", routerSpec.Name, "routerID", router.UUID, "status", router.Status, "requeueAfter", routerRequeueAfter) + return ctrl.Result{RequeueAfter: routerRequeueAfter}, nil + } + + return r.reconcileRouterInterfaces(ctx, clusterScope, routerSpec, router) +} + +// adoptRouter returns the pre-existing router the spec entry references by UUID. +// +// An adopted router is one CAPCS attaches and detaches interfaces on, but never creates or +// deletes itself. Resolving it is a plain read, so it never has to requeue. +func (r *CloudscaleClusterReconciler) adoptRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (*cloudscalesdk.Router, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.adoptRouter") + defer done() + + getCtx, cancel := context.WithTimeout(ctx, cloudscale.ReadTimeout) + router, err := clusterScope.CloudscaleClient.Routers.Get(getCtx, routerSpec.UUID) + cancel() + if err != nil { + return nil, fmt.Errorf("getting pre-existing router %s: %w", routerSpec.UUID, err) + } + if router.Zone.Slug != clusterScope.CloudscaleCluster.Spec.Zone { + return nil, fmt.Errorf("pre-existing router %s is in zone %q, expected zone %q", routerSpec.UUID, router.Zone.Slug, clusterScope.CloudscaleCluster.Spec.Zone) + } + // only log if we haven't recorded the status yet + if clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) == nil { + logger.Info("Discovered pre-existing router", "name", routerSpec.Name, "routerID", router.UUID) + } + return router, nil +} + +// ensureManagedRouter returns the router CAPCS owns for this spec entry, creating it when it +// does not exist yet. A non-zero result means the router is not available yet and the caller +// should requeue. +func (r *CloudscaleClusterReconciler) ensureManagedRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (*cloudscalesdk.Router, ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.ensureManagedRouter") + defer done() + + var routerID string + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name); rs != nil { + routerID = rs.RouterID + } + + tags := r.routerTags(clusterScope, routerSpec.Name) + router, resolvedRouterID, err := ensureResource(ctx, clusterScope, + routerID, + fmt.Sprintf("router/%s", routerSpec.Name), + clusterScope.CloudscaleClient.Routers, + func(r cloudscalesdk.Router) string { return r.UUID }, + tags, + ) + if err != nil { + return nil, ctrl.Result{}, fmt.Errorf("ensuring router: %w", err) + } + if resolvedRouterID != "" { + return router, ctrl.Result{}, nil + } + + logger.Info("Creating router", "name", routerSpec.Name) + createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) + router, err = clusterScope.CloudscaleClient.Routers.Create(createCtx, &cloudscalesdk.RouterCreateRequest{ + Name: routerSpec.Name, + InternetGateway: routerSpec.InternetGateway, + Zone: clusterScope.CloudscaleCluster.Spec.Zone, + Tags: new(tags), + }) + cancel() + if err != nil { + if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Router creation"); timedOut { + return nil, result, nil + } + return nil, ctrl.Result{}, fmt.Errorf("creating router: %w", err) + } + logger.Info("Created router", "name", routerSpec.Name, "routerID", router.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterCreated", "CreateRouter", + "Created router %s (%s) in zone %s", routerSpec.Name, router.UUID, router.Zone.Slug) + + return router, ctrl.Result{}, nil +} + +// requeueOnTimeout turns a timed-out write into a requeue instead of an error: the call +// may well have landed, so the next reconcile re-reads rather than retrying blindly. +func (r *CloudscaleClusterReconciler) requeueOnTimeout(clusterScope *scope.ClusterScope, err error, what string) (ctrl.Result, bool) { + if !cloudscale.IsTimeoutError(err) { + return ctrl.Result{}, false + } + clusterScope.Info(what+" timed out, waiting before retry", "requeueAfter", routerRequeueAfter) + return ctrl.Result{RequeueAfter: routerRequeueAfter}, true +} + +// reconcileRouterInterfaces attaches the router to every network in its spec it is not +// already attached to, and points those subnets' gateways at the interface where the spec +// asks for it. +func (r *CloudscaleClusterReconciler) reconcileRouterInterfaces( + ctx context.Context, + clusterScope *scope.ClusterScope, + routerSpec infrastructurev1beta2.RouterSpec, + router *cloudscalesdk.Router, +) (ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouterInterfaces") + defer done() + + // A router attaches to any given network at most once, so indexing by network is + // unambiguous. The index by UUID is what an adopted interface is looked up through. + liveByNetwork := make(map[string]cloudscalesdk.RouterInterface, len(router.Interfaces)) + liveByUUID := make(map[string]cloudscalesdk.RouterInterface, len(router.Interfaces)) + for _, iface := range router.Interfaces { + liveByUUID[iface.UUID] = iface + liveByNetwork[iface.Network.UUID] = iface + } + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + if rs == nil { + return ctrl.Result{}, fmt.Errorf("router %q not found in status", routerSpec.Name) + } + + for _, ifaceSpec := range routerSpec.Interfaces { + logger := logger.WithValues("name", routerSpec.Name, "routerID", router.UUID, "network", ifaceSpec.Network, "specAddress", ifaceSpec.Address) + + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) + if ns == nil { + return ctrl.Result{}, fmt.Errorf("network %q not found in status for router %q interface", ifaceSpec.Network, routerSpec.Name) + } + + // We always attach at the address the spec asks for, unless the router turns out + // to already be attached to this network, in which case the interface keeps the + // address it has: an attachment cannot be moved. + address := ifaceSpec.Address + live, attached := liveByNetwork[ns.NetworkID] + + switch { + case ifaceSpec.UUID != "": + adopted, err := r.adoptRouterInterface(clusterScope, logger, routerSpec, ifaceSpec, ns, liveByUUID) + if err != nil { + return ctrl.Result{}, err + } + address = adopted + case attached: + claimed, err := r.claimRouterInterface(clusterScope, logger, routerSpec, rs, ifaceSpec, ns, live) + if err != nil { + return ctrl.Result{}, err + } + address = claimed + default: + result, err := r.createRouterInterface(ctx, clusterScope, logger, routerSpec, router, ifaceSpec, ns) + if err != nil || !result.IsZero() { + return result, err + } + } + + if result, err := r.reconcileSubnetGateway(ctx, clusterScope, logger, routerSpec, ifaceSpec, ns, address); err != nil || !result.IsZero() { + return result, err + } + } + + return ctrl.Result{}, nil +} + +// adoptRouterInterface records the pre-existing interface the spec entry references by +// UUID, and returns the address it holds. +func (r *CloudscaleClusterReconciler) adoptRouterInterface( + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, + liveByUUID map[string]cloudscalesdk.RouterInterface, +) (string, error) { + live, exists := liveByUUID[ifaceSpec.UUID] + if !exists { + return "", fmt.Errorf("router %q has no interface %s to adopt, it may have been detached outside of CAPCS", routerSpec.Name, ifaceSpec.UUID) + } + if live.Network.UUID != ns.NetworkID { + return "", fmt.Errorf("router %q interface %s is attached to network %s, not to network %q (%s)", + routerSpec.Name, ifaceSpec.UUID, live.Network.UUID, ifaceSpec.Network, ns.NetworkID) + } + + address := interfaceAddress(live, ns.SubnetID) + if changed := r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, live.UUID, false); changed { + logger.Info("Adopted pre-existing router interface", "interfaceID", live.UUID, "address", address) + } + return address, nil +} + +// claimRouterInterface records an already existing interface in the clusters' status. +func (r *CloudscaleClusterReconciler) claimRouterInterface( + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + rs *infrastructurev1beta2.RouterStatus, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, + live cloudscalesdk.RouterInterface, +) (string, error) { + // if the interface is already in the API, but not yet in our status and the router is not managed, this means + // that we adopted a router with a pre-existing interface, but that interface was not mentioned in the list with a UUID. + if prev := rs.GetInterfaceStatus(ns.Name); prev == nil && !rs.Managed { + return "", fmt.Errorf("pre-existing router %q is already attached to network %q by interface %s: set spec.routers[].interfaces[].uuid to adopt that interface, or detach it", + routerSpec.Name, ifaceSpec.Network, live.UUID) + } + + address := interfaceAddress(live, ns.SubnetID) + + changed := r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, live.UUID, true) + if changed && address != ifaceSpec.Address { + logger.Info("Router is already attached to network at a different address", "address", address) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeWarning, "RouterInterfaceAddressMismatch", "ReconcileRouterInterface", + "Router %s is attached to network %s at %q, not at the requested %q, which therefore has no effect", + routerSpec.Name, ifaceSpec.Network, address, ifaceSpec.Address) + } + return address, nil +} + +// createRouterInterface attaches the router to the network at the address the spec asks +// for. +func (r *CloudscaleClusterReconciler) createRouterInterface( + ctx context.Context, + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + router *cloudscalesdk.Router, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, +) (ctrl.Result, error) { + // Record the attachment before it exists. The response can be lost to a timeout while + // the interface gets created anyway, and this entry is what tells the next reconcile the + // interface is one we created. + r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, "", true) + + logger.Info("Creating interface for router") + createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) + createdIface, err := clusterScope.CloudscaleClient.Routers.CreateInterface(createCtx, router.UUID, cloudscalesdk.CreateInterfaceRequest{ + Network: ns.NetworkID, + Addresses: []cloudscalesdk.CreateAddressRequest{ + { + Subnet: ns.SubnetID, + Address: ifaceSpec.Address, + }, + }, + }) + cancel() + if err != nil { + if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Router interface creation"); timedOut { + return result, nil + } + return ctrl.Result{}, fmt.Errorf("creating interface for router %q network %q: %w", routerSpec.Name, ifaceSpec.Network, err) + } + + r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, createdIface.UUID, true) + logger.Info("Created interface for router", "interfaceID", createdIface.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterInterfaceCreated", "CreateRouterInterface", + "Attached router %s to network %s with address %s", routerSpec.Name, ifaceSpec.Network, ifaceSpec.Address) + return ctrl.Result{}, nil +} + +// reconcileSubnetGateway points the subnet's gateway at the address the router interface +// holds, making the router the default route for servers on that subnet. +func (r *CloudscaleClusterReconciler) reconcileSubnetGateway( + ctx context.Context, + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, + address string, +) (ctrl.Result, error) { + // no configuration of the subnet gateway requested + if !ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { + return ctrl.Result{}, nil + } + // no address set + if address == "" { + // The API always returns the interface's address, so this cannot normally + // happen. Guard anyway: writing the empty address would strip the subnet's + // default route from every server on it. + logger.Info("Router interface holds no address on the tracked subnet, leaving the subnet gateway untouched", "subnetID", ns.SubnetID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeWarning, "RouterInterfaceAddressUnknown", "ReconcileRouterInterface", + "Router %s is attached to network %s but holds no address on its subnet; the subnet gateway was left unchanged", + routerSpec.Name, ifaceSpec.Network) + return ctrl.Result{}, nil + } + // gateway address already correct + if ns.GatewayAddress == address { + return ctrl.Result{}, nil + } + + logger.Info("Configuring subnet gateway", "address", address) + updateCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) + err := clusterScope.CloudscaleClient.Subnets.Update(updateCtx, ns.SubnetID, &cloudscalesdk.SubnetUpdateRequest{ + GatewayAddress: address, + }) + cancel() + if err != nil { + if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Subnet gateway update"); timedOut { + return result, nil + } + return ctrl.Result{}, fmt.Errorf("updating subnet gateway for router %q network %q: %w", routerSpec.Name, ifaceSpec.Network, err) + } + r.setNetworkGatewayAddress(clusterScope, ifaceSpec.Network, address) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "SubnetGatewayConfigured", "ConfigureSubnetGateway", + "Configured subnet %s gateway to router interface address %s", ifaceSpec.Network, address) + return ctrl.Result{}, nil +} + +// interfaceAddress returns the address the interface holds on the given subnet, or "" when +// it holds none: a network can carry more than one subnet, and the router may be attached +// on a different one than CAPCS tracks. +func interfaceAddress(iface cloudscalesdk.RouterInterface, subnetID string) string { + for _, addr := range iface.Addresses { + if addr.Subnet.UUID == subnetID { + return addr.Address + } + } + return "" +} + +// deleteRouters removes the interfaces we created, then the routers it created. +// Pre-existing routers are left in place: only their managed interfaces go. +// On partial failure the affected routers stay in status so the next attempt retries +// exactly those. +func (r *CloudscaleClusterReconciler) deleteRouters(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.deleteRouters") + defer done() + + defer func() { + if reterr != nil { + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterErrorReason, fmt.Sprintf("Failed to delete routers: %v", reterr)) + } else { + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterDeletingReason, "Routers have been deleted") + } + }() + + var remaining []infrastructurev1beta2.RouterStatus + var errs []error + + for ri := range clusterScope.CloudscaleCluster.Status.Routers { + router := &clusterScope.CloudscaleCluster.Status.Routers[ri] + logger := logger.WithValues("router", router.Name, "routerID", router.RouterID) + + // Never provisioned, so there is nothing to clean up. + if router.RouterID == "" { + continue + } + + // Routers can only be deleted if it's private interfaces are deleted first. + if ifaceErrs := r.deleteRouterInterfaces(ctx, clusterScope, router); len(ifaceErrs) > 0 { + errs = append(errs, ifaceErrs...) + remaining = append(remaining, *router) + continue + } + + if !router.Managed { + logger.Info("Skipping pre-existing router deletion") + remaining = append(remaining, *router) + continue + } + + logger.Info("Deleting router") + deleteCtx, cancel := context.WithTimeout(ctx, cloudscale.DeleteTimeout) + err := clusterScope.CloudscaleClient.Routers.Delete(deleteCtx, router.RouterID) + cancel() + + isNotFound := cloudscale.IsNotFound(err) + + if err != nil && !isNotFound { + remaining = append(remaining, *router) + errs = append(errs, fmt.Errorf("deleting router %q: %w", router.Name, err)) + continue + } + + // Idempotent not-found: log and fall through to the event. + if isNotFound { + logger.Info("Router already deleted") + } + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterDeleted", "DeleteRouter", + "Deleted router %s (%s)", router.Name, router.RouterID) + } + clusterScope.CloudscaleCluster.Status.Routers = remaining + return errors.Join(errs...) +} + +// deleteRouterInterfaces deletes the interfaces CAPCS created on a router. +// +// The router is read once so the recorded interfaces can be reconciled against the live +// ones. +func (r *CloudscaleClusterReconciler) deleteRouterInterfaces( + ctx context.Context, + clusterScope *scope.ClusterScope, + rs *infrastructurev1beta2.RouterStatus, +) (errs []error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.deleteRouterInterfaces") + defer done() + + logger = logger.WithValues("router", rs.Name, "routerID", rs.RouterID) + + if !hasManagedInterface(rs) { + // Nothing of ours to detach, so the router does not have to be read at all. + return nil + } + + getCtx, cancel := context.WithTimeout(ctx, cloudscale.ReadTimeout) + router, err := clusterScope.CloudscaleClient.Routers.Get(getCtx, rs.RouterID) + cancel() + if cloudscale.IsNotFound(err) { + // The router is gone, and its interfaces went with it. + logger.Info("Router already deleted, no interfaces left to detach") + rs.Interfaces = nil + return nil + } + if err != nil { + return []error{fmt.Errorf("getting router %q to detach its interfaces: %w", rs.Name, err)} + } + + liveUUIDs := make(map[string]struct{}, len(router.Interfaces)) + liveByNetwork := make(map[string]string, len(router.Interfaces)) + for _, iface := range router.Interfaces { + liveUUIDs[iface.UUID] = struct{}{} + liveByNetwork[iface.Network.UUID] = iface.UUID + } + + var remaining []infrastructurev1beta2.RouterInterfaceStatus + for _, iface := range rs.Interfaces { + if !iface.Managed { + remaining = append(remaining, iface) + continue + } + + interfaceID := iface.InterfaceID + if interfaceID == "" { + // An attach whose response never arrived: the network is all we recorded. + if ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(iface.Network); ns != nil { + interfaceID = liveByNetwork[ns.NetworkID] + } + } + if _, live := liveUUIDs[interfaceID]; !live { + // Never attached, or already detached: either way there is nothing to do. + logger.Info("Router interface already detached", "network", iface.Network, "interfaceID", iface.InterfaceID) + continue + } + + logger.Info("Deleting router interface", "network", iface.Network, "interfaceID", interfaceID) + + delCtx, cancel := context.WithTimeout(ctx, cloudscale.DeleteTimeout) + err := clusterScope.CloudscaleClient.Routers.DeleteInterface(delCtx, rs.RouterID, interfaceID) + cancel() + if err != nil && !cloudscale.IsNotFound(err) { + errs = append(errs, fmt.Errorf("deleting router interface for %s/%s: %w", rs.Name, iface.Network, err)) + // Keep the resolved UUID, so a retry does not have to resolve it again. + iface.InterfaceID = interfaceID + remaining = append(remaining, iface) + continue + } + logger.Info("Deleted router interface", "network", iface.Network, "interfaceID", interfaceID) + } + rs.Interfaces = remaining + return errs +} + +// hasManagedInterface reports whether the router has any interface CAPCS has to detach. +func hasManagedInterface(rs *infrastructurev1beta2.RouterStatus) bool { + for _, iface := range rs.Interfaces { + if iface.Managed { + return true + } + } + return false +} + +// routerTags returns the ownership tags for a specific named router. +func (r *CloudscaleClusterReconciler) routerTags(clusterScope *scope.ClusterScope, routerName string) cloudscalesdk.TagMap { + return cloudscalesdk.TagMap{ + infrastructurev1beta2.NameCloudscaleProviderOwned + clusterScope.Cluster.Name: routerName, + } +} + +// setRouterStatus updates or appends the router status entry for the given spec entry. +// .Interfaces is only written by setRouterInterfaceStatus and pruned by deleteRouterInterfaces. +func (r *CloudscaleClusterReconciler) setRouterStatus(clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec, routerID string) { + managed := routerSpec.UUID == "" + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name); rs != nil { + rs.RouterID = routerID + rs.Managed = managed + return + } + clusterScope.CloudscaleCluster.Status.Routers = append(clusterScope.CloudscaleCluster.Status.Routers, infrastructurev1beta2.RouterStatus{ + Name: routerSpec.Name, + RouterID: routerID, + Managed: managed, + }) +} + +// setRouterInterfaceStatus upserts an interface entry on the named router's status. +func (r *CloudscaleClusterReconciler) setRouterInterfaceStatus(clusterScope *scope.ClusterScope, routerName, networkName, interfaceID string, managed bool) bool { + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerName) + if rs == nil { + return false + } + if iface := rs.GetInterfaceStatus(networkName); iface != nil { + if iface.InterfaceID == interfaceID && iface.Managed == managed { + return false + } + iface.InterfaceID = interfaceID + iface.Managed = managed + return true + } + rs.Interfaces = append(rs.Interfaces, infrastructurev1beta2.RouterInterfaceStatus{ + Network: networkName, + InterfaceID: interfaceID, + Managed: managed, + }) + return true +} diff --git a/internal/controller/cloudscalecluster_router_test.go b/internal/controller/cloudscalecluster_router_test.go new file mode 100644 index 0000000..0f0a04b --- /dev/null +++ b/internal/controller/cloudscalecluster_router_test.go @@ -0,0 +1,1099 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "net/url" + "os" + "testing" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/cluster-api/util/conditions" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/testutils" +) + +const ( + routerUUID = "router-uuid-123" + ifaceUUID = "iface-uuid-123" + routerAddress = "10.0.0.1" + subnetUUID = "subnet-uuid-123" +) + +// activeRouter is what the API returns for a provisioned router. +func activeRouter(uuid string, interfaces ...cloudscalesdk.RouterInterface) *cloudscalesdk.Router { + return &cloudscalesdk.Router{ + UUID: uuid, + Status: cloudscalesdk.RouterActive, + Interfaces: interfaces, + Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, + } +} + +// routerInterface is what the API returns for an interface attached to a network, +// holding one address on that network's subnet. +func routerInterface(uuid, address string) cloudscalesdk.RouterInterface { + return cloudscalesdk.RouterInterface{ + UUID: uuid, + Network: cloudscalesdk.NetworkStub{Name: "test", UUID: netUUID}, + Addresses: []cloudscalesdk.IPAddress{ + {Address: address, Subnet: cloudscalesdk.SubnetStub{UUID: subnetUUID}}, + }, + } +} + +// routerSpec is a managed router attached to the default "test" network. +func routerSpec() infrastructurev1beta2.RouterSpec { + return infrastructurev1beta2.RouterSpec{ + Name: "test-router", + InternetGateway: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "test", Address: routerAddress, ConfigureSubnetGateway: new(true)}, + }, + } +} + +// managedInterfaces is how a status for a single interface CAPCS created on the +// default "test" network looks like. +func managedInterfaces() []infrastructurev1beta2.RouterInterfaceStatus { + return []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: ifaceUUID, Managed: true}, + } +} + +// recordedEvents drains everything the reconciler has emitted so far. +func recordedEvents(r *CloudscaleClusterReconciler) []string { + var out []string + for { + select { + case e := <-r.recorder.(*events.FakeRecorder).Events: + out = append(out, e) + default: + return out + } + } +} + +// --- Managed router provisioning --- + +func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { + g := NewWithT(t) + + var interfaceRouterID string + var capturedCreateReq *cloudscalesdk.RouterCreateRequest + var capturedIfaceReq cloudscalesdk.CreateInterfaceRequest + var capturedSubnetUpdateReq *cloudscalesdk.SubnetUpdateRequest + routerService := &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + capturedCreateReq = req + return activeRouter(routerUUID), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + interfaceRouterID = uuid + capturedIfaceReq = req + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + capturedSubnetUpdateReq = req + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(routerSpec()), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(capturedCreateReq).ToNot(BeNil()) + g.Expect(capturedCreateReq.Name).To(Equal("test-router")) + g.Expect(capturedCreateReq.InternetGateway).To(BeTrue()) + g.Expect(capturedCreateReq.Zone).To(Equal(clusterScope.CloudscaleCluster.Spec.Zone)) + g.Expect(capturedCreateReq.Tags).To(Equal(new(cloudscalesdk.TagMap{ + infrastructurev1beta2.NameCloudscaleProviderOwned + "test-cluster": "test-router", + }))) + + g.Expect(interfaceRouterID).To(Equal(routerUUID)) + g.Expect(capturedIfaceReq.Network).To(Equal(netUUID)) + g.Expect(capturedIfaceReq.Addresses).To(HaveLen(1)) + g.Expect(capturedIfaceReq.Addresses[0].Subnet).To(Equal(subnetUUID)) + g.Expect(capturedIfaceReq.Addresses[0].Address).To(Equal("10.0.0.1")) + + g.Expect(capturedSubnetUpdateReq).ToNot(BeNil()) + g.Expect(capturedSubnetUpdateReq.GatewayAddress).To(Equal(capturedIfaceReq.Addresses[0].Address)) + g.Expect(capturedSubnetUpdateReq.DNSServers).To(BeNil()) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.RouterID).To(Equal(routerUUID)) + g.Expect(rs.Managed).To(BeTrue()) + // Keyed by the CAPCS network name, holding the interface UUID (not the network's). + g.Expect(rs.Interfaces).To(Equal(managedInterfaces())) + + // The exact, ordered events for the whole managed path. Order is what pins the + // create-then-configure sequencing, which presence checks alone would not catch. + g.Expect(recordedEvents(r)).To(Equal([]string{ + "Normal RouterCreated Created router test-router (router-uuid-123) in zone rma1", + "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1", + "Normal SubnetGatewayConfigured Configured subnet test gateway to router interface address 10.0.0.1", + })) +} + +func TestReconcileRouters_ConfiguresSubnetGateway(t *testing.T) { + g := NewWithT(t) + + var updatedSubnetID string + var capturedUpdate *cloudscalesdk.SubnetUpdateRequest + updateCalls := 0 + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + } + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + updateCalls++ + updatedSubnetID = id + capturedUpdate = req + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(routerSpec()), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(updateCalls).To(Equal(1)) + g.Expect(updatedSubnetID).To(Equal(subnetUUID)) + g.Expect(capturedUpdate.GatewayAddress).To(Equal("10.0.0.1")) + // Recorded in status so the next reconcile skips the update. + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns.GatewayAddress).To(Equal("10.0.0.1")) + + _, err = r.reconcileRouters(context.Background(), clusterScope) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(updateCalls).To(Equal(1)) +} + +func TestReconcileRouters_SkipsSubnetGatewayWhenNotOwner(t *testing.T) { + g := NewWithT(t) + + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + } + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + g.Fail("Subnets.Update should not be called when configureSubnetGateway is false") + return nil + }, + } + + spec := routerSpec() + spec.Interfaces[0].ConfigureSubnetGateway = new(false) + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) +} + +// --- Pre-existing (adopted) routers --- + +// TestReconcileRouters_AdoptedInterfaceOwnership verifies the reconciler does correctly +// detect interface ownership. +func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { + const preAttachedUUID = "pre-attached-iface-uuid" + + // router in a network CAPCS knows nothing about. + foreign := cloudscalesdk.RouterInterface{ + UUID: "foreign-iface-uuid", + Network: cloudscalesdk.NetworkStub{Name: "someone-elses-network", UUID: "foreign-net-uuid"}, + } + interfaceStatus := func(uuid string, managed bool) []infrastructurev1beta2.RouterInterfaceStatus { + return []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: uuid, Managed: managed}, + } + } + + const ( + mismatchEvent = `Warning RouterInterfaceAddressMismatch Router test-router is attached to network test at "10.0.0.42", not at the requested "10.0.0.1", which therefore has no effect` + ifaceCreatedEvent = "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1" + ) + gatewaySetEvent := func(address string) string { + return "Normal SubnetGatewayConfigured Configured subnet test gateway to router interface address " + address + } + + tests := []struct { + name string + // live is what the API reports the router is attached to. + live []cloudscalesdk.RouterInterface + // adopt is the interface uuid the spec entry declares, empty when the interface should be created + adopt string + // status is the interface status + status []infrastructurev1beta2.RouterInterfaceStatus + // networkGateway is the gateway address. + networkGateway string + + // wantErr is a substring of the error the reconcile has to fail with. The + // reconcile is run twice either way: a collision must stay a collision. + wantErr string + wantInterfaces []infrastructurev1beta2.RouterInterfaceStatus + wantCreate bool + wantGatewaySet string + // wantEvents is the exact, ordered set of events the reconcile must emit. + wantEvents []string + }{ + { + // Nothing on a spec network, so CAPCS attaches its own and owns it. The + // router's foreign interface is not recorded at all. + name: "unattached to the spec network", + live: []cloudscalesdk.RouterInterface{foreign}, + wantInterfaces: managedInterfaces(), + wantCreate: true, + wantGatewaySet: routerAddress, + wantEvents: []string{ifaceCreatedEvent, gatewaySetEvent(routerAddress)}, + }, + { + name: "pre-existing attachment at the spec address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, routerAddress)}, + networkGateway: routerAddress, + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + }, + { + name: "pre-existing attachment at another address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, + networkGateway: "10.0.0.42", + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + }, + { + name: "adopted by uuid is used and left alone", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, + adopt: preAttachedUUID, + wantInterfaces: interfaceStatus(preAttachedUUID, false), + wantGatewaySet: "10.0.0.42", + wantEvents: []string{gatewaySetEvent("10.0.0.42")}, + }, + { + name: "adopted uuid the router does not carry", + live: []cloudscalesdk.RouterInterface{foreign}, + adopt: preAttachedUUID, + wantErr: "has no interface " + preAttachedUUID + " to adopt", + }, + { + name: "adopted uuid attached to another network", + live: []cloudscalesdk.RouterInterface{foreign}, + adopt: foreign.UUID, + wantErr: "is attached to network foreign-net-uuid, not to network \"test\"", + }, + { + // An attach whose response was lost to a timeout: the entry written before + // the call is what tells CAPCS this interface is its own, and it is filled + // in as soon as the interface is seen. + name: "attachment recorded before its response arrived", + live: []cloudscalesdk.RouterInterface{routerInterface(ifaceUUID, routerAddress)}, + status: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + networkGateway: routerAddress, + wantInterfaces: managedInterfaces(), + }, + { + name: "replaced interface stays managed", + live: []cloudscalesdk.RouterInterface{routerInterface("replacement-iface-uuid", "10.0.0.42")}, + status: managedInterfaces(), + networkGateway: "10.0.0.42", + wantInterfaces: interfaceStatus("replacement-iface-uuid", true), + wantEvents: []string{mismatchEvent}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + created := false + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, tc.live...), nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + g.Fail("Create must not be called for a pre-existing router") + return nil, nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Expect(tc.wantCreate).To(BeTrue(), "CreateInterface called but the router is already attached") + g.Expect(uuid).To(Equal(routerUUID)) + created = true + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + + var gatewaySet string + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + g.Expect(tc.wantGatewaySet).ToNot(BeEmpty(), + "Subnets.Update must not be called; the gateway is already correct or the address is unknown") + g.Expect(id).To(Equal(subnetUUID)) + gatewaySet = req.GatewayAddress + return nil + }, + } + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + if tc.adopt != "" { + // The webhook forbids requesting an address for an adopted interface. + spec.Interfaces[0].UUID = tc.adopt + spec.Interfaces[0].Address = "" + } + + opts := []testutils.ClusterScopeOption{ + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithAdoptedNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + } + if tc.status != nil { + opts = append(opts, testutils.WithRouterStatus("test-router", routerUUID, false, tc.status)) + } + clusterScope := testutils.NewClusterScopeOpts(opts...) + clusterScope.CloudscaleCluster.Status.Networks[0].GatewayAddress = tc.networkGateway + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + if tc.wantErr != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.wantErr))) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(created).To(Equal(tc.wantCreate)) + g.Expect(gatewaySet).To(Equal(tc.wantGatewaySet)) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.Managed).To(BeFalse(), "an adopted router is never CAPCS-managed") + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + + // The recorded gateway tracks whatever was actually written, so the next + // reconcile knows whether there is anything left to do. + wantRecorded := tc.networkGateway + if tc.wantGatewaySet != "" { + wantRecorded = tc.wantGatewaySet + } + g.Expect(clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test").GatewayAddress).To(Equal(wantRecorded)) + + g.Expect(recordedEvents(r)).To(Equal(tc.wantEvents)) + + // Reconciling again must not revisit any of it, and must not talk itself + // out of a collision either. + _, err = r.reconcileRouters(context.Background(), clusterScope) + if tc.wantErr != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.wantErr))) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + }) + } +} + +func TestReconcileRouters_PreExistingZoneMismatchErrors(t *testing.T) { + g := NewWithT(t) + + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return &cloudscalesdk.Router{ + UUID: id, + Status: cloudscalesdk.RouterActive, + Zone: cloudscalesdk.ZoneStub{Slug: "lpg1"}, + }, nil + }, + } + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("expected zone")) + + // A hard error must be visible on the condition, not just returned. + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterErrorReason)) + g.Expect(cond.Message).To(ContainSubstring("expected zone")) +} + +// TestReconcileRouters_AttachesMultipleInterfaces tests multiple networks with interfaces. +func TestReconcileRouters_AttachesMultipleInterfaces(t *testing.T) { + g := NewWithT(t) + + const ( + transitNetUUID = "transit-net-uuid" + transitSubnetUUID = "transit-subnet-uuid" + ) + + attachedTo := map[string]cloudscalesdk.CreateAddressRequest{} + routerService := &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + return activeRouter(routerUUID), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Expect(req.Addresses).To(HaveLen(1)) + attachedTo[req.Network] = req.Addresses[0] + return &cloudscalesdk.RouterInterface{UUID: "iface-" + req.Network}, nil + }, + } + + gatewayUpdates := map[string]string{} + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + gatewayUpdates[id] = req.GatewayAddress + return nil + }, + } + + spec := routerSpec() + spec.Interfaces = append(spec.Interfaces, infrastructurev1beta2.RouterInterfaceSpec{ + Network: "transit", Address: "10.9.0.2", ConfigureSubnetGateway: new(false), + }) + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithPreExistingNetwork("transit", transitNetUUID, transitSubnetUUID, "10.9.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(attachedTo).To(Equal(map[string]cloudscalesdk.CreateAddressRequest{ + netUUID: {Subnet: subnetUUID, Address: routerAddress}, + transitNetUUID: {Subnet: transitSubnetUUID, Address: "10.9.0.2"}, + })) + // Only the gateway-owning interface rewrites a subnet; the transit subnet keeps + // whatever gateway its own router set. + g.Expect(gatewayUpdates).To(Equal(map[string]string{subnetUUID: routerAddress})) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.Interfaces).To(Equal([]infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: "iface-" + netUUID, Managed: true}, + {Network: "transit", InterfaceID: "iface-" + transitNetUUID, Managed: true}, + })) +} + +// TestReconcileRouters_UnprovisionedNetworkErrors guards the ordering contract: routers are +// reconciled after networks, so an interface pointing at a network that has no status yet +// means something upstream went wrong and must not be papered over. +func TestReconcileRouters_UnprovisionedNetworkErrors(t *testing.T) { + g := NewWithT(t) + + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Fail("CreateInterface must not be called for a network that is not provisioned") + return nil, nil + }, + } + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + spec.Interfaces[0].Network = "not-provisioned-yet" + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`network "not-provisioned-yet" not found in status`)) +} + +// --- Condition reporting --- + +func TestReconcileRouters_ConditionReflectsProgress(t *testing.T) { + tests := []struct { + name string + setup func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) + wantRequeue bool + wantStatus metav1.ConditionStatus + wantReason string + wantAttached bool + }{ + { + name: "no routers is ready but disabled", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + return &testutils.MockRouterService{}, nil + }, + wantStatus: metav1.ConditionTrue, + wantReason: infrastructurev1beta2.RouterDisabledReason, + }, + { + name: "router not active yet requeues without claiming ready", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + svc := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return &cloudscalesdk.Router{ + UUID: id, + Status: "unknown", + Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, + }, nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return nil, errors.New("CreateInterface must not be called while the router is unknown") + }, + } + return svc, []testutils.ClusterScopeOption{ + testutils.WithRouter(routerSpec()), + testutils.WithRouterStatus("test-router", routerUUID, true, nil), + } + }, + wantRequeue: true, + wantStatus: metav1.ConditionFalse, + wantReason: infrastructurev1beta2.RouterNotReadyReason, + }, + { + name: "provisioned router is ready", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + svc := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + } + return svc, []testutils.ClusterScopeOption{ + testutils.WithRouter(routerSpec()), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + } + }, + wantStatus: metav1.ConditionTrue, + wantReason: infrastructurev1beta2.RouterProvisionedReason, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + routerService, extraOpts := tc.setup() + opts := append([]testutils.ClusterScopeOption{ + testutils.WithRouterService(routerService), + testutils.WithSubnetService(&testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { return nil }, + }), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + }, extraOpts...) + clusterScope := testutils.NewClusterScopeOpts(opts...) + // The gateway is already in place, so these cases only exercise the condition. + clusterScope.CloudscaleCluster.Status.Networks[0].GatewayAddress = routerAddress + r := newTestReconciler() + + result, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + if tc.wantRequeue { + g.Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + } else { + g.Expect(result.RequeueAfter).To(BeZero()) + } + + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(tc.wantStatus)) + g.Expect(cond.Reason).To(Equal(tc.wantReason)) + }) + } +} + +// TestReconcileRouters_TimeoutsRequeue verifies all timeouts requeue to recover the state. +func TestReconcileRouters_TimeoutsRequeue(t *testing.T) { + timeout := func(op, path string) error { + return &url.Error{Op: op, URL: "https://api.example.com/v1/" + path, Err: os.ErrDeadlineExceeded} + } + + tests := []struct { + name string + routerService func() *testutils.MockRouterService + subnetService func() *testutils.MockSubnetService + seedStatus bool + setupClusterScope func(*scope.ClusterScope) + // wantInterfaces is the interface status the timed-out write must leave behind. + wantInterfaces []infrastructurev1beta2.RouterInterfaceStatus + wantProvisioned bool + }{ + { + name: "router create", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + return nil, timeout("Post", "routers") + }, + } + }, + wantProvisioned: false, + }, + { + name: "interface create", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return nil, timeout("Post", "routers/"+uuid+"/interfaces") + }, + } + }, + seedStatus: true, + // The attach may have landed even though its response did not, so the + // entry recorded before the call has to survive the requeue. Without it + // the next reconcile cannot tell the interface from one the router + // already carried. + wantInterfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + wantProvisioned: false, + }, + { + name: "subnet gateway update", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + } + }, + subnetService: func() *testutils.MockSubnetService { + return &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + return timeout("Patch", "subnets/"+id) + }, + } + }, + seedStatus: true, + // The attach itself succeeded, so its interface is recorded in full. + wantInterfaces: managedInterfaces(), + wantProvisioned: false, + }, + { + name: "interface create succeeds", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + }, + seedStatus: true, + setupClusterScope: testutils.WithControlPlaneEndpoint, + wantInterfaces: managedInterfaces(), + wantProvisioned: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { return nil }, + } + if tc.subnetService != nil { + subnetService = tc.subnetService() + } + + opts := []testutils.ClusterScopeOption{ + testutils.WithRouterService(tc.routerService()), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(routerSpec()), + } + if tc.seedStatus { + opts = append(opts, testutils.WithRouterStatus("test-router", routerUUID, true, nil)) + } + clusterScope := testutils.NewClusterScopeOpts(opts...) + + if tc.setupClusterScope != nil { + tc.setupClusterScope(clusterScope) + } + + r := newTestReconciler() + + result, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + if tc.wantProvisioned { + g.Expect(result.IsZero()).To(BeTrue()) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterProvisionedReason)) + } else { + g.Expect(result.RequeueAfter).To(Equal(routerRequeueAfter)) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterNotReadyReason)) + } + + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router"); rs != nil { + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + } + g.Expect(r.isInfrastructureProvisioned(clusterScope)).To(Equal(tc.wantProvisioned), "expected IsInfrastructureProvisioned to be %v", tc.wantProvisioned) + }) + } +} + +// --- Deletion --- + +func TestDeleteRouters_DeletesManagedRouterAndInterfaces(t *testing.T) { + g := NewWithT(t) + + var deletedRouterID string + deletedInterfaces := map[string]string{} + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + deletedRouterID = id + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + deletedInterfaces[routerUUID] = interfaceUUID + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(deletedInterfaces).To(Equal(map[string]string{routerUUID: ifaceUUID})) + g.Expect(deletedRouterID).To(Equal(routerUUID)) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(BeEmpty()) +} + +// TestDeleteRouters_PreservesForeignInterfacesOnAdoptedRouter guards the case where a +// user points CAPCS at a shared router: deleting the cluster must remove the interfaces +// CAPCS attached and nothing else, and must leave the router itself alone. +func TestDeleteRouters_PreservesForeignInterfacesOnAdoptedRouter(t *testing.T) { + g := NewWithT(t) + + preAttached := infrastructurev1beta2.RouterInterfaceStatus{ + Network: "shared", InterfaceID: "pre-attached-iface-uuid", Managed: false, + } + + var deletedInterfaces []string + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, + routerInterface(ifaceUUID, routerAddress), + cloudscalesdk.RouterInterface{ + UUID: preAttached.InterfaceID, + Network: cloudscalesdk.NetworkStub{Name: "shared", UUID: "shared-net-uuid"}, + }, + ), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Delete should not be called for a pre-existing router") + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + deletedInterfaces = append(deletedInterfaces, interfaceUUID) + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + // "shared" is the user's network, so CAPCS never deletes it. + // "test" is one CAPCS manages. + testutils.WithAdoptedNetwork("shared", "shared-net-uuid", "shared-subnet-uuid", "10.1.0.0/24"), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("shared-router", routerUUID, false, + append(managedInterfaces(), preAttached)), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + // Only the CAPCS-created interface is detached + g.Expect(deletedInterfaces).To(Equal([]string{ifaceUUID})) + // The adopted router stays in status, keeping the interface it came with. + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(HaveLen(1)) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Name).To(Equal("shared-router")) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Managed).To(BeFalse()) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Interfaces).To(Equal( + []infrastructurev1beta2.RouterInterfaceStatus{preAttached})) +} + +// TestDeleteRouters_ResolvesInterfacesAgainstLiveRouter ensures interfaces are deleted based on the current +// router's API response. +func TestDeleteRouters_ResolvesInterfacesAgainstLiveRouter(t *testing.T) { + tests := []struct { + name string + // live is what the API reports the router is attached to, nil for a router that + // no longer exists. + live []cloudscalesdk.RouterInterface + routerGone bool + interfaces []infrastructurev1beta2.RouterInterfaceStatus + + wantDetached []string + }{ + { + // Recorded before the attach existed, so the network is all there is to go on. + name: "attachment with no recorded uuid is resolved by network", + live: []cloudscalesdk.RouterInterface{routerInterface(ifaceUUID, routerAddress)}, + interfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + wantDetached: []string{ifaceUUID}, + }, + { + // The attach never landed after all: there is nothing to detach, and the + // entry must not hold the router's deletion back. + name: "attachment that never landed is dropped", + live: nil, + interfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + }, + { + name: "recorded interface that is already detached is dropped", + live: nil, + interfaces: managedInterfaces(), + }, + { + name: "router already gone takes its interfaces with it", + routerGone: true, + interfaces: managedInterfaces(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + var detached []string + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + if tc.routerGone { + return nil, &cloudscalesdk.ErrorResponse{StatusCode: 404} + } + return activeRouter(id, tc.live...), nil + }, + DeleteFn: func(ctx context.Context, id string) error { return nil }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + detached = append(detached, interfaceUUID) + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("test-router", routerUUID, true, tc.interfaces), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(detached).To(Equal(tc.wantDetached)) + // Whatever happened to the interfaces, the router itself is gone from status. + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(BeEmpty()) + }) + } +} + +func TestDeleteRouters_BasicScenarios(t *testing.T) { + tests := []struct { + name string + routerService func(g *WithT) *testutils.MockRouterService + statusSetup func() []infrastructurev1beta2.RouterStatus + wantErr bool + wantStatusLen int + }{ + { + name: "never provisioned router is dropped without any API call", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Delete should not be called for a router that was never created") + return nil + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", Managed: true}} + }, + wantStatusLen: 0, + }, + { + name: "already deleted router is ignored", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + return &cloudscalesdk.ErrorResponse{StatusCode: 404} + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", RouterID: routerUUID, Managed: true}} + }, + wantStatusLen: 0, + }, + { + name: "failed router deletion keeps it in status", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + return errors.New("boom") + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", RouterID: routerUUID, Managed: true}} + }, + wantErr: true, + wantStatusLen: 1, + }, + { + name: "failed interface deletion keeps the router for the next attempt", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Router must not be deleted while its interfaces are still attached") + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + return errors.New("boom") + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{ + Name: "test-router", RouterID: routerUUID, Managed: true, + Interfaces: managedInterfaces(), + }} + }, + wantErr: true, + wantStatusLen: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + clusterScope := testutils.NewClusterScopeOpts(testutils.WithRouterService(tc.routerService(g))) + clusterScope.CloudscaleCluster.Status.Routers = tc.statusSetup() + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + wantReason := infrastructurev1beta2.RouterDeletingReason + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + wantReason = infrastructurev1beta2.RouterErrorReason + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(HaveLen(tc.wantStatusLen)) + + // Deletion always leaves the condition False; only the reason distinguishes + // "torn down" from "could not tear down". + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(wantReason)) + }) + } +} diff --git a/internal/controller/cloudscalecluster_servergroup.go b/internal/controller/cloudscalecluster_servergroup.go index 300c60d..37d81bd 100644 --- a/internal/controller/cloudscalecluster_servergroup.go +++ b/internal/controller/cloudscalecluster_servergroup.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "k8s.io/apimachinery/pkg/util/sets" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/controller/cloudscalecluster_servergroup_test.go b/internal/controller/cloudscalecluster_servergroup_test.go index 890acec..e03f2b0 100644 --- a/internal/controller/cloudscalecluster_servergroup_test.go +++ b/internal/controller/cloudscalecluster_servergroup_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" "k8s.io/client-go/tools/events" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalemachine_reconcile_test.go b/internal/controller/cloudscalemachine_reconcile_test.go index b366193..f49a2ba 100644 --- a/internal/controller/cloudscalemachine_reconcile_test.go +++ b/internal/controller/cloudscalemachine_reconcile_test.go @@ -27,7 +27,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" @@ -60,10 +60,10 @@ func TestMachineReconcileNormal_ServerRunning_SetsProvisioned(t *testing.T) { UUID: id, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.5", Version: 4}, }, }, diff --git a/internal/controller/cloudscalemachine_server.go b/internal/controller/cloudscalemachine_server.go index 324d4a6..092afe7 100644 --- a/internal/controller/cloudscalemachine_server.go +++ b/internal/controller/cloudscalemachine_server.go @@ -22,7 +22,7 @@ import ( "maps" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -303,7 +303,7 @@ func (r *CloudscaleMachineReconciler) machineLookupTag(machineScope *scope.Machi // If spec.interfaces is empty, defaults to the first cluster network + a public interface // (runtime cross-resource resolution that the webhook cannot do). // Returns the interface requests and the IPFamily from the public interface (if any). -func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) (*[]cloudscalesdk.InterfaceRequest, *infrastructurev1beta2.IPFamily, error) { +func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) (*[]cloudscalesdk.ServerInterfaceRequest, *infrastructurev1beta2.IPFamily, error) { ifaceSpecs := machineScope.CloudscaleMachine.Spec.Interfaces // Runtime default: first cluster network + public DualStack interface @@ -312,19 +312,19 @@ func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope return nil, nil, fmt.Errorf("cluster has no networks provisioned yet") } firstNetwork := machineScope.CloudscaleCluster.Status.Networks[0] - return &[]cloudscalesdk.InterfaceRequest{ + return &[]cloudscalesdk.ServerInterfaceRequest{ {Network: InterfaceTypePublic}, {Network: firstNetwork.NetworkID}, }, nil, nil } // Build from spec - reqs := make([]cloudscalesdk.InterfaceRequest, 0, len(ifaceSpecs)) + reqs := make([]cloudscalesdk.ServerInterfaceRequest, 0, len(ifaceSpecs)) var ipFamily *infrastructurev1beta2.IPFamily for _, iface := range ifaceSpecs { switch { case iface.Type == InterfaceTypePublic: - reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: InterfaceTypePublic}) + reqs = append(reqs, cloudscalesdk.ServerInterfaceRequest{Network: InterfaceTypePublic}) ipFamily = iface.IPFamily case iface.Network != "": ns := machineScope.CloudscaleCluster.Status.GetNetworkStatus(iface.Network) @@ -334,7 +334,7 @@ func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope if ns.NetworkID == "" { return nil, nil, fmt.Errorf("network %q not yet provisioned", iface.Network) } - reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: ns.NetworkID}) + reqs = append(reqs, cloudscalesdk.ServerInterfaceRequest{Network: ns.NetworkID}) default: return nil, nil, fmt.Errorf("interface must have either type or network set") } diff --git a/internal/controller/cloudscalemachine_server_test.go b/internal/controller/cloudscalemachine_server_test.go index b871921..a6f869d 100644 --- a/internal/controller/cloudscalemachine_server_test.go +++ b/internal/controller/cloudscalemachine_server_test.go @@ -20,7 +20,7 @@ import ( "context" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" @@ -48,10 +48,10 @@ func TestReconcileServer_CreatesServer(t *testing.T) { Name: req.Name, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.5", Version: 4}, }, }, @@ -122,16 +122,16 @@ func TestReconcileServer_SetsAddresses(t *testing.T) { Name: req.Name, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "public", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "185.98.123.45", Version: 4}, }, }, { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.10", Version: 4}, }, }, diff --git a/internal/controller/cloudscalemachine_servergroup.go b/internal/controller/cloudscalemachine_servergroup.go index 63d6a87..969f691 100644 --- a/internal/controller/cloudscalemachine_servergroup.go +++ b/internal/controller/cloudscalemachine_servergroup.go @@ -22,7 +22,7 @@ import ( "sync" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" diff --git a/internal/controller/cloudscalemachine_servergroup_test.go b/internal/controller/cloudscalemachine_servergroup_test.go index 9ce1d41..0226e45 100644 --- a/internal/controller/cloudscalemachine_servergroup_test.go +++ b/internal/controller/cloudscalemachine_servergroup_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" diff --git a/internal/testutils/cluster_scope.go b/internal/testutils/cluster_scope.go index 73f1aaa..dafddf7 100644 --- a/internal/testutils/cluster_scope.go +++ b/internal/testutils/cluster_scope.go @@ -85,6 +85,11 @@ func WithSubnetService(svc cloudscale.SubnetService) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleClient.Subnets = svc } } +// WithRouterService wires a custom Router service into the client. +func WithRouterService(svc cloudscale.RouterService) ClusterScopeOption { + return func(cs *scope.ClusterScope) { cs.CloudscaleClient.Routers = svc } +} + // WithServerGroupService wires a custom ServerGroup service into the client. func WithServerGroupService(svc cloudscale.ServerGroupService) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleClient.ServerGroups = svc } @@ -135,7 +140,9 @@ func WithHealthMonitorParams(delayS, timeoutS, upThreshold, downThreshold int) C } } -// WithPreExistingNetwork adds a NetworkStatus entry so the scope thinks the network already exists. +// WithPreExistingNetwork adds a NetworkStatus entry so the scope thinks the network +// already exists. The network is CAPCS-managed, i.e. defined by cidr in spec.networks; +// use WithAdoptedNetwork for one referenced by uuid. func WithPreExistingNetwork(name, netID, subnetID, cidr string) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleCluster.Status.Networks = append(cs.CloudscaleCluster.Status.Networks, @@ -150,6 +157,58 @@ func WithPreExistingNetwork(name, netID, subnetID, cidr string) ClusterScopeOpti } } +// WithAdoptedNetwork adds a NetworkStatus entry for a network CAPCS does not own, the +// shape spec.networks[].uuid produces. Deletion leaves such a network in place, so an +// interface attached to it is never detached on CAPCS's account. +func WithAdoptedNetwork(name, netID, subnetID, cidr string) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Status.Networks = append(cs.CloudscaleCluster.Status.Networks, + infrastructurev1beta2.NetworkStatus{ + Name: name, + NetworkID: netID, + SubnetID: subnetID, + CIDR: cidr, + Managed: false, + }, + ) + } +} + +// WithRouter appends a router to the spec. +func WithRouter(routerSpec infrastructurev1beta2.RouterSpec) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Spec.Routers = append(cs.CloudscaleCluster.Spec.Routers, routerSpec) + } +} + +// WithRouterStatus appends a RouterStatus entry so the scope thinks the router already +// exists. interfaces lists the router's recorded interfaces on spec networks; pass nil +// for a router with no recorded interfaces. +func WithRouterStatus(name, routerID string, managed bool, interfaces []infrastructurev1beta2.RouterInterfaceStatus) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Status.Routers = append(cs.CloudscaleCluster.Status.Routers, + infrastructurev1beta2.RouterStatus{ + Name: name, + RouterID: routerID, + Managed: managed, + Interfaces: interfaces, + }, + ) + } +} + +// WithControlPlaneEndpoint sets up load balancer status and control plane endpoint +// for isInfrastructureProvisioned testing. Uses sensible defaults for test scenarios. +func WithControlPlaneEndpoint(clusterScope *scope.ClusterScope) { + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = new(bool) + *clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = true + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "test-lb-id" + clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID = "test-lb-pool-id" + clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID = "test-lb-listener-id" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = "10.20.30.40" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port = 6443 +} + // WithGeneration sets the generation (used for status conditions). func WithGeneration(gen int64) ClusterScopeOption { return func(cs *scope.ClusterScope) { diff --git a/internal/testutils/fixtures.go b/internal/testutils/fixtures.go index 0255889..126208f 100644 --- a/internal/testutils/fixtures.go +++ b/internal/testutils/fixtures.go @@ -1,7 +1,7 @@ package testutils import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index ae2d1c2..57ceacd 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -4,7 +4,7 @@ import ( "context" "errors" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) // --- Network Service Mock --- @@ -51,6 +51,7 @@ type MockSubnetService struct { GetFn func(ctx context.Context, id string) (*cloudscalesdk.Subnet, error) ListFn func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) DeleteFn func(ctx context.Context, id string) error + UpdateFn func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error } func (m *MockSubnetService) Create(ctx context.Context, req *cloudscalesdk.SubnetCreateRequest) (*cloudscalesdk.Subnet, error) { @@ -81,6 +82,66 @@ func (m *MockSubnetService) Delete(ctx context.Context, id string) error { return errors.New("mock(Delete) not configured") } +func (m *MockSubnetService) Update(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + if m.UpdateFn != nil { + return m.UpdateFn(ctx, id, req) + } + return errors.New("mock(Update) not configured") +} + +// --- Router Service Mock --- + +type MockRouterService struct { + CreateFn func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) + GetFn func(ctx context.Context, id string) (*cloudscalesdk.Router, error) + ListFn func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) + DeleteFn func(ctx context.Context, id string) error + CreateInterfaceFn func(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) + DeleteInterfaceFn func(ctx context.Context, routerUUID, interfaceUUID string) error +} + +func (m *MockRouterService) Create(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + if m.CreateFn != nil { + return m.CreateFn(ctx, req) + } + return nil, errors.New("mock(Create) not configured") +} + +func (m *MockRouterService) Get(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + if m.GetFn != nil { + return m.GetFn(ctx, id) + } + return nil, errors.New("mock(Get) not configured") +} + +func (m *MockRouterService) List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + if m.ListFn != nil { + return m.ListFn(ctx, modifiers...) + } + return nil, errors.New("mock(List) not configured") +} + +func (m *MockRouterService) Delete(ctx context.Context, id string) error { + if m.DeleteFn != nil { + return m.DeleteFn(ctx, id) + } + return errors.New("mock(Delete) not configured") +} + +func (m *MockRouterService) CreateInterface(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + if m.CreateInterfaceFn != nil { + return m.CreateInterfaceFn(ctx, routerUUID, req) + } + return nil, errors.New("mock(CreateInterface) not configured") +} + +func (m *MockRouterService) DeleteInterface(ctx context.Context, routerUUID, interfaceUUID string) error { + if m.DeleteInterfaceFn != nil { + return m.DeleteInterfaceFn(ctx, routerUUID, interfaceUUID) + } + return errors.New("mock(DeleteInterface) not configured") +} + // --- FloatingIP Service Mock --- type MockFloatingIPService struct { diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook.go b/internal/webhook/v1beta2/cloudscalecluster_webhook.go index 0552bc7..3b94a1d 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook.go @@ -19,7 +19,9 @@ package v1beta2 import ( "context" "fmt" - "net" + "maps" + "net/netip" + "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" @@ -85,6 +87,10 @@ func (d *CloudscaleClusterCustomDefaulter) Default(_ context.Context, cluster *i clusterSpecDefault(&cluster.Spec) + // Router interface addresses are derived from the networks' CIDRs and are immutable + // once set, so they are defaulted here rather than in the shared defaulter. + defaultRouterInterfaceAddresses(&cluster.Spec) + return nil } @@ -122,6 +128,108 @@ func clusterSpecDefault(spec *infrastructurev1beta2.CloudscaleClusterSpec) { } } +// defaultRouterInterfaceAddresses assigns an address to every router interface that +// does not declare one. +// Interfaces on a network referenced by uuid are left untouched: that subnet is only +// known at runtime, so the address cannot be derived here. +func defaultRouterInterfaceAddresses(spec *infrastructurev1beta2.CloudscaleClusterSpec) { + networks := indexNetworks(spec.Networks) + + usedByNetwork := make(map[string]map[string]struct{}) + for ri := range spec.Routers { + for ii := range spec.Routers[ri].Interfaces { + ifaceSpec := &spec.Routers[ri].Interfaces[ii] + if usedByNetwork[ifaceSpec.Network] == nil { + usedByNetwork[ifaceSpec.Network] = make(map[string]struct{}) + } + if ifaceSpec.Address != "" { + usedByNetwork[ifaceSpec.Network][ifaceSpec.Address] = struct{}{} + } + } + } + + for ri := range spec.Routers { + for ii := range spec.Routers[ri].Interfaces { + ifaceSpec := &spec.Routers[ri].Interfaces[ii] + + // Normalise here rather than relying on the CRD default, so the value is + // set no matter which path reached this defaulter. + if ifaceSpec.ConfigureSubnetGateway == nil { + ifaceSpec.ConfigureSubnetGateway = new(true) + } + // An adopted interface keeps the address it already holds, which is only + // known once the controller reads the router. + if ifaceSpec.Address != "" || ifaceSpec.UUID != "" { + continue + } + + used := usedByNetwork[ifaceSpec.Network] + + // The gateway owner takes the address the subnet already advertises. If + // another interface has claimed it, fall through and allocate instead. + gateway := networks[ifaceSpec.Network].GatewayAddress + if gateway != "" && *ifaceSpec.ConfigureSubnetGateway { + if _, taken := used[gateway]; !taken { + ifaceSpec.Address = gateway + used[gateway] = struct{}{} + continue + } + } + + // err is ignored: an unparseable CIDR leaves the zero Prefix, which + // nextFreeAddress skips. validateNetworks reports the malformed value. + prefix, _ := netip.ParsePrefix(networks[ifaceSpec.Network].CIDR) + if addr, ok := nextFreeAddress(prefix, used); ok { + ifaceSpec.Address = addr + used[addr] = struct{}{} + } + } + } +} + +// .101-.254 of every subnet is DHCP. +const ( + dhcpPoolFirstOctet = 101 + dhcpPoolLastOctet = 254 +) + +// unassignableReason says why addr cannot be given to a router interface inside prefix, or +// "" when it can. An address in the DHCP pool may later be handed to a node, and the +// network and broadcast addresses cannot be held by an interface at all. +func unassignableReason(prefix netip.Prefix, addr netip.Addr) string { + if !addr.Is4() { + return "" + } + switch lastOctet := addr.As4()[3]; { + case addr == prefix.Masked().Addr(): + return "it is the network address" + case !prefix.Contains(addr.Next()): + return "it is the broadcast address" + case lastOctet >= dhcpPoolFirstOctet && lastOctet <= dhcpPoolLastOctet: + return fmt.Sprintf("it is inside the DHCP pool .%d-.%d", dhcpPoolFirstOctet, dhcpPoolLastOctet) + } + return "" +} + +// nextFreeAddress returns the first assignable address inside prefix that is not already +// used. It reports false for an invalid prefix or an exhausted range. +func nextFreeAddress(prefix netip.Prefix, used map[string]struct{}) (string, bool) { + if !prefix.IsValid() { + return "", false + } + for addr := prefix.Masked().Addr().Next(); prefix.Contains(addr); addr = addr.Next() { + // Step over rather than stop: on a prefix wider than a /24 the DHCP pool recurs + // inside the range, so there can be assignable addresses above it. + if unassignableReason(prefix, addr) != "" { + continue + } + if _, taken := used[addr.String()]; !taken { + return addr.String(), true + } + } + return "", false +} + // +kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta2-cloudscalecluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=cloudscaleclusters,verbs=create;update,versions=v1beta2,name=vcloudscalecluster-v1beta2.kb.io,admissionReviewVersions=v1 // CloudscaleClusterCustomValidator struct is responsible for validating the CloudscaleCluster resource @@ -137,7 +245,7 @@ type CloudscaleClusterCustomValidator struct { func (v *CloudscaleClusterCustomValidator) ValidateCreate(_ context.Context, cluster *infrastructurev1beta2.CloudscaleCluster) (admission.Warnings, error) { cloudscaleclusterlog.Info("Validation for CloudscaleCluster upon creation", "name", cluster.GetName()) - allErrs := clusterSpecValidateCreate(cluster.Spec, v.RegionInfo, field.NewPath("spec")) + allErrs := clusterSpecValidateCreate(cluster.Spec, v.RegionInfo, modeCluster, field.NewPath("spec")) if len(allErrs) > 0 { return nil, apierrors.NewInvalid( @@ -148,7 +256,25 @@ func (v *CloudscaleClusterCustomValidator) ValidateCreate(_ context.Context, clu return nil, nil } -func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, regionInfo *cloudscale.RegionInfo, fldPath *field.Path) field.ErrorList { +// validationMode captures what a spec has already been through by the time it reaches the +// validator. A CloudscaleCluster arrives defaulted; a CloudscaleClusterTemplate does not, +// because defaultRouterInterfaceAddresses only runs on the cluster (see Default). +type validationMode struct { + // routerAddressesDefaulted is true when defaultRouterInterfaceAddresses has already run, + // so an empty router interface address means "could not be derived" rather than "will be + // derived when a cluster is stamped from this template". + routerAddressesDefaulted bool +} + +var ( + // modeCluster validates a CloudscaleCluster, i.e. a fully defaulted spec. + modeCluster = validationMode{routerAddressesDefaulted: true} + // modeTemplate validates a CloudscaleClusterTemplate, whose router interface addresses + // are only derived once the topology controller stamps out a CloudscaleCluster. + modeTemplate = validationMode{} +) + +func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, regionInfo *cloudscale.RegionInfo, mode validationMode, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList // Validate zone belongs to region @@ -164,6 +290,9 @@ func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, // Validate networks allErrs = append(allErrs, validateNetworks(spec.Networks, fldPath.Child("networks"))...) + // Validate routers + allErrs = append(allErrs, validateRouters(spec.Routers, spec.Networks, mode, fldPath.Child("routers"))...) + // Validate LB network reference if spec.ControlPlaneLoadBalancer.Network != "" { allErrs = append(allErrs, validateNetworkReference( @@ -213,6 +342,12 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old // Validate new networks (new entries must still pass creation validation) allErrs = append(allErrs, validateNetworks(newClusterSpec.Networks, field.NewPath("spec", "networks"))...) + // The updated router set must satisfy the creation rules + routersPath := field.NewPath("spec", "routers") + allErrs = append(allErrs, validateRouters(newClusterSpec.Routers, newClusterSpec.Networks, modeCluster, routersPath)...) + // ...and on top of that, nothing already there may change or disappear. + allErrs = append(allErrs, validateRouterDiff(oldClusterSpec.Routers, newClusterSpec.Routers, routersPath)...) + // LoadBalancer Enabled is immutable if ptr.Deref(newClusterSpec.ControlPlaneLoadBalancer.Enabled, true) != ptr.Deref(oldClusterSpec.ControlPlaneLoadBalancer.Enabled, true) { allErrs = append(allErrs, field.Forbidden( @@ -220,9 +355,8 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old "field is immutable after cluster creation")) } - // LB network is immutable once set - if oldClusterSpec.ControlPlaneLoadBalancer.Network != "" && - newClusterSpec.ControlPlaneLoadBalancer.Network != oldClusterSpec.ControlPlaneLoadBalancer.Network { + // LB network is immutable + if newClusterSpec.ControlPlaneLoadBalancer.Network != oldClusterSpec.ControlPlaneLoadBalancer.Network { allErrs = append(allErrs, field.Forbidden( field.NewPath("spec", "controlPlaneLoadBalancer", "network"), "field is immutable once set")) @@ -310,7 +444,7 @@ func validateNetworks(networks []infrastructurev1beta2.NetworkSpec, fldPath *fie // Validate CIDR format if hasCIDR { - if _, _, err := net.ParseCIDR(netSpec.CIDR); err != nil { + if _, err := netip.ParsePrefix(netSpec.CIDR); err != nil { allErrs = append(allErrs, field.Invalid(netPath.Child("cidr"), netSpec.CIDR, fmt.Sprintf("invalid CIDR: %v", err))) } @@ -324,7 +458,7 @@ func validateNetworks(networks []infrastructurev1beta2.NetworkSpec, fldPath *fie // Validate gateway is within CIDR if netSpec.GatewayAddress != "" && hasCIDR { - allErrs = append(allErrs, validateGatewayInCIDR( + allErrs = append(allErrs, validateAddressInCIDR( netSpec.CIDR, netSpec.GatewayAddress, netPath.Child("gatewayAddress"), @@ -384,6 +518,12 @@ func validateNetworkImmutability(oldNetworks, newNetworks []infrastructurev1beta "field is immutable after cluster creation")) } + if newNet.MTU != oldNet.MTU { + allErrs = append(allErrs, field.Forbidden( + newPath.Child("mtu"), + "field is immutable after cluster creation")) + } + if newNet.GatewayAddress != oldNet.GatewayAddress { allErrs = append(allErrs, field.Forbidden( newPath.Child("gatewayAddress"), @@ -394,6 +534,243 @@ func validateNetworkImmutability(oldNetworks, newNetworks []infrastructurev1beta return allErrs } +// indexNetworks maps each network name to its spec, so router validation and defaulting +// can resolve an interface's network reference in one place. +func indexNetworks(networks []infrastructurev1beta2.NetworkSpec) map[string]infrastructurev1beta2.NetworkSpec { + byName := make(map[string]infrastructurev1beta2.NetworkSpec, len(networks)) + for _, n := range networks { + byName[n.Name] = n + } + return byName +} + +// validateRouters runs every rule that applies to a router set, whether it was declared +// at creation or added later. ValidateUpdate calls it on the new set too, so additions +// are held to the same standard as originals without restating the rules. +func validateRouters(routers []infrastructurev1beta2.RouterSpec, networks []infrastructurev1beta2.NetworkSpec, mode validationMode, fldPath *field.Path) field.ErrorList { + byName := indexNetworks(networks) + var allErrs field.ErrorList + for i, routerSpec := range routers { + allErrs = append(allErrs, validateSingleRouter(routerSpec, byName, mode, fldPath.Index(i))...) + } + allErrs = append(allErrs, validateRouterNetworkAddressing(routers, byName, fldPath)...) + return allErrs +} + +// validateSingleRouter validates one router spec against the network index, using +// routerPath as the base field.Path for errors. +// +// Router names are not checked for uniqueness: spec.routers is a list-map keyed by name, so +// the API server rejects duplicates during schema validation, which runs before validating +// webhooks. +func validateSingleRouter( + routerSpec infrastructurev1beta2.RouterSpec, + networks map[string]infrastructurev1beta2.NetworkSpec, + mode validationMode, + routerPath *field.Path, +) field.ErrorList { + var allErrs field.ErrorList + + if routerSpec.UUID != "" && routerSpec.InternetGateway { + allErrs = append(allErrs, field.Invalid(routerPath, routerSpec.Name, + "uuid and internetGateway are mutually exclusive")) + } + + for j, ifaceSpec := range routerSpec.Interfaces { + allErrs = append(allErrs, validateRouterInterface(ifaceSpec, networks, routerSpec.UUID != "", mode, routerPath.Child("interfaces").Index(j))...) + } + + return allErrs +} + +// validateRouterInterface validates one router interface against the network index. On a +// CloudscaleCluster the defaulter has already filled in every address it could derive, so an +// empty address means it could not — either because the network is referenced by uuid (no +// CIDR at admission time) or because the CIDR is exhausted. Both need the user to be +// explicit. An interface adopted by uuid is the exception: its address is whatever the +// pre-existing interface already holds. +// +// On a CloudscaleClusterTemplate (mode.routerAddressesDefaulted false) an empty address on a +// network that has a CIDR is fine: it is derived per cluster, so a topology patch may change +// the CIDR and have the address follow. Only the uuid-network case still errors, since such +// a network never gains a CIDR and so could not be derived later either. +func validateRouterInterface( + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + networks map[string]infrastructurev1beta2.NetworkSpec, + routerAdopted bool, + mode validationMode, + ifacePath *field.Path, +) field.ErrorList { + network, netExists := networks[ifaceSpec.Network] + if !netExists { + return field.ErrorList{field.NotFound(ifacePath.Child("network"), ifaceSpec.Network)} + } + + if ifaceSpec.UUID != "" { + var allErrs field.ErrorList + if !routerAdopted { + allErrs = append(allErrs, field.Forbidden(ifacePath.Child("uuid"), + "only a router referenced by uuid carries pre-existing interfaces to adopt")) + } + if ifaceSpec.Address != "" { + allErrs = append(allErrs, field.Forbidden(ifacePath.Child("address"), + "the address of an adopted interface is read from the interface itself, so it cannot be requested")) + } + return allErrs + } + + if ifaceSpec.Address == "" { + if network.CIDR == "" { + return field.ErrorList{field.Required(ifacePath.Child("address"), fmt.Sprintf( + "must be set explicitly: network %q is referenced by uuid, so its subnet is not known yet", + ifaceSpec.Network))} + } + if !mode.routerAddressesDefaulted { + // Derived from the network's CIDR when a cluster is stamped out of this template. + return nil + } + return field.ErrorList{field.Required(ifacePath.Child("address"), fmt.Sprintf( + "no free address left in %s for network %q", network.CIDR, ifaceSpec.Network))} + } + + // Containment can only be checked for a managed network; an uuid network's subnet + // is resolved by the controller at runtime. + if network.CIDR == "" { + return nil + } + + addrPath := ifacePath.Child("address") + if errs := validateAddressInCIDR(network.CIDR, ifaceSpec.Address, addrPath); len(errs) > 0 { + return errs + } + + prefix, _ := netip.ParsePrefix(network.CIDR) + ip, _ := netip.ParseAddr(ifaceSpec.Address) + if reason := unassignableReason(prefix, ip); reason != "" { + return field.ErrorList{field.Invalid(addrPath, ifaceSpec.Address, + fmt.Sprintf("cannot be assigned to a router interface: %s of %s", reason, network.CIDR))} + } + return nil +} + +// validateRouterNetworkAddressing enforces the rules that only make sense when all +// interfaces on a network are considered together: at most one interface may set the +// subnet gateway, explicit addresses must be unique per network, and the gateway owner's +// address must equal the network's declared gatewayAddress. +func validateRouterNetworkAddressing(routers []infrastructurev1beta2.RouterSpec, networks map[string]infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + subnetGateways := make(map[string][]*field.Path) + addrSeen := make(map[string]map[string]bool) + + for i, router := range routers { + for j, iface := range router.Interfaces { + ifacePath := fldPath.Index(i).Child("interfaces").Index(j) + + if ptr.Deref(iface.ConfigureSubnetGateway, true) { + subnetGateways[iface.Network] = append(subnetGateways[iface.Network], ifacePath.Child("configureSubnetGateway")) + + // The controller points the subnet's gateway at this interface's + // address, so a differing gatewayAddress would silently never take + // effect. An address is only empty on an adopted interface, whose + // address is not known until the controller reads it. + if gateway := networks[iface.Network].GatewayAddress; gateway != "" && iface.Address != "" && iface.Address != gateway { + allErrs = append(allErrs, field.Invalid(ifacePath.Child("address"), iface.Address, + fmt.Sprintf("interface configures the subnet gateway of network %q, so its address must equal the network's gatewayAddress %q", iface.Network, gateway))) + } + } + if iface.Address != "" { + if addrSeen[iface.Network] == nil { + addrSeen[iface.Network] = make(map[string]bool) + } + if addrSeen[iface.Network][iface.Address] { + allErrs = append(allErrs, field.Duplicate(ifacePath.Child("address"), iface.Address)) + } else { + addrSeen[iface.Network][iface.Address] = true + } + } + } + } + + // Report networks with more than one gateway, in a stable order. + for _, network := range slices.Sorted(maps.Keys(subnetGateways)) { + paths := subnetGateways[network] + if len(paths) <= 1 { + continue + } + for _, p := range paths { + allErrs = append(allErrs, field.Invalid(p, true, + fmt.Sprintf("network %q has multiple router interfaces configuring the subnet gateway; only one interface may set configureSubnetGateway=true", network))) + } + } + + return allErrs +} + +// validateRouterDiff enforces immutability for spec.routers on update: routers and their +// interfaces may be added, but never removed or altered. Whether the additions themselves +// are valid is validateRouters' job, which ValidateUpdate runs over the new set. +func validateRouterDiff(oldRouters, newRouters []infrastructurev1beta2.RouterSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + newByName := make(map[string]infrastructurev1beta2.RouterSpec, len(newRouters)) + newIndexByName := make(map[string]int, len(newRouters)) + for i, newRouter := range newRouters { + newByName[newRouter.Name] = newRouter + newIndexByName[newRouter.Name] = i + } + + for _, oldRouter := range oldRouters { + newRouter, exists := newByName[oldRouter.Name] + if !exists { + allErrs = append(allErrs, field.Forbidden(fldPath, + fmt.Sprintf("removing router %q is not allowed", oldRouter.Name))) + continue + } + + newPath := fldPath.Index(newIndexByName[oldRouter.Name]) + + if newRouter.UUID != oldRouter.UUID { + allErrs = append(allErrs, field.Forbidden(newPath.Child("uuid"), + "field is immutable after cluster creation")) + } + if newRouter.InternetGateway != oldRouter.InternetGateway { + allErrs = append(allErrs, field.Forbidden(newPath.Child("internetGateway"), + "field is immutable after cluster creation")) + } + + newIfaceByNetwork := make(map[string]infrastructurev1beta2.RouterInterfaceSpec, len(newRouter.Interfaces)) + for _, iface := range newRouter.Interfaces { + newIfaceByNetwork[iface.Network] = iface + } + + for _, oldIface := range oldRouter.Interfaces { + newIface, kept := newIfaceByNetwork[oldIface.Network] + if !kept { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("removing interface for network %q is not allowed", oldIface.Network))) + continue + } + if ptr.Deref(newIface.ConfigureSubnetGateway, true) != ptr.Deref(oldIface.ConfigureSubnetGateway, true) { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("configureSubnetGateway for network %q is immutable after cluster creation", newIface.Network))) + } + if newIface.Address != oldIface.Address { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("address for network %q is immutable after cluster creation", newIface.Network))) + } + // Flipping uuid would hand ownership of a live interface over or away: + // CAPCS detaches what it created and leaves adopted interfaces in place. + if newIface.UUID != oldIface.UUID { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("uuid for network %q is immutable after cluster creation", newIface.Network))) + } + } + } + + return allErrs +} + // validateFloatingIPRequiresLBOrPreExisting rejects managed floating IPs when the load balancer is disabled. // cloudscale.ch floating IPs require a dummy interface on the target server. // With a pre-existing FIP, the user knows the address upfront and can configure @@ -483,8 +860,10 @@ func validateLBPoolMemberNetworkResolvable(spec infrastructurev1beta2.Cloudscale return nil } + // Reported on the parent: either child field satisfies the requirement, so pinning + // the error on one of them would point the user at an arbitrary half of the fix. allErrs = append(allErrs, field.Required( - fldPath.Child("controlPlaneLoadBalancer", "network"), + fldPath.Child("controlPlaneLoadBalancer"), "must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) return allErrs @@ -502,9 +881,9 @@ func validateFloatingIP(fip *infrastructurev1beta2.FloatingIPSpec, fldPath *fiel "exactly one of ipFamily or ip must be specified")) } - if hasIP && net.ParseIP(fip.Address) == nil { + if _, err := netip.ParseAddr(fip.Address); hasIP && err != nil { allErrs = append(allErrs, field.Invalid(fldPath.Child("ip"), fip.Address, - "must be a valid IP address")) + fmt.Sprintf("must be a valid IP address: %s", err))) } return allErrs @@ -544,25 +923,26 @@ func validateFloatingIPImmutability(oldFIP, newFIP *infrastructurev1beta2.Floati return allErrs } -// validateGatewayInCIDR validates that the gateway address is within the specified CIDR. -func validateGatewayInCIDR(cidr, gateway string, fldPath *field.Path) field.ErrorList { +// validateAddressInCIDR validates that an address is within the specified CIDR. Used +// for both subnet gateways and router interface addresses. +func validateAddressInCIDR(cidr, address string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - _, ipNet, err := net.ParseCIDR(cidr) + prefix, err := netip.ParsePrefix(cidr) if err != nil { // CIDR validation should have caught this earlier return allErrs } - gatewayIP := net.ParseIP(gateway) - if gatewayIP == nil { - allErrs = append(allErrs, field.Invalid(fldPath, gateway, "invalid IP address")) + ip, err := netip.ParseAddr(address) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, address, fmt.Sprintf("invalid IP address: %s", err))) return allErrs } - if !ipNet.Contains(gatewayIP) { - allErrs = append(allErrs, field.Invalid(fldPath, gateway, - fmt.Sprintf("gateway must be within CIDR %s", cidr))) + if !prefix.Contains(ip) { + allErrs = append(allErrs, field.Invalid(fldPath, address, + fmt.Sprintf("must be within CIDR %s", cidr))) } return allErrs diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go index dc61b70..ac80dc4 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta2 import ( + "fmt" + "net/netip" "testing" . "github.com/onsi/gomega" @@ -53,6 +55,66 @@ func newClusterWebhookTestObjects() ( // CloudscaleCluster Defaulting Webhook // ============================================================================ +// routerNetworks is the network set the router defaulting cases start from. +func routerNetworks(cidr string) []infrastructurev1beta2.NetworkSpec { + return []infrastructurev1beta2.NetworkSpec{{Name: "main", CIDR: cidr}} +} + +// routerWith builds a single router named "r" attached to the given interfaces. +func routerWith(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{Name: "r", InternetGateway: true, Interfaces: ifaces}} +} + +// routerIface builds an interface on network "main"; an empty address leaves it for the +// defaulter to fill in. +func routerIface(address string) infrastructurev1beta2.RouterInterfaceSpec { + return infrastructurev1beta2.RouterInterfaceSpec{Network: "main", Address: address} +} + +// adoptedRouterWith builds a single router named "r" that adopts a pre-existing router by +// uuid, which is the only kind that can carry interfaces to adopt. +func adoptedRouterWith(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{Name: "r", UUID: "router-uuid", Interfaces: ifaces}} +} + +// routerIfaceAdopted builds an interface on network "main" that adopts a pre-existing +// interface, whose address is read off that interface rather than requested. +func routerIfaceAdopted() infrastructurev1beta2.RouterInterfaceSpec { + iface := routerIface("") + iface.UUID = "iface-uuid" + return iface +} + +// routerIfaceNoGateway is a routerIface that leaves the subnet gateway to someone else. +func routerIfaceNoGateway(address string) infrastructurev1beta2.RouterInterfaceSpec { + iface := routerIface(address) + iface.ConfigureSubnetGateway = new(false) + return iface +} + +// routersEach builds one router per interface, named r0, r1, ... Interfaces that share a +// network have to sit on separate routers: spec.routers[].interfaces is a list-map keyed +// by network, so a single router can hold at most one interface per network. +func routersEach(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { + routers := make([]infrastructurev1beta2.RouterSpec, len(ifaces)) + for i, iface := range ifaces { + routers[i] = infrastructurev1beta2.RouterSpec{ + Name: fmt.Sprintf("r%d", i), + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{iface}, + } + } + return routers +} + +// setupRouterCluster fills in the region/zone a valid cluster needs and attaches the +// given networks and routers. +func setupRouterCluster(c *infrastructurev1beta2.CloudscaleCluster, networks []infrastructurev1beta2.NetworkSpec, routers []infrastructurev1beta2.RouterSpec) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = networks + c.Spec.Routers = routers +} + func TestClusterDefaulting(t *testing.T) { cases := []struct { name string @@ -187,6 +249,145 @@ func TestClusterDefaulting(t *testing.T) { g.Expect(c.Spec.FloatingIP.Address).To(Equal("1.2.3.4")) }, }, + { + name: "interface address and configureSubnetGateway are both defaulted", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = routerWith(routerIface("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + iface := c.Spec.Routers[0].Interfaces[0] + g.Expect(iface.Address).To(Equal("10.0.0.1")) + g.Expect(iface.ConfigureSubnetGateway).To(Equal(new(true))) + }, + }, + { + name: "two routers on one network get distinct addresses", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = routersEach(routerIface(""), routerIfaceNoGateway("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) + }, + }, + { + name: "explicit address is never handed out twice", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) + }, + }, + { + name: "gateway owner takes the network's gatewayAddress", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: "10.0.0.0/24", GatewayAddress: "10.0.0.9"}, + } + c.Spec.Routers = routerWith(routerIface("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.9")) + }, + }, + { + name: "non-gateway interface does not take the network's gatewayAddress", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: "10.0.0.0/24", GatewayAddress: "10.0.0.9"}, + } + iface := routerIface("") + iface.ConfigureSubnetGateway = new(false) + c.Spec.Routers = routerWith(iface) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + }, + }, + { + name: "CIDR with host bits allocates from the masked network address", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.5/24") + c.Spec.Routers = routerWith(routerIface("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + }, + }, + { + // A network referenced by uuid has no CIDR here, so there is nothing to derive + // an address from. It must be left empty rather than filled with garbage. + name: "interface on a uuid network is left empty", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}} + c.Spec.Routers = routerWith(routerIface("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) + }, + }, + { + // An adopted interface holds whatever address it already has, which no + // amount of CIDR arithmetic here can know. + name: "adopted interface is left empty even on a CIDR network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = adoptedRouterWith(routerIfaceAdopted()) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) + // Everything else is still defaulted as usual. + g.Expect(c.Spec.Routers[0].Interfaces[0].ConfigureSubnetGateway).To(HaveValue(BeTrue())) + }, + }, + { + // The address an adopted interface does not take must stay available. + name: "adopted interface does not consume an address", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = routersEach(routerIfaceAdopted(), routerIface("")) + c.Spec.Routers[0].UUID = "router-uuid" + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) + g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.1")) + }, + }, + { + // A /30 holds .0 through .3, of which only .1 and .2 are assignable: .0 is the + // network address and .3 the broadcast address. The third router has nowhere + // to go. + name: "exhausted CIDR leaves the surplus interface empty", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/30") + c.Spec.Routers = routersEach(routerIface(""), + routerIfaceNoGateway(""), routerIfaceNoGateway("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) + g.Expect(c.Spec.Routers[2].Interfaces[0].Address).To(BeEmpty()) + // The broadcast address must go to nobody rather than to the last router. + for _, router := range c.Spec.Routers { + g.Expect(router.Interfaces[0].Address).ToNot(Equal("10.0.0.3")) + } + }, + }, } for _, tc := range cases { @@ -222,6 +423,7 @@ func TestClusterDefaulting_AllDefaultsApplied(t *testing.T) { g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.TimeoutS).To(Equal(3)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.UpThreshold).To(Equal(2)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.DownThreshold).To(Equal(3)) + g.Expect(obj.Spec.Routers).To(BeEmpty()) } // ============================================================================ @@ -370,7 +572,7 @@ func TestClusterValidateCreate(t *testing.T) { c.Spec.ControlPlaneLoadBalancer.Network = "" }, wantErr: true, - wantSubstrings: []string{"controlPlaneLoadBalancer.network"}, + wantSubstrings: []string{"controlPlaneLoadBalancer"}, }, { name: "LB.Network references unknown network", @@ -462,6 +664,179 @@ func TestClusterValidateCreate(t *testing.T) { c.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{Address: "1.2.3.4"} }, }, + { + name: "valid router", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface("10.0.0.1"))) + }, + }, + { + name: "nil configureSubnetGateway is treated as true", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + iface := routerIface("10.0.0.1") + iface.ConfigureSubnetGateway = nil + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(iface)) + }, + }, + { + name: "router interface address inside the DHCP pool", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.150" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.150"))) + }, + wantErr: true, + wantSubstrings: []string{"DHCP pool .101-.254", "cannot be assigned to a router interface"}, + }, + { + name: "router interface on the last address below the DHCP pool", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.100" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.100"))) + }, + }, + { + name: "router interface on the broadcast address", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.255" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.255"))) + }, + wantErr: true, + wantSubstrings: []string{"it is the broadcast address"}, + }, + { + name: "router interface address outside the network CIDR", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface("192.168.1.1"))) + }, + wantErr: true, + wantSubstrings: []string{"must be within CIDR"}, + }, + { + name: "router interface referencing an unknown network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + iface := routerIface("10.0.0.1") + iface.Network = "nope" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(iface)) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].network"}, + }, + { + name: "duplicate addresses on the same network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), + routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("10.0.0.1"))) + }, + wantErr: true, + wantSubstrings: []string{"Duplicate value", "10.0.0.1"}, + }, + { + name: "two subnet gateway owners on the same network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), + routersEach(routerIface("10.0.0.1"), routerIface("10.0.0.2"))) + }, + wantErr: true, + wantSubstrings: []string{"only one interface may set configureSubnetGateway=true"}, + }, + { + name: "same network on two different routers is allowed", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), + routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("10.0.0.2"))) + }, + }, + { + name: "gateway owner address contradicts the network's gatewayAddress", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.5" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.1"))) + }, + wantErr: true, + wantSubstrings: []string{"must equal the network's gatewayAddress", "10.0.0.5"}, + }, + { + name: "gateway owner address matching the network's gatewayAddress is valid", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.5" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.5"))) + }, + }, + { + name: "uuid and internetGateway are mutually exclusive", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + routers := routerWith(routerIface("10.0.0.1")) + routers[0].UUID = "router-uuid" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routers) + }, + wantErr: true, + wantSubstrings: []string{"uuid and internetGateway are mutually exclusive"}, + }, + { + name: "adopted interface is accepted without an address", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, + []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, + adoptedRouterWith(routerIfaceAdopted())) + }, + }, + { + name: "adopted interface with a requested address rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + iface := routerIfaceAdopted() + iface.Address = "10.0.0.1" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), adoptedRouterWith(iface)) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].address", "cannot be requested"}, + }, + { + name: "adopted interface on a managed router rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIfaceAdopted())) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].uuid", "referenced by uuid"}, + }, + { + name: "adopted gateway owner does not have to match the network's gatewayAddress", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.5" + setupRouterCluster(c, networks, adoptedRouterWith(routerIfaceAdopted())) + }, + }, + { + name: "interface on a uuid network with an explicit address is accepted", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, + []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, + routerWith(routerIface("192.168.77.1"))) + }, + }, + { + name: "address-less interface on a uuid network rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, + []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, + routerWith(routerIface(""))) + }, + wantErr: true, + wantSubstrings: []string{"must be set explicitly", "referenced by uuid"}, + }, + { + name: "address-less interface on an exhausted CIDR rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface(""))) + }, + wantErr: true, + wantSubstrings: []string{"no free address left"}, + }, } for _, tc := range cases { @@ -483,6 +858,52 @@ func TestClusterValidateCreate(t *testing.T) { } } +// TestClusterDefaultThenValidate_NonOwnerSquattingGatewayAddress runs both webhooks in +// the order the API server does. A non-owner interface claiming the network's declared +// gatewayAddress forces the defaulter to allocate the gateway owner a different address, +// which the validator must then reject as contradicting the network. +func TestClusterDefaultThenValidate_NonOwnerSquattingGatewayAddress(t *testing.T) { + g := NewWithT(t) + + obj, _, validator, defaulter := newClusterWebhookTestObjects() + + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.1" + squatter := routerIface("10.0.0.1") + squatter.ConfigureSubnetGateway = new(false) + owner := routerIface("") + owner.ConfigureSubnetGateway = new(true) + setupRouterCluster(obj, networks, []infrastructurev1beta2.RouterSpec{ + {Name: "r", Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{squatter}}, + {Name: "r2", Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{owner}}, + }) + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + // The declared gateway address was taken, so the owner got the next free one. + g.Expect(obj.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("must equal the network's gatewayAddress")) +} + +func TestClusterDefaulting_InterfaceAddedLater(t *testing.T) { + g := NewWithT(t) + + obj, _, _, defaulter := newClusterWebhookTestObjects() + setupRouterCluster(obj, routerNetworks("10.0.0.0/24"), routerWith(routerIface("10.0.0.1"))) + // A second router shows up later and asks for an address on the same network. + obj.Spec.Routers = append(obj.Spec.Routers, infrastructurev1beta2.RouterSpec{ + Name: "later", + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{routerIfaceNoGateway("")}, + }) + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + + g.Expect(obj.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + g.Expect(obj.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) +} + // ============================================================================ // CloudscaleCluster Validating Webhook - Update // ============================================================================ @@ -509,6 +930,20 @@ func setupUpdateTestObjects() ( return } +// seedRouter puts the same valid router on both sides of an update, so cases only need +// to express the change they are testing. The address sits inside defaultSubnetCIDR. +func seedRouter(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + router := infrastructurev1beta2.RouterSpec{ + Name: "r", + InternetGateway: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "main", Address: "172.18.0.1", ConfigureSubnetGateway: new(true)}, + }, + } + oldObj.Spec.Routers = []infrastructurev1beta2.RouterSpec{*router.DeepCopy()} + obj.Spec.Routers = []infrastructurev1beta2.RouterSpec{*router.DeepCopy()} +} + func TestClusterValidateUpdate(t *testing.T) { cases := []struct { name string @@ -575,9 +1010,22 @@ func TestClusterValidateUpdate(t *testing.T) { mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { oldObj.Spec.ControlPlaneLoadBalancer.Network = "main" obj.Spec.ControlPlaneLoadBalancer.Network = "other" + otherNet := infrastructurev1beta2.NetworkSpec{Name: "other", CIDR: "10.1.0.0/24"} + obj.Spec.Networks = append(obj.Spec.Networks, otherNet) }, wantErr: true, - wantSubstrings: []string{"controlPlaneLoadBalancer.network"}, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "immutable"}, + }, + { + name: "LB.Network change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + oldObj.Spec.ControlPlaneLoadBalancer.Network = "" + obj.Spec.ControlPlaneLoadBalancer.Network = "other" + otherNet := infrastructurev1beta2.NetworkSpec{Name: "other", CIDR: "10.1.0.0/24"} + obj.Spec.Networks = append(obj.Spec.Networks, otherNet) + }, + wantErr: true, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "immutable"}, }, { name: "ControlPlaneEndpoint host change rejected", @@ -635,6 +1083,90 @@ func TestClusterValidateUpdate(t *testing.T) { wantErr: true, wantSubstrings: []string{"spec.region", "spec.zone", "cidr"}, }, + { + name: "router unchanged", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + }, + }, + { + name: "router uuid change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].UUID = "other-uuid" + obj.Spec.Routers[0].InternetGateway = false + }, + wantErr: true, + wantSubstrings: []string{"routers[0].uuid", "immutable"}, + }, + { + name: "router internetGateway change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].InternetGateway = false + }, + wantErr: true, + wantSubstrings: []string{"routers[0].internetGateway", "immutable"}, + }, + { + name: "router interface address change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces[0].Address = "172.18.0.9" + }, + wantErr: true, + wantSubstrings: []string{"address for network", "immutable"}, + }, + { + name: "router interface uuid change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces[0].UUID = "iface-uuid" + }, + wantErr: true, + wantSubstrings: []string{"uuid for network", "immutable"}, + }, + { + name: "router interface configureSubnetGateway change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces[0].ConfigureSubnetGateway = new(false) + }, + wantErr: true, + wantSubstrings: []string{"configureSubnetGateway for network", "immutable"}, + }, + { + name: "router interface removal rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces = nil + }, + wantErr: true, + wantSubstrings: []string{"removing interface for network"}, + }, + { + name: "router removal rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers = nil + }, + wantErr: true, + wantSubstrings: []string{"removing router"}, + }, + { + name: "router added on update is validated", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers = append(obj.Spec.Routers, infrastructurev1beta2.RouterSpec{ + Name: "second", + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "main", Address: "192.168.1.1", ConfigureSubnetGateway: new(false)}, + }, + }) + }, + wantErr: true, + wantSubstrings: []string{"routers[1].interfaces[0].address", "must be within CIDR"}, + }, } for _, tc := range cases { @@ -765,10 +1297,58 @@ func TestClusterValidateDelete_AlwaysSucceeds(t *testing.T) { } // ============================================================================ -// Unit tests for validateGatewayInCIDR +// Unit tests for unassignableReason +// ============================================================================ + +// TestUnassignableReason pins the boundaries of the rule that the defaulter and the +// validator share. They belong here rather than in the defaulting table, where reaching +// the DHCP pool would mean claiming a hundred addresses first. +func TestUnassignableReason(t *testing.T) { + cases := []struct { + name string + prefix string + addr string + want string + }{ + {name: "first host address", prefix: "10.0.0.0/24", addr: "10.0.0.1"}, + {name: "last address below the pool", prefix: "10.0.0.0/24", addr: "10.0.0.100"}, + {name: "first address of the pool", prefix: "10.0.0.0/24", addr: "10.0.0.101", want: "DHCP pool"}, + {name: "last address of the pool", prefix: "10.0.0.0/24", addr: "10.0.0.254", want: "DHCP pool"}, + {name: "broadcast address", prefix: "10.0.0.0/24", addr: "10.0.0.255", want: "broadcast address"}, + {name: "network address", prefix: "10.0.0.0/24", addr: "10.0.0.0", want: "network address"}, + // The pool is a last-octet range, so it recurs in a prefix wider than a /24 while + // the broadcast address belongs to the prefix as a whole. + {name: "pool recurs inside a /16", prefix: "10.0.0.0/16", addr: "10.0.3.150", want: "DHCP pool"}, + {name: "x.x.n.255 is a host address in a /16", prefix: "10.0.0.0/16", addr: "10.0.3.255"}, + {name: "broadcast of a /16", prefix: "10.0.0.0/16", addr: "10.0.255.255", want: "broadcast address"}, + // IPv6 has none of these concepts, so its last address is assignable. + {name: "IPv6 last address", prefix: "2001:db8::/126", addr: "2001:db8::3"}, + {name: "IPv6 subnet-router anycast", prefix: "2001:db8::/126", addr: "2001:db8::"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + prefix, err := netip.ParsePrefix(tc.prefix) + g.Expect(err).ToNot(HaveOccurred()) + addr, err := netip.ParseAddr(tc.addr) + g.Expect(err).ToNot(HaveOccurred()) + + got := unassignableReason(prefix, addr) + if tc.want == "" { + g.Expect(got).To(BeEmpty(), "expected %s to be assignable in %s", tc.addr, tc.prefix) + return + } + g.Expect(got).To(ContainSubstring(tc.want)) + }) + } +} + +// ============================================================================ +// Unit tests for validateAddressInCIDR // ============================================================================ -func TestValidateGatewayInCIDR(t *testing.T) { +func TestValidateAddressInCIDR(t *testing.T) { path := field.NewPath("spec", "networks", "gatewayAddress") cases := []struct { name string @@ -787,7 +1367,7 @@ func TestValidateGatewayInCIDR(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR(tc.cidr, tc.gateway, path) + errs := validateAddressInCIDR(tc.cidr, tc.gateway, path) g.Expect(errs).To(HaveLen(tc.wantErrs)) if tc.wantDetail != "" { g.Expect(errs[0].Detail).To(ContainSubstring(tc.wantDetail)) diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go index 1233b74..56fcef0 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go @@ -75,7 +75,7 @@ type CloudscaleClusterTemplateCustomValidator struct { func (v *CloudscaleClusterTemplateCustomValidator) ValidateCreate(_ context.Context, clusterTemplate *infrastructurev1beta2.CloudscaleClusterTemplate) (admission.Warnings, error) { cloudscaleclustertemplatelog.Info("Validation for CloudscaleClusterTemplate upon creation", "name", clusterTemplate.GetName()) - allErrs := clusterSpecValidateCreate(clusterTemplate.Spec.Template.Spec, v.RegionInfo, field.NewPath("spec", "template", "spec")) + allErrs := clusterSpecValidateCreate(clusterTemplate.Spec.Template.Spec, v.RegionInfo, modeTemplate, field.NewPath("spec", "template", "spec")) if len(allErrs) > 0 { return nil, apierrors.NewInvalid( diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go index d5d890f..188beeb 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go @@ -100,6 +100,18 @@ func TestClusterTemplateDefaulting_ExplicitNetworksNotOverridden(t *testing.T) { g.Expect(obj.Spec.Template.Spec.Networks[0].CIDR).To(Equal("10.1.0.0/16")) } +// Router interface addresses must not be defaulted on the template. +func TestClusterTemplateDefaulting_RouterAddressesNotDefaulted(t *testing.T) { + g := NewWithT(t) + obj, _, _, defaulter := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.0.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + g.Expect(obj.Spec.Template.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) +} + func TestClusterTemplateDefaulting_LBEnabledToTrue(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterTemplateWebhookTestObjects() @@ -388,7 +400,7 @@ func TestClusterTemplateValidateCreate_PublicLBWithMultipleNetworksRequiresExpli _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.network")) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer")) } func TestClusterTemplateValidateCreate_LBNetworkReferenceInvalid(t *testing.T) { @@ -510,6 +522,109 @@ func TestClusterTemplateValidateCreate_PreExistingFloatingIPWithoutLBAllowed(t * g.Expect(err).NotTo(HaveOccurred()) } +// A template's router interface addresses are derived per cluster, not here, so leaving +// one empty is what the ClusterClass flow relies on: a topology patch may replace the +// network's CIDR and the address follows it. +func TestClusterTemplateValidateCreate_RouterInterfaceAddressDeferred(t *testing.T) { + g := NewWithT(t) + obj, _, validator, defaulter := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +// A uuid network never gains a CIDR, so no later defaulting could derive an address for it +// either. +func TestClusterTemplateValidateCreate_RouterInterfaceOnUUIDNetworkStillRequiresAddress(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}} + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("must be set explicitly")) + g.Expect(err.Error()).To(ContainSubstring("referenced by uuid")) +} + +func TestClusterTemplateValidateCreate_RouterInterfaceAddressOutsideCIDRRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("10.20.0.1")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("must be within CIDR 10.10.0.0/24")) +} + +func TestClusterTemplateValidateCreate_RouterInterfaceAddressInDHCPPoolRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("10.10.0.150")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("DHCP pool")) +} + +func TestClusterTemplateValidateCreate_RouterInterfaceUnknownNetworkRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "other", CIDR: "10.10.0.0/24"}} + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("Not found")) +} + +func TestClusterTemplateThenClusterDefaulting_DerivesRouterAddress(t *testing.T) { + for _, tc := range []struct { + name string + patchedCIDR string + wantAddress string + }{ + {name: "template CIDR untouched", patchedCIDR: "10.10.0.0/24", wantAddress: "10.10.0.1"}, + {name: "CIDR replaced by a topology patch", patchedCIDR: "10.20.0.0/24", wantAddress: "10.20.0.1"}, + {name: "CIDR narrowed by a topology patch", patchedCIDR: "172.16.5.0/25", wantAddress: "172.16.5.1"}, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + tmpl, _, tmplValidator, tmplDefaulter := newClusterTemplateWebhookTestObjects() + tmpl.Spec.Template.Spec.Region = RegionRma + tmpl.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") + tmpl.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + g.Expect(tmplDefaulter.Default(ctx, tmpl)).To(Succeed()) + _, err := tmplValidator.ValidateCreate(ctx, tmpl) + g.Expect(err).NotTo(HaveOccurred()) + + // What the topology controller does: copy the template spec, apply the + // patches, then let the cluster webhooks run on the result. + cluster, _, validator, defaulter := newClusterWebhookTestObjects() + cluster.Name = "my-cluster-abcde" + cluster.Spec = *tmpl.Spec.Template.Spec.DeepCopy() + cluster.Spec.Networks[0].CIDR = tc.patchedCIDR + + g.Expect(defaulter.Default(ctx, cluster)).To(Succeed()) + _, err = validator.ValidateCreate(ctx, cluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cluster.Spec.Routers[0].Interfaces[0].Address).To(Equal(tc.wantAddress)) + }) + } +} + // ============================================================================ // Tests for CloudscaleClusterTemplate Validating Webhook - Update // ============================================================================ diff --git a/templates/cluster-class-router-nat.yaml b/templates/cluster-class-router-nat.yaml new file mode 100644 index 0000000..7cd0a83 --- /dev/null +++ b/templates/cluster-class-router-nat.yaml @@ -0,0 +1,377 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: router-nat + namespace: "${NAMESPACE}" +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: router-nat-control-plane + machineInfrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 900 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyLessThanOrEqualTo: 33% + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + name: router-nat-cluster + workers: + machineDeployments: + - class: worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: router-nat-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 600 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyInRange: "[0-2]" + variables: + - name: region + required: true + schema: + openAPIV3Schema: + type: string + default: lpg + enum: + - lpg + - rma + - name: credentialsRefName + required: true + schema: + openAPIV3Schema: + type: string + default: "${CLUSTER_NAME}-credentials" + # networkName is the logical name of the cluster's private network. It is referenced + # from the router interface and from every machine's interface list, so it is patched + # into all three templates from this single variable. Give each cluster its own value + # to keep the networks and routers apart in the cloudscale.ch control panel. + - name: networkName + required: true + schema: + openAPIV3Schema: + type: string + default: node-net + # The router interface address is deliberately not part of the templates: the + # CloudscaleCluster admission webhook derives it from this CIDR when the topology + # controller stamps out the cluster, so changing the CIDR moves the address with it. + - name: networkCIDR + required: true + schema: + openAPIV3Schema: + type: string + default: 10.10.0.0/24 + - name: routerName + required: true + schema: + openAPIV3Schema: + type: string + default: router + - name: cloudscaleControlPlaneMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleControlPlaneMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleControlPlaneMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleWorkerMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleWorkerMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleWorkerMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleSSHKeys + required: true + schema: + openAPIV3Schema: + type: array + items: + type: string + - name: cloudscaleControlPlaneServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + - name: cloudscaleWorkerServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + patches: + - name: CloudscaleClusterTemplate + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/region" + valueFrom: + variable: region + - op: replace + path: "/spec/template/spec/credentialsRef/name" + valueFrom: + variable: credentialsRefName + # The whole list is replaced rather than its individual members, so networkName + # stays a single source of truth instead of a literal repeated per index. + - op: replace + path: "/spec/template/spec/networks" + valueFrom: + template: | + - name: {{ .networkName }} + cidr: {{ .networkCIDR }} + - op: replace + path: "/spec/template/spec/routers" + valueFrom: + template: | + - name: {{ .routerName }} + internetGateway: true + interfaces: + - network: {{ .networkName }} + - name: CloudscaleMachineTemplateControlPlane + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleControlPlaneMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleControlPlaneMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleControlPlaneMachineRootVolumeSize + # Private only: no {type: public} entry, so the nodes reach the internet + # exclusively through the router's SNAT. + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleControlPlaneServerGroupName }} + - name: CloudscaleMachineTemplateWorker + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleWorkerMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleWorkerMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleWorkerMachineRootVolumeSize + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleWorkerServerGroupName }} + - name: KubeadmControlPlaneTemplate + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/kubeadmConfigSpec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} + - name: KubeadmConfigTemplate + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleClusterTemplate +metadata: + name: router-nat-cluster + namespace: "${NAMESPACE}" +spec: + template: + metadata: + labels: {} + spec: + # Every field below is overwritten by the CloudscaleClusterTemplate patches above, so + # nothing here reaches a cluster: the values exist only so the patches' `op: replace` + # paths resolve. + region: lpg + credentialsRef: + name: "${CLUSTER_NAME}-credentials" + networks: + - name: node-net + cidr: 10.10.0.0/24 + routers: + - name: router + internetGateway: true + interfaces: + - network: node-net +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: router-nat-machinetemplate + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: flex-4-2 + image: custom:ubuntu-2404-kube-v1.36.0 + rootVolumeSize: 20 + interfaces: + - network: node-net +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: router-nat-control-plane + namespace: "${NAMESPACE}" +spec: + template: + spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: router-nat-worker + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] diff --git a/templates/cluster-template-public-lb-private-nodes.yaml b/templates/cluster-template-public-lb-private-nodes.yaml index e8f2208..a13279b 100644 --- a/templates/cluster-template-public-lb-private-nodes.yaml +++ b/templates/cluster-template-public-lb-private-nodes.yaml @@ -40,6 +40,12 @@ spec: networks: - name: "${CLUSTER_NAME}" uuid: "${CLOUDSCALE_NETWORK_UUID}" # Pre-existing network (must have NAT for internet egress) + routers: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_ROUTER_UUID}" # Pre-existing router with internet gateway + interfaces: + - uuid: "${CLOUDSCALE_ROUTER_INTERFACE_UUID}" + network: "${CLUSTER_NAME}" credentialsRef: name: "${CLUSTER_NAME}-credentials" --- diff --git a/templates/cluster-template-router-nat.yaml b/templates/cluster-template-router-nat.yaml new file mode 100644 index 0000000..3640e3f --- /dev/null +++ b/templates/cluster-template-router-nat.yaml @@ -0,0 +1,165 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + region: "${CLOUDSCALE_REGION}" + networks: + - name: "${CLUSTER_NAME}" + cidr: "10.10.0.0/24" + mtu: 1500 + routers: + - name: "${CLUSTER_NAME}-router" + internetGateway: true # enables SNAT for outbound internet access + interfaces: + - network: "${CLUSTER_NAME}" + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: "${KUBERNETES_VERSION}" + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-control-plane" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-md-0" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/templates/cluster-template-topology-router-nat.yaml b/templates/cluster-template-topology-router-nat.yaml new file mode 100644 index 0000000..fdfe80a --- /dev/null +++ b/templates/cluster-template-topology-router-nat.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + topology: + classRef: + name: router-nat + namespace: "${NAMESPACE}" + version: "${KUBERNETES_VERSION}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT:=2} + variables: + - name: region + value: "${CLOUDSCALE_REGION}" + # networkName and routerName carry the cluster name so that the networks and routers + # of clusters sharing this ClusterClass stay distinguishable in the cloudscale.ch + # control panel. + - name: networkName + value: "${CLUSTER_NAME}" + - name: networkCIDR + value: "10.10.0.0/24" + - name: routerName + value: "${CLUSTER_NAME}-router" + - name: cloudscaleControlPlaneMachineFlavor + value: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + - name: cloudscaleControlPlaneMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleControlPlaneMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleControlPlaneServerGroupName + value: "${CLUSTER_NAME}-cp" + - name: cloudscaleWorkerMachineFlavor + value: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + - name: cloudscaleWorkerMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleWorkerMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleWorkerServerGroupName + value: "${CLUSTER_NAME}-mp-0" + - name: credentialsRefName + value: "${CLUSTER_NAME}-credentials" + - name: cloudscaleSSHKeys + value: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/test/e2e/config/cloudscale.yaml b/test/e2e/config/cloudscale.yaml index a257c70..bf08685 100644 --- a/test/e2e/config/cloudscale.yaml +++ b/test/e2e/config/cloudscale.yaml @@ -55,8 +55,11 @@ providers: - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-md-remediation.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-pre-existing-network.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-public-lb-private-nodes.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-topology.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-topology-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/clusterclass-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-upgrades.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/metadata.yaml" targetName: metadata.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml new file mode 100644 index 0000000..f841c87 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml +patches: + - path: router-nat.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml new file mode 100644 index 0000000..b28f62f --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + networks: + - name: "${CLUSTER_NAME}" + cidr: "${CLOUDSCALE_NETWORK_CIDR}" + routers: + - name: "${CLUSTER_NAME}-router" + internetGateway: true + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml new file mode 100644 index 0000000..ff3e22f --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale + cni: cilium +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + topology: + classRef: + name: router-nat + namespace: "${NAMESPACE}" + version: "${KUBERNETES_VERSION}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT:=2} + variables: + - name: region + value: "${CLOUDSCALE_REGION}" + # networkName and routerName carry the cluster name so that the networks and routers + # of clusters sharing this ClusterClass stay distinguishable in the cloudscale.ch + # control panel. + - name: networkName + value: "${CLUSTER_NAME}" + - name: networkCIDR + value: "${CLOUDSCALE_NETWORK_CIDR}" + - name: routerName + value: "${CLUSTER_NAME}-router" + - name: cloudscaleControlPlaneMachineFlavor + value: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + - name: cloudscaleControlPlaneMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleControlPlaneMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleControlPlaneServerGroupName + value: "${CLUSTER_NAME}-cp" + - name: cloudscaleWorkerMachineFlavor + value: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + - name: cloudscaleWorkerMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleWorkerMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleWorkerServerGroupName + value: "${CLUSTER_NAME}-mp-0" + - name: credentialsRefName + value: "${CLUSTER_NAME}-credentials" + - name: cloudscaleSSHKeys + value: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml new file mode 100644 index 0000000..7ef8f01 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-topology.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml new file mode 100644 index 0000000..f3bd16b --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml @@ -0,0 +1,369 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: router-nat + namespace: "${NAMESPACE}" +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: router-nat-control-plane + machineInfrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 900 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyLessThanOrEqualTo: 33% + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + name: router-nat-cluster + workers: + machineDeployments: + - class: worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: router-nat-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 600 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyInRange: "[0-2]" + variables: + - name: region + required: true + schema: + openAPIV3Schema: + type: string + default: lpg + enum: + - lpg + - rma + - name: credentialsRefName + required: true + schema: + openAPIV3Schema: + type: string + default: "${CLUSTER_NAME}-credentials" + - name: networkName + required: true + schema: + openAPIV3Schema: + type: string + default: node-net + - name: networkCIDR + required: true + schema: + openAPIV3Schema: + type: string + default: 10.10.0.0/24 + - name: routerName + required: true + schema: + openAPIV3Schema: + type: string + default: router + - name: cloudscaleControlPlaneMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleControlPlaneMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleControlPlaneMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleWorkerMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleWorkerMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleWorkerMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleSSHKeys + required: true + schema: + openAPIV3Schema: + type: array + items: + type: string + - name: cloudscaleControlPlaneServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + - name: cloudscaleWorkerServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + patches: + - name: CloudscaleClusterTemplate + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/region" + valueFrom: + variable: region + - op: replace + path: "/spec/template/spec/credentialsRef/name" + valueFrom: + variable: credentialsRefName + # The whole list is replaced rather than its individual members, so networkName + # stays a single source of truth instead of a literal repeated per index. + - op: replace + path: "/spec/template/spec/networks" + valueFrom: + template: | + - name: {{ .networkName }} + cidr: {{ .networkCIDR }} + - op: replace + path: "/spec/template/spec/routers" + valueFrom: + template: | + - name: {{ .routerName }} + internetGateway: true + interfaces: + - network: {{ .networkName }} + - name: CloudscaleMachineTemplateControlPlane + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleControlPlaneMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleControlPlaneMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleControlPlaneMachineRootVolumeSize + # Private only: no {type: public} entry, so the nodes reach the internet + # exclusively through the router's SNAT. + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleControlPlaneServerGroupName }} + - name: CloudscaleMachineTemplateWorker + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleWorkerMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleWorkerMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleWorkerMachineRootVolumeSize + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleWorkerServerGroupName }} + - name: KubeadmControlPlaneTemplate + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/kubeadmConfigSpec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} + - name: KubeadmConfigTemplate + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleClusterTemplate +metadata: + name: router-nat-cluster + namespace: "${NAMESPACE}" +spec: + template: + metadata: + labels: {} + spec: + # Every field below is overwritten by the CloudscaleClusterTemplate patches above, so + # nothing here reaches a cluster. + region: lpg + credentialsRef: + name: "${CLUSTER_NAME}-credentials" + networks: + - name: node-net + cidr: 10.10.0.0/24 + routers: + - name: router + internetGateway: true + interfaces: + - network: node-net +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: router-nat-machinetemplate + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: flex-4-2 + image: custom:ubuntu-2404-kube-v1.36.0 + rootVolumeSize: 20 + interfaces: + - network: node-net +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: router-nat-control-plane + namespace: "${NAMESPACE}" +spec: + template: + spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: router-nat-worker + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml new file mode 100644 index 0000000..71396c4 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-class.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml deleted file mode 100644 index 0f86d1e..0000000 --- a/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml +++ /dev/null @@ -1,311 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: KubeadmConfigTemplate -metadata: - name: quick-start-worker - namespace: ${NAMESPACE} -spec: - template: - spec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - users: - - groups: sudo - name: capi - shell: /bin/bash - sshAuthorizedKeys: [] - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: cluster.x-k8s.io/v1beta2 -kind: ClusterClass -metadata: - name: quick-start - namespace: ${NAMESPACE} -spec: - controlPlane: - healthCheck: - checks: - nodeStartupTimeoutSeconds: 900 - unhealthyNodeConditions: - - status: Unknown - timeoutSeconds: 300 - type: Ready - - status: "False" - timeoutSeconds: 300 - type: Ready - remediation: - triggerIf: - unhealthyLessThanOrEqualTo: 33% - machineInfrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - name: quick-start-machinetemplate - templateRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: KubeadmControlPlaneTemplate - name: quick-start-control-plane - infrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleClusterTemplate - name: quick-start-cluster - patches: - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/region - valueFrom: - variable: region - - op: replace - path: /spec/template/spec/credentialsRef/name - valueFrom: - variable: credentialsRefName - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleClusterTemplate - matchResources: - infrastructureCluster: true - name: CloudscaleClusterTemplate - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/flavor - valueFrom: - variable: cloudscaleControlPlaneMachineFlavor - - op: replace - path: /spec/template/spec/image - valueFrom: - variable: cloudscaleControlPlaneMachineImage - - op: replace - path: /spec/template/spec/rootVolumeSize - valueFrom: - variable: cloudscaleControlPlaneMachineRootVolumeSize - - op: add - path: /spec/template/spec/serverGroup - valueFrom: - template: | - name: {{ .cloudscaleControlPlaneServerGroupName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - matchResources: - controlPlane: true - name: CloudscaleMachineTemplateControlPlane - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/flavor - valueFrom: - variable: cloudscaleWorkerMachineFlavor - - op: replace - path: /spec/template/spec/image - valueFrom: - variable: cloudscaleWorkerMachineImage - - op: replace - path: /spec/template/spec/rootVolumeSize - valueFrom: - variable: cloudscaleWorkerMachineRootVolumeSize - - op: add - path: /spec/template/spec/serverGroup - valueFrom: - template: | - name: {{ .cloudscaleWorkerServerGroupName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - matchResources: - machineDeploymentClass: - names: - - worker - name: CloudscaleMachineTemplateWorker - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/kubeadmConfigSpec/users - valueFrom: - template: | - - name: capi - groups: sudo - shell: "/bin/bash" - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - name: KubeadmControlPlaneTemplate - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/users - valueFrom: - template: | - - name: capi - groups: sudo - shell: "/bin/bash" - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} - selector: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: KubeadmConfigTemplate - matchResources: - machineDeploymentClass: - names: - - worker - name: KubeadmConfigTemplate - variables: - - name: region - required: true - schema: - openAPIV3Schema: - default: lpg - enum: - - lpg - - rma - type: string - - name: credentialsRefName - required: true - schema: - openAPIV3Schema: - default: ${CLUSTER_NAME}-credentials - type: string - - name: cloudscaleControlPlaneMachineFlavor - required: true - schema: - openAPIV3Schema: - default: flex-4-2 - type: string - - name: cloudscaleControlPlaneMachineImage - required: true - schema: - openAPIV3Schema: - default: custom:ubuntu-2404-kube-v1.36.0 - type: string - - name: cloudscaleControlPlaneMachineRootVolumeSize - required: true - schema: - openAPIV3Schema: - default: 20 - type: integer - - name: cloudscaleWorkerMachineFlavor - required: true - schema: - openAPIV3Schema: - default: flex-4-2 - type: string - - name: cloudscaleWorkerMachineImage - required: true - schema: - openAPIV3Schema: - default: custom:ubuntu-2404-kube-v1.36.0 - type: string - - name: cloudscaleWorkerMachineRootVolumeSize - required: true - schema: - openAPIV3Schema: - default: 20 - type: integer - - name: cloudscaleSSHKeys - required: true - schema: - openAPIV3Schema: - items: - type: string - type: array - - name: cloudscaleControlPlaneServerGroupName - required: true - schema: - openAPIV3Schema: - type: string - - name: cloudscaleWorkerServerGroupName - required: true - schema: - openAPIV3Schema: - type: string - workers: - machineDeployments: - - bootstrap: - templateRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: KubeadmConfigTemplate - name: quick-start-worker - class: worker - healthCheck: - checks: - nodeStartupTimeoutSeconds: 600 - unhealthyNodeConditions: - - status: Unknown - timeoutSeconds: 300 - type: Ready - - status: "False" - timeoutSeconds: 300 - type: Ready - remediation: - triggerIf: - unhealthyInRange: '[0-2]' - infrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - name: quick-start-machinetemplate ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta2 -kind: KubeadmControlPlaneTemplate -metadata: - name: quick-start-control-plane - namespace: ${NAMESPACE} -spec: - template: - spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - users: - - groups: sudo - name: capi - shell: /bin/bash - sshAuthorizedKeys: [] - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: CloudscaleClusterTemplate -metadata: - name: quick-start-cluster - namespace: ${NAMESPACE} -spec: - template: - metadata: - labels: {} - spec: - credentialsRef: - name: ${CLUSTER_NAME}-credentials - region: lpg ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: CloudscaleMachineTemplate -metadata: - name: quick-start-machinetemplate - namespace: ${NAMESPACE} -spec: - template: - spec: - flavor: flex-4-2 - image: custom:ubuntu-2404-kube-v1.36.0 - rootVolumeSize: 20 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6a8cf4b..4afa990 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -40,6 +40,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new(""), + ClusterName: scenarioClusterName("lifecycle"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -57,6 +58,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("ha"), + ClusterName: scenarioClusterName("ha"), ControlPlaneMachineCount: new(int64(3)), WorkerMachineCount: new(int64(2)), PostMachinesProvisioned: validateCloudscaleResources, @@ -76,6 +78,53 @@ var _ = Describe("Workload cluster-class topology", Label("topology"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("topology"), + ClusterName: scenarioClusterName("topology"), + ControlPlaneMachineCount: new(int64(1)), + WorkerMachineCount: new(int64(1)), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) +}) + +// Managed router tests use a router with internetGateway=true: nodes have no public +// interface at all and reach the internet only through the router's SNAT. These are the only +// specs that prove the router path end to end — if the router or its subnet gateway were +// wrong, the nodes would never finish bootstrapping. +var _ = Describe("Managed router NAT", Label("router-nat"), func() { + // Flat template: the network and router are spelled out in the CloudscaleCluster. + Context("With managed router providing NAT", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: new("cloudscale-ch-cloudscale"), + Flavor: new("router-nat"), + ClusterName: scenarioClusterName("router-nat"), + ControlPlaneMachineCount: new(int64(1)), + WorkerMachineCount: new(int64(1)), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) + + // Same topology via the router-nat ClusterClass. This additionally covers the + // CloudscaleClusterTemplate leaving the router interface address open for the + // CloudscaleCluster webhook to derive from the patched network CIDR. + Context("With managed router providing NAT via ClusterClass", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: new("cloudscale-ch-cloudscale"), + Flavor: new("topology-router-nat"), + ClusterName: scenarioClusterName("topology-router-nat"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -106,6 +155,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("pre-existing-network"), + ClusterName: scenarioClusterName("pre-existing-net"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -124,6 +174,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("public-lb-private-nodes"), + ClusterName: scenarioClusterName("public-lb-private-nodes"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -142,6 +193,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("fip"), + ClusterName: scenarioClusterName("fip"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index fad8612..f957332 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -20,17 +20,25 @@ package e2e import ( "context" + "fmt" + "slices" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" infrav1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" ) +// scenarioClusterName returns a randomised cluster name prefixed with the scenario under test. +func scenarioClusterName(scenario string) *string { + return new(fmt.Sprintf("%s-%s", scenario, util.RandomString(6))) +} + // validateCloudscaleResources validates that cloudscale-specific resources are properly created. // This is called by QuickStartSpec after machines are provisioned. func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, clusterName string) { @@ -55,6 +63,45 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste Expect(net.SubnetID).NotTo(BeEmpty(), "Network %d (%s) should have SubnetID", i, net.Name) } + // Validate router resources (if any are configured). Without these the router path is + // only covered implicitly, by the nodes failing to bootstrap when SNAT is broken. + for _, routerSpec := range cloudscaleCluster.Spec.Routers { + rs := cloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + Expect(rs).NotTo(BeNil(), "Router %s should be tracked in status", routerSpec.Name) + Expect(rs.RouterID).NotTo(BeEmpty(), "Router %s should have RouterID", routerSpec.Name) + // A router adopted by uuid must never be reported as CAPCS-managed: managed is what + // decides whether teardown deletes it. + Expect(rs.Managed).To(Equal(routerSpec.UUID == ""), + "Router %s managed flag should reflect whether it was adopted by uuid", routerSpec.Name) + + for _, ifaceSpec := range routerSpec.Interfaces { + ifaceStatus := rs.GetInterfaceStatus(ifaceSpec.Network) + Expect(ifaceStatus).NotTo(BeNil(), "Router %s should have an interface on network %s", routerSpec.Name, ifaceSpec.Network) + Expect(ifaceStatus.InterfaceID).NotTo(BeEmpty(), "Router %s interface on network %s should have InterfaceID", routerSpec.Name, ifaceSpec.Network) + Expect(ifaceStatus.Managed).To(Equal(ifaceSpec.UUID == ""), + "Router %s interface on network %s managed flag should reflect whether it was adopted by uuid", + routerSpec.Name, ifaceSpec.Network) + + if !ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { + continue + } + ns := cloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) + Expect(ns).NotTo(BeNil(), "Network %s should be tracked in status", ifaceSpec.Network) + if ifaceSpec.Address != "" { + // The address the webhook derived is the one the subnet gateway must end up + // pointing at. Checking only for non-emptiness would let a gateway pointed at + // the wrong host in the subnet pass. + Expect(ns.GatewayAddress).To(Equal(ifaceSpec.Address), + "Network %s subnet gateway should be router %s's interface address", ifaceSpec.Network, routerSpec.Name) + } else { + // Adopted interfaces and interfaces on uuid-referenced networks carry no + // spec address: it is only known once the controller reads the router. + Expect(ns.GatewayAddress).NotTo(BeEmpty(), + "Network %s should have its subnet gateway pointed at router %s", ifaceSpec.Network, routerSpec.Name) + } + } + } + // Validate load balancer resources (if enabled - default is true) if ptr.Deref(cloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { Expect(cloudscaleCluster.Status.LoadBalancerID).NotTo(BeEmpty(), "LoadBalancerID should be set") @@ -92,4 +139,24 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste // Validate addresses Expect(machine.Status.Addresses).NotTo(BeEmpty(), "Machine %s should have addresses", machine.Name) } + + // A managed router with internetGateway=true is the cluster's only path to the internet, so + // the nodes must have no public interface of their own. + hasNATRouter := slices.ContainsFunc(cloudscaleCluster.Spec.Routers, func(r infrav1beta2.RouterSpec) bool { + return r.InternetGateway + }) + if hasNATRouter { + By("Validating nodes are private-only behind the router") + + for _, machine := range machineList.Items { + for _, iface := range machine.Spec.Interfaces { + Expect(iface.Type).NotTo(Equal("public"), + "Machine %s should have no public interface: the router provides internet access via SNAT", machine.Name) + } + for _, addr := range machine.Status.Addresses { + Expect(addr.Type).NotTo(Equal(clusterv1.MachineExternalIP), + "Machine %s should report no external IP: the router provides internet access via SNAT", machine.Name) + } + } + } } From 1ac1cb3e9ab5dfade1ca469e9c0988dfb31075c5 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Wed, 19 Aug 2026 17:28:16 +0200 Subject: [PATCH 2/5] feat: resolve LB pool members via poolMemberNetwork Add spec.controlPlaneLoadBalancer.poolMemberNetwork so the LB pool members' subnet can be selected independently of the VIP placement, e.g. a public VIP with private control-plane nodes or a private VIP on a dedicated access network. getPoolMemberSubnetID now prefers an explicit poolMemberNetwork and errors while the referenced network is not provisioned yet, falling back to the previous behaviour (LB network, else the first network) when it is unset. The webhook validates the network reference on create and update, treats the field as strictly immutable (it cannot be set after creation either, since the controller does not migrate live pool members between subnets), and validateLBPoolMemberNetworkResolvable now accepts either network or poolMemberNetwork when multiple networks are defined. --- api/v1beta2/cloudscalecluster_types.go | 15 +++++- ...e.cluster.x-k8s.io_cloudscaleclusters.yaml | 15 +++++- ...r.x-k8s.io_cloudscaleclustertemplates.yaml | 15 +++++- .../cloudscalecluster_loadbalancer.go | 23 +++++---- .../cloudscalecluster_loadbalancer_test.go | 48 ++++++++++++++++++ .../v1beta2/cloudscalecluster_webhook.go | 34 ++++++++++--- .../v1beta2/cloudscalecluster_webhook_test.go | 49 ++++++++++++++++++- 7 files changed, 176 insertions(+), 23 deletions(-) diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index f967d3e..1aeb513 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -251,11 +251,22 @@ type LoadBalancerSpec struct { // Network places the LB VIP on a private network (internal LB). // References spec.networks[].name. Omit for a public LB. - // When multiple networks are defined this field is required so the LB - // pool members can be registered against a specific subnet. + // When multiple networks are defined, either this field or PoolMemberNetwork + // is required so the LB pool members can be registered against a specific subnet. // +optional Network string `json:"network,omitempty"` + // PoolMemberNetwork selects the network whose subnet the LB pool members are + // registered on (i.e. the network the control-plane machines attach to). + // References spec.networks[].name. When unset, the controller resolves the member + // subnet from Network, or from the first network if that is unset too; the field + // itself is left empty rather than defaulted. Set it when the control-plane nodes + // live on a different network than the VIP (Network), e.g. a public VIP with + // private control-plane nodes, or a private VIP on a dedicated access network. + // Strictly immutable: it cannot be changed, nor set after creation. + // +optional + PoolMemberNetwork string `json:"poolMemberNetwork,omitempty"` + // HealthMonitor configures the load balancer health monitor. // +optional HealthMonitor HealthMonitorSpec `json:"healthMonitor,omitempty"` diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index b2c4c3e..f48256c 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -154,8 +154,19 @@ spec: description: |- Network places the LB VIP on a private network (internal LB). References spec.networks[].name. Omit for a public LB. - When multiple networks are defined this field is required so the LB - pool members can be registered against a specific subnet. + When multiple networks are defined, either this field or PoolMemberNetwork + is required so the LB pool members can be registered against a specific subnet. + type: string + poolMemberNetwork: + description: |- + PoolMemberNetwork selects the network whose subnet the LB pool members are + registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. When unset, the controller resolves the member + subnet from Network, or from the first network if that is unset too; the field + itself is left empty rather than defaulted. Set it when the control-plane nodes + live on a different network than the VIP (Network), e.g. a public VIP with + private control-plane nodes, or a private VIP on a dedicated access network. + Strictly immutable: it cannot be changed, nor set after creation. type: string type: object credentialsRef: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml index 9ede64f..30b300d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml @@ -184,8 +184,19 @@ spec: description: |- Network places the LB VIP on a private network (internal LB). References spec.networks[].name. Omit for a public LB. - When multiple networks are defined this field is required so the LB - pool members can be registered against a specific subnet. + When multiple networks are defined, either this field or PoolMemberNetwork + is required so the LB pool members can be registered against a specific subnet. + type: string + poolMemberNetwork: + description: |- + PoolMemberNetwork selects the network whose subnet the LB pool members are + registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. When unset, the controller resolves the member + subnet from Network, or from the first network if that is unset too; the field + itself is left empty rather than defaulted. Set it when the control-plane nodes + live on a different network than the VIP (Network), e.g. a public VIP with + private control-plane nodes, or a private VIP on a dedicated access network. + Strictly immutable: it cannot be changed, nor set after creation. type: string type: object credentialsRef: diff --git a/internal/controller/cloudscalecluster_loadbalancer.go b/internal/controller/cloudscalecluster_loadbalancer.go index 41d57b9..c803e0e 100644 --- a/internal/controller/cloudscalecluster_loadbalancer.go +++ b/internal/controller/cloudscalecluster_loadbalancer.go @@ -207,7 +207,7 @@ func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterSc // Place LB on a private network if specified, otherwise public VIP if lbSpec.Network != "" { - subnetID, err := lbPrivateNetworkSubnetID(clusterScope) + subnetID, err := safeUseSubnet(clusterScope, clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network) if err != nil { return ctrl.Result{}, err } @@ -517,8 +517,14 @@ func (r *CloudscaleClusterReconciler) getMemberSubnetCIDR(clusterScope *scope.Cl // 2. .Network's subnet if specified // 3. first network's subnet func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope.ClusterScope) (string, error) { - if clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network != "" { - return lbPrivateNetworkSubnetID(clusterScope) + lbSpec := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer + + if lbSpec.PoolMemberNetwork != "" { + return safeUseSubnet(clusterScope, lbSpec.PoolMemberNetwork) + } + + if lbSpec.Network != "" { + return safeUseSubnet(clusterScope, lbSpec.Network) } networks := clusterScope.CloudscaleCluster.Status.Networks @@ -531,14 +537,11 @@ func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope. return networks[0].SubnetID, nil } -// lbPrivateNetworkSubnetID returns the subnet UUID of the private network that the LB -// VIP is placed on (spec.controlPlaneLoadBalancer.network). Caller must verify that -// spec.controlPlaneLoadBalancer.network is non-empty before calling. -func lbPrivateNetworkSubnetID(clusterScope *scope.ClusterScope) (string, error) { - name := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network - ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(name) +// safeUseSubnet returns the subnetID of the specified network (by name) only if the network is already successfully provisioned. +func safeUseSubnet(clusterScope *scope.ClusterScope, networkName string) (string, error) { + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(networkName) if ns == nil || ns.SubnetID == "" { - return "", fmt.Errorf("network %q not yet provisioned for LB VIP placement", name) + return "", fmt.Errorf("network %q not yet provisioned", networkName) } return ns.SubnetID, nil } diff --git a/internal/controller/cloudscalecluster_loadbalancer_test.go b/internal/controller/cloudscalecluster_loadbalancer_test.go index b2e44fe..08b6a43 100644 --- a/internal/controller/cloudscalecluster_loadbalancer_test.go +++ b/internal/controller/cloudscalecluster_loadbalancer_test.go @@ -890,3 +890,51 @@ func TestDeleteLoadBalancerMember_RemovesFromStatus(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(clusterScope.CloudscaleCluster.Status.LoadBalancerMemberIDs).To(Equal([]string{"keep-uuid"})) } + +// ============================================================================ +// Tests for getPoolMemberSubnetID +// ============================================================================ + +func TestGetPoolMemberSubnetID(t *testing.T) { + cases := []struct { + name string + lbNetwork string + poolMemberNetwork string + want string + wantErr string + }{ + {name: "poolMemberNetwork wins over network", lbNetwork: "vip", poolMemberNetwork: "nodes", want: "subnet-nodes"}, + {name: "poolMemberNetwork with public VIP", poolMemberNetwork: "nodes", want: "subnet-nodes"}, + {name: "falls back to network", lbNetwork: "vip", want: "subnet-vip"}, + {name: "falls back to first network", want: "subnet-vip"}, + {name: "unprovisioned poolMemberNetwork errors", poolMemberNetwork: "missing", wantErr: "not yet provisioned"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + clusterScope := newLBClusterScope() + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "vip", CIDR: "10.0.0.0/24"}, + {Name: "nodes", CIDR: "10.1.0.0/24"}, + } + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "vip", NetworkID: "net-vip", SubnetID: "subnet-vip", CIDR: "10.0.0.0/24", Managed: true}, + {Name: "nodes", NetworkID: "net-nodes", SubnetID: "subnet-nodes", CIDR: "10.1.0.0/24", Managed: true}, + } + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network = tc.lbNetwork + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = tc.poolMemberNetwork + + subnetID, err := newTestReconciler().getPoolMemberSubnetID(clusterScope) + + if tc.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(subnetID).To(Equal(tc.want)) + }) + } +} diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook.go b/internal/webhook/v1beta2/cloudscalecluster_webhook.go index 3b94a1d..0e96d30 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook.go @@ -302,6 +302,15 @@ func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, )...) } + // Validate LB pool member network reference + if spec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { + allErrs = append(allErrs, validateNetworkReference( + spec.ControlPlaneLoadBalancer.PoolMemberNetwork, + spec.Networks, + fldPath.Child("controlPlaneLoadBalancer", "poolMemberNetwork"), + )...) + } + // Validate floating IP if spec.FloatingIP != nil { allErrs = append(allErrs, validateFloatingIP(spec.FloatingIP, fldPath.Child("floatingIP"))...) @@ -379,6 +388,15 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old )...) } + // Validate LB pool member network reference (for new or existing) + if newClusterSpec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { + allErrs = append(allErrs, validateNetworkReference( + newClusterSpec.ControlPlaneLoadBalancer.PoolMemberNetwork, + newClusterSpec.Networks, + field.NewPath("spec", "controlPlaneLoadBalancer", "poolMemberNetwork"), + )...) + } + // ControlPlaneEndpoint is immutable once set if oldClusterSpec.ControlPlaneEndpoint.Host != "" { if newClusterSpec.ControlPlaneEndpoint.Host != oldClusterSpec.ControlPlaneEndpoint.Host { @@ -795,6 +813,9 @@ func validateFloatingIPRequiresLBOrPreExisting(spec infrastructurev1beta2.Clouds // validateLBImmutability forbids changes to LB fields that are baked into the LB at creation. // Algorithm, Flavor, APIServerPort and the HealthMonitor settings cannot be reissued // to an existing cloudscale.ch LB, so changing them in spec would silently lie to the user. +// PoolMemberNetwork is stricter than the sibling Network field (immutable *once set*): +// even setting it on a cluster that left it empty is forbidden, because the controller +// does not migrate live pool members from one subnet to another. func validateLBImmutability(oldLB, newLB *infrastructurev1beta2.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList @@ -809,6 +830,7 @@ func validateLBImmutability(oldLB, newLB *infrastructurev1beta2.LoadBalancerSpec forbidIfChanged("algorithm", oldLB.Algorithm, newLB.Algorithm) forbidIfChanged("flavor", oldLB.Flavor, newLB.Flavor) forbidIfChanged("apiServerPort", oldLB.APIServerPort, newLB.APIServerPort) + forbidIfChanged("poolMemberNetwork", oldLB.PoolMemberNetwork, newLB.PoolMemberNetwork) hmPath := fldPath.Child("healthMonitor") hmForbid := func(child string, oldV, newV int) { @@ -843,17 +865,17 @@ func validateFloatingIPRequiresPublicLB(spec infrastructurev1beta2.CloudscaleClu return allErrs } -// validateLBPoolMemberNetworkResolvable requires controlPlaneLoadBalancer.network to be set -// when there are multiple networks and the LB is public. Without an explicit network the -// controller would default the LB pool members' subnet to networks[0], which silently -// breaks clusters whose machines join a different network. +// validateLBPoolMemberNetworkResolvable requires either controlPlaneLoadBalancer.network +// or controlPlaneLoadBalancer.poolMemberNetwork to be set when there are multiple networks +// and the LB is public. With neither, the controller falls back to networks[0] for the pool +// members' subnet, which silently breaks clusters whose machines join a different network. func validateLBPoolMemberNetworkResolvable(spec infrastructurev1beta2.CloudscaleClusterSpec, fldPath *field.Path) field.ErrorList { var allErrs = make(field.ErrorList, 0, 1) if !ptr.Deref(spec.ControlPlaneLoadBalancer.Enabled, true) { return nil } - if spec.ControlPlaneLoadBalancer.Network != "" { + if spec.ControlPlaneLoadBalancer.Network != "" || spec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { return nil } if len(spec.Networks) <= 1 { @@ -864,7 +886,7 @@ func validateLBPoolMemberNetworkResolvable(spec infrastructurev1beta2.Cloudscale // the error on one of them would point the user at an arbitrary half of the fix. allErrs = append(allErrs, field.Required( fldPath.Child("controlPlaneLoadBalancer"), - "must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) + "controlPlaneLoadBalancer.network or controlPlaneLoadBalancer.poolMemberNetwork must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) return allErrs } diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go index ac80dc4..87741ca 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go @@ -572,7 +572,31 @@ func TestClusterValidateCreate(t *testing.T) { c.Spec.ControlPlaneLoadBalancer.Network = "" }, wantErr: true, - wantSubstrings: []string{"controlPlaneLoadBalancer"}, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "controlPlaneLoadBalancer.poolMemberNetwork"}, + }, + { + name: "public LB with multiple networks accepts poolMemberNetwork instead", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + {Name: "aux", CIDR: "10.1.0.0/24"}, + } + c.Spec.ControlPlaneLoadBalancer.Network = "" + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "aux" + }, + }, + { + name: "LB.PoolMemberNetwork references unknown network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", CIDR: defaultSubnetCIDR}} + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "nope" + }, + wantErr: true, + wantSubstrings: []string{"controlPlaneLoadBalancer.poolMemberNetwork"}, }, { name: "LB.Network references unknown network", @@ -1234,6 +1258,13 @@ func TestClusterValidateUpdate_LBFieldsImmutable(t *testing.T) { mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { c.Spec.ControlPlaneLoadBalancer.APIServerPort = 8443 }, errPath: "controlPlaneLoadBalancer.apiServerPort", }, + { + name: "PoolMemberNetwork", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "aux" + }, + errPath: "controlPlaneLoadBalancer.poolMemberNetwork", + }, { name: "HealthMonitor.DelayS", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { @@ -1284,6 +1315,22 @@ func TestClusterValidateUpdate_LBFieldsImmutable(t *testing.T) { } } +// TestClusterValidateUpdate_PoolMemberNetworkStrictlyImmutable pins the stricter rule +// for poolMemberNetwork: unlike the sibling network field (immutable *once set*), even +// setting it on a cluster that left it empty is rejected. The controller does not move +// live pool members between subnets. +func TestClusterValidateUpdate_PoolMemberNetworkStrictlyImmutable(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + oldObj.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "" + obj.Spec.ControlPlaneLoadBalancer = *oldObj.Spec.ControlPlaneLoadBalancer.DeepCopy() + obj.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "main" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.poolMemberNetwork")) +} + // ============================================================================ // CloudscaleCluster Validating Webhook - Delete // ============================================================================ From 18935bad61945ae3ee4f3a0c60dff719eeccd7e9 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Mon, 7 Sep 2026 14:32:59 +0200 Subject: [PATCH 3/5] fix(flavors): remove CLUSTER_NAME variable from clusterclass doesnt make sense to require CLUSTER_NAME in the cluster class, instead the credentialsRefName variable needs to be set --- templates/cluster-class-router-nat.yaml | 3 +-- templates/cluster-class.yaml | 3 +-- .../clusterclass-quick-start/cluster-class.yaml | 3 +-- .../clusterclass-router-nat/cluster-class.yaml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/templates/cluster-class-router-nat.yaml b/templates/cluster-class-router-nat.yaml index 7cd0a83..5b35502 100644 --- a/templates/cluster-class-router-nat.yaml +++ b/templates/cluster-class-router-nat.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" # networkName is the logical name of the cluster's private network. It is referenced # from the router interface and from every machine's interface list, so it is patched # into all three templates from this single variable. Give each cluster its own value @@ -301,7 +300,7 @@ spec: # paths resolve. region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "router-nat-credentials" networks: - name: node-net cidr: 10.10.0.0/24 diff --git a/templates/cluster-class.yaml b/templates/cluster-class.yaml index ad0bd90..9ffa629 100644 --- a/templates/cluster-class.yaml +++ b/templates/cluster-class.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" - name: cloudscaleControlPlaneMachineFlavor required: true schema: @@ -245,7 +244,7 @@ spec: spec: region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "quick-start-credentials" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml index ad0bd90..9ffa629 100644 --- a/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" - name: cloudscaleControlPlaneMachineFlavor required: true schema: @@ -245,7 +244,7 @@ spec: spec: region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "quick-start-credentials" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml index f3bd16b..42e4ee2 100644 --- a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" - name: networkName required: true schema: @@ -293,7 +292,7 @@ spec: # nothing here reaches a cluster. region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "router-nat-credentials" networks: - name: node-net cidr: 10.10.0.0/24 From f510d901b519239cee8a9ce8ce1578d49a594c14 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Mon, 7 Sep 2026 17:25:39 +0200 Subject: [PATCH 4/5] docs: reduce property comments to improve readability --- api/v1beta2/cloudscalecluster_types.go | 46 ++++++----------- ...e.cluster.x-k8s.io_cloudscaleclusters.yaml | 51 +++++++------------ ...r.x-k8s.io_cloudscaleclustertemplates.yaml | 41 +++++---------- 3 files changed, 46 insertions(+), 92 deletions(-) diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index 1aeb513..1e641cd 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -158,9 +158,8 @@ type RouterSpec struct { Name string `json:"name"` // UUID references a pre-existing router by UUID. - // CAPCS attaches and detaches the interfaces it created but does not create or delete - // the router, and leaves interfaces it did not create alone. Interfaces the router - // already carries must be adopted explicitly via interfaces[].uuid. + // CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + // Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. // Mutually exclusive with InternetGateway. Immutable after creation. // +optional UUID string `json:"uuid,omitempty"` @@ -185,31 +184,23 @@ type RouterInterfaceSpec struct { Network string `json:"network"` // UUID adopts a pre-existing interface of the router instead of attaching a new one. - // The interface must already be attached to the referenced network. CAPCS reads its - // address but never creates or detaches it. - // Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS - // creates has no pre-existing interfaces. Mutually exclusive with Address. + // CAPCS reads its address but never creates or detaches it. + // Only valid on an adopted router. Mutually exclusive with Address. // Immutable after creation. // +optional UUID string `json:"uuid,omitempty"` - // Address is the IP requested for this router interface within the referenced - // network's subnet. The API requires an explicit address per interface, so the - // admission webhook defaults it: the interface with configureSubnetGateway=true gets - // the network's gatewayAddress, or the first free address in the network's CIDR if - // that is unset. Must be set explicitly when the network is referenced by uuid — its - // subnet is not known at admission time. Must be left empty when this interface is - // adopted by uuid: its address is whatever the pre-existing interface holds. - // If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once - // the subnet gateway is configured. Immutable after creation. + // Address is the IP requested for this router interface within the referenced network's subnet. + // The admission webhook defaults it based on configureSubnetGateway and network settings. + // Must be set when the network is referenced by UUID (subnet unknown at admission time). + // Must be empty when this interface is adopted by UUID. + // Immutable after creation. // +optional Address string `json:"address,omitempty"` // ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - // to this router interface's assigned IP, making the router the default route for - // servers on that subnet. - // Set to false for transit/backbone networks where a different router (e.g. the - // internet-gateway router) owns the subnet gateway. + // to this router interface's assigned IP, making the router the default route. + // Set to false for transit/backbone networks where another router owns the subnet gateway. // Immutable after creation. // +kubebuilder:default=true // +optional @@ -256,13 +247,9 @@ type LoadBalancerSpec struct { // +optional Network string `json:"network,omitempty"` - // PoolMemberNetwork selects the network whose subnet the LB pool members are - // registered on (i.e. the network the control-plane machines attach to). - // References spec.networks[].name. When unset, the controller resolves the member - // subnet from Network, or from the first network if that is unset too; the field - // itself is left empty rather than defaulted. Set it when the control-plane nodes - // live on a different network than the VIP (Network), e.g. a public VIP with - // private control-plane nodes, or a private VIP on a dedicated access network. + // PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + // References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + // When unset, the controller resolves the member subnet from Network or the first network. // Strictly immutable: it cannot be changed, nor set after creation. // +optional PoolMemberNetwork string `json:"poolMemberNetwork,omitempty"` @@ -433,13 +420,10 @@ type RouterInterfaceStatus struct { Network string `json:"network"` // InterfaceID is the router interface UUID. - // Empty only while an attach is in flight: the entry is written before the create call, - // so an attach whose response was lost to a timeout is still recognized as CAPCS-created. // +optional InterfaceID string `json:"interfaceID,omitempty"` - // Managed indicates whether CAPCS created this interface, i.e. whether the spec entry - // left uuid empty. Interfaces adopted by uuid are never detached on teardown. + // Managed indicates whether CAPCS created this interface (spec.uuid was empty). Managed bool `json:"managed"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index f48256c..1b87396 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -159,13 +159,9 @@ spec: type: string poolMemberNetwork: description: |- - PoolMemberNetwork selects the network whose subnet the LB pool members are - registered on (i.e. the network the control-plane machines attach to). - References spec.networks[].name. When unset, the controller resolves the member - subnet from Network, or from the first network if that is unset too; the field - itself is left empty rather than defaulted. Set it when the control-plane nodes - live on a different network than the VIP (Network), e.g. a public VIP with - private control-plane nodes, or a private VIP on a dedicated access network. + PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + When unset, the controller resolves the member subnet from Network or the first network. Strictly immutable: it cannot be changed, nor set after creation. type: string type: object @@ -292,24 +288,18 @@ spec: properties: address: description: |- - Address is the IP requested for this router interface within the referenced - network's subnet. The API requires an explicit address per interface, so the - admission webhook defaults it: the interface with configureSubnetGateway=true gets - the network's gatewayAddress, or the first free address in the network's CIDR if - that is unset. Must be set explicitly when the network is referenced by uuid — its - subnet is not known at admission time. Must be left empty when this interface is - adopted by uuid: its address is whatever the pre-existing interface holds. - If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once - the subnet gateway is configured. Immutable after creation. + Address is the IP requested for this router interface within the referenced network's subnet. + The admission webhook defaults it based on configureSubnetGateway and network settings. + Must be set when the network is referenced by UUID (subnet unknown at admission time). + Must be empty when this interface is adopted by UUID. + Immutable after creation. type: string configureSubnetGateway: default: true description: |- ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - to this router interface's assigned IP, making the router the default route for - servers on that subnet. - Set to false for transit/backbone networks where a different router (e.g. the - internet-gateway router) owns the subnet gateway. + to this router interface's assigned IP, making the router the default route. + Set to false for transit/backbone networks where another router owns the subnet gateway. Immutable after creation. type: boolean network: @@ -318,10 +308,8 @@ spec: uuid: description: |- UUID adopts a pre-existing interface of the router instead of attaching a new one. - The interface must already be attached to the referenced network. CAPCS reads its - address but never creates or detaches it. - Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS - creates has no pre-existing interfaces. Mutually exclusive with Address. + CAPCS reads its address but never creates or detaches it. + Only valid on an adopted router. Mutually exclusive with Address. Immutable after creation. type: string required: @@ -344,9 +332,8 @@ spec: uuid: description: |- UUID references a pre-existing router by UUID. - CAPCS attaches and detaches the interfaces it created but does not create or delete - the router, and leaves interfaces it did not create alone. Interfaces the router - already carries must be adopted explicitly via interfaces[].uuid. + CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. Mutually exclusive with InternetGateway. Immutable after creation. type: string required: @@ -526,15 +513,11 @@ spec: description: RouterInterfaceStatus tracks one router interface. properties: interfaceID: - description: |- - InterfaceID is the router interface UUID. - Empty only while an attach is in flight: the entry is written before the create call, - so an attach whose response was lost to a timeout is still recognized as CAPCS-created. + description: InterfaceID is the router interface UUID. type: string managed: - description: |- - Managed indicates whether CAPCS created this interface, i.e. whether the spec entry - left uuid empty. Interfaces adopted by uuid are never detached on teardown. + description: Managed indicates whether CAPCS created this + interface (spec.uuid was empty). type: boolean network: description: Network matches spec.networks[].name. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml index 30b300d..471b40e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml @@ -189,13 +189,9 @@ spec: type: string poolMemberNetwork: description: |- - PoolMemberNetwork selects the network whose subnet the LB pool members are - registered on (i.e. the network the control-plane machines attach to). - References spec.networks[].name. When unset, the controller resolves the member - subnet from Network, or from the first network if that is unset too; the field - itself is left empty rather than defaulted. Set it when the control-plane nodes - live on a different network than the VIP (Network), e.g. a public VIP with - private control-plane nodes, or a private VIP on a dedicated access network. + PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + When unset, the controller resolves the member subnet from Network or the first network. Strictly immutable: it cannot be changed, nor set after creation. type: string type: object @@ -322,24 +318,18 @@ spec: properties: address: description: |- - Address is the IP requested for this router interface within the referenced - network's subnet. The API requires an explicit address per interface, so the - admission webhook defaults it: the interface with configureSubnetGateway=true gets - the network's gatewayAddress, or the first free address in the network's CIDR if - that is unset. Must be set explicitly when the network is referenced by uuid — its - subnet is not known at admission time. Must be left empty when this interface is - adopted by uuid: its address is whatever the pre-existing interface holds. - If configureSubnetGateway=true, the controller writes the resulting value to status.networks[].gatewayAddress once - the subnet gateway is configured. Immutable after creation. + Address is the IP requested for this router interface within the referenced network's subnet. + The admission webhook defaults it based on configureSubnetGateway and network settings. + Must be set when the network is referenced by UUID (subnet unknown at admission time). + Must be empty when this interface is adopted by UUID. + Immutable after creation. type: string configureSubnetGateway: default: true description: |- ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - to this router interface's assigned IP, making the router the default route for - servers on that subnet. - Set to false for transit/backbone networks where a different router (e.g. the - internet-gateway router) owns the subnet gateway. + to this router interface's assigned IP, making the router the default route. + Set to false for transit/backbone networks where another router owns the subnet gateway. Immutable after creation. type: boolean network: @@ -348,10 +338,8 @@ spec: uuid: description: |- UUID adopts a pre-existing interface of the router instead of attaching a new one. - The interface must already be attached to the referenced network. CAPCS reads its - address but never creates or detaches it. - Only valid on an adopted router (spec.routers[].uuid set), since a router CAPCS - creates has no pre-existing interfaces. Mutually exclusive with Address. + CAPCS reads its address but never creates or detaches it. + Only valid on an adopted router. Mutually exclusive with Address. Immutable after creation. type: string required: @@ -375,9 +363,8 @@ spec: uuid: description: |- UUID references a pre-existing router by UUID. - CAPCS attaches and detaches the interfaces it created but does not create or delete - the router, and leaves interfaces it did not create alone. Interfaces the router - already carries must be adopted explicitly via interfaces[].uuid. + CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. Mutually exclusive with InternetGateway. Immutable after creation. type: string required: From 3146c32a0e2bcec8ac4c956f20abb72869029741 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Fri, 18 Sep 2026 14:59:27 +0200 Subject: [PATCH 5/5] remove configureSubnetGateway and address defaulting --- api/v1beta2/cloudscalecluster_types.go | 12 +- api/v1beta2/zz_generated.deepcopy.go | 9 +- ...e.cluster.x-k8s.io_cloudscaleclusters.yaml | 12 +- ...r.x-k8s.io_cloudscaleclustertemplates.yaml | 12 +- .../cloudscalecluster_controller.go | 11 - .../controller/cloudscalecluster_network.go | 7 - .../controller/cloudscalecluster_router.go | 64 +--- .../cloudscalecluster_router_test.go | 169 +-------- .../v1beta2/cloudscalecluster_webhook.go | 274 +++----------- .../v1beta2/cloudscalecluster_webhook_test.go | 336 +----------------- .../cloudscaleclustertemplate_webhook.go | 7 +- .../cloudscaleclustertemplate_webhook_test.go | 79 ---- templates/cluster-class-router-nat.yaml | 13 +- templates/cluster-template-router-nat.yaml | 2 + test/e2e/config/cloudscale.yaml | 1 + .../router-nat.yaml | 2 + .../cluster-topology.yaml | 2 + .../cluster-class.yaml | 10 + test/e2e/e2e_test.go | 5 +- test/e2e/helpers.go | 18 - 20 files changed, 108 insertions(+), 937 deletions(-) diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index 1e641cd..d3d5f59 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -191,20 +191,10 @@ type RouterInterfaceSpec struct { UUID string `json:"uuid,omitempty"` // Address is the IP requested for this router interface within the referenced network's subnet. - // The admission webhook defaults it based on configureSubnetGateway and network settings. - // Must be set when the network is referenced by UUID (subnet unknown at admission time). - // Must be empty when this interface is adopted by UUID. + // Must be set explicitly. Must be empty when this interface is adopted by UUID. // Immutable after creation. // +optional Address string `json:"address,omitempty"` - - // ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - // to this router interface's assigned IP, making the router the default route. - // Set to false for transit/backbone networks where another router owns the subnet gateway. - // Immutable after creation. - // +kubebuilder:default=true - // +optional - ConfigureSubnetGateway *bool `json:"configureSubnetGateway,omitempty"` } // LoadBalancerSpec defines the load balancer configuration for the control plane. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 24e26b2..9095330 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -685,11 +685,6 @@ func (in *NodeInfo) DeepCopy() *NodeInfo { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RouterInterfaceSpec) DeepCopyInto(out *RouterInterfaceSpec) { *out = *in - if in.ConfigureSubnetGateway != nil { - in, out := &in.ConfigureSubnetGateway, &out.ConfigureSubnetGateway - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterInterfaceSpec. @@ -723,9 +718,7 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { if in.Interfaces != nil { in, out := &in.Interfaces, &out.Interfaces *out = make([]RouterInterfaceSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + copy(*out, *in) } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index 1b87396..e128102 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -289,19 +289,9 @@ spec: address: description: |- Address is the IP requested for this router interface within the referenced network's subnet. - The admission webhook defaults it based on configureSubnetGateway and network settings. - Must be set when the network is referenced by UUID (subnet unknown at admission time). - Must be empty when this interface is adopted by UUID. + Must be set explicitly. Must be empty when this interface is adopted by UUID. Immutable after creation. type: string - configureSubnetGateway: - default: true - description: |- - ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - to this router interface's assigned IP, making the router the default route. - Set to false for transit/backbone networks where another router owns the subnet gateway. - Immutable after creation. - type: boolean network: description: Network references spec.networks[].name. type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml index 471b40e..f5fae3d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml @@ -319,19 +319,9 @@ spec: address: description: |- Address is the IP requested for this router interface within the referenced network's subnet. - The admission webhook defaults it based on configureSubnetGateway and network settings. - Must be set when the network is referenced by UUID (subnet unknown at admission time). - Must be empty when this interface is adopted by UUID. + Must be set explicitly. Must be empty when this interface is adopted by UUID. Immutable after creation. type: string - configureSubnetGateway: - default: true - description: |- - ConfigureSubnetGateway, when true (default), sets the subnet's gatewayAddress - to this router interface's assigned IP, making the router the default route. - Set to false for transit/backbone networks where another router owns the subnet gateway. - Immutable after creation. - type: boolean network: description: Network references spec.networks[].name. type: string diff --git a/internal/controller/cloudscalecluster_controller.go b/internal/controller/cloudscalecluster_controller.go index ab31e8c..af8ef31 100644 --- a/internal/controller/cloudscalecluster_controller.go +++ b/internal/controller/cloudscalecluster_controller.go @@ -336,17 +336,6 @@ func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope * if iface == nil || iface.InterfaceID == "" { return false } - // When ConfigureSubnetGateway is true, the subnet gateway must be configured - if ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { - ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) - if ns == nil || ns.GatewayAddress == "" { - return false - } - // If the spec has an address, verify the gateway matches it - if ifaceSpec.Address != "" && ns.GatewayAddress != ifaceSpec.Address { - return false - } - } } } diff --git a/internal/controller/cloudscalecluster_network.go b/internal/controller/cloudscalecluster_network.go index 207954d..01e7bb6 100644 --- a/internal/controller/cloudscalecluster_network.go +++ b/internal/controller/cloudscalecluster_network.go @@ -301,13 +301,6 @@ func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.Clust }) } -// setNetworkGatewayAddress records the configured subnet gateway IP in the network status entry. -func (r *CloudscaleClusterReconciler) setNetworkGatewayAddress(clusterScope *scope.ClusterScope, networkName, gatewayAddress string) { - if ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(networkName); ns != nil { - ns.GatewayAddress = gatewayAddress - } -} - // networkTags returns the tags for a specific named network, combining cluster ownership with network name. func (r *CloudscaleClusterReconciler) networkTags(clusterScope *scope.ClusterScope, networkName string) cloudscalesdk.TagMap { tags := cloudscalesdk.TagMap{ diff --git a/internal/controller/cloudscalecluster_router.go b/internal/controller/cloudscalecluster_router.go index f4a1e14..f708b1e 100644 --- a/internal/controller/cloudscalecluster_router.go +++ b/internal/controller/cloudscalecluster_router.go @@ -26,7 +26,6 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" @@ -225,35 +224,25 @@ func (r *CloudscaleClusterReconciler) reconcileRouterInterfaces( return ctrl.Result{}, fmt.Errorf("network %q not found in status for router %q interface", ifaceSpec.Network, routerSpec.Name) } - // We always attach at the address the spec asks for, unless the router turns out - // to already be attached to this network, in which case the interface keeps the - // address it has: an attachment cannot be moved. - address := ifaceSpec.Address live, attached := liveByNetwork[ns.NetworkID] switch { case ifaceSpec.UUID != "": - adopted, err := r.adoptRouterInterface(clusterScope, logger, routerSpec, ifaceSpec, ns, liveByUUID) + _, err := r.adoptRouterInterface(clusterScope, logger, routerSpec, ifaceSpec, ns, liveByUUID) if err != nil { return ctrl.Result{}, err } - address = adopted case attached: - claimed, err := r.claimRouterInterface(clusterScope, logger, routerSpec, rs, ifaceSpec, ns, live) + _, err := r.claimRouterInterface(clusterScope, logger, routerSpec, rs, ifaceSpec, ns, live) if err != nil { return ctrl.Result{}, err } - address = claimed default: result, err := r.createRouterInterface(ctx, clusterScope, logger, routerSpec, router, ifaceSpec, ns) if err != nil || !result.IsZero() { return result, err } } - - if result, err := r.reconcileSubnetGateway(ctx, clusterScope, logger, routerSpec, ifaceSpec, ns, address); err != nil || !result.IsZero() { - return result, err - } } return ctrl.Result{}, nil @@ -356,55 +345,6 @@ func (r *CloudscaleClusterReconciler) createRouterInterface( return ctrl.Result{}, nil } -// reconcileSubnetGateway points the subnet's gateway at the address the router interface -// holds, making the router the default route for servers on that subnet. -func (r *CloudscaleClusterReconciler) reconcileSubnetGateway( - ctx context.Context, - clusterScope *scope.ClusterScope, - logger logr.Logger, - routerSpec infrastructurev1beta2.RouterSpec, - ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, - ns *infrastructurev1beta2.NetworkStatus, - address string, -) (ctrl.Result, error) { - // no configuration of the subnet gateway requested - if !ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { - return ctrl.Result{}, nil - } - // no address set - if address == "" { - // The API always returns the interface's address, so this cannot normally - // happen. Guard anyway: writing the empty address would strip the subnet's - // default route from every server on it. - logger.Info("Router interface holds no address on the tracked subnet, leaving the subnet gateway untouched", "subnetID", ns.SubnetID) - r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeWarning, "RouterInterfaceAddressUnknown", "ReconcileRouterInterface", - "Router %s is attached to network %s but holds no address on its subnet; the subnet gateway was left unchanged", - routerSpec.Name, ifaceSpec.Network) - return ctrl.Result{}, nil - } - // gateway address already correct - if ns.GatewayAddress == address { - return ctrl.Result{}, nil - } - - logger.Info("Configuring subnet gateway", "address", address) - updateCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) - err := clusterScope.CloudscaleClient.Subnets.Update(updateCtx, ns.SubnetID, &cloudscalesdk.SubnetUpdateRequest{ - GatewayAddress: address, - }) - cancel() - if err != nil { - if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Subnet gateway update"); timedOut { - return result, nil - } - return ctrl.Result{}, fmt.Errorf("updating subnet gateway for router %q network %q: %w", routerSpec.Name, ifaceSpec.Network, err) - } - r.setNetworkGatewayAddress(clusterScope, ifaceSpec.Network, address) - r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "SubnetGatewayConfigured", "ConfigureSubnetGateway", - "Configured subnet %s gateway to router interface address %s", ifaceSpec.Network, address) - return ctrl.Result{}, nil -} - // interfaceAddress returns the address the interface holds on the given subnet, or "" when // it holds none: a network can carry more than one subnet, and the router may be attached // on a different one than CAPCS tracks. diff --git a/internal/controller/cloudscalecluster_router_test.go b/internal/controller/cloudscalecluster_router_test.go index 0f0a04b..49b9839 100644 --- a/internal/controller/cloudscalecluster_router_test.go +++ b/internal/controller/cloudscalecluster_router_test.go @@ -69,7 +69,7 @@ func routerSpec() infrastructurev1beta2.RouterSpec { Name: "test-router", InternetGateway: true, Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ - {Network: "test", Address: routerAddress, ConfigureSubnetGateway: new(true)}, + {Network: "test", Address: routerAddress}, }, } } @@ -103,7 +103,6 @@ func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { var interfaceRouterID string var capturedCreateReq *cloudscalesdk.RouterCreateRequest var capturedIfaceReq cloudscalesdk.CreateInterfaceRequest - var capturedSubnetUpdateReq *cloudscalesdk.SubnetUpdateRequest routerService := &testutils.MockRouterService{ ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { return nil, nil @@ -118,12 +117,7 @@ func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil }, } - subnetService := &testutils.MockSubnetService{ - UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { - capturedSubnetUpdateReq = req - return nil - }, - } + subnetService := &testutils.MockSubnetService{} clusterScope := testutils.NewClusterScopeOpts( testutils.WithRouterService(routerService), @@ -151,10 +145,6 @@ func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { g.Expect(capturedIfaceReq.Addresses[0].Subnet).To(Equal(subnetUUID)) g.Expect(capturedIfaceReq.Addresses[0].Address).To(Equal("10.0.0.1")) - g.Expect(capturedSubnetUpdateReq).ToNot(BeNil()) - g.Expect(capturedSubnetUpdateReq.GatewayAddress).To(Equal(capturedIfaceReq.Addresses[0].Address)) - g.Expect(capturedSubnetUpdateReq.DNSServers).To(BeNil()) - rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") g.Expect(rs).ToNot(BeNil()) g.Expect(rs.RouterID).To(Equal(routerUUID)) @@ -162,91 +152,13 @@ func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { // Keyed by the CAPCS network name, holding the interface UUID (not the network's). g.Expect(rs.Interfaces).To(Equal(managedInterfaces())) - // The exact, ordered events for the whole managed path. Order is what pins the - // create-then-configure sequencing, which presence checks alone would not catch. + // The exact, ordered events for the managed path. g.Expect(recordedEvents(r)).To(Equal([]string{ "Normal RouterCreated Created router test-router (router-uuid-123) in zone rma1", "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1", - "Normal SubnetGatewayConfigured Configured subnet test gateway to router interface address 10.0.0.1", })) } -func TestReconcileRouters_ConfiguresSubnetGateway(t *testing.T) { - g := NewWithT(t) - - var updatedSubnetID string - var capturedUpdate *cloudscalesdk.SubnetUpdateRequest - updateCalls := 0 - routerService := &testutils.MockRouterService{ - GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { - return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil - }, - } - subnetService := &testutils.MockSubnetService{ - UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { - updateCalls++ - updatedSubnetID = id - capturedUpdate = req - return nil - }, - } - - clusterScope := testutils.NewClusterScopeOpts( - testutils.WithRouterService(routerService), - testutils.WithSubnetService(subnetService), - testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), - testutils.WithRouter(routerSpec()), - testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), - ) - r := newTestReconciler() - - _, err := r.reconcileRouters(context.Background(), clusterScope) - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(updateCalls).To(Equal(1)) - g.Expect(updatedSubnetID).To(Equal(subnetUUID)) - g.Expect(capturedUpdate.GatewayAddress).To(Equal("10.0.0.1")) - // Recorded in status so the next reconcile skips the update. - ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") - g.Expect(ns.GatewayAddress).To(Equal("10.0.0.1")) - - _, err = r.reconcileRouters(context.Background(), clusterScope) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(updateCalls).To(Equal(1)) -} - -func TestReconcileRouters_SkipsSubnetGatewayWhenNotOwner(t *testing.T) { - g := NewWithT(t) - - routerService := &testutils.MockRouterService{ - GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { - return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil - }, - } - subnetService := &testutils.MockSubnetService{ - UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { - g.Fail("Subnets.Update should not be called when configureSubnetGateway is false") - return nil - }, - } - - spec := routerSpec() - spec.Interfaces[0].ConfigureSubnetGateway = new(false) - - clusterScope := testutils.NewClusterScopeOpts( - testutils.WithRouterService(routerService), - testutils.WithSubnetService(subnetService), - testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), - testutils.WithRouter(spec), - testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), - ) - r := newTestReconciler() - - _, err := r.reconcileRouters(context.Background(), clusterScope) - - g.Expect(err).ToNot(HaveOccurred()) -} - // --- Pre-existing (adopted) routers --- // TestReconcileRouters_AdoptedInterfaceOwnership verifies the reconciler does correctly @@ -269,9 +181,6 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { mismatchEvent = `Warning RouterInterfaceAddressMismatch Router test-router is attached to network test at "10.0.0.42", not at the requested "10.0.0.1", which therefore has no effect` ifaceCreatedEvent = "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1" ) - gatewaySetEvent := func(address string) string { - return "Normal SubnetGatewayConfigured Configured subnet test gateway to router interface address " + address - } tests := []struct { name string @@ -281,15 +190,12 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { adopt string // status is the interface status status []infrastructurev1beta2.RouterInterfaceStatus - // networkGateway is the gateway address. - networkGateway string // wantErr is a substring of the error the reconcile has to fail with. The // reconcile is run twice either way: a collision must stay a collision. wantErr string wantInterfaces []infrastructurev1beta2.RouterInterfaceStatus wantCreate bool - wantGatewaySet string // wantEvents is the exact, ordered set of events the reconcile must emit. wantEvents []string }{ @@ -300,28 +206,23 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { live: []cloudscalesdk.RouterInterface{foreign}, wantInterfaces: managedInterfaces(), wantCreate: true, - wantGatewaySet: routerAddress, - wantEvents: []string{ifaceCreatedEvent, gatewaySetEvent(routerAddress)}, + wantEvents: []string{ifaceCreatedEvent}, }, { - name: "pre-existing attachment at the spec address is a collision", - live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, routerAddress)}, - networkGateway: routerAddress, - wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + name: "pre-existing attachment at the spec address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, routerAddress)}, + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", }, { - name: "pre-existing attachment at another address is a collision", - live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, - networkGateway: "10.0.0.42", - wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + name: "pre-existing attachment at another address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", }, { name: "adopted by uuid is used and left alone", live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, adopt: preAttachedUUID, wantInterfaces: interfaceStatus(preAttachedUUID, false), - wantGatewaySet: "10.0.0.42", - wantEvents: []string{gatewaySetEvent("10.0.0.42")}, }, { name: "adopted uuid the router does not carry", @@ -342,14 +243,12 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { name: "attachment recorded before its response arrived", live: []cloudscalesdk.RouterInterface{routerInterface(ifaceUUID, routerAddress)}, status: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, - networkGateway: routerAddress, wantInterfaces: managedInterfaces(), }, { name: "replaced interface stays managed", live: []cloudscalesdk.RouterInterface{routerInterface("replacement-iface-uuid", "10.0.0.42")}, status: managedInterfaces(), - networkGateway: "10.0.0.42", wantInterfaces: interfaceStatus("replacement-iface-uuid", true), wantEvents: []string{mismatchEvent}, }, @@ -376,16 +275,7 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { }, } - var gatewaySet string - subnetService := &testutils.MockSubnetService{ - UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { - g.Expect(tc.wantGatewaySet).ToNot(BeEmpty(), - "Subnets.Update must not be called; the gateway is already correct or the address is unknown") - g.Expect(id).To(Equal(subnetUUID)) - gatewaySet = req.GatewayAddress - return nil - }, - } + subnetService := &testutils.MockSubnetService{} spec := routerSpec() spec.UUID = routerUUID @@ -406,7 +296,6 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { opts = append(opts, testutils.WithRouterStatus("test-router", routerUUID, false, tc.status)) } clusterScope := testutils.NewClusterScopeOpts(opts...) - clusterScope.CloudscaleCluster.Status.Networks[0].GatewayAddress = tc.networkGateway r := newTestReconciler() _, err := r.reconcileRouters(context.Background(), clusterScope) @@ -417,21 +306,12 @@ func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) } g.Expect(created).To(Equal(tc.wantCreate)) - g.Expect(gatewaySet).To(Equal(tc.wantGatewaySet)) rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") g.Expect(rs).ToNot(BeNil()) g.Expect(rs.Managed).To(BeFalse(), "an adopted router is never CAPCS-managed") g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) - // The recorded gateway tracks whatever was actually written, so the next - // reconcile knows whether there is anything left to do. - wantRecorded := tc.networkGateway - if tc.wantGatewaySet != "" { - wantRecorded = tc.wantGatewaySet - } - g.Expect(clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test").GatewayAddress).To(Equal(wantRecorded)) - g.Expect(recordedEvents(r)).To(Equal(tc.wantEvents)) // Reconciling again must not revisit any of it, and must not talk itself @@ -518,7 +398,7 @@ func TestReconcileRouters_AttachesMultipleInterfaces(t *testing.T) { spec := routerSpec() spec.Interfaces = append(spec.Interfaces, infrastructurev1beta2.RouterInterfaceSpec{ - Network: "transit", Address: "10.9.0.2", ConfigureSubnetGateway: new(false), + Network: "transit", Address: "10.9.0.2", }) clusterScope := testutils.NewClusterScopeOpts( @@ -537,9 +417,6 @@ func TestReconcileRouters_AttachesMultipleInterfaces(t *testing.T) { netUUID: {Subnet: subnetUUID, Address: routerAddress}, transitNetUUID: {Subnet: transitSubnetUUID, Address: "10.9.0.2"}, })) - // Only the gateway-owning interface rewrites a subnet; the transit subnet keeps - // whatever gateway its own router set. - g.Expect(gatewayUpdates).To(Equal(map[string]string{subnetUUID: routerAddress})) rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") g.Expect(rs).ToNot(BeNil()) @@ -728,27 +605,7 @@ func TestReconcileRouters_TimeoutsRequeue(t *testing.T) { wantInterfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, wantProvisioned: false, }, - { - name: "subnet gateway update", - routerService: func() *testutils.MockRouterService { - return &testutils.MockRouterService{ - GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { - return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil - }, - } - }, - subnetService: func() *testutils.MockSubnetService { - return &testutils.MockSubnetService{ - UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { - return timeout("Patch", "subnets/"+id) - }, - } - }, - seedStatus: true, - // The attach itself succeeded, so its interface is recorded in full. - wantInterfaces: managedInterfaces(), - wantProvisioned: false, - }, + { name: "interface create succeeds", routerService: func() *testutils.MockRouterService { diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook.go b/internal/webhook/v1beta2/cloudscalecluster_webhook.go index 0e96d30..bc87672 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook.go @@ -19,9 +19,7 @@ package v1beta2 import ( "context" "fmt" - "maps" "net/netip" - "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" @@ -87,10 +85,6 @@ func (d *CloudscaleClusterCustomDefaulter) Default(_ context.Context, cluster *i clusterSpecDefault(&cluster.Spec) - // Router interface addresses are derived from the networks' CIDRs and are immutable - // once set, so they are defaulted here rather than in the shared defaulter. - defaultRouterInterfaceAddresses(&cluster.Spec) - return nil } @@ -128,108 +122,6 @@ func clusterSpecDefault(spec *infrastructurev1beta2.CloudscaleClusterSpec) { } } -// defaultRouterInterfaceAddresses assigns an address to every router interface that -// does not declare one. -// Interfaces on a network referenced by uuid are left untouched: that subnet is only -// known at runtime, so the address cannot be derived here. -func defaultRouterInterfaceAddresses(spec *infrastructurev1beta2.CloudscaleClusterSpec) { - networks := indexNetworks(spec.Networks) - - usedByNetwork := make(map[string]map[string]struct{}) - for ri := range spec.Routers { - for ii := range spec.Routers[ri].Interfaces { - ifaceSpec := &spec.Routers[ri].Interfaces[ii] - if usedByNetwork[ifaceSpec.Network] == nil { - usedByNetwork[ifaceSpec.Network] = make(map[string]struct{}) - } - if ifaceSpec.Address != "" { - usedByNetwork[ifaceSpec.Network][ifaceSpec.Address] = struct{}{} - } - } - } - - for ri := range spec.Routers { - for ii := range spec.Routers[ri].Interfaces { - ifaceSpec := &spec.Routers[ri].Interfaces[ii] - - // Normalise here rather than relying on the CRD default, so the value is - // set no matter which path reached this defaulter. - if ifaceSpec.ConfigureSubnetGateway == nil { - ifaceSpec.ConfigureSubnetGateway = new(true) - } - // An adopted interface keeps the address it already holds, which is only - // known once the controller reads the router. - if ifaceSpec.Address != "" || ifaceSpec.UUID != "" { - continue - } - - used := usedByNetwork[ifaceSpec.Network] - - // The gateway owner takes the address the subnet already advertises. If - // another interface has claimed it, fall through and allocate instead. - gateway := networks[ifaceSpec.Network].GatewayAddress - if gateway != "" && *ifaceSpec.ConfigureSubnetGateway { - if _, taken := used[gateway]; !taken { - ifaceSpec.Address = gateway - used[gateway] = struct{}{} - continue - } - } - - // err is ignored: an unparseable CIDR leaves the zero Prefix, which - // nextFreeAddress skips. validateNetworks reports the malformed value. - prefix, _ := netip.ParsePrefix(networks[ifaceSpec.Network].CIDR) - if addr, ok := nextFreeAddress(prefix, used); ok { - ifaceSpec.Address = addr - used[addr] = struct{}{} - } - } - } -} - -// .101-.254 of every subnet is DHCP. -const ( - dhcpPoolFirstOctet = 101 - dhcpPoolLastOctet = 254 -) - -// unassignableReason says why addr cannot be given to a router interface inside prefix, or -// "" when it can. An address in the DHCP pool may later be handed to a node, and the -// network and broadcast addresses cannot be held by an interface at all. -func unassignableReason(prefix netip.Prefix, addr netip.Addr) string { - if !addr.Is4() { - return "" - } - switch lastOctet := addr.As4()[3]; { - case addr == prefix.Masked().Addr(): - return "it is the network address" - case !prefix.Contains(addr.Next()): - return "it is the broadcast address" - case lastOctet >= dhcpPoolFirstOctet && lastOctet <= dhcpPoolLastOctet: - return fmt.Sprintf("it is inside the DHCP pool .%d-.%d", dhcpPoolFirstOctet, dhcpPoolLastOctet) - } - return "" -} - -// nextFreeAddress returns the first assignable address inside prefix that is not already -// used. It reports false for an invalid prefix or an exhausted range. -func nextFreeAddress(prefix netip.Prefix, used map[string]struct{}) (string, bool) { - if !prefix.IsValid() { - return "", false - } - for addr := prefix.Masked().Addr().Next(); prefix.Contains(addr); addr = addr.Next() { - // Step over rather than stop: on a prefix wider than a /24 the DHCP pool recurs - // inside the range, so there can be assignable addresses above it. - if unassignableReason(prefix, addr) != "" { - continue - } - if _, taken := used[addr.String()]; !taken { - return addr.String(), true - } - } - return "", false -} - // +kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta2-cloudscalecluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=cloudscaleclusters,verbs=create;update,versions=v1beta2,name=vcloudscalecluster-v1beta2.kb.io,admissionReviewVersions=v1 // CloudscaleClusterCustomValidator struct is responsible for validating the CloudscaleCluster resource @@ -245,36 +137,19 @@ type CloudscaleClusterCustomValidator struct { func (v *CloudscaleClusterCustomValidator) ValidateCreate(_ context.Context, cluster *infrastructurev1beta2.CloudscaleCluster) (admission.Warnings, error) { cloudscaleclusterlog.Info("Validation for CloudscaleCluster upon creation", "name", cluster.GetName()) - allErrs := clusterSpecValidateCreate(cluster.Spec, v.RegionInfo, modeCluster, field.NewPath("spec")) + allErrs := clusterSpecValidateCreate(cluster.Spec, v.RegionInfo, field.NewPath("spec")) + warnings := checkSNATConfiguration(cluster.Spec) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, cluster.Name, allErrs) } - return nil, nil -} - -// validationMode captures what a spec has already been through by the time it reaches the -// validator. A CloudscaleCluster arrives defaulted; a CloudscaleClusterTemplate does not, -// because defaultRouterInterfaceAddresses only runs on the cluster (see Default). -type validationMode struct { - // routerAddressesDefaulted is true when defaultRouterInterfaceAddresses has already run, - // so an empty router interface address means "could not be derived" rather than "will be - // derived when a cluster is stamped from this template". - routerAddressesDefaulted bool + return warnings, nil } -var ( - // modeCluster validates a CloudscaleCluster, i.e. a fully defaulted spec. - modeCluster = validationMode{routerAddressesDefaulted: true} - // modeTemplate validates a CloudscaleClusterTemplate, whose router interface addresses - // are only derived once the topology controller stamps out a CloudscaleCluster. - modeTemplate = validationMode{} -) - -func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, regionInfo *cloudscale.RegionInfo, mode validationMode, fldPath *field.Path) field.ErrorList { +func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, regionInfo *cloudscale.RegionInfo, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList // Validate zone belongs to region @@ -291,7 +166,7 @@ func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, allErrs = append(allErrs, validateNetworks(spec.Networks, fldPath.Child("networks"))...) // Validate routers - allErrs = append(allErrs, validateRouters(spec.Routers, spec.Networks, mode, fldPath.Child("routers"))...) + allErrs = append(allErrs, validateRouters(spec.Routers, spec.Networks, fldPath.Child("routers"))...) // Validate LB network reference if spec.ControlPlaneLoadBalancer.Network != "" { @@ -327,6 +202,7 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old cloudscaleclusterlog.Info("Validation for CloudscaleCluster upon update", "name", newCluster.GetName()) var allErrs field.ErrorList + warnings := checkSNATConfiguration(newCluster.Spec) newClusterSpec := newCluster.Spec oldClusterSpec := oldCluster.Spec @@ -353,7 +229,7 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old // The updated router set must satisfy the creation rules routersPath := field.NewPath("spec", "routers") - allErrs = append(allErrs, validateRouters(newClusterSpec.Routers, newClusterSpec.Networks, modeCluster, routersPath)...) + allErrs = append(allErrs, validateRouters(newClusterSpec.Routers, newClusterSpec.Networks, routersPath)...) // ...and on top of that, nothing already there may change or disappear. allErrs = append(allErrs, validateRouterDiff(oldClusterSpec.Routers, newClusterSpec.Routers, routersPath)...) @@ -424,12 +300,12 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old allErrs = append(allErrs, validateLBPoolMemberNetworkResolvable(newClusterSpec, field.NewPath("spec"))...) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, newCluster.Name, allErrs) } - return nil, nil + return warnings, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type CloudscaleCluster. @@ -565,13 +441,12 @@ func indexNetworks(networks []infrastructurev1beta2.NetworkSpec) map[string]infr // validateRouters runs every rule that applies to a router set, whether it was declared // at creation or added later. ValidateUpdate calls it on the new set too, so additions // are held to the same standard as originals without restating the rules. -func validateRouters(routers []infrastructurev1beta2.RouterSpec, networks []infrastructurev1beta2.NetworkSpec, mode validationMode, fldPath *field.Path) field.ErrorList { +func validateRouters(routers []infrastructurev1beta2.RouterSpec, networks []infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { byName := indexNetworks(networks) var allErrs field.ErrorList for i, routerSpec := range routers { - allErrs = append(allErrs, validateSingleRouter(routerSpec, byName, mode, fldPath.Index(i))...) + allErrs = append(allErrs, validateSingleRouter(routerSpec, byName, fldPath.Index(i))...) } - allErrs = append(allErrs, validateRouterNetworkAddressing(routers, byName, fldPath)...) return allErrs } @@ -584,7 +459,6 @@ func validateRouters(routers []infrastructurev1beta2.RouterSpec, networks []infr func validateSingleRouter( routerSpec infrastructurev1beta2.RouterSpec, networks map[string]infrastructurev1beta2.NetworkSpec, - mode validationMode, routerPath *field.Path, ) field.ErrorList { var allErrs field.ErrorList @@ -595,28 +469,19 @@ func validateSingleRouter( } for j, ifaceSpec := range routerSpec.Interfaces { - allErrs = append(allErrs, validateRouterInterface(ifaceSpec, networks, routerSpec.UUID != "", mode, routerPath.Child("interfaces").Index(j))...) + allErrs = append(allErrs, validateRouterInterface(ifaceSpec, networks, routerSpec.UUID != "", routerPath.Child("interfaces").Index(j))...) } return allErrs } -// validateRouterInterface validates one router interface against the network index. On a -// CloudscaleCluster the defaulter has already filled in every address it could derive, so an -// empty address means it could not — either because the network is referenced by uuid (no -// CIDR at admission time) or because the CIDR is exhausted. Both need the user to be -// explicit. An interface adopted by uuid is the exception: its address is whatever the -// pre-existing interface already holds. -// -// On a CloudscaleClusterTemplate (mode.routerAddressesDefaulted false) an empty address on a -// network that has a CIDR is fine: it is derived per cluster, so a topology patch may change -// the CIDR and have the address follow. Only the uuid-network case still errors, since such -// a network never gains a CIDR and so could not be derived later either. +// validateRouterInterface validates one router interface against the network index. +// An adopted interface (by UUID) carries its address from the pre-existing interface. +// All other interfaces must have an explicit address set. func validateRouterInterface( ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, networks map[string]infrastructurev1beta2.NetworkSpec, routerAdopted bool, - mode validationMode, ifacePath *field.Path, ) field.ErrorList { network, netExists := networks[ifaceSpec.Network] @@ -638,17 +503,8 @@ func validateRouterInterface( } if ifaceSpec.Address == "" { - if network.CIDR == "" { - return field.ErrorList{field.Required(ifacePath.Child("address"), fmt.Sprintf( - "must be set explicitly: network %q is referenced by uuid, so its subnet is not known yet", - ifaceSpec.Network))} - } - if !mode.routerAddressesDefaulted { - // Derived from the network's CIDR when a cluster is stamped out of this template. - return nil - } - return field.ErrorList{field.Required(ifacePath.Child("address"), fmt.Sprintf( - "no free address left in %s for network %q", network.CIDR, ifaceSpec.Network))} + return field.ErrorList{field.Required(ifacePath.Child("address"), + "router interface address must be set explicitly")} } // Containment can only be checked for a managed network; an uuid network's subnet @@ -662,69 +518,9 @@ func validateRouterInterface( return errs } - prefix, _ := netip.ParsePrefix(network.CIDR) - ip, _ := netip.ParseAddr(ifaceSpec.Address) - if reason := unassignableReason(prefix, ip); reason != "" { - return field.ErrorList{field.Invalid(addrPath, ifaceSpec.Address, - fmt.Sprintf("cannot be assigned to a router interface: %s of %s", reason, network.CIDR))} - } return nil } -// validateRouterNetworkAddressing enforces the rules that only make sense when all -// interfaces on a network are considered together: at most one interface may set the -// subnet gateway, explicit addresses must be unique per network, and the gateway owner's -// address must equal the network's declared gatewayAddress. -func validateRouterNetworkAddressing(routers []infrastructurev1beta2.RouterSpec, networks map[string]infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - - subnetGateways := make(map[string][]*field.Path) - addrSeen := make(map[string]map[string]bool) - - for i, router := range routers { - for j, iface := range router.Interfaces { - ifacePath := fldPath.Index(i).Child("interfaces").Index(j) - - if ptr.Deref(iface.ConfigureSubnetGateway, true) { - subnetGateways[iface.Network] = append(subnetGateways[iface.Network], ifacePath.Child("configureSubnetGateway")) - - // The controller points the subnet's gateway at this interface's - // address, so a differing gatewayAddress would silently never take - // effect. An address is only empty on an adopted interface, whose - // address is not known until the controller reads it. - if gateway := networks[iface.Network].GatewayAddress; gateway != "" && iface.Address != "" && iface.Address != gateway { - allErrs = append(allErrs, field.Invalid(ifacePath.Child("address"), iface.Address, - fmt.Sprintf("interface configures the subnet gateway of network %q, so its address must equal the network's gatewayAddress %q", iface.Network, gateway))) - } - } - if iface.Address != "" { - if addrSeen[iface.Network] == nil { - addrSeen[iface.Network] = make(map[string]bool) - } - if addrSeen[iface.Network][iface.Address] { - allErrs = append(allErrs, field.Duplicate(ifacePath.Child("address"), iface.Address)) - } else { - addrSeen[iface.Network][iface.Address] = true - } - } - } - } - - // Report networks with more than one gateway, in a stable order. - for _, network := range slices.Sorted(maps.Keys(subnetGateways)) { - paths := subnetGateways[network] - if len(paths) <= 1 { - continue - } - for _, p := range paths { - allErrs = append(allErrs, field.Invalid(p, true, - fmt.Sprintf("network %q has multiple router interfaces configuring the subnet gateway; only one interface may set configureSubnetGateway=true", network))) - } - } - - return allErrs -} - // validateRouterDiff enforces immutability for spec.routers on update: routers and their // interfaces may be added, but never removed or altered. Whether the additions themselves // are valid is validateRouters' job, which ValidateUpdate runs over the new set. @@ -769,10 +565,6 @@ func validateRouterDiff(oldRouters, newRouters []infrastructurev1beta2.RouterSpe fmt.Sprintf("removing interface for network %q is not allowed", oldIface.Network))) continue } - if ptr.Deref(newIface.ConfigureSubnetGateway, true) != ptr.Deref(oldIface.ConfigureSubnetGateway, true) { - allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), - fmt.Sprintf("configureSubnetGateway for network %q is immutable after cluster creation", newIface.Network))) - } if newIface.Address != oldIface.Address { allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), fmt.Sprintf("address for network %q is immutable after cluster creation", newIface.Network))) @@ -969,3 +761,33 @@ func validateAddressInCIDR(cidr, address string, fldPath *field.Path) field.Erro return allErrs } + +// checkSNATConfiguration returns warnings when a router with internetGateway enabled +// does not have a gatewayAddress configured on its attached networks. +func checkSNATConfiguration(spec infrastructurev1beta2.CloudscaleClusterSpec) admission.Warnings { + var warnings admission.Warnings + for _, router := range spec.Routers { + if !router.InternetGateway { + continue + } + hasGateway := false + for _, iface := range router.Interfaces { + for _, network := range spec.Networks { + if network.Name == iface.Network && network.GatewayAddress != "" { + hasGateway = true + break + } + } + if hasGateway { + break + } + } + if !hasGateway { + warnings = append(warnings, fmt.Sprintf( + "Router %q has internetGateway enabled but no gatewayAddress is set on its attached networks. "+ + "Set networks[].gatewayAddress to enable SNAT for outbound internet access.", + router.Name)) + } + } + return warnings +} diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go index 87741ca..83687b0 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go @@ -17,8 +17,6 @@ limitations under the License. package v1beta2 import ( - "fmt" - "net/netip" "testing" . "github.com/onsi/gomega" @@ -85,27 +83,7 @@ func routerIfaceAdopted() infrastructurev1beta2.RouterInterfaceSpec { return iface } -// routerIfaceNoGateway is a routerIface that leaves the subnet gateway to someone else. -func routerIfaceNoGateway(address string) infrastructurev1beta2.RouterInterfaceSpec { - iface := routerIface(address) - iface.ConfigureSubnetGateway = new(false) - return iface -} - // routersEach builds one router per interface, named r0, r1, ... Interfaces that share a -// network have to sit on separate routers: spec.routers[].interfaces is a list-map keyed -// by network, so a single router can hold at most one interface per network. -func routersEach(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { - routers := make([]infrastructurev1beta2.RouterSpec, len(ifaces)) - for i, iface := range ifaces { - routers[i] = infrastructurev1beta2.RouterSpec{ - Name: fmt.Sprintf("r%d", i), - Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{iface}, - } - } - return routers -} - // setupRouterCluster fills in the region/zone a valid cluster needs and attaches the // given networks and routers. func setupRouterCluster(c *infrastructurev1beta2.CloudscaleCluster, networks []infrastructurev1beta2.NetworkSpec, routers []infrastructurev1beta2.RouterSpec) { @@ -250,79 +228,14 @@ func TestClusterDefaulting(t *testing.T) { }, }, { - name: "interface address and configureSubnetGateway are both defaulted", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.0/24") - c.Spec.Routers = routerWith(routerIface("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - iface := c.Spec.Routers[0].Interfaces[0] - g.Expect(iface.Address).To(Equal("10.0.0.1")) - g.Expect(iface.ConfigureSubnetGateway).To(Equal(new(true))) - }, - }, - { - name: "two routers on one network get distinct addresses", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.0/24") - c.Spec.Routers = routersEach(routerIface(""), routerIfaceNoGateway("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) - g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) - }, - }, - { - name: "explicit address is never handed out twice", + name: "explicit address is preserved", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { c.Spec.Region = RegionRma c.Spec.Networks = routerNetworks("10.0.0.0/24") - c.Spec.Routers = routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) - g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) - }, - }, - { - name: "gateway owner takes the network's gatewayAddress", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ - {Name: "main", CIDR: "10.0.0.0/24", GatewayAddress: "10.0.0.9"}, - } - c.Spec.Routers = routerWith(routerIface("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.9")) - }, - }, - { - name: "non-gateway interface does not take the network's gatewayAddress", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ - {Name: "main", CIDR: "10.0.0.0/24", GatewayAddress: "10.0.0.9"}, - } - iface := routerIface("") - iface.ConfigureSubnetGateway = new(false) - c.Spec.Routers = routerWith(iface) + c.Spec.Routers = routerWith(routerIface("10.0.0.5")) }, assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) - }, - }, - { - name: "CIDR with host bits allocates from the masked network address", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.5/24") - c.Spec.Routers = routerWith(routerIface("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.5")) }, }, { @@ -338,56 +251,6 @@ func TestClusterDefaulting(t *testing.T) { g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) }, }, - { - // An adopted interface holds whatever address it already has, which no - // amount of CIDR arithmetic here can know. - name: "adopted interface is left empty even on a CIDR network", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.0/24") - c.Spec.Routers = adoptedRouterWith(routerIfaceAdopted()) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) - // Everything else is still defaulted as usual. - g.Expect(c.Spec.Routers[0].Interfaces[0].ConfigureSubnetGateway).To(HaveValue(BeTrue())) - }, - }, - { - // The address an adopted interface does not take must stay available. - name: "adopted interface does not consume an address", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.0/24") - c.Spec.Routers = routersEach(routerIfaceAdopted(), routerIface("")) - c.Spec.Routers[0].UUID = "router-uuid" - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) - g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.1")) - }, - }, - { - // A /30 holds .0 through .3, of which only .1 and .2 are assignable: .0 is the - // network address and .3 the broadcast address. The third router has nowhere - // to go. - name: "exhausted CIDR leaves the surplus interface empty", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - c.Spec.Region = RegionRma - c.Spec.Networks = routerNetworks("10.0.0.0/30") - c.Spec.Routers = routersEach(routerIface(""), - routerIfaceNoGateway(""), routerIfaceNoGateway("")) - }, - assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { - g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) - g.Expect(c.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) - g.Expect(c.Spec.Routers[2].Interfaces[0].Address).To(BeEmpty()) - // The broadcast address must go to nobody rather than to the last router. - for _, router := range c.Spec.Routers { - g.Expect(router.Interfaces[0].Address).ToNot(Equal("10.0.0.3")) - } - }, - }, } for _, tc := range cases { @@ -694,42 +557,7 @@ func TestClusterValidateCreate(t *testing.T) { setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface("10.0.0.1"))) }, }, - { - name: "nil configureSubnetGateway is treated as true", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - iface := routerIface("10.0.0.1") - iface.ConfigureSubnetGateway = nil - setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(iface)) - }, - }, - { - name: "router interface address inside the DHCP pool", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - networks := routerNetworks("10.0.0.0/24") - networks[0].GatewayAddress = "10.0.0.150" - setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.150"))) - }, - wantErr: true, - wantSubstrings: []string{"DHCP pool .101-.254", "cannot be assigned to a router interface"}, - }, - { - name: "router interface on the last address below the DHCP pool", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - networks := routerNetworks("10.0.0.0/24") - networks[0].GatewayAddress = "10.0.0.100" - setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.100"))) - }, - }, - { - name: "router interface on the broadcast address", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - networks := routerNetworks("10.0.0.0/24") - networks[0].GatewayAddress = "10.0.0.255" - setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.255"))) - }, - wantErr: true, - wantSubstrings: []string{"it is the broadcast address"}, - }, + { name: "router interface address outside the network CIDR", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { @@ -748,41 +576,6 @@ func TestClusterValidateCreate(t *testing.T) { wantErr: true, wantSubstrings: []string{"routers[0].interfaces[0].network"}, }, - { - name: "duplicate addresses on the same network", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - setupRouterCluster(c, routerNetworks("10.0.0.0/24"), - routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("10.0.0.1"))) - }, - wantErr: true, - wantSubstrings: []string{"Duplicate value", "10.0.0.1"}, - }, - { - name: "two subnet gateway owners on the same network", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - setupRouterCluster(c, routerNetworks("10.0.0.0/24"), - routersEach(routerIface("10.0.0.1"), routerIface("10.0.0.2"))) - }, - wantErr: true, - wantSubstrings: []string{"only one interface may set configureSubnetGateway=true"}, - }, - { - name: "same network on two different routers is allowed", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - setupRouterCluster(c, routerNetworks("10.0.0.0/24"), - routersEach(routerIface("10.0.0.1"), routerIfaceNoGateway("10.0.0.2"))) - }, - }, - { - name: "gateway owner address contradicts the network's gatewayAddress", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - networks := routerNetworks("10.0.0.0/24") - networks[0].GatewayAddress = "10.0.0.5" - setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.1"))) - }, - wantErr: true, - wantSubstrings: []string{"must equal the network's gatewayAddress", "10.0.0.5"}, - }, { name: "gateway owner address matching the network's gatewayAddress is valid", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { @@ -844,22 +637,12 @@ func TestClusterValidateCreate(t *testing.T) { }, }, { - name: "address-less interface on a uuid network rejected", - mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { - setupRouterCluster(c, - []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, - routerWith(routerIface(""))) - }, - wantErr: true, - wantSubstrings: []string{"must be set explicitly", "referenced by uuid"}, - }, - { - name: "address-less interface on an exhausted CIDR rejected", + name: "address-less interface rejected", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface(""))) }, wantErr: true, - wantSubstrings: []string{"no free address left"}, + wantSubstrings: []string{"must be set explicitly"}, }, } @@ -882,52 +665,6 @@ func TestClusterValidateCreate(t *testing.T) { } } -// TestClusterDefaultThenValidate_NonOwnerSquattingGatewayAddress runs both webhooks in -// the order the API server does. A non-owner interface claiming the network's declared -// gatewayAddress forces the defaulter to allocate the gateway owner a different address, -// which the validator must then reject as contradicting the network. -func TestClusterDefaultThenValidate_NonOwnerSquattingGatewayAddress(t *testing.T) { - g := NewWithT(t) - - obj, _, validator, defaulter := newClusterWebhookTestObjects() - - networks := routerNetworks("10.0.0.0/24") - networks[0].GatewayAddress = "10.0.0.1" - squatter := routerIface("10.0.0.1") - squatter.ConfigureSubnetGateway = new(false) - owner := routerIface("") - owner.ConfigureSubnetGateway = new(true) - setupRouterCluster(obj, networks, []infrastructurev1beta2.RouterSpec{ - {Name: "r", Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{squatter}}, - {Name: "r2", Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{owner}}, - }) - - g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - // The declared gateway address was taken, so the owner got the next free one. - g.Expect(obj.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) - - _, err := validator.ValidateCreate(ctx, obj) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("must equal the network's gatewayAddress")) -} - -func TestClusterDefaulting_InterfaceAddedLater(t *testing.T) { - g := NewWithT(t) - - obj, _, _, defaulter := newClusterWebhookTestObjects() - setupRouterCluster(obj, routerNetworks("10.0.0.0/24"), routerWith(routerIface("10.0.0.1"))) - // A second router shows up later and asks for an address on the same network. - obj.Spec.Routers = append(obj.Spec.Routers, infrastructurev1beta2.RouterSpec{ - Name: "later", - Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{routerIfaceNoGateway("")}, - }) - - g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - - g.Expect(obj.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.1")) - g.Expect(obj.Spec.Routers[1].Interfaces[0].Address).To(Equal("10.0.0.2")) -} - // ============================================================================ // CloudscaleCluster Validating Webhook - Update // ============================================================================ @@ -961,7 +698,7 @@ func seedRouter(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { Name: "r", InternetGateway: true, Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ - {Network: "main", Address: "172.18.0.1", ConfigureSubnetGateway: new(true)}, + {Network: "main", Address: "172.18.0.1"}, }, } oldObj.Spec.Routers = []infrastructurev1beta2.RouterSpec{*router.DeepCopy()} @@ -1150,15 +887,6 @@ func TestClusterValidateUpdate(t *testing.T) { wantErr: true, wantSubstrings: []string{"uuid for network", "immutable"}, }, - { - name: "router interface configureSubnetGateway change rejected", - mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { - seedRouter(oldObj, obj) - obj.Spec.Routers[0].Interfaces[0].ConfigureSubnetGateway = new(false) - }, - wantErr: true, - wantSubstrings: []string{"configureSubnetGateway for network", "immutable"}, - }, { name: "router interface removal rejected", mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { @@ -1184,7 +912,7 @@ func TestClusterValidateUpdate(t *testing.T) { obj.Spec.Routers = append(obj.Spec.Routers, infrastructurev1beta2.RouterSpec{ Name: "second", Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ - {Network: "main", Address: "192.168.1.1", ConfigureSubnetGateway: new(false)}, + {Network: "main", Address: "192.168.1.1"}, }, }) }, @@ -1343,54 +1071,6 @@ func TestClusterValidateDelete_AlwaysSucceeds(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) } -// ============================================================================ -// Unit tests for unassignableReason -// ============================================================================ - -// TestUnassignableReason pins the boundaries of the rule that the defaulter and the -// validator share. They belong here rather than in the defaulting table, where reaching -// the DHCP pool would mean claiming a hundred addresses first. -func TestUnassignableReason(t *testing.T) { - cases := []struct { - name string - prefix string - addr string - want string - }{ - {name: "first host address", prefix: "10.0.0.0/24", addr: "10.0.0.1"}, - {name: "last address below the pool", prefix: "10.0.0.0/24", addr: "10.0.0.100"}, - {name: "first address of the pool", prefix: "10.0.0.0/24", addr: "10.0.0.101", want: "DHCP pool"}, - {name: "last address of the pool", prefix: "10.0.0.0/24", addr: "10.0.0.254", want: "DHCP pool"}, - {name: "broadcast address", prefix: "10.0.0.0/24", addr: "10.0.0.255", want: "broadcast address"}, - {name: "network address", prefix: "10.0.0.0/24", addr: "10.0.0.0", want: "network address"}, - // The pool is a last-octet range, so it recurs in a prefix wider than a /24 while - // the broadcast address belongs to the prefix as a whole. - {name: "pool recurs inside a /16", prefix: "10.0.0.0/16", addr: "10.0.3.150", want: "DHCP pool"}, - {name: "x.x.n.255 is a host address in a /16", prefix: "10.0.0.0/16", addr: "10.0.3.255"}, - {name: "broadcast of a /16", prefix: "10.0.0.0/16", addr: "10.0.255.255", want: "broadcast address"}, - // IPv6 has none of these concepts, so its last address is assignable. - {name: "IPv6 last address", prefix: "2001:db8::/126", addr: "2001:db8::3"}, - {name: "IPv6 subnet-router anycast", prefix: "2001:db8::/126", addr: "2001:db8::"}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - prefix, err := netip.ParsePrefix(tc.prefix) - g.Expect(err).ToNot(HaveOccurred()) - addr, err := netip.ParseAddr(tc.addr) - g.Expect(err).ToNot(HaveOccurred()) - - got := unassignableReason(prefix, addr) - if tc.want == "" { - g.Expect(got).To(BeEmpty(), "expected %s to be assignable in %s", tc.addr, tc.prefix) - return - } - g.Expect(got).To(ContainSubstring(tc.want)) - }) - } -} - // ============================================================================ // Unit tests for validateAddressInCIDR // ============================================================================ diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go index 56fcef0..78a8070 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go @@ -75,15 +75,16 @@ type CloudscaleClusterTemplateCustomValidator struct { func (v *CloudscaleClusterTemplateCustomValidator) ValidateCreate(_ context.Context, clusterTemplate *infrastructurev1beta2.CloudscaleClusterTemplate) (admission.Warnings, error) { cloudscaleclustertemplatelog.Info("Validation for CloudscaleClusterTemplate upon creation", "name", clusterTemplate.GetName()) - allErrs := clusterSpecValidateCreate(clusterTemplate.Spec.Template.Spec, v.RegionInfo, modeTemplate, field.NewPath("spec", "template", "spec")) + allErrs := clusterSpecValidateCreate(clusterTemplate.Spec.Template.Spec, v.RegionInfo, field.NewPath("spec", "template", "spec")) + warnings := checkSNATConfiguration(clusterTemplate.Spec.Template.Spec) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleClusterTemplate"}, clusterTemplate.Name, allErrs) } - return nil, nil + return warnings, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type CloudscaleClusterTemplate. diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go index 188beeb..4165121 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go @@ -522,37 +522,6 @@ func TestClusterTemplateValidateCreate_PreExistingFloatingIPWithoutLBAllowed(t * g.Expect(err).NotTo(HaveOccurred()) } -// A template's router interface addresses are derived per cluster, not here, so leaving -// one empty is what the ClusterClass flow relies on: a topology patch may replace the -// network's CIDR and the address follows it. -func TestClusterTemplateValidateCreate_RouterInterfaceAddressDeferred(t *testing.T) { - g := NewWithT(t) - obj, _, validator, defaulter := newClusterTemplateWebhookTestObjects() - obj.Spec.Template.Spec.Region = RegionRma - obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") - obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) - - g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - - _, err := validator.ValidateCreate(ctx, obj) - g.Expect(err).NotTo(HaveOccurred()) -} - -// A uuid network never gains a CIDR, so no later defaulting could derive an address for it -// either. -func TestClusterTemplateValidateCreate_RouterInterfaceOnUUIDNetworkStillRequiresAddress(t *testing.T) { - g := NewWithT(t) - obj, _, validator, _ := newClusterTemplateWebhookTestObjects() - obj.Spec.Template.Spec.Region = RegionRma - obj.Spec.Template.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}} - obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) - - _, err := validator.ValidateCreate(ctx, obj) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("must be set explicitly")) - g.Expect(err.Error()).To(ContainSubstring("referenced by uuid")) -} - func TestClusterTemplateValidateCreate_RouterInterfaceAddressOutsideCIDRRejected(t *testing.T) { g := NewWithT(t) obj, _, validator, _ := newClusterTemplateWebhookTestObjects() @@ -565,18 +534,6 @@ func TestClusterTemplateValidateCreate_RouterInterfaceAddressOutsideCIDRRejected g.Expect(err.Error()).To(ContainSubstring("must be within CIDR 10.10.0.0/24")) } -func TestClusterTemplateValidateCreate_RouterInterfaceAddressInDHCPPoolRejected(t *testing.T) { - g := NewWithT(t) - obj, _, validator, _ := newClusterTemplateWebhookTestObjects() - obj.Spec.Template.Spec.Region = RegionRma - obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") - obj.Spec.Template.Spec.Routers = routerWith(routerIface("10.10.0.150")) - - _, err := validator.ValidateCreate(ctx, obj) - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("DHCP pool")) -} - func TestClusterTemplateValidateCreate_RouterInterfaceUnknownNetworkRejected(t *testing.T) { g := NewWithT(t) obj, _, validator, _ := newClusterTemplateWebhookTestObjects() @@ -589,42 +546,6 @@ func TestClusterTemplateValidateCreate_RouterInterfaceUnknownNetworkRejected(t * g.Expect(err.Error()).To(ContainSubstring("Not found")) } -func TestClusterTemplateThenClusterDefaulting_DerivesRouterAddress(t *testing.T) { - for _, tc := range []struct { - name string - patchedCIDR string - wantAddress string - }{ - {name: "template CIDR untouched", patchedCIDR: "10.10.0.0/24", wantAddress: "10.10.0.1"}, - {name: "CIDR replaced by a topology patch", patchedCIDR: "10.20.0.0/24", wantAddress: "10.20.0.1"}, - {name: "CIDR narrowed by a topology patch", patchedCIDR: "172.16.5.0/25", wantAddress: "172.16.5.1"}, - } { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - tmpl, _, tmplValidator, tmplDefaulter := newClusterTemplateWebhookTestObjects() - tmpl.Spec.Template.Spec.Region = RegionRma - tmpl.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") - tmpl.Spec.Template.Spec.Routers = routerWith(routerIface("")) - - g.Expect(tmplDefaulter.Default(ctx, tmpl)).To(Succeed()) - _, err := tmplValidator.ValidateCreate(ctx, tmpl) - g.Expect(err).NotTo(HaveOccurred()) - - // What the topology controller does: copy the template spec, apply the - // patches, then let the cluster webhooks run on the result. - cluster, _, validator, defaulter := newClusterWebhookTestObjects() - cluster.Name = "my-cluster-abcde" - cluster.Spec = *tmpl.Spec.Template.Spec.DeepCopy() - cluster.Spec.Networks[0].CIDR = tc.patchedCIDR - - g.Expect(defaulter.Default(ctx, cluster)).To(Succeed()) - _, err = validator.ValidateCreate(ctx, cluster) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(cluster.Spec.Routers[0].Interfaces[0].Address).To(Equal(tc.wantAddress)) - }) - } -} - // ============================================================================ // Tests for CloudscaleClusterTemplate Validating Webhook - Update // ============================================================================ diff --git a/templates/cluster-class-router-nat.yaml b/templates/cluster-class-router-nat.yaml index 5b35502..8ad6bb0 100644 --- a/templates/cluster-class-router-nat.yaml +++ b/templates/cluster-class-router-nat.yaml @@ -83,15 +83,18 @@ spec: openAPIV3Schema: type: string default: node-net - # The router interface address is deliberately not part of the templates: the - # CloudscaleCluster admission webhook derives it from this CIDR when the topology - # controller stamps out the cluster, so changing the CIDR moves the address with it. - name: networkCIDR required: true schema: openAPIV3Schema: type: string default: 10.10.0.0/24 + - name: routerInterfaceAddress + required: true + schema: + openAPIV3Schema: + type: string + default: "10.10.0.1" - name: routerName required: true schema: @@ -176,6 +179,7 @@ spec: template: | - name: {{ .networkName }} cidr: {{ .networkCIDR }} + gatewayAddress: {{ .routerInterfaceAddress }} - op: replace path: "/spec/template/spec/routers" valueFrom: @@ -184,6 +188,7 @@ spec: internetGateway: true interfaces: - network: {{ .networkName }} + address: {{ .routerInterfaceAddress }} - name: CloudscaleMachineTemplateControlPlane definitions: - selector: @@ -304,11 +309,13 @@ spec: networks: - name: node-net cidr: 10.10.0.0/24 + gatewayAddress: 10.10.0.1 routers: - name: router internetGateway: true interfaces: - network: node-net + address: 10.10.0.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/templates/cluster-template-router-nat.yaml b/templates/cluster-template-router-nat.yaml index 3640e3f..cd08ae5 100644 --- a/templates/cluster-template-router-nat.yaml +++ b/templates/cluster-template-router-nat.yaml @@ -40,12 +40,14 @@ spec: networks: - name: "${CLUSTER_NAME}" cidr: "10.10.0.0/24" + gatewayAddress: "10.10.0.1" mtu: 1500 routers: - name: "${CLUSTER_NAME}-router" internetGateway: true # enables SNAT for outbound internet access interfaces: - network: "${CLUSTER_NAME}" + address: "10.10.0.1" credentialsRef: name: "${CLUSTER_NAME}-credentials" --- diff --git a/test/e2e/config/cloudscale.yaml b/test/e2e/config/cloudscale.yaml index bf08685..25d6344 100644 --- a/test/e2e/config/cloudscale.yaml +++ b/test/e2e/config/cloudscale.yaml @@ -79,6 +79,7 @@ variables: # CLOUDSCALE_SSH_PUBLIC_KEY: Set via environment variable CLOUDSCALE_ROOT_VOLUME_SIZE: "20" CLOUDSCALE_NETWORK_CIDR: "172.18.0.0/24" + CLOUDSCALE_ROUTER_INTERFACE_ADDRESS: "172.18.0.1" # Machine template names for upgrade tests CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "k8s-upgrade-control-plane" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml index b28f62f..21a2d41 100644 --- a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml @@ -8,11 +8,13 @@ spec: networks: - name: "${CLUSTER_NAME}" cidr: "${CLOUDSCALE_NETWORK_CIDR}" + gatewayAddress: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" routers: - name: "${CLUSTER_NAME}-router" internetGateway: true interfaces: - network: "${CLUSTER_NAME}" + address: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml index ff3e22f..6f4ec2b 100644 --- a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml @@ -44,6 +44,8 @@ spec: value: "${CLUSTER_NAME}" - name: networkCIDR value: "${CLOUDSCALE_NETWORK_CIDR}" + - name: routerInterfaceAddress + value: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" - name: routerName value: "${CLUSTER_NAME}-router" - name: cloudscaleControlPlaneMachineFlavor diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml index 42e4ee2..5797569 100644 --- a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml @@ -85,6 +85,12 @@ spec: openAPIV3Schema: type: string default: 10.10.0.0/24 + - name: routerInterfaceAddress + required: true + schema: + openAPIV3Schema: + type: string + default: "10.10.0.1" - name: routerName required: true schema: @@ -169,6 +175,7 @@ spec: template: | - name: {{ .networkName }} cidr: {{ .networkCIDR }} + gatewayAddress: {{ .routerInterfaceAddress }} - op: replace path: "/spec/template/spec/routers" valueFrom: @@ -177,6 +184,7 @@ spec: internetGateway: true interfaces: - network: {{ .networkName }} + address: {{ .routerInterfaceAddress }} - name: CloudscaleMachineTemplateControlPlane definitions: - selector: @@ -296,11 +304,13 @@ spec: networks: - name: node-net cidr: 10.10.0.0/24 + gatewayAddress: 10.10.0.1 routers: - name: router internetGateway: true interfaces: - network: node-net + address: 10.10.0.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4afa990..62cea57 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -111,9 +111,8 @@ var _ = Describe("Managed router NAT", Label("router-nat"), func() { }) }) - // Same topology via the router-nat ClusterClass. This additionally covers the - // CloudscaleClusterTemplate leaving the router interface address open for the - // CloudscaleCluster webhook to derive from the patched network CIDR. + // Same topology via the router-nat ClusterClass. This additionally covers patching the + // network gateway and router interface address from the ClusterClass variable. Context("With managed router providing NAT via ClusterClass", func() { capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { return capi_e2e.QuickStartSpecInput{ diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index f957332..5c8350d 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -81,24 +81,6 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste Expect(ifaceStatus.Managed).To(Equal(ifaceSpec.UUID == ""), "Router %s interface on network %s managed flag should reflect whether it was adopted by uuid", routerSpec.Name, ifaceSpec.Network) - - if !ptr.Deref(ifaceSpec.ConfigureSubnetGateway, true) { - continue - } - ns := cloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) - Expect(ns).NotTo(BeNil(), "Network %s should be tracked in status", ifaceSpec.Network) - if ifaceSpec.Address != "" { - // The address the webhook derived is the one the subnet gateway must end up - // pointing at. Checking only for non-emptiness would let a gateway pointed at - // the wrong host in the subnet pass. - Expect(ns.GatewayAddress).To(Equal(ifaceSpec.Address), - "Network %s subnet gateway should be router %s's interface address", ifaceSpec.Network, routerSpec.Name) - } else { - // Adopted interfaces and interfaces on uuid-referenced networks carry no - // spec address: it is only known once the controller reads the router. - Expect(ns.GatewayAddress).NotTo(BeEmpty(), - "Network %s should have its subnet gateway pointed at router %s", ifaceSpec.Network, routerSpec.Name) - } } }