diff --git a/.github/workflows/e2e-weekly.yml b/.github/workflows/e2e-weekly.yml index b7673a7..0cf7456 100644 --- a/.github/workflows/e2e-weekly.yml +++ b/.github/workflows/e2e-weekly.yml @@ -42,7 +42,7 @@ jobs: TAG: e2e-weekly-${{ github.sha }} run: | make test-e2e \ - GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || topology || conformance || pre-existing-networking" \ + GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || topology || conformance || pre-existing-networking || router-nat" \ KUBETEST_CONFIGURATION=./data/kubetest/conformance-fast.yaml - name: Install regctl diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 389ef01..5cc2489 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -17,6 +17,7 @@ on: - test-e2e-self-hosted - test-e2e-md-remediation - test-e2e-pre-existing-networking + - test-e2e-router-nat - test-e2e-topology - test-e2e-upgrade - test-e2e-conformance diff --git a/.gitignore b/.gitignore index 4afe9fd..f93a469 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ go.work _artifacts/ test/e2e/config/*.generated.yaml test/e2e/data/infrastructure-cloudscale/main/cluster-template*.yaml +test/e2e/data/infrastructure-cloudscale/main/clusterclass*.yaml # release dist/ diff --git a/AGENTS.md b/AGENTS.md index 43672eb..fc8e791 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,10 +109,12 @@ For the prose architecture sketch see [`docs/development.md`](docs/development.m - The e2e overlay `test/infrastructure/cloudscale/clusterclass-quick-start/cluster-class.yaml` intentionally duplicates `templates/cluster-class.yaml`. Keep them in sync, but do not attempt to merge or deduplicate them — both copies are required. + The same applies to `clusterclass-router-nat/cluster-class.yaml` and + `templates/cluster-class-router-nat.yaml`; those two are byte-identical. ## Cloudscale SDK usage -- Do not `import "github.com/cloudscale-ch/cloudscale-go-sdk/v9"` outside +- Do not `import "github.com/cloudscale-ch/cloudscale-go-sdk/v10"` outside `internal/cloudscale/`. Controllers and webhooks talk to the SDK through the service interfaces on `cloudscale.Client` (`internal/cloudscale/client.go:32`). diff --git a/Makefile b/Makefile index 5eca562..5617378 100644 --- a/Makefile +++ b/Makefile @@ -162,9 +162,12 @@ E2E_CLUSTER_TEMPLATES := cluster-template \ cluster-template-md-remediation \ cluster-template-pre-existing-network \ cluster-template-public-lb-private-nodes \ + cluster-template-router-nat \ cluster-template-topology \ + cluster-template-topology-router-nat \ cluster-template-upgrades \ - clusterclass-quick-start + clusterclass-quick-start \ + clusterclass-router-nat .PHONY: generate-e2e-templates generate-e2e-templates: $(KUSTOMIZE) generate-e2e-cni generate-e2e-ccm ## Generate e2e cluster templates using kustomize overlays @@ -285,6 +288,19 @@ test-e2e-topology: $(GINKGO) generate-e2e-templates generate-e2e-config docker-b -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) +.PHONY: test-e2e-router-nat +test-e2e-router-nat: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run managed-router NAT e2e tests + $(GINKGO) -v --trace --tags=e2e \ + --nodes=$(GINKGO_NODES) \ + --label-filter="router-nat" \ + --timeout=90m \ + --output-dir="$(E2E_ARTIFACTS_FOLDER)" --junit-report="junit.e2e_router_nat.xml" \ + ./test/e2e -- \ + -e2e.config=$(E2E_CONF_FILE) \ + -e2e.artifacts-folder=$(E2E_ARTIFACTS_FOLDER) \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + .PHONY: test-e2e-conformance test-e2e-conformance: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run K8s conformance e2e tests $(GINKGO) -v --trace --tags=e2e \ diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index f55a010..d3d5f59 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -72,6 +72,14 @@ type CloudscaleClusterSpec struct { // +optional Networks []NetworkSpec `json:"networks,omitempty"` + // Routers define routers for this cluster. + // Each router can be attached to one or more networks via its interfaces. + // If empty, no router is provisioned. + // +listType=map + // +listMapKey=name + // +optional + Routers []RouterSpec `json:"routers,omitempty"` + // ControlPlaneLoadBalancer configures the load balancer for the control plane. // +optional ControlPlaneLoadBalancer LoadBalancerSpec `json:"controlPlaneLoadBalancer,omitzero"` @@ -127,16 +135,68 @@ type NetworkSpec struct { // +optional CIDR string `json:"cidr,omitempty"` + // MTU defines the MTU size for this network. + // Immutable after creation + // +kubebuilder:validation:Minimum=1400 + // +kubebuilder:validation:Maximum=9000 + // +optional + MTU int `json:"mtu,omitempty"` + // GatewayAddress is the gateway IP address for the subnet. - // Only applicable when CIDR is set (managed network). - // By default, no gateway is configured on the subnet. This ensures - // that outbound internet traffic uses the public network interface. - // Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - // a NAT gateway or similar infrastructure on the private network. + // Only applicable when CIDR is set (managed network). Immutable after creation. // +optional GatewayAddress string `json:"gatewayAddress,omitempty"` } +// RouterSpec defines a router managed or adopted for this cluster. +// Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. +type RouterSpec struct { + // Name identifies this router within the cluster. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // UUID references a pre-existing router by UUID. + // CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + // Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. + // Mutually exclusive with InternetGateway. Immutable after creation. + // +optional + UUID string `json:"uuid,omitempty"` + + // InternetGateway enables SNAT for outbound internet access on a managed router. + // Only valid when UUID is not set. Immutable after creation. + // +optional + InternetGateway bool `json:"internetGateway,omitempty"` + + // Interfaces lists the networks this router is attached to. + // A router attaches to any given network at most once. + // +listType=map + // +listMapKey=network + // +optional + Interfaces []RouterInterfaceSpec `json:"interfaces,omitempty"` +} + +// RouterInterfaceSpec defines a single network attachment for a router. +type RouterInterfaceSpec struct { + // Network references spec.networks[].name. + // +kubebuilder:validation:Required + Network string `json:"network"` + + // UUID adopts a pre-existing interface of the router instead of attaching a new one. + // CAPCS reads its address but never creates or detaches it. + // Only valid on an adopted router. Mutually exclusive with Address. + // Immutable after creation. + // +optional + UUID string `json:"uuid,omitempty"` + + // Address is the IP requested for this router interface within the referenced network's subnet. + // Must be set explicitly. Must be empty when this interface is adopted by UUID. + // Immutable after creation. + // +optional + Address string `json:"address,omitempty"` +} + // LoadBalancerSpec defines the load balancer configuration for the control plane. type LoadBalancerSpec struct { // Enabled controls whether a load balancer is created for the control plane. @@ -172,11 +232,18 @@ type LoadBalancerSpec struct { // Network places the LB VIP on a private network (internal LB). // References spec.networks[].name. Omit for a public LB. - // When multiple networks are defined this field is required so the LB - // pool members can be registered against a specific subnet. + // When multiple networks are defined, either this field or PoolMemberNetwork + // is required so the LB pool members can be registered against a specific subnet. // +optional Network string `json:"network,omitempty"` + // PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + // References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + // When unset, the controller resolves the member subnet from Network or the first network. + // Strictly immutable: it cannot be changed, nor set after creation. + // +optional + PoolMemberNetwork string `json:"poolMemberNetwork,omitempty"` + // HealthMonitor configures the load balancer health monitor. // +optional HealthMonitor HealthMonitorSpec `json:"healthMonitor,omitempty"` @@ -248,6 +315,12 @@ type CloudscaleClusterStatus struct { // +optional Networks []NetworkStatus `json:"networks,omitempty"` + // Routers track the provisioned state of each router defined in spec.routers. + // +listType=map + // +listMapKey=name + // +optional + Routers []RouterStatus `json:"routers,omitempty"` + // FloatingIP is the cloudscale.ch floating IP. // +optional FloatingIP string `json:"floatingIP,omitempty"` @@ -297,9 +370,62 @@ type NetworkStatus struct { // +optional CIDR string `json:"cidr,omitempty"` + // MTU is the MTU of this network. + // +optional + MTU int `json:"mtu,omitempty"` + // Managed indicates whether CAPCS manages this network's lifecycle. // false for pre-existing networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). Managed bool `json:"managed"` + + // GatewayAddress is the subnet gateway IP that CAPCS has configured on the router interface. + // Empty until the router interface is attached and the subnet gateway is updated. + // +optional + GatewayAddress string `json:"gatewayAddress,omitempty"` +} + +// RouterStatus tracks the provisioned state of a single router. +type RouterStatus struct { + // Name matches spec.routers[].name. + Name string `json:"name"` + + // RouterID is the router UUID. + // +optional + RouterID string `json:"routerID,omitempty"` + + // Interfaces tracks this router's interfaces, at most one per network. + // +listType=map + // +listMapKey=network + // +optional + Interfaces []RouterInterfaceStatus `json:"interfaces,omitempty"` + + // Managed indicates whether CAPCS manages this router's lifecycle. + // false for pre-existing routers (referenced by UUID), true for CAPCS-created routers. + Managed bool `json:"managed"` +} + +// RouterInterfaceStatus tracks one router interface. +type RouterInterfaceStatus struct { + // Network matches spec.networks[].name. + Network string `json:"network"` + + // InterfaceID is the router interface UUID. + // +optional + InterfaceID string `json:"interfaceID,omitempty"` + + // Managed indicates whether CAPCS created this interface (spec.uuid was empty). + Managed bool `json:"managed"` +} + +// GetInterfaceStatus returns the RouterInterfaceStatus for the given network name, or +// nil if the router has no recorded interface on that network. +func (rs *RouterStatus) GetInterfaceStatus(network string) *RouterInterfaceStatus { + for i := range rs.Interfaces { + if rs.Interfaces[i].Network == network { + return &rs.Interfaces[i] + } + } + return nil } // ClusterInitializationStatus contains v1beta2 initialization tracking for CloudscaleCluster. @@ -320,6 +446,16 @@ func (s *CloudscaleClusterStatus) GetNetworkStatus(name string) *NetworkStatus { return nil } +// GetRouterStatus returns the RouterStatus for the given router name, or nil if not found. +func (s *CloudscaleClusterStatus) GetRouterStatus(name string) *RouterStatus { + for i := range s.Routers { + if s.Routers[i].Name == name { + return &s.Routers[i] + } + } + return nil +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=cloudscaleclusters,scope=Namespaced,categories=cluster-api diff --git a/api/v1beta2/condition_types.go b/api/v1beta2/condition_types.go index d82e359..2c83d13 100644 --- a/api/v1beta2/condition_types.go +++ b/api/v1beta2/condition_types.go @@ -30,6 +30,10 @@ const ( // True when the pause annotation is present on the resource or parent cluster. PausedCondition = "Paused" + // RouterReadyCondition indicates all spec.routers are provisioned, active, and have their + // interfaces attached. Also set to True when no routers are defined. + RouterReadyCondition = "RouterReady" + // FloatingIPReadyCondition indicates whether the floating IP has been provisioned and assigned. // Only applicable to CloudscaleCluster when spec.floatingIP is set. FloatingIPReadyCondition = "FloatingIPReady" @@ -65,6 +69,21 @@ const ( // LoadBalancerDeletingReason indicates the load balancer is being deleted. LoadBalancerDeletingReason = "LoadBalancerDeleting" + // RouterProvisionedReason indicates all routers and their interfaces are provisioned. + RouterProvisionedReason = "RouterProvisioned" + + // RouterNotReadyReason indicates a router exists but its status is not yet active. + RouterNotReadyReason = "RouterNotReady" + + // RouterErrorReason indicates an error occurred during router operations. + RouterErrorReason = "RouterError" + + // RouterDisabledReason indicates spec.routers is empty. + RouterDisabledReason = "RouterDisabled" + + // RouterDeletingReason indicates routers are being deleted. + RouterDeletingReason = "RouterDeleting" + // FloatingIPProvisionedReason indicates the floating IP has been successfully provisioned and assigned. FloatingIPProvisionedReason = "FloatingIPProvisioned" diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index ef98f12..9095330 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -96,6 +96,13 @@ func (in *CloudscaleClusterSpec) DeepCopyInto(out *CloudscaleClusterSpec) { *out = make([]NetworkSpec, len(*in)) copy(*out, *in) } + if in.Routers != nil { + in, out := &in.Routers, &out.Routers + *out = make([]RouterSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.ControlPlaneLoadBalancer.DeepCopyInto(&out.ControlPlaneLoadBalancer) if in.FloatingIP != nil { in, out := &in.FloatingIP, &out.FloatingIP @@ -127,6 +134,13 @@ func (in *CloudscaleClusterStatus) DeepCopyInto(out *CloudscaleClusterStatus) { *out = make([]NetworkStatus, len(*in)) copy(*out, *in) } + if in.Routers != nil { + in, out := &in.Routers, &out.Routers + *out = make([]RouterStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.LoadBalancerMemberIDs != nil { in, out := &in.LoadBalancerMemberIDs, &out.LoadBalancerMemberIDs *out = make([]string, len(*in)) @@ -668,6 +682,76 @@ func (in *NodeInfo) DeepCopy() *NodeInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterInterfaceSpec) DeepCopyInto(out *RouterInterfaceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterInterfaceSpec. +func (in *RouterInterfaceSpec) DeepCopy() *RouterInterfaceSpec { + if in == nil { + return nil + } + out := new(RouterInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterInterfaceStatus) DeepCopyInto(out *RouterInterfaceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterInterfaceStatus. +func (in *RouterInterfaceStatus) DeepCopy() *RouterInterfaceStatus { + if in == nil { + return nil + } + out := new(RouterInterfaceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { + *out = *in + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]RouterInterfaceSpec, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterSpec. +func (in *RouterSpec) DeepCopy() *RouterSpec { + if in == nil { + return nil + } + out := new(RouterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouterStatus) DeepCopyInto(out *RouterStatus) { + *out = *in + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]RouterInterfaceStatus, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterStatus. +func (in *RouterStatus) DeepCopy() *RouterStatus { + if in == nil { + return nil + } + out := new(RouterStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 55938f1..e051d06 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,7 +29,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" - "github.com/cloudscale-ch/cloudscale-go-sdk/v9/instrumentation" + "github.com/cloudscale-ch/cloudscale-go-sdk/v10/instrumentation" "go.opentelemetry.io/otel" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/runtime" diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index ea6d5f6..e128102 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -154,8 +154,15 @@ spec: description: |- Network places the LB VIP on a private network (internal LB). References spec.networks[].name. Omit for a public LB. - When multiple networks are defined this field is required so the LB - pool members can be registered against a specific subnet. + When multiple networks are defined, either this field or PoolMemberNetwork + is required so the LB pool members can be registered against a specific subnet. + type: string + poolMemberNetwork: + description: |- + PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + When unset, the controller resolves the member subnet from Network or the first network. + Strictly immutable: it cannot be changed, nor set after creation. type: string type: object credentialsRef: @@ -223,12 +230,15 @@ spec: gatewayAddress: description: |- GatewayAddress is the gateway IP address for the subnet. - Only applicable when CIDR is set (managed network). - By default, no gateway is configured on the subnet. This ensures - that outbound internet traffic uses the public network interface. - Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - a NAT gateway or similar infrastructure on the private network. + Only applicable when CIDR is set (managed network). Immutable after creation. type: string + mtu: + description: |- + MTU defines the MTU size for this network. + Immutable after creation + maximum: 9000 + minimum: 1400 + type: integer name: description: |- Name identifies this network within the cluster. @@ -258,6 +268,71 @@ spec: - rma - lpg type: string + routers: + description: |- + Routers define routers for this cluster. + Each router can be attached to one or more networks via its interfaces. + If empty, no router is provisioned. + items: + description: |- + RouterSpec defines a router managed or adopted for this cluster. + Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. + properties: + interfaces: + description: |- + Interfaces lists the networks this router is attached to. + A router attaches to any given network at most once. + items: + description: RouterInterfaceSpec defines a single network + attachment for a router. + properties: + address: + description: |- + Address is the IP requested for this router interface within the referenced network's subnet. + Must be set explicitly. Must be empty when this interface is adopted by UUID. + Immutable after creation. + type: string + network: + description: Network references spec.networks[].name. + type: string + uuid: + description: |- + UUID adopts a pre-existing interface of the router instead of attaching a new one. + CAPCS reads its address but never creates or detaches it. + Only valid on an adopted router. Mutually exclusive with Address. + Immutable after creation. + type: string + required: + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + internetGateway: + description: |- + InternetGateway enables SNAT for outbound internet access on a managed router. + Only valid when UUID is not set. Immutable after creation. + type: boolean + name: + description: Name identifies this router within the cluster. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + uuid: + description: |- + UUID references a pre-existing router by UUID. + CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. + Mutually exclusive with InternetGateway. Immutable after creation. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map zone: description: |- Zone is the cloudscale.ch zone within Region. @@ -378,11 +453,19 @@ spec: CIDR is the subnet CIDR block. Set from spec for managed networks or discovered from the API for pre-existing networks. type: string + gatewayAddress: + description: |- + GatewayAddress is the subnet gateway IP that CAPCS has configured on the router interface. + Empty until the router interface is attached and the subnet gateway is updated. + type: string managed: description: |- Managed indicates whether CAPCS manages this network's lifecycle. false for pre-existing networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). type: boolean + mtu: + description: MTU is the MTU of this network. + type: integer name: description: Name matches the logical name from spec.networks[].name. type: string @@ -406,6 +489,56 @@ spec: format: int64 minimum: 1 type: integer + routers: + description: Routers track the provisioned state of each router defined + in spec.routers. + items: + description: RouterStatus tracks the provisioned state of a single + router. + properties: + interfaces: + description: Interfaces tracks this router's interfaces, at + most one per network. + items: + description: RouterInterfaceStatus tracks one router interface. + properties: + interfaceID: + description: InterfaceID is the router interface UUID. + type: string + managed: + description: Managed indicates whether CAPCS created this + interface (spec.uuid was empty). + type: boolean + network: + description: Network matches spec.networks[].name. + type: string + required: + - managed + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + managed: + description: |- + Managed indicates whether CAPCS manages this router's lifecycle. + false for pre-existing routers (referenced by UUID), true for CAPCS-created routers. + type: boolean + name: + description: Name matches spec.routers[].name. + type: string + routerID: + description: RouterID is the router UUID. + type: string + required: + - managed + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object required: - spec diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml index 3e003da..f5fae3d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclustertemplates.yaml @@ -184,8 +184,15 @@ spec: description: |- Network places the LB VIP on a private network (internal LB). References spec.networks[].name. Omit for a public LB. - When multiple networks are defined this field is required so the LB - pool members can be registered against a specific subnet. + When multiple networks are defined, either this field or PoolMemberNetwork + is required so the LB pool members can be registered against a specific subnet. + type: string + poolMemberNetwork: + description: |- + PoolMemberNetwork selects the network whose subnet the LB pool members are registered on (i.e. the network the control-plane machines attach to). + References spec.networks[].name. Set when control-plane nodes live on a different network than the VIP (Network). + When unset, the controller resolves the member subnet from Network or the first network. + Strictly immutable: it cannot be changed, nor set after creation. type: string type: object credentialsRef: @@ -253,12 +260,15 @@ spec: gatewayAddress: description: |- GatewayAddress is the gateway IP address for the subnet. - Only applicable when CIDR is set (managed network). - By default, no gateway is configured on the subnet. This ensures - that outbound internet traffic uses the public network interface. - Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - a NAT gateway or similar infrastructure on the private network. + Only applicable when CIDR is set (managed network). Immutable after creation. type: string + mtu: + description: |- + MTU defines the MTU size for this network. + Immutable after creation + maximum: 9000 + minimum: 1400 + type: integer name: description: |- Name identifies this network within the cluster. @@ -288,6 +298,72 @@ spec: - rma - lpg type: string + routers: + description: |- + Routers define routers for this cluster. + Each router can be attached to one or more networks via its interfaces. + If empty, no router is provisioned. + items: + description: |- + RouterSpec defines a router managed or adopted for this cluster. + Set UUID to adopt a pre-existing router; leave it empty to have CAPCS create one. + properties: + interfaces: + description: |- + Interfaces lists the networks this router is attached to. + A router attaches to any given network at most once. + items: + description: RouterInterfaceSpec defines a single + network attachment for a router. + properties: + address: + description: |- + Address is the IP requested for this router interface within the referenced network's subnet. + Must be set explicitly. Must be empty when this interface is adopted by UUID. + Immutable after creation. + type: string + network: + description: Network references spec.networks[].name. + type: string + uuid: + description: |- + UUID adopts a pre-existing interface of the router instead of attaching a new one. + CAPCS reads its address but never creates or detaches it. + Only valid on an adopted router. Mutually exclusive with Address. + Immutable after creation. + type: string + required: + - network + type: object + type: array + x-kubernetes-list-map-keys: + - network + x-kubernetes-list-type: map + internetGateway: + description: |- + InternetGateway enables SNAT for outbound internet access on a managed router. + Only valid when UUID is not set. Immutable after creation. + type: boolean + name: + description: Name identifies this router within the + cluster. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + uuid: + description: |- + UUID references a pre-existing router by UUID. + CAPCS attaches/detaches its interfaces but does not manage the router lifecycle. + Interfaces already attached to the router must be adopted explicitly via interfaces[].uuid. + Mutually exclusive with InternetGateway. Immutable after creation. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map zone: description: |- Zone is the cloudscale.ch zone within Region. diff --git a/docs/getting-started.md b/docs/getting-started.md index 7828e14..3e6df50 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -113,44 +113,63 @@ Set them in your shell, or keep them in `clusterctl.yaml` alongside the token. ## 6. Pick a cluster template flavor -| Flavor | Network | Control plane endpoint | Node connectivity | Requires | -|---------------------------|--------------------------|------------------------|-------------------|------------------------------------------------------| -| *(default)* | Managed, `172.18.0.0/24` | Public LB, DualStack | Public + cluster | — | -| `fip` | Pre-existing | Floating IP, IPv4 | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | -| `pre-existing-network` | Pre-existing | Public LB, DualStack | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | -| `public-lb-private-nodes` | Pre-existing + NAT | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID`, with a NAT gateway set up | -| `topology` | Managed, `172.18.0.0/24` | Public LB, DualStack | Public + cluster | `CLUSTER_TOPOLOGY=true` feature gate | +| Flavor | Network | Router/sNAT | Control plane endpoint | Node connectivity | Requires | +|---------------------------|--------------|--------------|------------------------|-------------------|------------------------------------------------------| +| *(default)* | Managed | not required | Public LB, DualStack | Public + cluster | — | +| `fip` | Pre-existing | not required | Floating IP, IPv4 | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | +| `pre-existing-network` | Pre-existing | not required | Public LB, DualStack | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | +| `public-lb-private-nodes` | Pre-existing | pre-existing | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID`, with a NAT gateway set up | +| `router-nat` | Managed | managed | Public LB | Private only | — | +| `topology` | Managed | not required | Public LB, DualStack | Public + cluster | `CLUSTER_TOPOLOGY=true` feature gate | +| `topology-router-nat` | Managed | managed | Public LB | Private only | `CLUSTER_TOPOLOGY=true` feature gate | These flavors just show possible configurations. You're encouraged to copy and adjust them to your needs. -If you do need a NAT gateway set up for your cluster, set up a private network and contact cloudscale support with the -following details: +## Using Templates with Topology/ClusterClass -- Desired name of the gateway -- UUID of the network (in control panel details) -- Optional: desired IP of the gateway in the network +The flavors with a `topology` prefix generate a `Cluster` that references a +[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) in `spec.topology.classRef`. -### Using the topology flavor - -The `topology` flavor generates a `Cluster` that references the `quick-start` -[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) -shipped in `templates/cluster-class.yaml`. Apply it once per namespace before -generating any topology cluster: +To use a topology flavor, you first need to apply its referenced ClusterClass: ```bash +# adjust CLUSTER_CLASS_FLAVOR to the respective cluster class yaml file. clusterctl generate yaml \ - --from https://raw.githubusercontent.com/cloudscale-ch/cluster-api-provider-cloudscale/main/templates/cluster-class.yaml \ + --from https://raw.githubusercontent.com/cloudscale-ch/cluster-api-provider-cloudscale/main/templates/{CLUSTER_CLASS_FLAVOR}.yaml \ | kubectl apply -f - ``` -Per-cluster overrides go under `spec.topology.variables`. See `templates/cluster-class.yaml` for the full -variable list and defaults. +Per-cluster overrides go under `spec.topology.variables`. The available variables depend on which ClusterClass flavor you use. + +**Common variables** (both `topology` and `topology-router-nat`): + +| Variable | Type | Required | Default | Description | +|-----------------------------------------------|------------------|----------|-----------------------------------|--------------------------------------------------------------------------------------| +| `region` | string | yes | `lpg` | cloudscale region (`lpg` or `rma`) | +| `credentialsRefName` | string | yes | — | Name of the Secret holding API credentials (must match the Secret created in step 7) | +| `cloudscaleControlPlaneMachineFlavor` | string | yes | `flex-4-2` | Flavor for control plane nodes | +| `cloudscaleControlPlaneMachineImage` | string | yes | `custom:ubuntu-2404-kube-v1.36.0` | Image for control plane nodes | +| `cloudscaleControlPlaneMachineRootVolumeSize` | integer | yes | `20` | Root volume size in GB for control plane | +| `cloudscaleWorkerMachineFlavor` | string | yes | `flex-4-2` | Flavor for worker nodes | +| `cloudscaleWorkerMachineImage` | string | yes | `custom:ubuntu-2404-kube-v1.36.0` | Image for worker nodes | +| `cloudscaleWorkerMachineRootVolumeSize` | integer | yes | `20` | Root volume size in GB for workers | +| `cloudscaleSSHKeys` | array of strings | yes | — | SSH public keys for all nodes | +| `cloudscaleControlPlaneServerGroupName` | string | yes | — | Server group name for control plane | +| `cloudscaleWorkerServerGroupName` | string | yes | — | Server group name for workers | + +*(Source: [`templates/cluster-class.yaml`](../templates/cluster-class.yaml))* + +The `topology-router-nat` flavor adds these variables: -Caveat: the `quick-start` ClusterClass exposes a narrower configurability than the traditional templates — no floating -IP, no pre-existing network, no private load balancer, and pod/service CIDRs are pinned to -`192.168.0.0/16` / `10.96.0.0/12`. Adjust the template for your use-case. +| Variable | Type | Required | Default | Description | +|---------------|--------|----------|----------------|------------------------------------------------------------------------------| +| `networkName` | string | yes | `node-net` | Logical name for the private network | +| `networkCIDR` | string | yes | `10.10.0.0/24` | CIDR for the private network (router interface address is derived from this) | +| `routerName` | string | yes | `router` | Name of the managed router | -Use `--flavor topology` in the next step if you did use ClusterClass. +*(Source: [`templates/cluster-class-router-nat.yaml`](../templates/cluster-class-router-nat.yaml))* + +Use `--flavor topology` or `--flavor topology-router-nat` in the next step if you did use ClusterClass. ## 7. Generate and apply the workload cluster @@ -178,16 +197,26 @@ Watch progress: clusterctl describe cluster my-cluster ``` +Wait for the control plane to come up enough to hand out a kubeconfig. This +doesn't need a CCM or CNI yet: + +```bash +kubectl wait --for=condition=ControlPlaneInitialized cluster/my-cluster --timeout=15m +``` + ## 8. Install the cloudscale CCM CAPCS only provisions infrastructure. For a working cluster the [cloudscale CCM](https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager) must be installed on the **workload** cluster. -CAPCS provides a [`ClusterResourceSet` you apply on the **management** cluster](https://cluster-api.sigs.k8s.io/tasks/cluster-resource-set). -CAPI's CRS controller then deploys it into any workload cluster labelled `ccm: cloudscale` — every CAPCS template sets this label. +CAPCS provides a [`ClusterResourceSet` you apply on the +**management** cluster](https://cluster-api.sigs.k8s.io/tasks/cluster-resource-set). +CAPI's CRS controller then deploys it into any workload cluster labelled `ccm: cloudscale` — every CAPCS template sets +this label. -**N.B.:** the CCM also needs a CLOUDSCALE_API_TOKEN in the environment set. It's recommended to use a different token for the CCM and CAPCS. +**N.B.:** the CCM also needs a CLOUDSCALE_API_TOKEN in the environment set. It's recommended to use a different token +for the CCM and CAPCS. ```bash # the namespace must match the namespace of the workload cluster definition @@ -202,17 +231,16 @@ the CCM starts and removes the taint — that is expected. ## 9. Install a CNI (Cilium) -Fetch the workload cluster's kubeconfig and point `KUBECONFIG` at it: +Fetch the workload cluster's kubeconfig: ```bash clusterctl get kubeconfig my-cluster > ~/.kube/my-cluster.yaml -export KUBECONFIG=~/.kube/my-cluster.yaml ``` Install Cilium with defaults: ```bash -cilium install +KUBECONFIG=~/.kube/my-cluster.yaml cilium install ``` Nodes become `Ready` once Cilium is up. Any standard CNI works — Cilium is just @@ -222,26 +250,33 @@ the example. ```bash # management cluster -clusterctl describe cluster +clusterctl describe cluster my-cluster # workload cluster -kubectl get nodes -kubectl -n kube-system get pods +KUBECONFIG=~/.kube/my-cluster.yaml kubectl get nodes +KUBECONFIG=~/.kube/my-cluster.yaml kubectl -n kube-system get pods ``` All nodes should report `Ready`. In `kube-system` you should see `cloudscale-cloud-controller-manager-*` on each control plane node and `cilium-*` on every node. +Now wait for the cluster to report fully available. + +```bash +kubectl wait --for=condition=Available cluster/my-cluster --timeout=15m +``` + If something is stuck, see [troubleshooting](troubleshooting.md). ## Optional: persistent storage (cloudscale CSI) For `PersistentVolumeClaim` support, install the -[cloudscale CSI driver](https://github.com/cloudscale-ch/csi-cloudscale) on the -**workload** cluster. With `KUBECONFIG` still pointing at the workload cluster: +[cloudscale CSI driver](https://github.com/cloudscale-ch/csi-cloudscale) on the **workload** cluster: ```bash +export KUBECONFIG=~/.kube/my-cluster.yaml + kubectl -n kube-system create secret generic cloudscale \ --from-literal=access-token="$CLOUDSCALE_API_TOKEN" diff --git a/go.mod b/go.mod index 909a9fa..9feaa61 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cloudscale-ch/cluster-api-provider-cloudscale go 1.27.0 require ( - github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0 + github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0 github.com/go-logr/logr v1.4.4 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 @@ -105,10 +105,10 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.68.0 // indirect - github.com/prometheus/procfs v0.20.1 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/go.sum b/go.sum index a5658bd..ce38838 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0 h1:i9xlttrksSDd8HHyj8ezYRUuVxiBPUAeJ2rZS9DFAnM= -github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0/go.mod h1:cTPjnvp+5J6XVSgHl8UdBxokNye8lKv/MJxx5/rYk40= +github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0 h1:Ir0/Ei9mU1OWBPDb95THR5H/Cu6WiXL1W/O67KS5YMM= +github.com/cloudscale-ch/cloudscale-go-sdk/v10 v10.0.0/go.mod h1:PbmyqFmr70walcN/nbphTlznb5XadWR4fRRmbd5tV+s= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -166,8 +166,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -227,14 +227,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.68.0 h1:8rQJvQmYltsR2L7h8Zw0Iyj8WYNNmpwikoQTZXwfVeA= -github.com/prometheus/common v0.68.0/go.mod h1:4soH+U8yJSROk7OJ//hmTiWKsxapv6zRGgTt3keN8gQ= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= diff --git a/internal/cloudscale/client.go b/internal/cloudscale/client.go index f1f4e05..5e8b658 100644 --- a/internal/cloudscale/client.go +++ b/internal/cloudscale/client.go @@ -25,13 +25,14 @@ import ( "strings" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "golang.org/x/oauth2" ) type Client struct { Networks NetworkService Subnets SubnetService + Routers RouterService Regions RegionService Servers ServerService ServerGroups ServerGroupService @@ -109,6 +110,7 @@ func NewClient(token, version string, transport http.RoundTripper) *Client { return &Client{ Networks: sdkClient.Networks, Subnets: sdkClient.Subnets, + Routers: sdkClient.Routers, Regions: sdkClient.Regions, Servers: sdkClient.Servers, ServerGroups: sdkClient.ServerGroups, diff --git a/internal/cloudscale/client_test.go b/internal/cloudscale/client_test.go index 5dc9e16..88b1db4 100644 --- a/internal/cloudscale/client_test.go +++ b/internal/cloudscale/client_test.go @@ -22,7 +22,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" ) diff --git a/internal/cloudscale/flavors.go b/internal/cloudscale/flavors.go index e9b54c9..54b0ca7 100644 --- a/internal/cloudscale/flavors.go +++ b/internal/cloudscale/flavors.go @@ -3,7 +3,7 @@ package cloudscale import ( "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) diff --git a/internal/cloudscale/flavors_test.go b/internal/cloudscale/flavors_test.go index d8b557a..d3c614d 100644 --- a/internal/cloudscale/flavors_test.go +++ b/internal/cloudscale/flavors_test.go @@ -3,7 +3,7 @@ package cloudscale import ( "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/internal/cloudscale/regions.go b/internal/cloudscale/regions.go index e0e891b..10627eb 100644 --- a/internal/cloudscale/regions.go +++ b/internal/cloudscale/regions.go @@ -17,7 +17,7 @@ limitations under the License. package cloudscale import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) // RegionInfo holds cloudscale.ch region and zone information for validation. diff --git a/internal/cloudscale/regions_test.go b/internal/cloudscale/regions_test.go index bbeeeed..159d919 100644 --- a/internal/cloudscale/regions_test.go +++ b/internal/cloudscale/regions_test.go @@ -19,7 +19,7 @@ package cloudscale import ( "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" ) diff --git a/internal/cloudscale/services.go b/internal/cloudscale/services.go index 464d0f5..7c2c28a 100644 --- a/internal/cloudscale/services.go +++ b/internal/cloudscale/services.go @@ -19,7 +19,7 @@ package cloudscale import ( "context" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) type NetworkService interface { @@ -34,6 +34,16 @@ type SubnetService interface { Get(ctx context.Context, id string) (*cloudscalesdk.Subnet, error) List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) Delete(ctx context.Context, id string) error + Update(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error +} + +type RouterService interface { + Create(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) + Get(ctx context.Context, id string) (*cloudscalesdk.Router, error) + List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) + Delete(ctx context.Context, id string) error + CreateInterface(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) + DeleteInterface(ctx context.Context, routerUUID, interfaceUUID string) error } type RegionService interface { diff --git a/internal/controller/cloudscale_services.go b/internal/controller/cloudscale_services.go index e7b6062..c3d21cc 100644 --- a/internal/controller/cloudscale_services.go +++ b/internal/controller/cloudscale_services.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" diff --git a/internal/controller/cloudscale_services_test.go b/internal/controller/cloudscale_services_test.go index 860fe8a..e6cc896 100644 --- a/internal/controller/cloudscale_services_test.go +++ b/internal/controller/cloudscale_services_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "github.com/go-logr/logr" . "github.com/onsi/gomega" diff --git a/internal/controller/cloudscale_tags.go b/internal/controller/cloudscale_tags.go index 3edffa3..8dd5801 100644 --- a/internal/controller/cloudscale_tags.go +++ b/internal/controller/cloudscale_tags.go @@ -1,7 +1,7 @@ package controller import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" ) diff --git a/internal/controller/cloudscalecluster_controller.go b/internal/controller/cloudscalecluster_controller.go index e1ac6d8..af8ef31 100644 --- a/internal/controller/cloudscalecluster_controller.go +++ b/internal/controller/cloudscalecluster_controller.go @@ -176,6 +176,14 @@ func (r *CloudscaleClusterReconciler) reconcileNormal(ctx context.Context, clust return result, nil } + result, err = r.reconcileRouters(ctx, clusterScope) + if err != nil { + return ctrl.Result{}, fmt.Errorf("reconciling routers: %w", err) + } + if !result.IsZero() { + return result, nil + } + lb, result, err := r.reconcileLoadBalancer(ctx, clusterScope) if err != nil { return ctrl.Result{}, fmt.Errorf("reconciling load balancer: %w", err) @@ -241,6 +249,10 @@ func (r *CloudscaleClusterReconciler) reconcileDelete(ctx context.Context, clust return ctrl.Result{}, fmt.Errorf("deleting server groups: %w", err) } + if err := r.deleteRouters(ctx, clusterScope); err != nil { + return ctrl.Result{}, fmt.Errorf("deleting routers: %w", err) + } + if err := r.deleteNetwork(ctx, clusterScope); err != nil { if errors.Is(err, errNetworkNotReady) { return ctrl.Result{RequeueAfter: 10 * time.Second}, nil @@ -296,8 +308,9 @@ func (r *CloudscaleClusterReconciler) reconcileControlPlaneEndpoint(clusterScope } // isInfrastructureProvisioned returns true if all cluster infrastructure is ready. -// This includes all Networks+Subnets, Load Balancer (with pool and listener) if enabled, -// Floating IP if configured, and Control Plane Endpoint. +// This includes all Networks+Subnets, Routers (with all interfaces attached), Load +// Balancer (with pool and listener) if enabled, Floating IP if configured, and +// Control Plane Endpoint. func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope *scope.ClusterScope) bool { // All networks must have both network and subnet IDs if len(clusterScope.CloudscaleCluster.Status.Networks) == 0 { @@ -309,6 +322,23 @@ func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope * } } + // All routers must be resolved and all their spec interfaces attached. + for _, routerSpec := range clusterScope.CloudscaleCluster.Spec.Routers { + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + if rs == nil || rs.RouterID == "" { + return false + } + for _, ifaceSpec := range routerSpec.Interfaces { + iface := rs.GetInterfaceStatus(ifaceSpec.Network) + // interfaces are first created with an empty interfaceID in the status (see createRouterInterface) + // to avoid potential issues with timeouts during interface creation. An interface in status is only fully set up + // if there's an interfaceID set. + if iface == nil || iface.InterfaceID == "" { + return false + } + } + } + // Load balancer, pool, and listener must exist (if LB is enabled) if ptr.Deref(clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { if clusterScope.CloudscaleCluster.Status.LoadBalancerID == "" || @@ -338,6 +368,7 @@ func (r *CloudscaleClusterReconciler) setReadyCondition(clusterScope *scope.Clus // Check all sub-conditions subConditions := []string{ infrastructurev1beta2.NetworkReadyCondition, + infrastructurev1beta2.RouterReadyCondition, infrastructurev1beta2.LoadBalancerReadyCondition, infrastructurev1beta2.FloatingIPReadyCondition, } diff --git a/internal/controller/cloudscalecluster_controller_test.go b/internal/controller/cloudscalecluster_controller_test.go index 117e4e6..cd68ea8 100644 --- a/internal/controller/cloudscalecluster_controller_test.go +++ b/internal/controller/cloudscalecluster_controller_test.go @@ -148,10 +148,20 @@ func TestReconcileControlPlaneEndpoint_ExternalControlPlaneNoSource(t *testing.T // TestIsInfrastructureProvisioned exercises the readiness predicate used by // reconcileNormal to decide when to flip Status.Initialization.Provisioned. func TestIsInfrastructureProvisioned(t *testing.T) { + // routerSpec is a single router "r" attached to the given network. + routerSpec := func(network string) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{ + Name: "r", + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{{Network: network}}, + }} + } + cases := []struct { name string lbEnabled bool networks []infrastructurev1beta2.NetworkStatus + routerSpecs []infrastructurev1beta2.RouterSpec + routerStatuses []infrastructurev1beta2.RouterStatus endpointHost string endpointPort int32 lbID string @@ -192,6 +202,41 @@ func TestIsInfrastructureProvisioned(t *testing.T) { lbEnabled: false, networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, }, + { + // A router with no status at all: it has not even been resolved yet. + name: "router not yet provisioned", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + }, + { + // The router exists, but the interface the spec asks for is not attached, so + // nodes on that subnet still have no route out. + name: "router provisioned but interface not attached", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + routerStatuses: []infrastructurev1beta2.RouterStatus{{Name: "r", RouterID: "router-123", Managed: true}}, + }, + { + name: "router provisioned with its interface attached", + lbEnabled: false, + networks: []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, GatewayAddress: "10.0.0.1"}}, + endpointHost: "1.2.3.4", + endpointPort: 6443, + routerSpecs: routerSpec("test"), + routerStatuses: []infrastructurev1beta2.RouterStatus{{ + Name: "r", RouterID: "router-123", Managed: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: "iface-123", Managed: true}, + }, + }}, + wantProvisioned: true, + }, { name: "Floating IP requested but status floating IP empty", lbEnabled: false, @@ -225,9 +270,11 @@ func TestIsInfrastructureProvisioned(t *testing.T) { Enabled: new(tc.lbEnabled), }, FloatingIP: tc.floatingIP, + Routers: tc.routerSpecs, }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ Networks: tc.networks, + Routers: tc.routerStatuses, LoadBalancerID: tc.lbID, LoadBalancerPoolID: tc.lbPoolID, LoadBalancerListenerID: tc.lbListenerID, @@ -255,6 +302,7 @@ func TestSetReadyCondition(t *testing.T) { name: "all sub-conditions True yields Ready=True", subConditions: []metav1.Condition{ {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.LoadBalancerReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.FloatingIPReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, }, @@ -275,12 +323,23 @@ func TestSetReadyCondition(t *testing.T) { name: "LoadBalancer=False propagates reason and message", subConditions: []metav1.Condition{ {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, {Type: infrastructurev1beta2.LoadBalancerReadyCondition, Status: metav1.ConditionFalse, Reason: infrastructurev1beta2.LoadBalancerNotReadyReason, Message: "Load balancer is not running", ObservedGeneration: 1}, }, expectStatus: metav1.ConditionFalse, expectReason: infrastructurev1beta2.LoadBalancerNotReadyReason, expectMsgContains: "Load balancer is not running", }, + { + name: "Router=False propagates reason and message", + subConditions: []metav1.Condition{ + {Type: infrastructurev1beta2.NetworkReadyCondition, Status: metav1.ConditionTrue, Reason: "Provisioned", ObservedGeneration: 1}, + {Type: infrastructurev1beta2.RouterReadyCondition, Status: metav1.ConditionFalse, Reason: infrastructurev1beta2.RouterNotReadyReason, Message: "Waiting for routers to become available", ObservedGeneration: 1}, + }, + expectStatus: metav1.ConditionFalse, + expectReason: infrastructurev1beta2.RouterNotReadyReason, + expectMsgContains: "Waiting for routers to become available", + }, { name: "no sub-conditions yields Ready=False/NotReady", subConditions: nil, diff --git a/internal/controller/cloudscalecluster_floatingip.go b/internal/controller/cloudscalecluster_floatingip.go index 43199a3..68c8aad 100644 --- a/internal/controller/cloudscalecluster_floatingip.go +++ b/internal/controller/cloudscalecluster_floatingip.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" diff --git a/internal/controller/cloudscalecluster_floatingip_test.go b/internal/controller/cloudscalecluster_floatingip_test.go index 3a80d4e..a5db7e8 100644 --- a/internal/controller/cloudscalecluster_floatingip_test.go +++ b/internal/controller/cloudscalecluster_floatingip_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalecluster_loadbalancer.go b/internal/controller/cloudscalecluster_loadbalancer.go index 8ff11ae..c803e0e 100644 --- a/internal/controller/cloudscalecluster_loadbalancer.go +++ b/internal/controller/cloudscalecluster_loadbalancer.go @@ -23,7 +23,7 @@ import ( "slices" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -207,7 +207,7 @@ func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterSc // Place LB on a private network if specified, otherwise public VIP if lbSpec.Network != "" { - subnetID, err := lbPrivateNetworkSubnetID(clusterScope) + subnetID, err := safeUseSubnet(clusterScope, clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network) if err != nil { return ctrl.Result{}, err } @@ -513,11 +513,18 @@ func (r *CloudscaleClusterReconciler) getMemberSubnetCIDR(clusterScope *scope.Cl } // getPoolMemberSubnetID determines the subnet UUID for LB pool members. -// If the LB is on a private network, use that network's subnet. -// Otherwise (public LB), use the first network's subnet. +// 1. .PoolMemberNetwork's subnet if specified +// 2. .Network's subnet if specified +// 3. first network's subnet func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope.ClusterScope) (string, error) { - if clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network != "" { - return lbPrivateNetworkSubnetID(clusterScope) + lbSpec := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer + + if lbSpec.PoolMemberNetwork != "" { + return safeUseSubnet(clusterScope, lbSpec.PoolMemberNetwork) + } + + if lbSpec.Network != "" { + return safeUseSubnet(clusterScope, lbSpec.Network) } networks := clusterScope.CloudscaleCluster.Status.Networks @@ -530,14 +537,11 @@ func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope. return networks[0].SubnetID, nil } -// lbPrivateNetworkSubnetID returns the subnet UUID of the private network that the LB -// VIP is placed on (spec.controlPlaneLoadBalancer.network). Caller must verify that -// spec.controlPlaneLoadBalancer.network is non-empty before calling. -func lbPrivateNetworkSubnetID(clusterScope *scope.ClusterScope) (string, error) { - name := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network - ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(name) +// safeUseSubnet returns the subnetID of the specified network (by name) only if the network is already successfully provisioned. +func safeUseSubnet(clusterScope *scope.ClusterScope, networkName string) (string, error) { + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(networkName) if ns == nil || ns.SubnetID == "" { - return "", fmt.Errorf("network %q not yet provisioned for LB VIP placement", name) + return "", fmt.Errorf("network %q not yet provisioned", networkName) } return ns.SubnetID, nil } diff --git a/internal/controller/cloudscalecluster_loadbalancer_test.go b/internal/controller/cloudscalecluster_loadbalancer_test.go index e71315c..08b6a43 100644 --- a/internal/controller/cloudscalecluster_loadbalancer_test.go +++ b/internal/controller/cloudscalecluster_loadbalancer_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -890,3 +890,51 @@ func TestDeleteLoadBalancerMember_RemovesFromStatus(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(clusterScope.CloudscaleCluster.Status.LoadBalancerMemberIDs).To(Equal([]string{"keep-uuid"})) } + +// ============================================================================ +// Tests for getPoolMemberSubnetID +// ============================================================================ + +func TestGetPoolMemberSubnetID(t *testing.T) { + cases := []struct { + name string + lbNetwork string + poolMemberNetwork string + want string + wantErr string + }{ + {name: "poolMemberNetwork wins over network", lbNetwork: "vip", poolMemberNetwork: "nodes", want: "subnet-nodes"}, + {name: "poolMemberNetwork with public VIP", poolMemberNetwork: "nodes", want: "subnet-nodes"}, + {name: "falls back to network", lbNetwork: "vip", want: "subnet-vip"}, + {name: "falls back to first network", want: "subnet-vip"}, + {name: "unprovisioned poolMemberNetwork errors", poolMemberNetwork: "missing", wantErr: "not yet provisioned"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + clusterScope := newLBClusterScope() + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "vip", CIDR: "10.0.0.0/24"}, + {Name: "nodes", CIDR: "10.1.0.0/24"}, + } + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "vip", NetworkID: "net-vip", SubnetID: "subnet-vip", CIDR: "10.0.0.0/24", Managed: true}, + {Name: "nodes", NetworkID: "net-nodes", SubnetID: "subnet-nodes", CIDR: "10.1.0.0/24", Managed: true}, + } + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network = tc.lbNetwork + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = tc.poolMemberNetwork + + subnetID, err := newTestReconciler().getPoolMemberSubnetID(clusterScope) + + if tc.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(subnetID).To(Equal(tc.want)) + }) + } +} diff --git a/internal/controller/cloudscalecluster_network.go b/internal/controller/cloudscalecluster_network.go index b4fcd91..01e7bb6 100644 --- a/internal/controller/cloudscalecluster_network.go +++ b/internal/controller/cloudscalecluster_network.go @@ -23,7 +23,7 @@ import ( "strings" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -102,7 +102,7 @@ func (r *CloudscaleClusterReconciler) reconcilePreExistingNetwork(ctx context.Co return fmt.Errorf("pre-existing network %s has no subnets", netSpec.UUID) } - r.setNetworkStatus(clusterScope, netSpec.Name, network.UUID, network.Subnets[0].UUID, network.Subnets[0].CIDR, false) + r.setNetworkStatus(clusterScope, netSpec.Name, network.UUID, network.Subnets[0].UUID, network.Subnets[0].CIDR, false, network.MTU) clusterScope.Info("Discovered pre-existing network", "name", netSpec.Name, "networkID", network.UUID, "subnetID", network.Subnets[0].UUID, "cidr", network.Subnets[0].CIDR) return nil @@ -120,7 +120,7 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex tags := r.networkTags(clusterScope, netSpec.Name) - _, resolvedNetworkID, err := ensureResource(ctx, clusterScope, + network, resolvedNetworkID, err := ensureResource(ctx, clusterScope, networkID, fmt.Sprintf("network/%s", netSpec.Name), clusterScope.CloudscaleClient.Networks, @@ -136,12 +136,16 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex clusterScope.Info("Creating network", "name", netSpec.Name) createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) defer cancel() - network, err := clusterScope.CloudscaleClient.Networks.Create(createCtx, &cloudscalesdk.NetworkCreateRequest{ + req := &cloudscalesdk.NetworkCreateRequest{ Name: netSpec.Name, AutoCreateIPV4Subnet: new(false), Zone: clusterScope.CloudscaleCluster.Spec.Zone, Tags: new(tags), - }) + } + if netSpec.MTU > 0 { + req.MTU = netSpec.MTU + } + network, err = clusterScope.CloudscaleClient.Networks.Create(createCtx, req) if err != nil { if cloudscale.IsTimeoutError(err) { clusterScope.Info("Network creation timed out, waiting before retry", "requeueAfter", createNetworkTimeoutRequeueAfter) @@ -196,7 +200,7 @@ func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Contex "Created subnet %s (%s) with CIDR %s", netSpec.Name, subnet.UUID, netSpec.CIDR) } - r.setNetworkStatus(clusterScope, netSpec.Name, resolvedNetworkID, resolvedSubnetID, netSpec.CIDR, true) + r.setNetworkStatus(clusterScope, netSpec.Name, resolvedNetworkID, resolvedSubnetID, netSpec.CIDR, true, network.MTU) return ctrl.Result{}, nil } @@ -276,13 +280,14 @@ func isLBPoolMembersError(err error) bool { } // setNetworkStatus updates or appends the network status entry for the given name. -func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.ClusterScope, name, networkID, subnetID, cidr string, managed bool) { +func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.ClusterScope, name, networkID, subnetID, cidr string, managed bool, mtu int) { for i, ns := range clusterScope.CloudscaleCluster.Status.Networks { if ns.Name == name { clusterScope.CloudscaleCluster.Status.Networks[i].NetworkID = networkID clusterScope.CloudscaleCluster.Status.Networks[i].SubnetID = subnetID clusterScope.CloudscaleCluster.Status.Networks[i].CIDR = cidr clusterScope.CloudscaleCluster.Status.Networks[i].Managed = managed + clusterScope.CloudscaleCluster.Status.Networks[i].MTU = mtu return } } @@ -292,6 +297,7 @@ func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.Clust SubnetID: subnetID, CIDR: cidr, Managed: managed, + MTU: mtu, }) } diff --git a/internal/controller/cloudscalecluster_network_test.go b/internal/controller/cloudscalecluster_network_test.go index 37e6154..4459525 100644 --- a/internal/controller/cloudscalecluster_network_test.go +++ b/internal/controller/cloudscalecluster_network_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" @@ -134,6 +134,34 @@ func TestReconcileNetwork_CreatesResources(t *testing.T) { g.Expect(capturedSubReq.GatewayAddress).To(Equal("10.0.0.254")) }, }, + { + name: "explicit mtu", + spec: func() (*testutils.MockNetworkService, *testutils.MockSubnetService, *scope.ClusterScope) { + networkService := &testutils.MockNetworkService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Network, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.NetworkCreateRequest) (*cloudscalesdk.Network, error) { + return &cloudscalesdk.Network{UUID: netUUID, Name: req.Name}, nil + }, + } + subnetService := &testutils.MockSubnetService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.SubnetCreateRequest) (*cloudscalesdk.Subnet, error) { + return &cloudscalesdk.Subnet{UUID: "subnet-uuid-123", CIDR: req.CIDR}, nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts(testutils.WithNetworkService(networkService), testutils.WithSubnetService(subnetService)) + clusterScope.CloudscaleCluster.Spec.Networks[0].MTU = 1500 + return networkService, subnetService, clusterScope + }, + want: func(g *WithT, capturedNetReq *cloudscalesdk.NetworkCreateRequest, capturedSubReq *cloudscalesdk.SubnetCreateRequest, clusterScope *scope.ClusterScope) { + g.Expect(capturedNetReq.MTU).To(Equal(1500)) + }, + }, } for _, tc := range tests { diff --git a/internal/controller/cloudscalecluster_reconcile_test.go b/internal/controller/cloudscalecluster_reconcile_test.go index 9781db0..563cdf3 100644 --- a/internal/controller/cloudscalecluster_reconcile_test.go +++ b/internal/controller/cloudscalecluster_reconcile_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalecluster_router.go b/internal/controller/cloudscalecluster_router.go new file mode 100644 index 0000000..f708b1e --- /dev/null +++ b/internal/controller/cloudscalecluster_router.go @@ -0,0 +1,555 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "time" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/observability" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" +) + +// routerRequeueAfter is how long to wait before looking again at a router that is still +// settling: a create or attach that timed out, or a router that is not active yet. +const routerRequeueAfter = 5 * time.Second + +// reconcileRouters orchestrates router provisioning for all routers +// defined in spec.routers. A single RouterReadyCondition covers all routers. +func (r *CloudscaleClusterReconciler) reconcileRouters(ctx context.Context, clusterScope *scope.ClusterScope) (result ctrl.Result, reterr error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouters") + defer done() + + defer func() { + switch { + case reterr != nil: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterErrorReason, reterr.Error()) + case !result.IsZero(): + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterNotReadyReason, "Waiting for routers to become available") + case len(clusterScope.CloudscaleCluster.Spec.Routers) == 0: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.RouterDisabledReason, "") + default: + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.RouterProvisionedReason, "") + } + }() + + logger.Info("Reconciling routers") + + // Process all routers in one reconcile call by returning the lowest requeueAfter time (or just a non-zero result). + // This ensures routers get created in parallel and avoids longer waiting times if there is more than one router + // specified. + for _, routerSpec := range clusterScope.CloudscaleCluster.Spec.Routers { + res, err := r.reconcileRouter(ctx, clusterScope, routerSpec) + if err != nil { + return ctrl.Result{}, fmt.Errorf("reconciling router %q: %w", routerSpec.Name, err) + } + noResultYet := result.IsZero() + lowerRequeueAfter := res.RequeueAfter < result.RequeueAfter + if !res.IsZero() && (noResultYet || lowerRequeueAfter) { + result = res + } + } + return result, nil +} + +// reconcileRouter brings one router to the state its spec asks for: resolve or create it, +// record it in status, and attach its interfaces once it is active. +func (r *CloudscaleClusterReconciler) reconcileRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouter") + defer done() + + var ( + router *cloudscalesdk.Router + result ctrl.Result + err error + ) + if routerSpec.UUID != "" { + router, err = r.adoptRouter(ctx, clusterScope, routerSpec) + } else { + router, result, err = r.ensureManagedRouter(ctx, clusterScope, routerSpec) + } + if err != nil || !result.IsZero() { + return result, err + } + + r.setRouterStatus(clusterScope, routerSpec, router.UUID) + + if router.Status != cloudscalesdk.RouterActive { + logger.Info("Router is not active yet, waiting before retry", + "name", routerSpec.Name, "routerID", router.UUID, "status", router.Status, "requeueAfter", routerRequeueAfter) + return ctrl.Result{RequeueAfter: routerRequeueAfter}, nil + } + + return r.reconcileRouterInterfaces(ctx, clusterScope, routerSpec, router) +} + +// adoptRouter returns the pre-existing router the spec entry references by UUID. +// +// An adopted router is one CAPCS attaches and detaches interfaces on, but never creates or +// deletes itself. Resolving it is a plain read, so it never has to requeue. +func (r *CloudscaleClusterReconciler) adoptRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (*cloudscalesdk.Router, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.adoptRouter") + defer done() + + getCtx, cancel := context.WithTimeout(ctx, cloudscale.ReadTimeout) + router, err := clusterScope.CloudscaleClient.Routers.Get(getCtx, routerSpec.UUID) + cancel() + if err != nil { + return nil, fmt.Errorf("getting pre-existing router %s: %w", routerSpec.UUID, err) + } + if router.Zone.Slug != clusterScope.CloudscaleCluster.Spec.Zone { + return nil, fmt.Errorf("pre-existing router %s is in zone %q, expected zone %q", routerSpec.UUID, router.Zone.Slug, clusterScope.CloudscaleCluster.Spec.Zone) + } + // only log if we haven't recorded the status yet + if clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) == nil { + logger.Info("Discovered pre-existing router", "name", routerSpec.Name, "routerID", router.UUID) + } + return router, nil +} + +// ensureManagedRouter returns the router CAPCS owns for this spec entry, creating it when it +// does not exist yet. A non-zero result means the router is not available yet and the caller +// should requeue. +func (r *CloudscaleClusterReconciler) ensureManagedRouter(ctx context.Context, clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec) (*cloudscalesdk.Router, ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.ensureManagedRouter") + defer done() + + var routerID string + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name); rs != nil { + routerID = rs.RouterID + } + + tags := r.routerTags(clusterScope, routerSpec.Name) + router, resolvedRouterID, err := ensureResource(ctx, clusterScope, + routerID, + fmt.Sprintf("router/%s", routerSpec.Name), + clusterScope.CloudscaleClient.Routers, + func(r cloudscalesdk.Router) string { return r.UUID }, + tags, + ) + if err != nil { + return nil, ctrl.Result{}, fmt.Errorf("ensuring router: %w", err) + } + if resolvedRouterID != "" { + return router, ctrl.Result{}, nil + } + + logger.Info("Creating router", "name", routerSpec.Name) + createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) + router, err = clusterScope.CloudscaleClient.Routers.Create(createCtx, &cloudscalesdk.RouterCreateRequest{ + Name: routerSpec.Name, + InternetGateway: routerSpec.InternetGateway, + Zone: clusterScope.CloudscaleCluster.Spec.Zone, + Tags: new(tags), + }) + cancel() + if err != nil { + if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Router creation"); timedOut { + return nil, result, nil + } + return nil, ctrl.Result{}, fmt.Errorf("creating router: %w", err) + } + logger.Info("Created router", "name", routerSpec.Name, "routerID", router.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterCreated", "CreateRouter", + "Created router %s (%s) in zone %s", routerSpec.Name, router.UUID, router.Zone.Slug) + + return router, ctrl.Result{}, nil +} + +// requeueOnTimeout turns a timed-out write into a requeue instead of an error: the call +// may well have landed, so the next reconcile re-reads rather than retrying blindly. +func (r *CloudscaleClusterReconciler) requeueOnTimeout(clusterScope *scope.ClusterScope, err error, what string) (ctrl.Result, bool) { + if !cloudscale.IsTimeoutError(err) { + return ctrl.Result{}, false + } + clusterScope.Info(what+" timed out, waiting before retry", "requeueAfter", routerRequeueAfter) + return ctrl.Result{RequeueAfter: routerRequeueAfter}, true +} + +// reconcileRouterInterfaces attaches the router to every network in its spec it is not +// already attached to, and points those subnets' gateways at the interface where the spec +// asks for it. +func (r *CloudscaleClusterReconciler) reconcileRouterInterfaces( + ctx context.Context, + clusterScope *scope.ClusterScope, + routerSpec infrastructurev1beta2.RouterSpec, + router *cloudscalesdk.Router, +) (ctrl.Result, error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.reconcileRouterInterfaces") + defer done() + + // A router attaches to any given network at most once, so indexing by network is + // unambiguous. The index by UUID is what an adopted interface is looked up through. + liveByNetwork := make(map[string]cloudscalesdk.RouterInterface, len(router.Interfaces)) + liveByUUID := make(map[string]cloudscalesdk.RouterInterface, len(router.Interfaces)) + for _, iface := range router.Interfaces { + liveByUUID[iface.UUID] = iface + liveByNetwork[iface.Network.UUID] = iface + } + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + if rs == nil { + return ctrl.Result{}, fmt.Errorf("router %q not found in status", routerSpec.Name) + } + + for _, ifaceSpec := range routerSpec.Interfaces { + logger := logger.WithValues("name", routerSpec.Name, "routerID", router.UUID, "network", ifaceSpec.Network, "specAddress", ifaceSpec.Address) + + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(ifaceSpec.Network) + if ns == nil { + return ctrl.Result{}, fmt.Errorf("network %q not found in status for router %q interface", ifaceSpec.Network, routerSpec.Name) + } + + live, attached := liveByNetwork[ns.NetworkID] + + switch { + case ifaceSpec.UUID != "": + _, err := r.adoptRouterInterface(clusterScope, logger, routerSpec, ifaceSpec, ns, liveByUUID) + if err != nil { + return ctrl.Result{}, err + } + case attached: + _, err := r.claimRouterInterface(clusterScope, logger, routerSpec, rs, ifaceSpec, ns, live) + if err != nil { + return ctrl.Result{}, err + } + default: + result, err := r.createRouterInterface(ctx, clusterScope, logger, routerSpec, router, ifaceSpec, ns) + if err != nil || !result.IsZero() { + return result, err + } + } + } + + return ctrl.Result{}, nil +} + +// adoptRouterInterface records the pre-existing interface the spec entry references by +// UUID, and returns the address it holds. +func (r *CloudscaleClusterReconciler) adoptRouterInterface( + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, + liveByUUID map[string]cloudscalesdk.RouterInterface, +) (string, error) { + live, exists := liveByUUID[ifaceSpec.UUID] + if !exists { + return "", fmt.Errorf("router %q has no interface %s to adopt, it may have been detached outside of CAPCS", routerSpec.Name, ifaceSpec.UUID) + } + if live.Network.UUID != ns.NetworkID { + return "", fmt.Errorf("router %q interface %s is attached to network %s, not to network %q (%s)", + routerSpec.Name, ifaceSpec.UUID, live.Network.UUID, ifaceSpec.Network, ns.NetworkID) + } + + address := interfaceAddress(live, ns.SubnetID) + if changed := r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, live.UUID, false); changed { + logger.Info("Adopted pre-existing router interface", "interfaceID", live.UUID, "address", address) + } + return address, nil +} + +// claimRouterInterface records an already existing interface in the clusters' status. +func (r *CloudscaleClusterReconciler) claimRouterInterface( + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + rs *infrastructurev1beta2.RouterStatus, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, + live cloudscalesdk.RouterInterface, +) (string, error) { + // if the interface is already in the API, but not yet in our status and the router is not managed, this means + // that we adopted a router with a pre-existing interface, but that interface was not mentioned in the list with a UUID. + if prev := rs.GetInterfaceStatus(ns.Name); prev == nil && !rs.Managed { + return "", fmt.Errorf("pre-existing router %q is already attached to network %q by interface %s: set spec.routers[].interfaces[].uuid to adopt that interface, or detach it", + routerSpec.Name, ifaceSpec.Network, live.UUID) + } + + address := interfaceAddress(live, ns.SubnetID) + + changed := r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, live.UUID, true) + if changed && address != ifaceSpec.Address { + logger.Info("Router is already attached to network at a different address", "address", address) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeWarning, "RouterInterfaceAddressMismatch", "ReconcileRouterInterface", + "Router %s is attached to network %s at %q, not at the requested %q, which therefore has no effect", + routerSpec.Name, ifaceSpec.Network, address, ifaceSpec.Address) + } + return address, nil +} + +// createRouterInterface attaches the router to the network at the address the spec asks +// for. +func (r *CloudscaleClusterReconciler) createRouterInterface( + ctx context.Context, + clusterScope *scope.ClusterScope, + logger logr.Logger, + routerSpec infrastructurev1beta2.RouterSpec, + router *cloudscalesdk.Router, + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + ns *infrastructurev1beta2.NetworkStatus, +) (ctrl.Result, error) { + // Record the attachment before it exists. The response can be lost to a timeout while + // the interface gets created anyway, and this entry is what tells the next reconcile the + // interface is one we created. + r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, "", true) + + logger.Info("Creating interface for router") + createCtx, cancel := context.WithTimeout(ctx, cloudscale.WriteTimeout) + createdIface, err := clusterScope.CloudscaleClient.Routers.CreateInterface(createCtx, router.UUID, cloudscalesdk.CreateInterfaceRequest{ + Network: ns.NetworkID, + Addresses: []cloudscalesdk.CreateAddressRequest{ + { + Subnet: ns.SubnetID, + Address: ifaceSpec.Address, + }, + }, + }) + cancel() + if err != nil { + if result, timedOut := r.requeueOnTimeout(clusterScope, err, "Router interface creation"); timedOut { + return result, nil + } + return ctrl.Result{}, fmt.Errorf("creating interface for router %q network %q: %w", routerSpec.Name, ifaceSpec.Network, err) + } + + r.setRouterInterfaceStatus(clusterScope, routerSpec.Name, ns.Name, createdIface.UUID, true) + logger.Info("Created interface for router", "interfaceID", createdIface.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterInterfaceCreated", "CreateRouterInterface", + "Attached router %s to network %s with address %s", routerSpec.Name, ifaceSpec.Network, ifaceSpec.Address) + return ctrl.Result{}, nil +} + +// interfaceAddress returns the address the interface holds on the given subnet, or "" when +// it holds none: a network can carry more than one subnet, and the router may be attached +// on a different one than CAPCS tracks. +func interfaceAddress(iface cloudscalesdk.RouterInterface, subnetID string) string { + for _, addr := range iface.Addresses { + if addr.Subnet.UUID == subnetID { + return addr.Address + } + } + return "" +} + +// deleteRouters removes the interfaces we created, then the routers it created. +// Pre-existing routers are left in place: only their managed interfaces go. +// On partial failure the affected routers stay in status so the next attempt retries +// exactly those. +func (r *CloudscaleClusterReconciler) deleteRouters(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.deleteRouters") + defer done() + + defer func() { + if reterr != nil { + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterErrorReason, fmt.Sprintf("Failed to delete routers: %v", reterr)) + } else { + r.setCondition(clusterScope, infrastructurev1beta2.RouterReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.RouterDeletingReason, "Routers have been deleted") + } + }() + + var remaining []infrastructurev1beta2.RouterStatus + var errs []error + + for ri := range clusterScope.CloudscaleCluster.Status.Routers { + router := &clusterScope.CloudscaleCluster.Status.Routers[ri] + logger := logger.WithValues("router", router.Name, "routerID", router.RouterID) + + // Never provisioned, so there is nothing to clean up. + if router.RouterID == "" { + continue + } + + // Routers can only be deleted if it's private interfaces are deleted first. + if ifaceErrs := r.deleteRouterInterfaces(ctx, clusterScope, router); len(ifaceErrs) > 0 { + errs = append(errs, ifaceErrs...) + remaining = append(remaining, *router) + continue + } + + if !router.Managed { + logger.Info("Skipping pre-existing router deletion") + remaining = append(remaining, *router) + continue + } + + logger.Info("Deleting router") + deleteCtx, cancel := context.WithTimeout(ctx, cloudscale.DeleteTimeout) + err := clusterScope.CloudscaleClient.Routers.Delete(deleteCtx, router.RouterID) + cancel() + + isNotFound := cloudscale.IsNotFound(err) + + if err != nil && !isNotFound { + remaining = append(remaining, *router) + errs = append(errs, fmt.Errorf("deleting router %q: %w", router.Name, err)) + continue + } + + // Idempotent not-found: log and fall through to the event. + if isNotFound { + logger.Info("Router already deleted") + } + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "RouterDeleted", "DeleteRouter", + "Deleted router %s (%s)", router.Name, router.RouterID) + } + clusterScope.CloudscaleCluster.Status.Routers = remaining + return errors.Join(errs...) +} + +// deleteRouterInterfaces deletes the interfaces CAPCS created on a router. +// +// The router is read once so the recorded interfaces can be reconciled against the live +// ones. +func (r *CloudscaleClusterReconciler) deleteRouterInterfaces( + ctx context.Context, + clusterScope *scope.ClusterScope, + rs *infrastructurev1beta2.RouterStatus, +) (errs []error) { + ctx, logger, done := observability.StartSpanWithLogger(ctx, "controllers.CloudscaleClusterReconciler.deleteRouterInterfaces") + defer done() + + logger = logger.WithValues("router", rs.Name, "routerID", rs.RouterID) + + if !hasManagedInterface(rs) { + // Nothing of ours to detach, so the router does not have to be read at all. + return nil + } + + getCtx, cancel := context.WithTimeout(ctx, cloudscale.ReadTimeout) + router, err := clusterScope.CloudscaleClient.Routers.Get(getCtx, rs.RouterID) + cancel() + if cloudscale.IsNotFound(err) { + // The router is gone, and its interfaces went with it. + logger.Info("Router already deleted, no interfaces left to detach") + rs.Interfaces = nil + return nil + } + if err != nil { + return []error{fmt.Errorf("getting router %q to detach its interfaces: %w", rs.Name, err)} + } + + liveUUIDs := make(map[string]struct{}, len(router.Interfaces)) + liveByNetwork := make(map[string]string, len(router.Interfaces)) + for _, iface := range router.Interfaces { + liveUUIDs[iface.UUID] = struct{}{} + liveByNetwork[iface.Network.UUID] = iface.UUID + } + + var remaining []infrastructurev1beta2.RouterInterfaceStatus + for _, iface := range rs.Interfaces { + if !iface.Managed { + remaining = append(remaining, iface) + continue + } + + interfaceID := iface.InterfaceID + if interfaceID == "" { + // An attach whose response never arrived: the network is all we recorded. + if ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(iface.Network); ns != nil { + interfaceID = liveByNetwork[ns.NetworkID] + } + } + if _, live := liveUUIDs[interfaceID]; !live { + // Never attached, or already detached: either way there is nothing to do. + logger.Info("Router interface already detached", "network", iface.Network, "interfaceID", iface.InterfaceID) + continue + } + + logger.Info("Deleting router interface", "network", iface.Network, "interfaceID", interfaceID) + + delCtx, cancel := context.WithTimeout(ctx, cloudscale.DeleteTimeout) + err := clusterScope.CloudscaleClient.Routers.DeleteInterface(delCtx, rs.RouterID, interfaceID) + cancel() + if err != nil && !cloudscale.IsNotFound(err) { + errs = append(errs, fmt.Errorf("deleting router interface for %s/%s: %w", rs.Name, iface.Network, err)) + // Keep the resolved UUID, so a retry does not have to resolve it again. + iface.InterfaceID = interfaceID + remaining = append(remaining, iface) + continue + } + logger.Info("Deleted router interface", "network", iface.Network, "interfaceID", interfaceID) + } + rs.Interfaces = remaining + return errs +} + +// hasManagedInterface reports whether the router has any interface CAPCS has to detach. +func hasManagedInterface(rs *infrastructurev1beta2.RouterStatus) bool { + for _, iface := range rs.Interfaces { + if iface.Managed { + return true + } + } + return false +} + +// routerTags returns the ownership tags for a specific named router. +func (r *CloudscaleClusterReconciler) routerTags(clusterScope *scope.ClusterScope, routerName string) cloudscalesdk.TagMap { + return cloudscalesdk.TagMap{ + infrastructurev1beta2.NameCloudscaleProviderOwned + clusterScope.Cluster.Name: routerName, + } +} + +// setRouterStatus updates or appends the router status entry for the given spec entry. +// .Interfaces is only written by setRouterInterfaceStatus and pruned by deleteRouterInterfaces. +func (r *CloudscaleClusterReconciler) setRouterStatus(clusterScope *scope.ClusterScope, routerSpec infrastructurev1beta2.RouterSpec, routerID string) { + managed := routerSpec.UUID == "" + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerSpec.Name); rs != nil { + rs.RouterID = routerID + rs.Managed = managed + return + } + clusterScope.CloudscaleCluster.Status.Routers = append(clusterScope.CloudscaleCluster.Status.Routers, infrastructurev1beta2.RouterStatus{ + Name: routerSpec.Name, + RouterID: routerID, + Managed: managed, + }) +} + +// setRouterInterfaceStatus upserts an interface entry on the named router's status. +func (r *CloudscaleClusterReconciler) setRouterInterfaceStatus(clusterScope *scope.ClusterScope, routerName, networkName, interfaceID string, managed bool) bool { + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus(routerName) + if rs == nil { + return false + } + if iface := rs.GetInterfaceStatus(networkName); iface != nil { + if iface.InterfaceID == interfaceID && iface.Managed == managed { + return false + } + iface.InterfaceID = interfaceID + iface.Managed = managed + return true + } + rs.Interfaces = append(rs.Interfaces, infrastructurev1beta2.RouterInterfaceStatus{ + Network: networkName, + InterfaceID: interfaceID, + Managed: managed, + }) + return true +} diff --git a/internal/controller/cloudscalecluster_router_test.go b/internal/controller/cloudscalecluster_router_test.go new file mode 100644 index 0000000..49b9839 --- /dev/null +++ b/internal/controller/cloudscalecluster_router_test.go @@ -0,0 +1,956 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "net/url" + "os" + "testing" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/cluster-api/util/conditions" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/testutils" +) + +const ( + routerUUID = "router-uuid-123" + ifaceUUID = "iface-uuid-123" + routerAddress = "10.0.0.1" + subnetUUID = "subnet-uuid-123" +) + +// activeRouter is what the API returns for a provisioned router. +func activeRouter(uuid string, interfaces ...cloudscalesdk.RouterInterface) *cloudscalesdk.Router { + return &cloudscalesdk.Router{ + UUID: uuid, + Status: cloudscalesdk.RouterActive, + Interfaces: interfaces, + Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, + } +} + +// routerInterface is what the API returns for an interface attached to a network, +// holding one address on that network's subnet. +func routerInterface(uuid, address string) cloudscalesdk.RouterInterface { + return cloudscalesdk.RouterInterface{ + UUID: uuid, + Network: cloudscalesdk.NetworkStub{Name: "test", UUID: netUUID}, + Addresses: []cloudscalesdk.IPAddress{ + {Address: address, Subnet: cloudscalesdk.SubnetStub{UUID: subnetUUID}}, + }, + } +} + +// routerSpec is a managed router attached to the default "test" network. +func routerSpec() infrastructurev1beta2.RouterSpec { + return infrastructurev1beta2.RouterSpec{ + Name: "test-router", + InternetGateway: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "test", Address: routerAddress}, + }, + } +} + +// managedInterfaces is how a status for a single interface CAPCS created on the +// default "test" network looks like. +func managedInterfaces() []infrastructurev1beta2.RouterInterfaceStatus { + return []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: ifaceUUID, Managed: true}, + } +} + +// recordedEvents drains everything the reconciler has emitted so far. +func recordedEvents(r *CloudscaleClusterReconciler) []string { + var out []string + for { + select { + case e := <-r.recorder.(*events.FakeRecorder).Events: + out = append(out, e) + default: + return out + } + } +} + +// --- Managed router provisioning --- + +func TestReconcileRouters_CreatesRouterAndInterface(t *testing.T) { + g := NewWithT(t) + + var interfaceRouterID string + var capturedCreateReq *cloudscalesdk.RouterCreateRequest + var capturedIfaceReq cloudscalesdk.CreateInterfaceRequest + routerService := &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + capturedCreateReq = req + return activeRouter(routerUUID), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + interfaceRouterID = uuid + capturedIfaceReq = req + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + subnetService := &testutils.MockSubnetService{} + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(routerSpec()), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(capturedCreateReq).ToNot(BeNil()) + g.Expect(capturedCreateReq.Name).To(Equal("test-router")) + g.Expect(capturedCreateReq.InternetGateway).To(BeTrue()) + g.Expect(capturedCreateReq.Zone).To(Equal(clusterScope.CloudscaleCluster.Spec.Zone)) + g.Expect(capturedCreateReq.Tags).To(Equal(new(cloudscalesdk.TagMap{ + infrastructurev1beta2.NameCloudscaleProviderOwned + "test-cluster": "test-router", + }))) + + g.Expect(interfaceRouterID).To(Equal(routerUUID)) + g.Expect(capturedIfaceReq.Network).To(Equal(netUUID)) + g.Expect(capturedIfaceReq.Addresses).To(HaveLen(1)) + g.Expect(capturedIfaceReq.Addresses[0].Subnet).To(Equal(subnetUUID)) + g.Expect(capturedIfaceReq.Addresses[0].Address).To(Equal("10.0.0.1")) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.RouterID).To(Equal(routerUUID)) + g.Expect(rs.Managed).To(BeTrue()) + // Keyed by the CAPCS network name, holding the interface UUID (not the network's). + g.Expect(rs.Interfaces).To(Equal(managedInterfaces())) + + // The exact, ordered events for the managed path. + g.Expect(recordedEvents(r)).To(Equal([]string{ + "Normal RouterCreated Created router test-router (router-uuid-123) in zone rma1", + "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1", + })) +} + +// --- Pre-existing (adopted) routers --- + +// TestReconcileRouters_AdoptedInterfaceOwnership verifies the reconciler does correctly +// detect interface ownership. +func TestReconcileRouters_AdoptedInterfaceOwnership(t *testing.T) { + const preAttachedUUID = "pre-attached-iface-uuid" + + // router in a network CAPCS knows nothing about. + foreign := cloudscalesdk.RouterInterface{ + UUID: "foreign-iface-uuid", + Network: cloudscalesdk.NetworkStub{Name: "someone-elses-network", UUID: "foreign-net-uuid"}, + } + interfaceStatus := func(uuid string, managed bool) []infrastructurev1beta2.RouterInterfaceStatus { + return []infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: uuid, Managed: managed}, + } + } + + const ( + mismatchEvent = `Warning RouterInterfaceAddressMismatch Router test-router is attached to network test at "10.0.0.42", not at the requested "10.0.0.1", which therefore has no effect` + ifaceCreatedEvent = "Normal RouterInterfaceCreated Attached router test-router to network test with address 10.0.0.1" + ) + + tests := []struct { + name string + // live is what the API reports the router is attached to. + live []cloudscalesdk.RouterInterface + // adopt is the interface uuid the spec entry declares, empty when the interface should be created + adopt string + // status is the interface status + status []infrastructurev1beta2.RouterInterfaceStatus + + // wantErr is a substring of the error the reconcile has to fail with. The + // reconcile is run twice either way: a collision must stay a collision. + wantErr string + wantInterfaces []infrastructurev1beta2.RouterInterfaceStatus + wantCreate bool + // wantEvents is the exact, ordered set of events the reconcile must emit. + wantEvents []string + }{ + { + // Nothing on a spec network, so CAPCS attaches its own and owns it. The + // router's foreign interface is not recorded at all. + name: "unattached to the spec network", + live: []cloudscalesdk.RouterInterface{foreign}, + wantInterfaces: managedInterfaces(), + wantCreate: true, + wantEvents: []string{ifaceCreatedEvent}, + }, + { + name: "pre-existing attachment at the spec address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, routerAddress)}, + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + }, + { + name: "pre-existing attachment at another address is a collision", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, + wantErr: "set spec.routers[].interfaces[].uuid to adopt that interface", + }, + { + name: "adopted by uuid is used and left alone", + live: []cloudscalesdk.RouterInterface{routerInterface(preAttachedUUID, "10.0.0.42")}, + adopt: preAttachedUUID, + wantInterfaces: interfaceStatus(preAttachedUUID, false), + }, + { + name: "adopted uuid the router does not carry", + live: []cloudscalesdk.RouterInterface{foreign}, + adopt: preAttachedUUID, + wantErr: "has no interface " + preAttachedUUID + " to adopt", + }, + { + name: "adopted uuid attached to another network", + live: []cloudscalesdk.RouterInterface{foreign}, + adopt: foreign.UUID, + wantErr: "is attached to network foreign-net-uuid, not to network \"test\"", + }, + { + // An attach whose response was lost to a timeout: the entry written before + // the call is what tells CAPCS this interface is its own, and it is filled + // in as soon as the interface is seen. + name: "attachment recorded before its response arrived", + live: []cloudscalesdk.RouterInterface{routerInterface(ifaceUUID, routerAddress)}, + status: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + wantInterfaces: managedInterfaces(), + }, + { + name: "replaced interface stays managed", + live: []cloudscalesdk.RouterInterface{routerInterface("replacement-iface-uuid", "10.0.0.42")}, + status: managedInterfaces(), + wantInterfaces: interfaceStatus("replacement-iface-uuid", true), + wantEvents: []string{mismatchEvent}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + created := false + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, tc.live...), nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + g.Fail("Create must not be called for a pre-existing router") + return nil, nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Expect(tc.wantCreate).To(BeTrue(), "CreateInterface called but the router is already attached") + g.Expect(uuid).To(Equal(routerUUID)) + created = true + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + + subnetService := &testutils.MockSubnetService{} + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + if tc.adopt != "" { + // The webhook forbids requesting an address for an adopted interface. + spec.Interfaces[0].UUID = tc.adopt + spec.Interfaces[0].Address = "" + } + + opts := []testutils.ClusterScopeOption{ + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithAdoptedNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + } + if tc.status != nil { + opts = append(opts, testutils.WithRouterStatus("test-router", routerUUID, false, tc.status)) + } + clusterScope := testutils.NewClusterScopeOpts(opts...) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + if tc.wantErr != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.wantErr))) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(created).To(Equal(tc.wantCreate)) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.Managed).To(BeFalse(), "an adopted router is never CAPCS-managed") + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + + g.Expect(recordedEvents(r)).To(Equal(tc.wantEvents)) + + // Reconciling again must not revisit any of it, and must not talk itself + // out of a collision either. + _, err = r.reconcileRouters(context.Background(), clusterScope) + if tc.wantErr != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.wantErr))) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + }) + } +} + +func TestReconcileRouters_PreExistingZoneMismatchErrors(t *testing.T) { + g := NewWithT(t) + + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return &cloudscalesdk.Router{ + UUID: id, + Status: cloudscalesdk.RouterActive, + Zone: cloudscalesdk.ZoneStub{Slug: "lpg1"}, + }, nil + }, + } + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("expected zone")) + + // A hard error must be visible on the condition, not just returned. + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterErrorReason)) + g.Expect(cond.Message).To(ContainSubstring("expected zone")) +} + +// TestReconcileRouters_AttachesMultipleInterfaces tests multiple networks with interfaces. +func TestReconcileRouters_AttachesMultipleInterfaces(t *testing.T) { + g := NewWithT(t) + + const ( + transitNetUUID = "transit-net-uuid" + transitSubnetUUID = "transit-subnet-uuid" + ) + + attachedTo := map[string]cloudscalesdk.CreateAddressRequest{} + routerService := &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + return activeRouter(routerUUID), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Expect(req.Addresses).To(HaveLen(1)) + attachedTo[req.Network] = req.Addresses[0] + return &cloudscalesdk.RouterInterface{UUID: "iface-" + req.Network}, nil + }, + } + + gatewayUpdates := map[string]string{} + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + gatewayUpdates[id] = req.GatewayAddress + return nil + }, + } + + spec := routerSpec() + spec.Interfaces = append(spec.Interfaces, infrastructurev1beta2.RouterInterfaceSpec{ + Network: "transit", Address: "10.9.0.2", + }) + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithPreExistingNetwork("transit", transitNetUUID, transitSubnetUUID, "10.9.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(attachedTo).To(Equal(map[string]cloudscalesdk.CreateAddressRequest{ + netUUID: {Subnet: subnetUUID, Address: routerAddress}, + transitNetUUID: {Subnet: transitSubnetUUID, Address: "10.9.0.2"}, + })) + + rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router") + g.Expect(rs).ToNot(BeNil()) + g.Expect(rs.Interfaces).To(Equal([]infrastructurev1beta2.RouterInterfaceStatus{ + {Network: "test", InterfaceID: "iface-" + netUUID, Managed: true}, + {Network: "transit", InterfaceID: "iface-" + transitNetUUID, Managed: true}, + })) +} + +// TestReconcileRouters_UnprovisionedNetworkErrors guards the ordering contract: routers are +// reconciled after networks, so an interface pointing at a network that has no status yet +// means something upstream went wrong and must not be papered over. +func TestReconcileRouters_UnprovisionedNetworkErrors(t *testing.T) { + g := NewWithT(t) + + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + g.Fail("CreateInterface must not be called for a network that is not provisioned") + return nil, nil + }, + } + + spec := routerSpec() + spec.UUID = routerUUID + spec.InternetGateway = false + spec.Interfaces[0].Network = "not-provisioned-yet" + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(spec), + ) + r := newTestReconciler() + + _, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`network "not-provisioned-yet" not found in status`)) +} + +// --- Condition reporting --- + +func TestReconcileRouters_ConditionReflectsProgress(t *testing.T) { + tests := []struct { + name string + setup func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) + wantRequeue bool + wantStatus metav1.ConditionStatus + wantReason string + wantAttached bool + }{ + { + name: "no routers is ready but disabled", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + return &testutils.MockRouterService{}, nil + }, + wantStatus: metav1.ConditionTrue, + wantReason: infrastructurev1beta2.RouterDisabledReason, + }, + { + name: "router not active yet requeues without claiming ready", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + svc := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return &cloudscalesdk.Router{ + UUID: id, + Status: "unknown", + Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, + }, nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return nil, errors.New("CreateInterface must not be called while the router is unknown") + }, + } + return svc, []testutils.ClusterScopeOption{ + testutils.WithRouter(routerSpec()), + testutils.WithRouterStatus("test-router", routerUUID, true, nil), + } + }, + wantRequeue: true, + wantStatus: metav1.ConditionFalse, + wantReason: infrastructurev1beta2.RouterNotReadyReason, + }, + { + name: "provisioned router is ready", + setup: func() (*testutils.MockRouterService, []testutils.ClusterScopeOption) { + svc := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + } + return svc, []testutils.ClusterScopeOption{ + testutils.WithRouter(routerSpec()), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + } + }, + wantStatus: metav1.ConditionTrue, + wantReason: infrastructurev1beta2.RouterProvisionedReason, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + routerService, extraOpts := tc.setup() + opts := append([]testutils.ClusterScopeOption{ + testutils.WithRouterService(routerService), + testutils.WithSubnetService(&testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { return nil }, + }), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + }, extraOpts...) + clusterScope := testutils.NewClusterScopeOpts(opts...) + // The gateway is already in place, so these cases only exercise the condition. + clusterScope.CloudscaleCluster.Status.Networks[0].GatewayAddress = routerAddress + r := newTestReconciler() + + result, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + if tc.wantRequeue { + g.Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + } else { + g.Expect(result.RequeueAfter).To(BeZero()) + } + + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(tc.wantStatus)) + g.Expect(cond.Reason).To(Equal(tc.wantReason)) + }) + } +} + +// TestReconcileRouters_TimeoutsRequeue verifies all timeouts requeue to recover the state. +func TestReconcileRouters_TimeoutsRequeue(t *testing.T) { + timeout := func(op, path string) error { + return &url.Error{Op: op, URL: "https://api.example.com/v1/" + path, Err: os.ErrDeadlineExceeded} + } + + tests := []struct { + name string + routerService func() *testutils.MockRouterService + subnetService func() *testutils.MockSubnetService + seedStatus bool + setupClusterScope func(*scope.ClusterScope) + // wantInterfaces is the interface status the timed-out write must leave behind. + wantInterfaces []infrastructurev1beta2.RouterInterfaceStatus + wantProvisioned bool + }{ + { + name: "router create", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + CreateFn: func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + return nil, timeout("Post", "routers") + }, + } + }, + wantProvisioned: false, + }, + { + name: "interface create", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return nil, timeout("Post", "routers/"+uuid+"/interfaces") + }, + } + }, + seedStatus: true, + // The attach may have landed even though its response did not, so the + // entry recorded before the call has to survive the requeue. Without it + // the next reconcile cannot tell the interface from one the router + // already carried. + wantInterfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + wantProvisioned: false, + }, + + { + name: "interface create succeeds", + routerService: func() *testutils.MockRouterService { + return &testutils.MockRouterService{ + ListFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + return nil, nil + }, + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + CreateInterfaceFn: func(ctx context.Context, uuid string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + return &cloudscalesdk.RouterInterface{UUID: ifaceUUID}, nil + }, + } + }, + seedStatus: true, + setupClusterScope: testutils.WithControlPlaneEndpoint, + wantInterfaces: managedInterfaces(), + wantProvisioned: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + subnetService := &testutils.MockSubnetService{ + UpdateFn: func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { return nil }, + } + if tc.subnetService != nil { + subnetService = tc.subnetService() + } + + opts := []testutils.ClusterScopeOption{ + testutils.WithRouterService(tc.routerService()), + testutils.WithSubnetService(subnetService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouter(routerSpec()), + } + if tc.seedStatus { + opts = append(opts, testutils.WithRouterStatus("test-router", routerUUID, true, nil)) + } + clusterScope := testutils.NewClusterScopeOpts(opts...) + + if tc.setupClusterScope != nil { + tc.setupClusterScope(clusterScope) + } + + r := newTestReconciler() + + result, err := r.reconcileRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + if tc.wantProvisioned { + g.Expect(result.IsZero()).To(BeTrue()) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterProvisionedReason)) + } else { + g.Expect(result.RequeueAfter).To(Equal(routerRequeueAfter)) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.RouterNotReadyReason)) + } + + if rs := clusterScope.CloudscaleCluster.Status.GetRouterStatus("test-router"); rs != nil { + g.Expect(rs.Interfaces).To(Equal(tc.wantInterfaces)) + } + g.Expect(r.isInfrastructureProvisioned(clusterScope)).To(Equal(tc.wantProvisioned), "expected IsInfrastructureProvisioned to be %v", tc.wantProvisioned) + }) + } +} + +// --- Deletion --- + +func TestDeleteRouters_DeletesManagedRouterAndInterfaces(t *testing.T) { + g := NewWithT(t) + + var deletedRouterID string + deletedInterfaces := map[string]string{} + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + deletedRouterID = id + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + deletedInterfaces[routerUUID] = interfaceUUID + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("test-router", routerUUID, true, managedInterfaces()), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(deletedInterfaces).To(Equal(map[string]string{routerUUID: ifaceUUID})) + g.Expect(deletedRouterID).To(Equal(routerUUID)) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(BeEmpty()) +} + +// TestDeleteRouters_PreservesForeignInterfacesOnAdoptedRouter guards the case where a +// user points CAPCS at a shared router: deleting the cluster must remove the interfaces +// CAPCS attached and nothing else, and must leave the router itself alone. +func TestDeleteRouters_PreservesForeignInterfacesOnAdoptedRouter(t *testing.T) { + g := NewWithT(t) + + preAttached := infrastructurev1beta2.RouterInterfaceStatus{ + Network: "shared", InterfaceID: "pre-attached-iface-uuid", Managed: false, + } + + var deletedInterfaces []string + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, + routerInterface(ifaceUUID, routerAddress), + cloudscalesdk.RouterInterface{ + UUID: preAttached.InterfaceID, + Network: cloudscalesdk.NetworkStub{Name: "shared", UUID: "shared-net-uuid"}, + }, + ), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Delete should not be called for a pre-existing router") + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + deletedInterfaces = append(deletedInterfaces, interfaceUUID) + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + // "shared" is the user's network, so CAPCS never deletes it. + // "test" is one CAPCS manages. + testutils.WithAdoptedNetwork("shared", "shared-net-uuid", "shared-subnet-uuid", "10.1.0.0/24"), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("shared-router", routerUUID, false, + append(managedInterfaces(), preAttached)), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + // Only the CAPCS-created interface is detached + g.Expect(deletedInterfaces).To(Equal([]string{ifaceUUID})) + // The adopted router stays in status, keeping the interface it came with. + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(HaveLen(1)) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Name).To(Equal("shared-router")) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Managed).To(BeFalse()) + g.Expect(clusterScope.CloudscaleCluster.Status.Routers[0].Interfaces).To(Equal( + []infrastructurev1beta2.RouterInterfaceStatus{preAttached})) +} + +// TestDeleteRouters_ResolvesInterfacesAgainstLiveRouter ensures interfaces are deleted based on the current +// router's API response. +func TestDeleteRouters_ResolvesInterfacesAgainstLiveRouter(t *testing.T) { + tests := []struct { + name string + // live is what the API reports the router is attached to, nil for a router that + // no longer exists. + live []cloudscalesdk.RouterInterface + routerGone bool + interfaces []infrastructurev1beta2.RouterInterfaceStatus + + wantDetached []string + }{ + { + // Recorded before the attach existed, so the network is all there is to go on. + name: "attachment with no recorded uuid is resolved by network", + live: []cloudscalesdk.RouterInterface{routerInterface(ifaceUUID, routerAddress)}, + interfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + wantDetached: []string{ifaceUUID}, + }, + { + // The attach never landed after all: there is nothing to detach, and the + // entry must not hold the router's deletion back. + name: "attachment that never landed is dropped", + live: nil, + interfaces: []infrastructurev1beta2.RouterInterfaceStatus{{Network: "test", Managed: true}}, + }, + { + name: "recorded interface that is already detached is dropped", + live: nil, + interfaces: managedInterfaces(), + }, + { + name: "router already gone takes its interfaces with it", + routerGone: true, + interfaces: managedInterfaces(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + var detached []string + routerService := &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + if tc.routerGone { + return nil, &cloudscalesdk.ErrorResponse{StatusCode: 404} + } + return activeRouter(id, tc.live...), nil + }, + DeleteFn: func(ctx context.Context, id string) error { return nil }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + detached = append(detached, interfaceUUID) + return nil + }, + } + + clusterScope := testutils.NewClusterScopeOpts( + testutils.WithRouterService(routerService), + testutils.WithPreExistingNetwork("test", netUUID, subnetUUID, "10.0.0.0/24"), + testutils.WithRouterStatus("test-router", routerUUID, true, tc.interfaces), + ) + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(detached).To(Equal(tc.wantDetached)) + // Whatever happened to the interfaces, the router itself is gone from status. + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(BeEmpty()) + }) + } +} + +func TestDeleteRouters_BasicScenarios(t *testing.T) { + tests := []struct { + name string + routerService func(g *WithT) *testutils.MockRouterService + statusSetup func() []infrastructurev1beta2.RouterStatus + wantErr bool + wantStatusLen int + }{ + { + name: "never provisioned router is dropped without any API call", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Delete should not be called for a router that was never created") + return nil + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", Managed: true}} + }, + wantStatusLen: 0, + }, + { + name: "already deleted router is ignored", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + return &cloudscalesdk.ErrorResponse{StatusCode: 404} + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", RouterID: routerUUID, Managed: true}} + }, + wantStatusLen: 0, + }, + { + name: "failed router deletion keeps it in status", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + DeleteFn: func(ctx context.Context, id string) error { + return errors.New("boom") + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{Name: "test-router", RouterID: routerUUID, Managed: true}} + }, + wantErr: true, + wantStatusLen: 1, + }, + { + name: "failed interface deletion keeps the router for the next attempt", + routerService: func(g *WithT) *testutils.MockRouterService { + return &testutils.MockRouterService{ + GetFn: func(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + return activeRouter(id, routerInterface(ifaceUUID, routerAddress)), nil + }, + DeleteFn: func(ctx context.Context, id string) error { + g.Fail("Router must not be deleted while its interfaces are still attached") + return nil + }, + DeleteInterfaceFn: func(ctx context.Context, routerUUID, interfaceUUID string) error { + return errors.New("boom") + }, + } + }, + statusSetup: func() []infrastructurev1beta2.RouterStatus { + return []infrastructurev1beta2.RouterStatus{{ + Name: "test-router", RouterID: routerUUID, Managed: true, + Interfaces: managedInterfaces(), + }} + }, + wantErr: true, + wantStatusLen: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + clusterScope := testutils.NewClusterScopeOpts(testutils.WithRouterService(tc.routerService(g))) + clusterScope.CloudscaleCluster.Status.Routers = tc.statusSetup() + r := newTestReconciler() + + err := r.deleteRouters(context.Background(), clusterScope) + + wantReason := infrastructurev1beta2.RouterDeletingReason + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + wantReason = infrastructurev1beta2.RouterErrorReason + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(clusterScope.CloudscaleCluster.Status.Routers).To(HaveLen(tc.wantStatusLen)) + + // Deletion always leaves the condition False; only the reason distinguishes + // "torn down" from "could not tear down". + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.RouterReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(wantReason)) + }) + } +} diff --git a/internal/controller/cloudscalecluster_servergroup.go b/internal/controller/cloudscalecluster_servergroup.go index 300c60d..37d81bd 100644 --- a/internal/controller/cloudscalecluster_servergroup.go +++ b/internal/controller/cloudscalecluster_servergroup.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" "k8s.io/apimachinery/pkg/util/sets" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/controller/cloudscalecluster_servergroup_test.go b/internal/controller/cloudscalecluster_servergroup_test.go index 890acec..e03f2b0 100644 --- a/internal/controller/cloudscalecluster_servergroup_test.go +++ b/internal/controller/cloudscalecluster_servergroup_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" "k8s.io/client-go/tools/events" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" diff --git a/internal/controller/cloudscalemachine_reconcile_test.go b/internal/controller/cloudscalemachine_reconcile_test.go index b366193..f49a2ba 100644 --- a/internal/controller/cloudscalemachine_reconcile_test.go +++ b/internal/controller/cloudscalemachine_reconcile_test.go @@ -27,7 +27,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" @@ -60,10 +60,10 @@ func TestMachineReconcileNormal_ServerRunning_SetsProvisioned(t *testing.T) { UUID: id, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.5", Version: 4}, }, }, diff --git a/internal/controller/cloudscalemachine_server.go b/internal/controller/cloudscalemachine_server.go index 324d4a6..092afe7 100644 --- a/internal/controller/cloudscalemachine_server.go +++ b/internal/controller/cloudscalemachine_server.go @@ -22,7 +22,7 @@ import ( "maps" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -303,7 +303,7 @@ func (r *CloudscaleMachineReconciler) machineLookupTag(machineScope *scope.Machi // If spec.interfaces is empty, defaults to the first cluster network + a public interface // (runtime cross-resource resolution that the webhook cannot do). // Returns the interface requests and the IPFamily from the public interface (if any). -func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) (*[]cloudscalesdk.InterfaceRequest, *infrastructurev1beta2.IPFamily, error) { +func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) (*[]cloudscalesdk.ServerInterfaceRequest, *infrastructurev1beta2.IPFamily, error) { ifaceSpecs := machineScope.CloudscaleMachine.Spec.Interfaces // Runtime default: first cluster network + public DualStack interface @@ -312,19 +312,19 @@ func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope return nil, nil, fmt.Errorf("cluster has no networks provisioned yet") } firstNetwork := machineScope.CloudscaleCluster.Status.Networks[0] - return &[]cloudscalesdk.InterfaceRequest{ + return &[]cloudscalesdk.ServerInterfaceRequest{ {Network: InterfaceTypePublic}, {Network: firstNetwork.NetworkID}, }, nil, nil } // Build from spec - reqs := make([]cloudscalesdk.InterfaceRequest, 0, len(ifaceSpecs)) + reqs := make([]cloudscalesdk.ServerInterfaceRequest, 0, len(ifaceSpecs)) var ipFamily *infrastructurev1beta2.IPFamily for _, iface := range ifaceSpecs { switch { case iface.Type == InterfaceTypePublic: - reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: InterfaceTypePublic}) + reqs = append(reqs, cloudscalesdk.ServerInterfaceRequest{Network: InterfaceTypePublic}) ipFamily = iface.IPFamily case iface.Network != "": ns := machineScope.CloudscaleCluster.Status.GetNetworkStatus(iface.Network) @@ -334,7 +334,7 @@ func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope if ns.NetworkID == "" { return nil, nil, fmt.Errorf("network %q not yet provisioned", iface.Network) } - reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: ns.NetworkID}) + reqs = append(reqs, cloudscalesdk.ServerInterfaceRequest{Network: ns.NetworkID}) default: return nil, nil, fmt.Errorf("interface must have either type or network set") } diff --git a/internal/controller/cloudscalemachine_server_test.go b/internal/controller/cloudscalemachine_server_test.go index b871921..a6f869d 100644 --- a/internal/controller/cloudscalemachine_server_test.go +++ b/internal/controller/cloudscalemachine_server_test.go @@ -20,7 +20,7 @@ import ( "context" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" @@ -48,10 +48,10 @@ func TestReconcileServer_CreatesServer(t *testing.T) { Name: req.Name, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.5", Version: 4}, }, }, @@ -122,16 +122,16 @@ func TestReconcileServer_SetsAddresses(t *testing.T) { Name: req.Name, Status: "running", Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}, - Interfaces: []cloudscalesdk.Interface{ + Interfaces: []cloudscalesdk.ServerInterface{ { Type: "public", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "185.98.123.45", Version: 4}, }, }, { Type: "private", - Addresses: []cloudscalesdk.Address{ + Addresses: []cloudscalesdk.ServerAddress{ {Address: "10.0.0.10", Version: 4}, }, }, diff --git a/internal/controller/cloudscalemachine_servergroup.go b/internal/controller/cloudscalemachine_servergroup.go index 63d6a87..969f691 100644 --- a/internal/controller/cloudscalemachine_servergroup.go +++ b/internal/controller/cloudscalemachine_servergroup.go @@ -22,7 +22,7 @@ import ( "sync" "time" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" diff --git a/internal/controller/cloudscalemachine_servergroup_test.go b/internal/controller/cloudscalemachine_servergroup_test.go index 9ce1d41..0226e45 100644 --- a/internal/controller/cloudscalemachine_servergroup_test.go +++ b/internal/controller/cloudscalemachine_servergroup_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" diff --git a/internal/testutils/cluster_scope.go b/internal/testutils/cluster_scope.go index 73f1aaa..dafddf7 100644 --- a/internal/testutils/cluster_scope.go +++ b/internal/testutils/cluster_scope.go @@ -85,6 +85,11 @@ func WithSubnetService(svc cloudscale.SubnetService) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleClient.Subnets = svc } } +// WithRouterService wires a custom Router service into the client. +func WithRouterService(svc cloudscale.RouterService) ClusterScopeOption { + return func(cs *scope.ClusterScope) { cs.CloudscaleClient.Routers = svc } +} + // WithServerGroupService wires a custom ServerGroup service into the client. func WithServerGroupService(svc cloudscale.ServerGroupService) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleClient.ServerGroups = svc } @@ -135,7 +140,9 @@ func WithHealthMonitorParams(delayS, timeoutS, upThreshold, downThreshold int) C } } -// WithPreExistingNetwork adds a NetworkStatus entry so the scope thinks the network already exists. +// WithPreExistingNetwork adds a NetworkStatus entry so the scope thinks the network +// already exists. The network is CAPCS-managed, i.e. defined by cidr in spec.networks; +// use WithAdoptedNetwork for one referenced by uuid. func WithPreExistingNetwork(name, netID, subnetID, cidr string) ClusterScopeOption { return func(cs *scope.ClusterScope) { cs.CloudscaleCluster.Status.Networks = append(cs.CloudscaleCluster.Status.Networks, @@ -150,6 +157,58 @@ func WithPreExistingNetwork(name, netID, subnetID, cidr string) ClusterScopeOpti } } +// WithAdoptedNetwork adds a NetworkStatus entry for a network CAPCS does not own, the +// shape spec.networks[].uuid produces. Deletion leaves such a network in place, so an +// interface attached to it is never detached on CAPCS's account. +func WithAdoptedNetwork(name, netID, subnetID, cidr string) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Status.Networks = append(cs.CloudscaleCluster.Status.Networks, + infrastructurev1beta2.NetworkStatus{ + Name: name, + NetworkID: netID, + SubnetID: subnetID, + CIDR: cidr, + Managed: false, + }, + ) + } +} + +// WithRouter appends a router to the spec. +func WithRouter(routerSpec infrastructurev1beta2.RouterSpec) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Spec.Routers = append(cs.CloudscaleCluster.Spec.Routers, routerSpec) + } +} + +// WithRouterStatus appends a RouterStatus entry so the scope thinks the router already +// exists. interfaces lists the router's recorded interfaces on spec networks; pass nil +// for a router with no recorded interfaces. +func WithRouterStatus(name, routerID string, managed bool, interfaces []infrastructurev1beta2.RouterInterfaceStatus) ClusterScopeOption { + return func(cs *scope.ClusterScope) { + cs.CloudscaleCluster.Status.Routers = append(cs.CloudscaleCluster.Status.Routers, + infrastructurev1beta2.RouterStatus{ + Name: name, + RouterID: routerID, + Managed: managed, + Interfaces: interfaces, + }, + ) + } +} + +// WithControlPlaneEndpoint sets up load balancer status and control plane endpoint +// for isInfrastructureProvisioned testing. Uses sensible defaults for test scenarios. +func WithControlPlaneEndpoint(clusterScope *scope.ClusterScope) { + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = new(bool) + *clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = true + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "test-lb-id" + clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID = "test-lb-pool-id" + clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID = "test-lb-listener-id" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = "10.20.30.40" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port = 6443 +} + // WithGeneration sets the generation (used for status conditions). func WithGeneration(gen int64) ClusterScopeOption { return func(cs *scope.ClusterScope) { diff --git a/internal/testutils/fixtures.go b/internal/testutils/fixtures.go index 0255889..126208f 100644 --- a/internal/testutils/fixtures.go +++ b/internal/testutils/fixtures.go @@ -1,7 +1,7 @@ package testutils import ( - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" diff --git a/internal/testutils/mocks.go b/internal/testutils/mocks.go index ae2d1c2..57ceacd 100644 --- a/internal/testutils/mocks.go +++ b/internal/testutils/mocks.go @@ -4,7 +4,7 @@ import ( "context" "errors" - cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v9" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v10" ) // --- Network Service Mock --- @@ -51,6 +51,7 @@ type MockSubnetService struct { GetFn func(ctx context.Context, id string) (*cloudscalesdk.Subnet, error) ListFn func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Subnet, error) DeleteFn func(ctx context.Context, id string) error + UpdateFn func(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error } func (m *MockSubnetService) Create(ctx context.Context, req *cloudscalesdk.SubnetCreateRequest) (*cloudscalesdk.Subnet, error) { @@ -81,6 +82,66 @@ func (m *MockSubnetService) Delete(ctx context.Context, id string) error { return errors.New("mock(Delete) not configured") } +func (m *MockSubnetService) Update(ctx context.Context, id string, req *cloudscalesdk.SubnetUpdateRequest) error { + if m.UpdateFn != nil { + return m.UpdateFn(ctx, id, req) + } + return errors.New("mock(Update) not configured") +} + +// --- Router Service Mock --- + +type MockRouterService struct { + CreateFn func(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) + GetFn func(ctx context.Context, id string) (*cloudscalesdk.Router, error) + ListFn func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) + DeleteFn func(ctx context.Context, id string) error + CreateInterfaceFn func(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) + DeleteInterfaceFn func(ctx context.Context, routerUUID, interfaceUUID string) error +} + +func (m *MockRouterService) Create(ctx context.Context, req *cloudscalesdk.RouterCreateRequest) (*cloudscalesdk.Router, error) { + if m.CreateFn != nil { + return m.CreateFn(ctx, req) + } + return nil, errors.New("mock(Create) not configured") +} + +func (m *MockRouterService) Get(ctx context.Context, id string) (*cloudscalesdk.Router, error) { + if m.GetFn != nil { + return m.GetFn(ctx, id) + } + return nil, errors.New("mock(Get) not configured") +} + +func (m *MockRouterService) List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Router, error) { + if m.ListFn != nil { + return m.ListFn(ctx, modifiers...) + } + return nil, errors.New("mock(List) not configured") +} + +func (m *MockRouterService) Delete(ctx context.Context, id string) error { + if m.DeleteFn != nil { + return m.DeleteFn(ctx, id) + } + return errors.New("mock(Delete) not configured") +} + +func (m *MockRouterService) CreateInterface(ctx context.Context, routerUUID string, req cloudscalesdk.CreateInterfaceRequest) (*cloudscalesdk.RouterInterface, error) { + if m.CreateInterfaceFn != nil { + return m.CreateInterfaceFn(ctx, routerUUID, req) + } + return nil, errors.New("mock(CreateInterface) not configured") +} + +func (m *MockRouterService) DeleteInterface(ctx context.Context, routerUUID, interfaceUUID string) error { + if m.DeleteInterfaceFn != nil { + return m.DeleteInterfaceFn(ctx, routerUUID, interfaceUUID) + } + return errors.New("mock(DeleteInterface) not configured") +} + // --- FloatingIP Service Mock --- type MockFloatingIPService struct { diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook.go b/internal/webhook/v1beta2/cloudscalecluster_webhook.go index 0552bc7..bc87672 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook.go @@ -19,7 +19,7 @@ package v1beta2 import ( "context" "fmt" - "net" + "net/netip" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" @@ -138,14 +138,15 @@ func (v *CloudscaleClusterCustomValidator) ValidateCreate(_ context.Context, clu cloudscaleclusterlog.Info("Validation for CloudscaleCluster upon creation", "name", cluster.GetName()) allErrs := clusterSpecValidateCreate(cluster.Spec, v.RegionInfo, field.NewPath("spec")) + warnings := checkSNATConfiguration(cluster.Spec) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, cluster.Name, allErrs) } - return nil, nil + return warnings, nil } func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, regionInfo *cloudscale.RegionInfo, fldPath *field.Path) field.ErrorList { @@ -164,6 +165,9 @@ func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, // Validate networks allErrs = append(allErrs, validateNetworks(spec.Networks, fldPath.Child("networks"))...) + // Validate routers + allErrs = append(allErrs, validateRouters(spec.Routers, spec.Networks, fldPath.Child("routers"))...) + // Validate LB network reference if spec.ControlPlaneLoadBalancer.Network != "" { allErrs = append(allErrs, validateNetworkReference( @@ -173,6 +177,15 @@ func clusterSpecValidateCreate(spec infrastructurev1beta2.CloudscaleClusterSpec, )...) } + // Validate LB pool member network reference + if spec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { + allErrs = append(allErrs, validateNetworkReference( + spec.ControlPlaneLoadBalancer.PoolMemberNetwork, + spec.Networks, + fldPath.Child("controlPlaneLoadBalancer", "poolMemberNetwork"), + )...) + } + // Validate floating IP if spec.FloatingIP != nil { allErrs = append(allErrs, validateFloatingIP(spec.FloatingIP, fldPath.Child("floatingIP"))...) @@ -189,6 +202,7 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old cloudscaleclusterlog.Info("Validation for CloudscaleCluster upon update", "name", newCluster.GetName()) var allErrs field.ErrorList + warnings := checkSNATConfiguration(newCluster.Spec) newClusterSpec := newCluster.Spec oldClusterSpec := oldCluster.Spec @@ -213,6 +227,12 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old // Validate new networks (new entries must still pass creation validation) allErrs = append(allErrs, validateNetworks(newClusterSpec.Networks, field.NewPath("spec", "networks"))...) + // The updated router set must satisfy the creation rules + routersPath := field.NewPath("spec", "routers") + allErrs = append(allErrs, validateRouters(newClusterSpec.Routers, newClusterSpec.Networks, routersPath)...) + // ...and on top of that, nothing already there may change or disappear. + allErrs = append(allErrs, validateRouterDiff(oldClusterSpec.Routers, newClusterSpec.Routers, routersPath)...) + // LoadBalancer Enabled is immutable if ptr.Deref(newClusterSpec.ControlPlaneLoadBalancer.Enabled, true) != ptr.Deref(oldClusterSpec.ControlPlaneLoadBalancer.Enabled, true) { allErrs = append(allErrs, field.Forbidden( @@ -220,9 +240,8 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old "field is immutable after cluster creation")) } - // LB network is immutable once set - if oldClusterSpec.ControlPlaneLoadBalancer.Network != "" && - newClusterSpec.ControlPlaneLoadBalancer.Network != oldClusterSpec.ControlPlaneLoadBalancer.Network { + // LB network is immutable + if newClusterSpec.ControlPlaneLoadBalancer.Network != oldClusterSpec.ControlPlaneLoadBalancer.Network { allErrs = append(allErrs, field.Forbidden( field.NewPath("spec", "controlPlaneLoadBalancer", "network"), "field is immutable once set")) @@ -245,6 +264,15 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old )...) } + // Validate LB pool member network reference (for new or existing) + if newClusterSpec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { + allErrs = append(allErrs, validateNetworkReference( + newClusterSpec.ControlPlaneLoadBalancer.PoolMemberNetwork, + newClusterSpec.Networks, + field.NewPath("spec", "controlPlaneLoadBalancer", "poolMemberNetwork"), + )...) + } + // ControlPlaneEndpoint is immutable once set if oldClusterSpec.ControlPlaneEndpoint.Host != "" { if newClusterSpec.ControlPlaneEndpoint.Host != oldClusterSpec.ControlPlaneEndpoint.Host { @@ -272,12 +300,12 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old allErrs = append(allErrs, validateLBPoolMemberNetworkResolvable(newClusterSpec, field.NewPath("spec"))...) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, newCluster.Name, allErrs) } - return nil, nil + return warnings, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type CloudscaleCluster. @@ -310,7 +338,7 @@ func validateNetworks(networks []infrastructurev1beta2.NetworkSpec, fldPath *fie // Validate CIDR format if hasCIDR { - if _, _, err := net.ParseCIDR(netSpec.CIDR); err != nil { + if _, err := netip.ParsePrefix(netSpec.CIDR); err != nil { allErrs = append(allErrs, field.Invalid(netPath.Child("cidr"), netSpec.CIDR, fmt.Sprintf("invalid CIDR: %v", err))) } @@ -324,7 +352,7 @@ func validateNetworks(networks []infrastructurev1beta2.NetworkSpec, fldPath *fie // Validate gateway is within CIDR if netSpec.GatewayAddress != "" && hasCIDR { - allErrs = append(allErrs, validateGatewayInCIDR( + allErrs = append(allErrs, validateAddressInCIDR( netSpec.CIDR, netSpec.GatewayAddress, netPath.Child("gatewayAddress"), @@ -384,6 +412,12 @@ func validateNetworkImmutability(oldNetworks, newNetworks []infrastructurev1beta "field is immutable after cluster creation")) } + if newNet.MTU != oldNet.MTU { + allErrs = append(allErrs, field.Forbidden( + newPath.Child("mtu"), + "field is immutable after cluster creation")) + } + if newNet.GatewayAddress != oldNet.GatewayAddress { allErrs = append(allErrs, field.Forbidden( newPath.Child("gatewayAddress"), @@ -394,6 +428,159 @@ func validateNetworkImmutability(oldNetworks, newNetworks []infrastructurev1beta return allErrs } +// indexNetworks maps each network name to its spec, so router validation and defaulting +// can resolve an interface's network reference in one place. +func indexNetworks(networks []infrastructurev1beta2.NetworkSpec) map[string]infrastructurev1beta2.NetworkSpec { + byName := make(map[string]infrastructurev1beta2.NetworkSpec, len(networks)) + for _, n := range networks { + byName[n.Name] = n + } + return byName +} + +// validateRouters runs every rule that applies to a router set, whether it was declared +// at creation or added later. ValidateUpdate calls it on the new set too, so additions +// are held to the same standard as originals without restating the rules. +func validateRouters(routers []infrastructurev1beta2.RouterSpec, networks []infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { + byName := indexNetworks(networks) + var allErrs field.ErrorList + for i, routerSpec := range routers { + allErrs = append(allErrs, validateSingleRouter(routerSpec, byName, fldPath.Index(i))...) + } + return allErrs +} + +// validateSingleRouter validates one router spec against the network index, using +// routerPath as the base field.Path for errors. +// +// Router names are not checked for uniqueness: spec.routers is a list-map keyed by name, so +// the API server rejects duplicates during schema validation, which runs before validating +// webhooks. +func validateSingleRouter( + routerSpec infrastructurev1beta2.RouterSpec, + networks map[string]infrastructurev1beta2.NetworkSpec, + routerPath *field.Path, +) field.ErrorList { + var allErrs field.ErrorList + + if routerSpec.UUID != "" && routerSpec.InternetGateway { + allErrs = append(allErrs, field.Invalid(routerPath, routerSpec.Name, + "uuid and internetGateway are mutually exclusive")) + } + + for j, ifaceSpec := range routerSpec.Interfaces { + allErrs = append(allErrs, validateRouterInterface(ifaceSpec, networks, routerSpec.UUID != "", routerPath.Child("interfaces").Index(j))...) + } + + return allErrs +} + +// validateRouterInterface validates one router interface against the network index. +// An adopted interface (by UUID) carries its address from the pre-existing interface. +// All other interfaces must have an explicit address set. +func validateRouterInterface( + ifaceSpec infrastructurev1beta2.RouterInterfaceSpec, + networks map[string]infrastructurev1beta2.NetworkSpec, + routerAdopted bool, + ifacePath *field.Path, +) field.ErrorList { + network, netExists := networks[ifaceSpec.Network] + if !netExists { + return field.ErrorList{field.NotFound(ifacePath.Child("network"), ifaceSpec.Network)} + } + + if ifaceSpec.UUID != "" { + var allErrs field.ErrorList + if !routerAdopted { + allErrs = append(allErrs, field.Forbidden(ifacePath.Child("uuid"), + "only a router referenced by uuid carries pre-existing interfaces to adopt")) + } + if ifaceSpec.Address != "" { + allErrs = append(allErrs, field.Forbidden(ifacePath.Child("address"), + "the address of an adopted interface is read from the interface itself, so it cannot be requested")) + } + return allErrs + } + + if ifaceSpec.Address == "" { + return field.ErrorList{field.Required(ifacePath.Child("address"), + "router interface address must be set explicitly")} + } + + // Containment can only be checked for a managed network; an uuid network's subnet + // is resolved by the controller at runtime. + if network.CIDR == "" { + return nil + } + + addrPath := ifacePath.Child("address") + if errs := validateAddressInCIDR(network.CIDR, ifaceSpec.Address, addrPath); len(errs) > 0 { + return errs + } + + return nil +} + +// validateRouterDiff enforces immutability for spec.routers on update: routers and their +// interfaces may be added, but never removed or altered. Whether the additions themselves +// are valid is validateRouters' job, which ValidateUpdate runs over the new set. +func validateRouterDiff(oldRouters, newRouters []infrastructurev1beta2.RouterSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + newByName := make(map[string]infrastructurev1beta2.RouterSpec, len(newRouters)) + newIndexByName := make(map[string]int, len(newRouters)) + for i, newRouter := range newRouters { + newByName[newRouter.Name] = newRouter + newIndexByName[newRouter.Name] = i + } + + for _, oldRouter := range oldRouters { + newRouter, exists := newByName[oldRouter.Name] + if !exists { + allErrs = append(allErrs, field.Forbidden(fldPath, + fmt.Sprintf("removing router %q is not allowed", oldRouter.Name))) + continue + } + + newPath := fldPath.Index(newIndexByName[oldRouter.Name]) + + if newRouter.UUID != oldRouter.UUID { + allErrs = append(allErrs, field.Forbidden(newPath.Child("uuid"), + "field is immutable after cluster creation")) + } + if newRouter.InternetGateway != oldRouter.InternetGateway { + allErrs = append(allErrs, field.Forbidden(newPath.Child("internetGateway"), + "field is immutable after cluster creation")) + } + + newIfaceByNetwork := make(map[string]infrastructurev1beta2.RouterInterfaceSpec, len(newRouter.Interfaces)) + for _, iface := range newRouter.Interfaces { + newIfaceByNetwork[iface.Network] = iface + } + + for _, oldIface := range oldRouter.Interfaces { + newIface, kept := newIfaceByNetwork[oldIface.Network] + if !kept { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("removing interface for network %q is not allowed", oldIface.Network))) + continue + } + if newIface.Address != oldIface.Address { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("address for network %q is immutable after cluster creation", newIface.Network))) + } + // Flipping uuid would hand ownership of a live interface over or away: + // CAPCS detaches what it created and leaves adopted interfaces in place. + if newIface.UUID != oldIface.UUID { + allErrs = append(allErrs, field.Forbidden(newPath.Child("interfaces"), + fmt.Sprintf("uuid for network %q is immutable after cluster creation", newIface.Network))) + } + } + } + + return allErrs +} + // validateFloatingIPRequiresLBOrPreExisting rejects managed floating IPs when the load balancer is disabled. // cloudscale.ch floating IPs require a dummy interface on the target server. // With a pre-existing FIP, the user knows the address upfront and can configure @@ -418,6 +605,9 @@ func validateFloatingIPRequiresLBOrPreExisting(spec infrastructurev1beta2.Clouds // validateLBImmutability forbids changes to LB fields that are baked into the LB at creation. // Algorithm, Flavor, APIServerPort and the HealthMonitor settings cannot be reissued // to an existing cloudscale.ch LB, so changing them in spec would silently lie to the user. +// PoolMemberNetwork is stricter than the sibling Network field (immutable *once set*): +// even setting it on a cluster that left it empty is forbidden, because the controller +// does not migrate live pool members from one subnet to another. func validateLBImmutability(oldLB, newLB *infrastructurev1beta2.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList @@ -432,6 +622,7 @@ func validateLBImmutability(oldLB, newLB *infrastructurev1beta2.LoadBalancerSpec forbidIfChanged("algorithm", oldLB.Algorithm, newLB.Algorithm) forbidIfChanged("flavor", oldLB.Flavor, newLB.Flavor) forbidIfChanged("apiServerPort", oldLB.APIServerPort, newLB.APIServerPort) + forbidIfChanged("poolMemberNetwork", oldLB.PoolMemberNetwork, newLB.PoolMemberNetwork) hmPath := fldPath.Child("healthMonitor") hmForbid := func(child string, oldV, newV int) { @@ -466,26 +657,28 @@ func validateFloatingIPRequiresPublicLB(spec infrastructurev1beta2.CloudscaleClu return allErrs } -// validateLBPoolMemberNetworkResolvable requires controlPlaneLoadBalancer.network to be set -// when there are multiple networks and the LB is public. Without an explicit network the -// controller would default the LB pool members' subnet to networks[0], which silently -// breaks clusters whose machines join a different network. +// validateLBPoolMemberNetworkResolvable requires either controlPlaneLoadBalancer.network +// or controlPlaneLoadBalancer.poolMemberNetwork to be set when there are multiple networks +// and the LB is public. With neither, the controller falls back to networks[0] for the pool +// members' subnet, which silently breaks clusters whose machines join a different network. func validateLBPoolMemberNetworkResolvable(spec infrastructurev1beta2.CloudscaleClusterSpec, fldPath *field.Path) field.ErrorList { var allErrs = make(field.ErrorList, 0, 1) if !ptr.Deref(spec.ControlPlaneLoadBalancer.Enabled, true) { return nil } - if spec.ControlPlaneLoadBalancer.Network != "" { + if spec.ControlPlaneLoadBalancer.Network != "" || spec.ControlPlaneLoadBalancer.PoolMemberNetwork != "" { return nil } if len(spec.Networks) <= 1 { return nil } + // Reported on the parent: either child field satisfies the requirement, so pinning + // the error on one of them would point the user at an arbitrary half of the fix. allErrs = append(allErrs, field.Required( - fldPath.Child("controlPlaneLoadBalancer", "network"), - "must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) + fldPath.Child("controlPlaneLoadBalancer"), + "controlPlaneLoadBalancer.network or controlPlaneLoadBalancer.poolMemberNetwork must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) return allErrs } @@ -502,9 +695,9 @@ func validateFloatingIP(fip *infrastructurev1beta2.FloatingIPSpec, fldPath *fiel "exactly one of ipFamily or ip must be specified")) } - if hasIP && net.ParseIP(fip.Address) == nil { + if _, err := netip.ParseAddr(fip.Address); hasIP && err != nil { allErrs = append(allErrs, field.Invalid(fldPath.Child("ip"), fip.Address, - "must be a valid IP address")) + fmt.Sprintf("must be a valid IP address: %s", err))) } return allErrs @@ -544,26 +737,57 @@ func validateFloatingIPImmutability(oldFIP, newFIP *infrastructurev1beta2.Floati return allErrs } -// validateGatewayInCIDR validates that the gateway address is within the specified CIDR. -func validateGatewayInCIDR(cidr, gateway string, fldPath *field.Path) field.ErrorList { +// validateAddressInCIDR validates that an address is within the specified CIDR. Used +// for both subnet gateways and router interface addresses. +func validateAddressInCIDR(cidr, address string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - _, ipNet, err := net.ParseCIDR(cidr) + prefix, err := netip.ParsePrefix(cidr) if err != nil { // CIDR validation should have caught this earlier return allErrs } - gatewayIP := net.ParseIP(gateway) - if gatewayIP == nil { - allErrs = append(allErrs, field.Invalid(fldPath, gateway, "invalid IP address")) + ip, err := netip.ParseAddr(address) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, address, fmt.Sprintf("invalid IP address: %s", err))) return allErrs } - if !ipNet.Contains(gatewayIP) { - allErrs = append(allErrs, field.Invalid(fldPath, gateway, - fmt.Sprintf("gateway must be within CIDR %s", cidr))) + if !prefix.Contains(ip) { + allErrs = append(allErrs, field.Invalid(fldPath, address, + fmt.Sprintf("must be within CIDR %s", cidr))) } return allErrs } + +// checkSNATConfiguration returns warnings when a router with internetGateway enabled +// does not have a gatewayAddress configured on its attached networks. +func checkSNATConfiguration(spec infrastructurev1beta2.CloudscaleClusterSpec) admission.Warnings { + var warnings admission.Warnings + for _, router := range spec.Routers { + if !router.InternetGateway { + continue + } + hasGateway := false + for _, iface := range router.Interfaces { + for _, network := range spec.Networks { + if network.Name == iface.Network && network.GatewayAddress != "" { + hasGateway = true + break + } + } + if hasGateway { + break + } + } + if !hasGateway { + warnings = append(warnings, fmt.Sprintf( + "Router %q has internetGateway enabled but no gatewayAddress is set on its attached networks. "+ + "Set networks[].gatewayAddress to enable SNAT for outbound internet access.", + router.Name)) + } + } + return warnings +} diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go index dc61b70..83687b0 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go @@ -53,6 +53,46 @@ func newClusterWebhookTestObjects() ( // CloudscaleCluster Defaulting Webhook // ============================================================================ +// routerNetworks is the network set the router defaulting cases start from. +func routerNetworks(cidr string) []infrastructurev1beta2.NetworkSpec { + return []infrastructurev1beta2.NetworkSpec{{Name: "main", CIDR: cidr}} +} + +// routerWith builds a single router named "r" attached to the given interfaces. +func routerWith(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{Name: "r", InternetGateway: true, Interfaces: ifaces}} +} + +// routerIface builds an interface on network "main"; an empty address leaves it for the +// defaulter to fill in. +func routerIface(address string) infrastructurev1beta2.RouterInterfaceSpec { + return infrastructurev1beta2.RouterInterfaceSpec{Network: "main", Address: address} +} + +// adoptedRouterWith builds a single router named "r" that adopts a pre-existing router by +// uuid, which is the only kind that can carry interfaces to adopt. +func adoptedRouterWith(ifaces ...infrastructurev1beta2.RouterInterfaceSpec) []infrastructurev1beta2.RouterSpec { + return []infrastructurev1beta2.RouterSpec{{Name: "r", UUID: "router-uuid", Interfaces: ifaces}} +} + +// routerIfaceAdopted builds an interface on network "main" that adopts a pre-existing +// interface, whose address is read off that interface rather than requested. +func routerIfaceAdopted() infrastructurev1beta2.RouterInterfaceSpec { + iface := routerIface("") + iface.UUID = "iface-uuid" + return iface +} + +// routersEach builds one router per interface, named r0, r1, ... Interfaces that share a +// setupRouterCluster fills in the region/zone a valid cluster needs and attaches the +// given networks and routers. +func setupRouterCluster(c *infrastructurev1beta2.CloudscaleCluster, networks []infrastructurev1beta2.NetworkSpec, routers []infrastructurev1beta2.RouterSpec) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = networks + c.Spec.Routers = routers +} + func TestClusterDefaulting(t *testing.T) { cases := []struct { name string @@ -187,6 +227,30 @@ func TestClusterDefaulting(t *testing.T) { g.Expect(c.Spec.FloatingIP.Address).To(Equal("1.2.3.4")) }, }, + { + name: "explicit address is preserved", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = routerNetworks("10.0.0.0/24") + c.Spec.Routers = routerWith(routerIface("10.0.0.5")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(Equal("10.0.0.5")) + }, + }, + { + // A network referenced by uuid has no CIDR here, so there is nothing to derive + // an address from. It must be left empty rather than filled with garbage. + name: "interface on a uuid network is left empty", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}} + c.Spec.Routers = routerWith(routerIface("")) + }, + assert: func(g *WithT, c *infrastructurev1beta2.CloudscaleCluster) { + g.Expect(c.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) + }, + }, } for _, tc := range cases { @@ -222,6 +286,7 @@ func TestClusterDefaulting_AllDefaultsApplied(t *testing.T) { g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.TimeoutS).To(Equal(3)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.UpThreshold).To(Equal(2)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.DownThreshold).To(Equal(3)) + g.Expect(obj.Spec.Routers).To(BeEmpty()) } // ============================================================================ @@ -370,7 +435,31 @@ func TestClusterValidateCreate(t *testing.T) { c.Spec.ControlPlaneLoadBalancer.Network = "" }, wantErr: true, - wantSubstrings: []string{"controlPlaneLoadBalancer.network"}, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "controlPlaneLoadBalancer.poolMemberNetwork"}, + }, + { + name: "public LB with multiple networks accepts poolMemberNetwork instead", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + {Name: "aux", CIDR: "10.1.0.0/24"}, + } + c.Spec.ControlPlaneLoadBalancer.Network = "" + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "aux" + }, + }, + { + name: "LB.PoolMemberNetwork references unknown network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.Region = RegionRma + c.Spec.Zone = ZoneRma1 + c.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "main", CIDR: defaultSubnetCIDR}} + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "nope" + }, + wantErr: true, + wantSubstrings: []string{"controlPlaneLoadBalancer.poolMemberNetwork"}, }, { name: "LB.Network references unknown network", @@ -462,6 +551,99 @@ func TestClusterValidateCreate(t *testing.T) { c.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{Address: "1.2.3.4"} }, }, + { + name: "valid router", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface("10.0.0.1"))) + }, + }, + + { + name: "router interface address outside the network CIDR", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface("192.168.1.1"))) + }, + wantErr: true, + wantSubstrings: []string{"must be within CIDR"}, + }, + { + name: "router interface referencing an unknown network", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + iface := routerIface("10.0.0.1") + iface.Network = "nope" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(iface)) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].network"}, + }, + { + name: "gateway owner address matching the network's gatewayAddress is valid", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.5" + setupRouterCluster(c, networks, routerWith(routerIface("10.0.0.5"))) + }, + }, + { + name: "uuid and internetGateway are mutually exclusive", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + routers := routerWith(routerIface("10.0.0.1")) + routers[0].UUID = "router-uuid" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routers) + }, + wantErr: true, + wantSubstrings: []string{"uuid and internetGateway are mutually exclusive"}, + }, + { + name: "adopted interface is accepted without an address", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, + []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, + adoptedRouterWith(routerIfaceAdopted())) + }, + }, + { + name: "adopted interface with a requested address rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + iface := routerIfaceAdopted() + iface.Address = "10.0.0.1" + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), adoptedRouterWith(iface)) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].address", "cannot be requested"}, + }, + { + name: "adopted interface on a managed router rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIfaceAdopted())) + }, + wantErr: true, + wantSubstrings: []string{"routers[0].interfaces[0].uuid", "referenced by uuid"}, + }, + { + name: "adopted gateway owner does not have to match the network's gatewayAddress", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + networks := routerNetworks("10.0.0.0/24") + networks[0].GatewayAddress = "10.0.0.5" + setupRouterCluster(c, networks, adoptedRouterWith(routerIfaceAdopted())) + }, + }, + { + name: "interface on a uuid network with an explicit address is accepted", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, + []infrastructurev1beta2.NetworkSpec{{Name: "main", UUID: "net-uuid"}}, + routerWith(routerIface("192.168.77.1"))) + }, + }, + { + name: "address-less interface rejected", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + setupRouterCluster(c, routerNetworks("10.0.0.0/24"), routerWith(routerIface(""))) + }, + wantErr: true, + wantSubstrings: []string{"must be set explicitly"}, + }, } for _, tc := range cases { @@ -509,6 +691,20 @@ func setupUpdateTestObjects() ( return } +// seedRouter puts the same valid router on both sides of an update, so cases only need +// to express the change they are testing. The address sits inside defaultSubnetCIDR. +func seedRouter(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + router := infrastructurev1beta2.RouterSpec{ + Name: "r", + InternetGateway: true, + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "main", Address: "172.18.0.1"}, + }, + } + oldObj.Spec.Routers = []infrastructurev1beta2.RouterSpec{*router.DeepCopy()} + obj.Spec.Routers = []infrastructurev1beta2.RouterSpec{*router.DeepCopy()} +} + func TestClusterValidateUpdate(t *testing.T) { cases := []struct { name string @@ -575,9 +771,22 @@ func TestClusterValidateUpdate(t *testing.T) { mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { oldObj.Spec.ControlPlaneLoadBalancer.Network = "main" obj.Spec.ControlPlaneLoadBalancer.Network = "other" + otherNet := infrastructurev1beta2.NetworkSpec{Name: "other", CIDR: "10.1.0.0/24"} + obj.Spec.Networks = append(obj.Spec.Networks, otherNet) }, wantErr: true, - wantSubstrings: []string{"controlPlaneLoadBalancer.network"}, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "immutable"}, + }, + { + name: "LB.Network change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + oldObj.Spec.ControlPlaneLoadBalancer.Network = "" + obj.Spec.ControlPlaneLoadBalancer.Network = "other" + otherNet := infrastructurev1beta2.NetworkSpec{Name: "other", CIDR: "10.1.0.0/24"} + obj.Spec.Networks = append(obj.Spec.Networks, otherNet) + }, + wantErr: true, + wantSubstrings: []string{"controlPlaneLoadBalancer.network", "immutable"}, }, { name: "ControlPlaneEndpoint host change rejected", @@ -635,6 +844,81 @@ func TestClusterValidateUpdate(t *testing.T) { wantErr: true, wantSubstrings: []string{"spec.region", "spec.zone", "cidr"}, }, + { + name: "router unchanged", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + }, + }, + { + name: "router uuid change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].UUID = "other-uuid" + obj.Spec.Routers[0].InternetGateway = false + }, + wantErr: true, + wantSubstrings: []string{"routers[0].uuid", "immutable"}, + }, + { + name: "router internetGateway change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].InternetGateway = false + }, + wantErr: true, + wantSubstrings: []string{"routers[0].internetGateway", "immutable"}, + }, + { + name: "router interface address change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces[0].Address = "172.18.0.9" + }, + wantErr: true, + wantSubstrings: []string{"address for network", "immutable"}, + }, + { + name: "router interface uuid change rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces[0].UUID = "iface-uuid" + }, + wantErr: true, + wantSubstrings: []string{"uuid for network", "immutable"}, + }, + { + name: "router interface removal rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers[0].Interfaces = nil + }, + wantErr: true, + wantSubstrings: []string{"removing interface for network"}, + }, + { + name: "router removal rejected", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers = nil + }, + wantErr: true, + wantSubstrings: []string{"removing router"}, + }, + { + name: "router added on update is validated", + mutate: func(oldObj, obj *infrastructurev1beta2.CloudscaleCluster) { + seedRouter(oldObj, obj) + obj.Spec.Routers = append(obj.Spec.Routers, infrastructurev1beta2.RouterSpec{ + Name: "second", + Interfaces: []infrastructurev1beta2.RouterInterfaceSpec{ + {Network: "main", Address: "192.168.1.1"}, + }, + }) + }, + wantErr: true, + wantSubstrings: []string{"routers[1].interfaces[0].address", "must be within CIDR"}, + }, } for _, tc := range cases { @@ -702,6 +986,13 @@ func TestClusterValidateUpdate_LBFieldsImmutable(t *testing.T) { mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { c.Spec.ControlPlaneLoadBalancer.APIServerPort = 8443 }, errPath: "controlPlaneLoadBalancer.apiServerPort", }, + { + name: "PoolMemberNetwork", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "aux" + }, + errPath: "controlPlaneLoadBalancer.poolMemberNetwork", + }, { name: "HealthMonitor.DelayS", mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { @@ -752,6 +1043,22 @@ func TestClusterValidateUpdate_LBFieldsImmutable(t *testing.T) { } } +// TestClusterValidateUpdate_PoolMemberNetworkStrictlyImmutable pins the stricter rule +// for poolMemberNetwork: unlike the sibling network field (immutable *once set*), even +// setting it on a cluster that left it empty is rejected. The controller does not move +// live pool members between subnets. +func TestClusterValidateUpdate_PoolMemberNetworkStrictlyImmutable(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + oldObj.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "" + obj.Spec.ControlPlaneLoadBalancer = *oldObj.Spec.ControlPlaneLoadBalancer.DeepCopy() + obj.Spec.ControlPlaneLoadBalancer.PoolMemberNetwork = "main" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.poolMemberNetwork")) +} + // ============================================================================ // CloudscaleCluster Validating Webhook - Delete // ============================================================================ @@ -765,10 +1072,10 @@ func TestClusterValidateDelete_AlwaysSucceeds(t *testing.T) { } // ============================================================================ -// Unit tests for validateGatewayInCIDR +// Unit tests for validateAddressInCIDR // ============================================================================ -func TestValidateGatewayInCIDR(t *testing.T) { +func TestValidateAddressInCIDR(t *testing.T) { path := field.NewPath("spec", "networks", "gatewayAddress") cases := []struct { name string @@ -787,7 +1094,7 @@ func TestValidateGatewayInCIDR(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR(tc.cidr, tc.gateway, path) + errs := validateAddressInCIDR(tc.cidr, tc.gateway, path) g.Expect(errs).To(HaveLen(tc.wantErrs)) if tc.wantDetail != "" { g.Expect(errs[0].Detail).To(ContainSubstring(tc.wantDetail)) diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go index 1233b74..78a8070 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook.go @@ -76,14 +76,15 @@ func (v *CloudscaleClusterTemplateCustomValidator) ValidateCreate(_ context.Cont cloudscaleclustertemplatelog.Info("Validation for CloudscaleClusterTemplate upon creation", "name", clusterTemplate.GetName()) allErrs := clusterSpecValidateCreate(clusterTemplate.Spec.Template.Spec, v.RegionInfo, field.NewPath("spec", "template", "spec")) + warnings := checkSNATConfiguration(clusterTemplate.Spec.Template.Spec) if len(allErrs) > 0 { - return nil, apierrors.NewInvalid( + return warnings, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleClusterTemplate"}, clusterTemplate.Name, allErrs) } - return nil, nil + return warnings, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type CloudscaleClusterTemplate. diff --git a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go index d5d890f..4165121 100644 --- a/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscaleclustertemplate_webhook_test.go @@ -100,6 +100,18 @@ func TestClusterTemplateDefaulting_ExplicitNetworksNotOverridden(t *testing.T) { g.Expect(obj.Spec.Template.Spec.Networks[0].CIDR).To(Equal("10.1.0.0/16")) } +// Router interface addresses must not be defaulted on the template. +func TestClusterTemplateDefaulting_RouterAddressesNotDefaulted(t *testing.T) { + g := NewWithT(t) + obj, _, _, defaulter := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.0.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + g.Expect(obj.Spec.Template.Spec.Routers[0].Interfaces[0].Address).To(BeEmpty()) +} + func TestClusterTemplateDefaulting_LBEnabledToTrue(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterTemplateWebhookTestObjects() @@ -388,7 +400,7 @@ func TestClusterTemplateValidateCreate_PublicLBWithMultipleNetworksRequiresExpli _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.network")) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer")) } func TestClusterTemplateValidateCreate_LBNetworkReferenceInvalid(t *testing.T) { @@ -510,6 +522,30 @@ func TestClusterTemplateValidateCreate_PreExistingFloatingIPWithoutLBAllowed(t * g.Expect(err).NotTo(HaveOccurred()) } +func TestClusterTemplateValidateCreate_RouterInterfaceAddressOutsideCIDRRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = routerNetworks("10.10.0.0/24") + obj.Spec.Template.Spec.Routers = routerWith(routerIface("10.20.0.1")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("must be within CIDR 10.10.0.0/24")) +} + +func TestClusterTemplateValidateCreate_RouterInterfaceUnknownNetworkRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterTemplateWebhookTestObjects() + obj.Spec.Template.Spec.Region = RegionRma + obj.Spec.Template.Spec.Networks = []infrastructurev1beta2.NetworkSpec{{Name: "other", CIDR: "10.10.0.0/24"}} + obj.Spec.Template.Spec.Routers = routerWith(routerIface("")) + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("Not found")) +} + // ============================================================================ // Tests for CloudscaleClusterTemplate Validating Webhook - Update // ============================================================================ diff --git a/templates/cluster-class-router-nat.yaml b/templates/cluster-class-router-nat.yaml new file mode 100644 index 0000000..8ad6bb0 --- /dev/null +++ b/templates/cluster-class-router-nat.yaml @@ -0,0 +1,383 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: router-nat + namespace: "${NAMESPACE}" +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: router-nat-control-plane + machineInfrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 900 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyLessThanOrEqualTo: 33% + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + name: router-nat-cluster + workers: + machineDeployments: + - class: worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: router-nat-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 600 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyInRange: "[0-2]" + variables: + - name: region + required: true + schema: + openAPIV3Schema: + type: string + default: lpg + enum: + - lpg + - rma + - name: credentialsRefName + required: true + schema: + openAPIV3Schema: + type: string + # networkName is the logical name of the cluster's private network. It is referenced + # from the router interface and from every machine's interface list, so it is patched + # into all three templates from this single variable. Give each cluster its own value + # to keep the networks and routers apart in the cloudscale.ch control panel. + - name: networkName + required: true + schema: + openAPIV3Schema: + type: string + default: node-net + - name: networkCIDR + required: true + schema: + openAPIV3Schema: + type: string + default: 10.10.0.0/24 + - name: routerInterfaceAddress + required: true + schema: + openAPIV3Schema: + type: string + default: "10.10.0.1" + - name: routerName + required: true + schema: + openAPIV3Schema: + type: string + default: router + - name: cloudscaleControlPlaneMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleControlPlaneMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleControlPlaneMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleWorkerMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleWorkerMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleWorkerMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleSSHKeys + required: true + schema: + openAPIV3Schema: + type: array + items: + type: string + - name: cloudscaleControlPlaneServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + - name: cloudscaleWorkerServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + patches: + - name: CloudscaleClusterTemplate + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/region" + valueFrom: + variable: region + - op: replace + path: "/spec/template/spec/credentialsRef/name" + valueFrom: + variable: credentialsRefName + # The whole list is replaced rather than its individual members, so networkName + # stays a single source of truth instead of a literal repeated per index. + - op: replace + path: "/spec/template/spec/networks" + valueFrom: + template: | + - name: {{ .networkName }} + cidr: {{ .networkCIDR }} + gatewayAddress: {{ .routerInterfaceAddress }} + - op: replace + path: "/spec/template/spec/routers" + valueFrom: + template: | + - name: {{ .routerName }} + internetGateway: true + interfaces: + - network: {{ .networkName }} + address: {{ .routerInterfaceAddress }} + - name: CloudscaleMachineTemplateControlPlane + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleControlPlaneMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleControlPlaneMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleControlPlaneMachineRootVolumeSize + # Private only: no {type: public} entry, so the nodes reach the internet + # exclusively through the router's SNAT. + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleControlPlaneServerGroupName }} + - name: CloudscaleMachineTemplateWorker + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleWorkerMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleWorkerMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleWorkerMachineRootVolumeSize + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleWorkerServerGroupName }} + - name: KubeadmControlPlaneTemplate + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/kubeadmConfigSpec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} + - name: KubeadmConfigTemplate + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleClusterTemplate +metadata: + name: router-nat-cluster + namespace: "${NAMESPACE}" +spec: + template: + metadata: + labels: {} + spec: + # Every field below is overwritten by the CloudscaleClusterTemplate patches above, so + # nothing here reaches a cluster: the values exist only so the patches' `op: replace` + # paths resolve. + region: lpg + credentialsRef: + name: "router-nat-credentials" + networks: + - name: node-net + cidr: 10.10.0.0/24 + gatewayAddress: 10.10.0.1 + routers: + - name: router + internetGateway: true + interfaces: + - network: node-net + address: 10.10.0.1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: router-nat-machinetemplate + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: flex-4-2 + image: custom:ubuntu-2404-kube-v1.36.0 + rootVolumeSize: 20 + interfaces: + - network: node-net +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: router-nat-control-plane + namespace: "${NAMESPACE}" +spec: + template: + spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: router-nat-worker + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] diff --git a/templates/cluster-class.yaml b/templates/cluster-class.yaml index ad0bd90..9ffa629 100644 --- a/templates/cluster-class.yaml +++ b/templates/cluster-class.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" - name: cloudscaleControlPlaneMachineFlavor required: true schema: @@ -245,7 +244,7 @@ spec: spec: region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "quick-start-credentials" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/templates/cluster-template-public-lb-private-nodes.yaml b/templates/cluster-template-public-lb-private-nodes.yaml index e8f2208..a13279b 100644 --- a/templates/cluster-template-public-lb-private-nodes.yaml +++ b/templates/cluster-template-public-lb-private-nodes.yaml @@ -40,6 +40,12 @@ spec: networks: - name: "${CLUSTER_NAME}" uuid: "${CLOUDSCALE_NETWORK_UUID}" # Pre-existing network (must have NAT for internet egress) + routers: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_ROUTER_UUID}" # Pre-existing router with internet gateway + interfaces: + - uuid: "${CLOUDSCALE_ROUTER_INTERFACE_UUID}" + network: "${CLUSTER_NAME}" credentialsRef: name: "${CLUSTER_NAME}-credentials" --- diff --git a/templates/cluster-template-router-nat.yaml b/templates/cluster-template-router-nat.yaml new file mode 100644 index 0000000..cd08ae5 --- /dev/null +++ b/templates/cluster-template-router-nat.yaml @@ -0,0 +1,167 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + region: "${CLOUDSCALE_REGION}" + networks: + - name: "${CLUSTER_NAME}" + cidr: "10.10.0.0/24" + gatewayAddress: "10.10.0.1" + mtu: 1500 + routers: + - name: "${CLUSTER_NAME}-router" + internetGateway: true # enables SNAT for outbound internet access + interfaces: + - network: "${CLUSTER_NAME}" + address: "10.10.0.1" + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: "${KUBERNETES_VERSION}" + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-control-plane" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-md-0" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/templates/cluster-template-topology-router-nat.yaml b/templates/cluster-template-topology-router-nat.yaml new file mode 100644 index 0000000..fdfe80a --- /dev/null +++ b/templates/cluster-template-topology-router-nat.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + topology: + classRef: + name: router-nat + namespace: "${NAMESPACE}" + version: "${KUBERNETES_VERSION}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT:=2} + variables: + - name: region + value: "${CLOUDSCALE_REGION}" + # networkName and routerName carry the cluster name so that the networks and routers + # of clusters sharing this ClusterClass stay distinguishable in the cloudscale.ch + # control panel. + - name: networkName + value: "${CLUSTER_NAME}" + - name: networkCIDR + value: "10.10.0.0/24" + - name: routerName + value: "${CLUSTER_NAME}-router" + - name: cloudscaleControlPlaneMachineFlavor + value: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + - name: cloudscaleControlPlaneMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleControlPlaneMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleControlPlaneServerGroupName + value: "${CLUSTER_NAME}-cp" + - name: cloudscaleWorkerMachineFlavor + value: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + - name: cloudscaleWorkerMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleWorkerMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleWorkerServerGroupName + value: "${CLUSTER_NAME}-mp-0" + - name: credentialsRefName + value: "${CLUSTER_NAME}-credentials" + - name: cloudscaleSSHKeys + value: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/test/e2e/config/cloudscale.yaml b/test/e2e/config/cloudscale.yaml index a257c70..25d6344 100644 --- a/test/e2e/config/cloudscale.yaml +++ b/test/e2e/config/cloudscale.yaml @@ -55,8 +55,11 @@ providers: - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-md-remediation.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-pre-existing-network.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-public-lb-private-nodes.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-topology.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-topology-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/clusterclass-router-nat.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-upgrades.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/metadata.yaml" targetName: metadata.yaml @@ -76,6 +79,7 @@ variables: # CLOUDSCALE_SSH_PUBLIC_KEY: Set via environment variable CLOUDSCALE_ROOT_VOLUME_SIZE: "20" CLOUDSCALE_NETWORK_CIDR: "172.18.0.0/24" + CLOUDSCALE_ROUTER_INTERFACE_ADDRESS: "172.18.0.1" # Machine template names for upgrade tests CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "k8s-upgrade-control-plane" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml new file mode 100644 index 0000000..f841c87 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml +patches: + - path: router-nat.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml new file mode 100644 index 0000000..21a2d41 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-router-nat/router-nat.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + networks: + - name: "${CLUSTER_NAME}" + cidr: "${CLOUDSCALE_NETWORK_CIDR}" + gatewayAddress: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" + routers: + - name: "${CLUSTER_NAME}-router" + internetGateway: true + interfaces: + - network: "${CLUSTER_NAME}" + address: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml new file mode 100644 index 0000000..6f4ec2b --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/cluster-topology.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale + cni: cilium +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + topology: + classRef: + name: router-nat + namespace: "${NAMESPACE}" + version: "${KUBERNETES_VERSION}" + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT:=2} + variables: + - name: region + value: "${CLOUDSCALE_REGION}" + # networkName and routerName carry the cluster name so that the networks and routers + # of clusters sharing this ClusterClass stay distinguishable in the cloudscale.ch + # control panel. + - name: networkName + value: "${CLUSTER_NAME}" + - name: networkCIDR + value: "${CLOUDSCALE_NETWORK_CIDR}" + - name: routerInterfaceAddress + value: "${CLOUDSCALE_ROUTER_INTERFACE_ADDRESS}" + - name: routerName + value: "${CLUSTER_NAME}-router" + - name: cloudscaleControlPlaneMachineFlavor + value: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + - name: cloudscaleControlPlaneMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleControlPlaneMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleControlPlaneServerGroupName + value: "${CLUSTER_NAME}-cp" + - name: cloudscaleWorkerMachineFlavor + value: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + - name: cloudscaleWorkerMachineImage + value: "${CLOUDSCALE_MACHINE_IMAGE}" + - name: cloudscaleWorkerMachineRootVolumeSize + value: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + - name: cloudscaleWorkerServerGroupName + value: "${CLUSTER_NAME}-mp-0" + - name: credentialsRefName + value: "${CLUSTER_NAME}-credentials" + - name: cloudscaleSSHKeys + value: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml new file mode 100644 index 0000000..7ef8f01 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-topology-router-nat/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-topology.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml index ad0bd90..9ffa629 100644 --- a/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-quick-start/cluster-class.yaml @@ -73,7 +73,6 @@ spec: schema: openAPIV3Schema: type: string - default: "${CLUSTER_NAME}-credentials" - name: cloudscaleControlPlaneMachineFlavor required: true schema: @@ -245,7 +244,7 @@ spec: spec: region: lpg credentialsRef: - name: "${CLUSTER_NAME}-credentials" + name: "quick-start-credentials" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml new file mode 100644 index 0000000..5797569 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/cluster-class.yaml @@ -0,0 +1,378 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: router-nat + namespace: "${NAMESPACE}" +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: router-nat-control-plane + machineInfrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 900 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyLessThanOrEqualTo: 33% + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + name: router-nat-cluster + workers: + machineDeployments: + - class: worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: router-nat-worker + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: router-nat-machinetemplate + healthCheck: + checks: + nodeStartupTimeoutSeconds: 600 + unhealthyNodeConditions: + - type: Ready + status: Unknown + timeoutSeconds: 300 + - type: Ready + status: "False" + timeoutSeconds: 300 + remediation: + triggerIf: + unhealthyInRange: "[0-2]" + variables: + - name: region + required: true + schema: + openAPIV3Schema: + type: string + default: lpg + enum: + - lpg + - rma + - name: credentialsRefName + required: true + schema: + openAPIV3Schema: + type: string + - name: networkName + required: true + schema: + openAPIV3Schema: + type: string + default: node-net + - name: networkCIDR + required: true + schema: + openAPIV3Schema: + type: string + default: 10.10.0.0/24 + - name: routerInterfaceAddress + required: true + schema: + openAPIV3Schema: + type: string + default: "10.10.0.1" + - name: routerName + required: true + schema: + openAPIV3Schema: + type: string + default: router + - name: cloudscaleControlPlaneMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleControlPlaneMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleControlPlaneMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleWorkerMachineFlavor + required: true + schema: + openAPIV3Schema: + type: string + default: flex-4-2 + - name: cloudscaleWorkerMachineImage + required: true + schema: + openAPIV3Schema: + type: string + default: custom:ubuntu-2404-kube-v1.36.0 + - name: cloudscaleWorkerMachineRootVolumeSize + required: true + schema: + openAPIV3Schema: + type: integer + default: 20 + - name: cloudscaleSSHKeys + required: true + schema: + openAPIV3Schema: + type: array + items: + type: string + - name: cloudscaleControlPlaneServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + - name: cloudscaleWorkerServerGroupName + required: true + schema: + openAPIV3Schema: + type: string + patches: + - name: CloudscaleClusterTemplate + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: replace + path: "/spec/template/spec/region" + valueFrom: + variable: region + - op: replace + path: "/spec/template/spec/credentialsRef/name" + valueFrom: + variable: credentialsRefName + # The whole list is replaced rather than its individual members, so networkName + # stays a single source of truth instead of a literal repeated per index. + - op: replace + path: "/spec/template/spec/networks" + valueFrom: + template: | + - name: {{ .networkName }} + cidr: {{ .networkCIDR }} + gatewayAddress: {{ .routerInterfaceAddress }} + - op: replace + path: "/spec/template/spec/routers" + valueFrom: + template: | + - name: {{ .routerName }} + internetGateway: true + interfaces: + - network: {{ .networkName }} + address: {{ .routerInterfaceAddress }} + - name: CloudscaleMachineTemplateControlPlane + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleControlPlaneMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleControlPlaneMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleControlPlaneMachineRootVolumeSize + # Private only: no {type: public} entry, so the nodes reach the internet + # exclusively through the router's SNAT. + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleControlPlaneServerGroupName }} + - name: CloudscaleMachineTemplateWorker + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/flavor" + valueFrom: + variable: cloudscaleWorkerMachineFlavor + - op: replace + path: "/spec/template/spec/image" + valueFrom: + variable: cloudscaleWorkerMachineImage + - op: replace + path: "/spec/template/spec/rootVolumeSize" + valueFrom: + variable: cloudscaleWorkerMachineRootVolumeSize + - op: replace + path: "/spec/template/spec/interfaces" + valueFrom: + template: | + - network: {{ .networkName }} + - op: add + path: "/spec/template/spec/serverGroup" + valueFrom: + template: | + name: {{ .cloudscaleWorkerServerGroupName }} + - name: KubeadmControlPlaneTemplate + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: "/spec/template/spec/kubeadmConfigSpec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} + - name: KubeadmConfigTemplate + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + jsonPatches: + - op: replace + path: "/spec/template/spec/users" + valueFrom: + template: | + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleClusterTemplate +metadata: + name: router-nat-cluster + namespace: "${NAMESPACE}" +spec: + template: + metadata: + labels: {} + spec: + # Every field below is overwritten by the CloudscaleClusterTemplate patches above, so + # nothing here reaches a cluster. + region: lpg + credentialsRef: + name: "router-nat-credentials" + networks: + - name: node-net + cidr: 10.10.0.0/24 + gatewayAddress: 10.10.0.1 + routers: + - name: router + internetGateway: true + interfaces: + - network: node-net + address: 10.10.0.1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: router-nat-machinetemplate + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: flex-4-2 + image: custom:ubuntu-2404-kube-v1.36.0 + rootVolumeSize: 20 + interfaces: + - network: node-net +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlaneTemplate +metadata: + name: router-nat-control-plane + namespace: "${NAMESPACE}" +spec: + template: + spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: router-nat-worker + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: sudo + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [] diff --git a/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml new file mode 100644 index 0000000..71396c4 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/clusterclass-router-nat/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-class.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml deleted file mode 100644 index 0f86d1e..0000000 --- a/test/e2e/data/infrastructure-cloudscale/main/clusterclass-quick-start.yaml +++ /dev/null @@ -1,311 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: KubeadmConfigTemplate -metadata: - name: quick-start-worker - namespace: ${NAMESPACE} -spec: - template: - spec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - users: - - groups: sudo - name: capi - shell: /bin/bash - sshAuthorizedKeys: [] - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: cluster.x-k8s.io/v1beta2 -kind: ClusterClass -metadata: - name: quick-start - namespace: ${NAMESPACE} -spec: - controlPlane: - healthCheck: - checks: - nodeStartupTimeoutSeconds: 900 - unhealthyNodeConditions: - - status: Unknown - timeoutSeconds: 300 - type: Ready - - status: "False" - timeoutSeconds: 300 - type: Ready - remediation: - triggerIf: - unhealthyLessThanOrEqualTo: 33% - machineInfrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - name: quick-start-machinetemplate - templateRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: KubeadmControlPlaneTemplate - name: quick-start-control-plane - infrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleClusterTemplate - name: quick-start-cluster - patches: - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/region - valueFrom: - variable: region - - op: replace - path: /spec/template/spec/credentialsRef/name - valueFrom: - variable: credentialsRefName - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleClusterTemplate - matchResources: - infrastructureCluster: true - name: CloudscaleClusterTemplate - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/flavor - valueFrom: - variable: cloudscaleControlPlaneMachineFlavor - - op: replace - path: /spec/template/spec/image - valueFrom: - variable: cloudscaleControlPlaneMachineImage - - op: replace - path: /spec/template/spec/rootVolumeSize - valueFrom: - variable: cloudscaleControlPlaneMachineRootVolumeSize - - op: add - path: /spec/template/spec/serverGroup - valueFrom: - template: | - name: {{ .cloudscaleControlPlaneServerGroupName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - matchResources: - controlPlane: true - name: CloudscaleMachineTemplateControlPlane - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/flavor - valueFrom: - variable: cloudscaleWorkerMachineFlavor - - op: replace - path: /spec/template/spec/image - valueFrom: - variable: cloudscaleWorkerMachineImage - - op: replace - path: /spec/template/spec/rootVolumeSize - valueFrom: - variable: cloudscaleWorkerMachineRootVolumeSize - - op: add - path: /spec/template/spec/serverGroup - valueFrom: - template: | - name: {{ .cloudscaleWorkerServerGroupName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - matchResources: - machineDeploymentClass: - names: - - worker - name: CloudscaleMachineTemplateWorker - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/kubeadmConfigSpec/users - valueFrom: - template: | - - name: capi - groups: sudo - shell: "/bin/bash" - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta2 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - name: KubeadmControlPlaneTemplate - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/users - valueFrom: - template: | - - name: capi - groups: sudo - shell: "/bin/bash" - sudo: "ALL=(ALL) NOPASSWD:ALL" - sshAuthorizedKeys: {{ .cloudscaleSSHKeys | toJson }} - selector: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: KubeadmConfigTemplate - matchResources: - machineDeploymentClass: - names: - - worker - name: KubeadmConfigTemplate - variables: - - name: region - required: true - schema: - openAPIV3Schema: - default: lpg - enum: - - lpg - - rma - type: string - - name: credentialsRefName - required: true - schema: - openAPIV3Schema: - default: ${CLUSTER_NAME}-credentials - type: string - - name: cloudscaleControlPlaneMachineFlavor - required: true - schema: - openAPIV3Schema: - default: flex-4-2 - type: string - - name: cloudscaleControlPlaneMachineImage - required: true - schema: - openAPIV3Schema: - default: custom:ubuntu-2404-kube-v1.36.0 - type: string - - name: cloudscaleControlPlaneMachineRootVolumeSize - required: true - schema: - openAPIV3Schema: - default: 20 - type: integer - - name: cloudscaleWorkerMachineFlavor - required: true - schema: - openAPIV3Schema: - default: flex-4-2 - type: string - - name: cloudscaleWorkerMachineImage - required: true - schema: - openAPIV3Schema: - default: custom:ubuntu-2404-kube-v1.36.0 - type: string - - name: cloudscaleWorkerMachineRootVolumeSize - required: true - schema: - openAPIV3Schema: - default: 20 - type: integer - - name: cloudscaleSSHKeys - required: true - schema: - openAPIV3Schema: - items: - type: string - type: array - - name: cloudscaleControlPlaneServerGroupName - required: true - schema: - openAPIV3Schema: - type: string - - name: cloudscaleWorkerServerGroupName - required: true - schema: - openAPIV3Schema: - type: string - workers: - machineDeployments: - - bootstrap: - templateRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: KubeadmConfigTemplate - name: quick-start-worker - class: worker - healthCheck: - checks: - nodeStartupTimeoutSeconds: 600 - unhealthyNodeConditions: - - status: Unknown - timeoutSeconds: 300 - type: Ready - - status: "False" - timeoutSeconds: 300 - type: Ready - remediation: - triggerIf: - unhealthyInRange: '[0-2]' - infrastructure: - templateRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - name: quick-start-machinetemplate ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta2 -kind: KubeadmControlPlaneTemplate -metadata: - name: quick-start-control-plane - namespace: ${NAMESPACE} -spec: - template: - spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - - name: cloud-provider - value: external - users: - - groups: sudo - name: capi - shell: /bin/bash - sshAuthorizedKeys: [] - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: CloudscaleClusterTemplate -metadata: - name: quick-start-cluster - namespace: ${NAMESPACE} -spec: - template: - metadata: - labels: {} - spec: - credentialsRef: - name: ${CLUSTER_NAME}-credentials - region: lpg ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: CloudscaleMachineTemplate -metadata: - name: quick-start-machinetemplate - namespace: ${NAMESPACE} -spec: - template: - spec: - flavor: flex-4-2 - image: custom:ubuntu-2404-kube-v1.36.0 - rootVolumeSize: 20 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6a8cf4b..62cea57 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -40,6 +40,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new(""), + ClusterName: scenarioClusterName("lifecycle"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -57,6 +58,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("ha"), + ClusterName: scenarioClusterName("ha"), ControlPlaneMachineCount: new(int64(3)), WorkerMachineCount: new(int64(2)), PostMachinesProvisioned: validateCloudscaleResources, @@ -76,6 +78,52 @@ var _ = Describe("Workload cluster-class topology", Label("topology"), func() { SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("topology"), + ClusterName: scenarioClusterName("topology"), + ControlPlaneMachineCount: new(int64(1)), + WorkerMachineCount: new(int64(1)), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) +}) + +// Managed router tests use a router with internetGateway=true: nodes have no public +// interface at all and reach the internet only through the router's SNAT. These are the only +// specs that prove the router path end to end — if the router or its subnet gateway were +// wrong, the nodes would never finish bootstrapping. +var _ = Describe("Managed router NAT", Label("router-nat"), func() { + // Flat template: the network and router are spelled out in the CloudscaleCluster. + Context("With managed router providing NAT", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: new("cloudscale-ch-cloudscale"), + Flavor: new("router-nat"), + ClusterName: scenarioClusterName("router-nat"), + ControlPlaneMachineCount: new(int64(1)), + WorkerMachineCount: new(int64(1)), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) + + // Same topology via the router-nat ClusterClass. This additionally covers patching the + // network gateway and router interface address from the ClusterClass variable. + Context("With managed router providing NAT via ClusterClass", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: new("cloudscale-ch-cloudscale"), + Flavor: new("topology-router-nat"), + ClusterName: scenarioClusterName("topology-router-nat"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -106,6 +154,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("pre-existing-network"), + ClusterName: scenarioClusterName("pre-existing-net"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -124,6 +173,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("public-lb-private-nodes"), + ClusterName: scenarioClusterName("public-lb-private-nodes"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, @@ -142,6 +192,7 @@ var _ = Describe("Pre-existing networking", Label("pre-existing-networking"), fu SkipCleanup: skipCleanup, InfrastructureProvider: new("cloudscale-ch-cloudscale"), Flavor: new("fip"), + ClusterName: scenarioClusterName("fip"), ControlPlaneMachineCount: new(int64(1)), WorkerMachineCount: new(int64(1)), PostMachinesProvisioned: validateCloudscaleResources, diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index fad8612..5c8350d 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -20,17 +20,25 @@ package e2e import ( "context" + "fmt" + "slices" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" infrav1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" ) +// scenarioClusterName returns a randomised cluster name prefixed with the scenario under test. +func scenarioClusterName(scenario string) *string { + return new(fmt.Sprintf("%s-%s", scenario, util.RandomString(6))) +} + // validateCloudscaleResources validates that cloudscale-specific resources are properly created. // This is called by QuickStartSpec after machines are provisioned. func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, clusterName string) { @@ -55,6 +63,27 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste Expect(net.SubnetID).NotTo(BeEmpty(), "Network %d (%s) should have SubnetID", i, net.Name) } + // Validate router resources (if any are configured). Without these the router path is + // only covered implicitly, by the nodes failing to bootstrap when SNAT is broken. + for _, routerSpec := range cloudscaleCluster.Spec.Routers { + rs := cloudscaleCluster.Status.GetRouterStatus(routerSpec.Name) + Expect(rs).NotTo(BeNil(), "Router %s should be tracked in status", routerSpec.Name) + Expect(rs.RouterID).NotTo(BeEmpty(), "Router %s should have RouterID", routerSpec.Name) + // A router adopted by uuid must never be reported as CAPCS-managed: managed is what + // decides whether teardown deletes it. + Expect(rs.Managed).To(Equal(routerSpec.UUID == ""), + "Router %s managed flag should reflect whether it was adopted by uuid", routerSpec.Name) + + for _, ifaceSpec := range routerSpec.Interfaces { + ifaceStatus := rs.GetInterfaceStatus(ifaceSpec.Network) + Expect(ifaceStatus).NotTo(BeNil(), "Router %s should have an interface on network %s", routerSpec.Name, ifaceSpec.Network) + Expect(ifaceStatus.InterfaceID).NotTo(BeEmpty(), "Router %s interface on network %s should have InterfaceID", routerSpec.Name, ifaceSpec.Network) + Expect(ifaceStatus.Managed).To(Equal(ifaceSpec.UUID == ""), + "Router %s interface on network %s managed flag should reflect whether it was adopted by uuid", + routerSpec.Name, ifaceSpec.Network) + } + } + // Validate load balancer resources (if enabled - default is true) if ptr.Deref(cloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { Expect(cloudscaleCluster.Status.LoadBalancerID).NotTo(BeEmpty(), "LoadBalancerID should be set") @@ -92,4 +121,24 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste // Validate addresses Expect(machine.Status.Addresses).NotTo(BeEmpty(), "Machine %s should have addresses", machine.Name) } + + // A managed router with internetGateway=true is the cluster's only path to the internet, so + // the nodes must have no public interface of their own. + hasNATRouter := slices.ContainsFunc(cloudscaleCluster.Spec.Routers, func(r infrav1beta2.RouterSpec) bool { + return r.InternetGateway + }) + if hasNATRouter { + By("Validating nodes are private-only behind the router") + + for _, machine := range machineList.Items { + for _, iface := range machine.Spec.Interfaces { + Expect(iface.Type).NotTo(Equal("public"), + "Machine %s should have no public interface: the router provides internet access via SNAT", machine.Name) + } + for _, addr := range machine.Status.Addresses { + Expect(addr.Type).NotTo(Equal(clusterv1.MachineExternalIP), + "Machine %s should report no external IP: the router provides internet access via SNAT", machine.Name) + } + } + } }