diff --git a/api/v1/egressgateway_types.go b/api/v1/egressgateway_types.go index c92dad1923..547b4642d9 100644 --- a/api/v1/egressgateway_types.go +++ b/api/v1/egressgateway_types.go @@ -62,6 +62,7 @@ type EGWDeploymentInitContainer struct { } // EgressGatewaySpec defines the desired state of EgressGateway +// +kubebuilder:validation:XValidation:rule="!(has(self.network) && has(self.externalNetworks) && size(self.externalNetworks) > 0)",message="network and externalNetworks are mutually exclusive; use network" type EgressGatewaySpec struct { // Replicas defines how many instances of the Egress Gateway pod will run. // +kubebuilder:validation:Minimum=0 @@ -79,9 +80,34 @@ type EgressGatewaySpec struct { // ExternalNetworks defines the external network names this Egress Gateway is // associated with. // ExternalNetworks must match existing external networks. + // Deprecated: superseded by Network and will be removed in a future release. + // Mutually exclusive with Network. // +optional ExternalNetworks []string `json:"externalNetworks,omitempty"` + // Network names a cluster-scoped Calico Network (projectcalico.org/v3) that + // the primary pod interface attaches to. When set, EgressGateway pods are + // annotated with cni.projectcalico.org/networks so the Calico CNI plumbs + // eth0 into that Network. + // Mutually exclusive with ExternalNetworks. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + Network string `json:"network,omitempty"` + + // AdditionalInterfaces declares secondary NICs to attach to each Egress + // Gateway pod, in addition to the primary eth0 interface. Each entry names + // an interface (which becomes the device name inside the pod) and + // describes how it is plumbed in via a Multus NetworkAttachmentDefinition. + // Requires Installation.spec.calicoNetwork.multiInterfaceMode=Multus and + // the Multus CNI to be installed in the cluster. + // +optional + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=9 + AdditionalInterfaces []AdditionalInterface `json:"additionalInterfaces,omitempty"` + // LogSeverity defines the logging level of the Egress Gateway. // +optional // +kubebuilder:default:=Info @@ -104,6 +130,59 @@ type EgressGatewaySpec struct { AWS *AWSEgressGateway `json:"aws,omitempty"` } +// AdditionalInterface describes a secondary network interface attached to +// each Egress Gateway pod, in addition to the primary eth0 interface. +type AdditionalInterface struct { + // Name is the interface name inside the pod (e.g. "eth1", "data0"). + // Must be a valid Linux interface name, must not be "eth0", and must be + // unique within additionalInterfaces. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:validation:Pattern=`^[a-z][a-z0-9-]*$` + // +kubebuilder:validation:XValidation:rule="self != 'eth0'",message="name must not be 'eth0'; eth0 is the primary interface" + Name string `json:"name"` + + // Attachment selects how this interface is plumbed into the pod. + // Exactly one attachment mechanism must be set. + // +kubebuilder:validation:Required + Attachment InterfaceAttachment `json:"attachment"` + + // IPPools optionally restricts the Calico IP pools the CNI may allocate + // from for this interface. If unset, IP allocation follows the + // configuration baked into the attachment (e.g. the NetworkAttachmentDefinition's + // CNI config). + // +optional + // +kubebuilder:validation:MaxItems=10 + IPPools []EgressGatewayIPPool `json:"ipPools,omitempty"` +} + +// InterfaceAttachment selects the mechanism that plumbs a secondary +// interface into an Egress Gateway pod. Exactly one arm must be set. +// +kubebuilder:validation:MinProperties=1 +// +kubebuilder:validation:MaxProperties=1 +type InterfaceAttachment struct { + // Multus attaches the interface via a Multus NetworkAttachmentDefinition. + // +optional + Multus *MultusAttachment `json:"multus,omitempty"` +} + +// MultusAttachment references a Multus NetworkAttachmentDefinition. +type MultusAttachment struct { + // Name of the NetworkAttachmentDefinition. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` + + // Namespace of the NetworkAttachmentDefinition. Defaults to the namespace + // of the EgressGateway. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Namespace string `json:"namespace,omitempty"` +} + // EgressGatewayDeploymentPodSpec is the Egress Gateway Deployment's PodSpec. type EgressGatewayDeploymentPodSpec struct { // InitContainers is a list of EGW init containers. @@ -297,6 +376,7 @@ type EgressGatewayStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=`.spec.network` // EgressGateway is the Schema for the egressgateways API type EgressGateway struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 7a26d77a2f..580158ac2d 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -411,6 +411,27 @@ func (in *AWSEgressGateway) DeepCopy() *AWSEgressGateway { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalInterface) DeepCopyInto(out *AdditionalInterface) { + *out = *in + in.Attachment.DeepCopyInto(&out.Attachment) + if in.IPPools != nil { + in, out := &in.IPPools, &out.IPPools + *out = make([]EgressGatewayIPPool, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalInterface. +func (in *AdditionalInterface) DeepCopy() *AdditionalInterface { + if in == nil { + return nil + } + out := new(AdditionalInterface) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdditionalLogSourceSpec) DeepCopyInto(out *AdditionalLogSourceSpec) { *out = *in @@ -3901,6 +3922,13 @@ func (in *EgressGatewaySpec) DeepCopyInto(out *EgressGatewaySpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.AdditionalInterfaces != nil { + in, out := &in.AdditionalInterfaces, &out.AdditionalInterfaces + *out = make([]AdditionalInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.LogSeverity != nil { in, out := &in.LogSeverity, &out.LogSeverity *out = new(LogSeverity) @@ -6110,6 +6138,26 @@ func (in *InstallationStatus) DeepCopy() *InstallationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InterfaceAttachment) DeepCopyInto(out *InterfaceAttachment) { + *out = *in + if in.Multus != nil { + in, out := &in.Multus, &out.Multus + *out = new(MultusAttachment) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InterfaceAttachment. +func (in *InterfaceAttachment) DeepCopy() *InterfaceAttachment { + if in == nil { + return nil + } + out := new(InterfaceAttachment) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IntrusionDetection) DeepCopyInto(out *IntrusionDetection) { *out = *in @@ -8044,6 +8092,21 @@ func (in *MonitorStatus) DeepCopy() *MonitorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultusAttachment) DeepCopyInto(out *MultusAttachment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultusAttachment. +func (in *MultusAttachment) DeepCopy() *MultusAttachment { + if in == nil { + return nil + } + out := new(MultusAttachment) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamespacedName) DeepCopyInto(out *NamespacedName) { *out = *in diff --git a/internal/controller/egressgateway_controller.go b/internal/controller/egressgateway_controller.go index fc647f17c2..edbf579222 100644 --- a/internal/controller/egressgateway_controller.go +++ b/internal/controller/egressgateway_controller.go @@ -34,6 +34,7 @@ type EgressGatewayReconciler struct { // +kubebuilder:rbac:groups=operator.tigera.io,resources=egressgateways,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=operator.tigera.io,resources=egressgateways/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch func (r *EgressGatewayReconciler) SetupWithManager(mgr ctrl.Manager, opts options.ControllerOptions) error { return egressgateway.Add(mgr, opts) diff --git a/pkg/controller/egressgateway/egressgateway_controller.go b/pkg/controller/egressgateway/egressgateway_controller.go index 1531baf3bf..dd5fae00ec 100644 --- a/pkg/controller/egressgateway/egressgateway_controller.go +++ b/pkg/controller/egressgateway/egressgateway_controller.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + netattachv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" @@ -64,7 +65,13 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } licenseAPIReady := &utils.ReadyFlag{} - reconciler := newReconciler(mgr, opts, licenseAPIReady) + multusEnabled, err := utils.MultusEnabled(opts.K8sClientset) + if err != nil { + log.Error(err, "Failed to detect Multus NetworkAttachmentDefinition CRD; assuming Multus is not installed") + multusEnabled = false + } + + reconciler := newReconciler(mgr, opts, licenseAPIReady, multusEnabled) c, err := ctrlruntime.NewController("egressgateway-controller", mgr, controller.Options{Reconciler: reconcile.Reconciler(reconciler)}) if err != nil { @@ -73,11 +80,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { go utils.WaitToAddLicenseKeyWatch(c, opts.K8sClientset, log, licenseAPIReady) - return add(mgr, c) + return add(mgr, c, multusEnabled) } // newReconciler returns a new *reconcile.Reconciler. -func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseAPIReady *utils.ReadyFlag) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseAPIReady *utils.ReadyFlag, multusEnabled bool) reconcile.Reconciler { r := &ReconcileEgressGateway{ client: mgr.GetClient(), scheme: mgr.GetScheme(), @@ -85,6 +92,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseA status: status.New(mgr.GetClient(), "egressgateway", opts.KubernetesVersion), clusterDomain: opts.ClusterDomain, licenseAPIReady: licenseAPIReady, + multusEnabled: multusEnabled, } r.status.Run(opts.ShutdownContext) return r @@ -94,7 +102,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseA // Watching namespaced resources must be avoided as the controller // can't differentiate if the request namespaced resource is an // Egress Gateway resource or not. -func add(_ manager.Manager, c ctrlruntime.Controller) error { +func add(_ manager.Manager, c ctrlruntime.Controller, multusEnabled bool) error { var err error // Watch for changes to primary resource Egress Gateway. @@ -117,6 +125,17 @@ func add(_ manager.Manager, c ctrlruntime.Controller) error { return fmt.Errorf("egressGateway-controller failed to watch FelixConfiguration resource: %w", err) } + // Watch NetworkAttachmentDefinitions when Multus is installed so that NAD + // create/delete events re-trigger reconciliation. Without this, a NAD + // referenced from additionalInterfaces[] would not pick up changes after + // the EgressGateway is first reconciled. + if multusEnabled { + err = c.WatchObject(&netattachv1.NetworkAttachmentDefinition{}, &handler.EnqueueRequestForObject{}) + if err != nil { + return fmt.Errorf("egressgateway-controller failed to watch NetworkAttachmentDefinition resource: %w", err) + } + } + return nil } @@ -133,6 +152,9 @@ type ReconcileEgressGateway struct { status status.StatusManager clusterDomain string licenseAPIReady *utils.ReadyFlag + // multusEnabled reports whether the Multus NetworkAttachmentDefinition + // CRD is installed in the cluster. Detected at controller startup. + multusEnabled bool } // Reconcile reads that state of the cluster for an EgressGateway object and makes changes @@ -348,7 +370,7 @@ func (r *ReconcileEgressGateway) reconcileEgressGateway(ctx context.Context, egw // update the EGW resource with default values. fillDefaults(egw, installationSpec) // Validate the EGW resource. - err := validateEgressGateway(ctx, r.client, egw) + warnings, err := validateEgressGateway(ctx, r.client, egw, installationSpec, r.multusEnabled) if err != nil { reqLogger.Error(err, fmt.Sprintf("Error validating Egress Gateway Name = %s, Namespace = %s", egw.Name, egw.Namespace)) r.status.SetDegraded(operatorv1.ResourceValidationError, @@ -356,6 +378,9 @@ func (r *ReconcileEgressGateway) reconcileEgressGateway(ctx context.Context, egw setDegraded(r.client, ctx, egw, reconcileErr, fmt.Sprintf("Error validating egress gateway err = %s", err.Error())) return err } + for _, w := range warnings { + reqLogger.Info("EgressGateway validation warning", "warning", w) + } if err = r.client.Patch(ctx, egw, preDefaultPatchFrom); err != nil { reqLogger.Error(err, fmt.Sprintf("Failed to write defaults to egress gateway Name = %s, Namespace = %s", egw.Name, egw.Namespace)) @@ -428,8 +453,14 @@ func getRequestedEgressGateway(egws []operatorv1.EgressGateway, request reconcil return nil, -1 } -// validateEgressGateway checks if the ippools specified are already present. -func validateEgressGateway(ctx context.Context, cli client.Client, egw *operatorv1.EgressGateway) error { +// validateEgressGateway checks if the ippools specified are already present and +// validates multi-NIC preconditions. Returns a slice of non-fatal warnings (e.g. +// references to NetworkAttachmentDefinitions that don't yet exist) alongside +// any error that should prevent the EGW being rendered. +func validateEgressGateway(ctx context.Context, cli client.Client, egw *operatorv1.EgressGateway, + installationSpec *operatorv1.InstallationSpec, multusEnabled bool, +) ([]string, error) { + var warnings []string nativeIP := operatorv1.NativeIPDisabled if egw.Spec.AWS != nil && egw.Spec.AWS.NativeIP != nil { nativeIP = *egw.Spec.AWS.NativeIP @@ -440,27 +471,27 @@ func validateEgressGateway(ctx context.Context, cli client.Client, egw *operator // If CIDR is specified, check if CIDR matches with any IPPool. // If Aws.NativeIP is enabled, check if the IPPool is backed by aws-subnet ID. if len(egw.Spec.IPPools) == 0 { - return fmt.Errorf("at least one IPPool must be specified") + return nil, fmt.Errorf("at least one IPPool must be specified") } for _, ippool := range egw.Spec.IPPools { err := validateIPPool(ctx, cli, ippool, nativeIP) if err != nil { - return err + return nil, err } } for _, externalNetwork := range egw.Spec.ExternalNetworks { err := validateExternalNetwork(ctx, cli, externalNetwork) if err != nil { - return err + return nil, err } } // Check if ElasticIPs are specified only if NativeIP is enabled. if egw.Spec.AWS != nil { if len(egw.Spec.AWS.ElasticIPs) > 0 && (*egw.Spec.AWS.NativeIP == operatorv1.NativeIPDisabled) { - return fmt.Errorf("NativeIP must be enabled when elastic IPs are used") + return nil, fmt.Errorf("NativeIP must be enabled when elastic IPs are used") } } @@ -468,23 +499,55 @@ func validateEgressGateway(ctx context.Context, cli client.Client, egw *operator if egw.Spec.EgressGatewayFailureDetection != nil { if egw.Spec.EgressGatewayFailureDetection.ICMPProbe == nil && egw.Spec.EgressGatewayFailureDetection.HTTPProbe == nil { - return fmt.Errorf("either ICMP or HTTP probe must be configured") + return nil, fmt.Errorf("either ICMP or HTTP probe must be configured") } // Check if ICMP and HTTP probe timeout is greater than interval. if egw.Spec.EgressGatewayFailureDetection.ICMPProbe != nil { if *egw.Spec.EgressGatewayFailureDetection.ICMPProbe.TimeoutSeconds < *egw.Spec.EgressGatewayFailureDetection.ICMPProbe.IntervalSeconds { - return fmt.Errorf("ICMP probe timeout must be greater than interval") + return nil, fmt.Errorf("ICMP probe timeout must be greater than interval") } } if egw.Spec.EgressGatewayFailureDetection.HTTPProbe != nil { if *egw.Spec.EgressGatewayFailureDetection.HTTPProbe.TimeoutSeconds < *egw.Spec.EgressGatewayFailureDetection.HTTPProbe.IntervalSeconds { - return fmt.Errorf("HTTP probe timeout must be greater than interval") + return nil, fmt.Errorf("HTTP probe timeout must be greater than interval") } } } - return nil + + if len(egw.Spec.AdditionalInterfaces) > 0 { + if installationSpec == nil || installationSpec.CalicoNetwork == nil || + installationSpec.CalicoNetwork.MultiInterfaceMode == nil || + *installationSpec.CalicoNetwork.MultiInterfaceMode != operatorv1.MultiInterfaceModeMultus { + return nil, fmt.Errorf("additionalInterfaces requires Installation.spec.calicoNetwork.multiInterfaceMode=Multus") + } + if !multusEnabled { + return nil, fmt.Errorf("additionalInterfaces requires Multus (NetworkAttachmentDefinition CRD) to be installed in the cluster") + } + for _, iface := range egw.Spec.AdditionalInterfaces { + if iface.Attachment.Multus == nil { + // Should be unreachable given CRD-level MinProperties=1, but + // guard so that future arms cannot regress silently. + return nil, fmt.Errorf("additionalInterfaces[%q]: attachment is not set", iface.Name) + } + ns := iface.Attachment.Multus.Namespace + if ns == "" { + ns = egw.Namespace + } + nad := &netattachv1.NetworkAttachmentDefinition{} + err := cli.Get(ctx, types.NamespacedName{Name: iface.Attachment.Multus.Name, Namespace: ns}, nad) + if err != nil { + if errors.IsNotFound(err) { + warnings = append(warnings, fmt.Sprintf("additionalInterfaces[%q]: NetworkAttachmentDefinition %s/%s not found", iface.Name, ns, iface.Attachment.Multus.Name)) + continue + } + return warnings, fmt.Errorf("additionalInterfaces[%q]: error querying NetworkAttachmentDefinition %s/%s: %w", iface.Name, ns, iface.Attachment.Multus.Name, err) + } + } + } + + return warnings, nil } // getEgressGateways returns the egress gateways in all namespaces or in the request's namespace. diff --git a/pkg/controller/egressgateway/egressgateway_controller_test.go b/pkg/controller/egressgateway/egressgateway_controller_test.go index 789de86408..3c46d935a7 100644 --- a/pkg/controller/egressgateway/egressgateway_controller_test.go +++ b/pkg/controller/egressgateway/egressgateway_controller_test.go @@ -722,6 +722,98 @@ var _ = Describe("Egress Gateway controller tests", func() { mockStatus.AssertExpectations(GinkgoT()) }) + It("Should reject additionalInterfaces when MultiInterfaceMode is not Multus", func() { + mockStatus.On("SetDegraded", operatorv1.ResourceValidationError, "Error validating egress gateway Name = calico-red, Namespace = calico-egress", "additionalInterfaces requires Installation.spec.calicoNetwork.multiInterfaceMode=Multus", mock.Anything, mock.Anything).Return() + Expect(c.Create(ctx, installation)).NotTo(HaveOccurred()) + egw := &operatorv1.EgressGateway{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-red", Namespace: "calico-egress"}, + Spec: operatorv1.EgressGatewaySpec{ + Replicas: ptr.To(int32(1)), + IPPools: []operatorv1.EgressGatewayIPPool{{Name: "ippool-1"}}, + AdditionalInterfaces: []operatorv1.AdditionalInterface{{ + Name: "data0", + Attachment: operatorv1.InterfaceAttachment{ + Multus: &operatorv1.MultusAttachment{Name: "finance-nad"}, + }, + }}, + Template: &operatorv1.EgressGatewayDeploymentPodTemplateSpec{Metadata: &operatorv1.EgressGatewayMetadata{Labels: map[string]string{"egress-code": "red"}}}, + }, + Status: operatorv1.EgressGatewayStatus{State: operatorv1.TigeraStatusReady}, + } + Expect(c.Create(ctx, egw)).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).Should(HaveOccurred()) + mockStatus.AssertExpectations(GinkgoT()) + }) + + It("Should reject additionalInterfaces when Multus CRD is not installed", func() { + mockStatus.On("SetDegraded", operatorv1.ResourceValidationError, "Error validating egress gateway Name = calico-red, Namespace = calico-egress", "additionalInterfaces requires Multus (NetworkAttachmentDefinition CRD) to be installed in the cluster", mock.Anything, mock.Anything).Return() + multus := operatorv1.MultiInterfaceModeMultus + installation.Spec.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multus} + Expect(c.Create(ctx, installation)).NotTo(HaveOccurred()) + egw := &operatorv1.EgressGateway{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-red", Namespace: "calico-egress"}, + Spec: operatorv1.EgressGatewaySpec{ + Replicas: ptr.To(int32(1)), + IPPools: []operatorv1.EgressGatewayIPPool{{Name: "ippool-1"}}, + AdditionalInterfaces: []operatorv1.AdditionalInterface{{ + Name: "data0", + Attachment: operatorv1.InterfaceAttachment{ + Multus: &operatorv1.MultusAttachment{Name: "finance-nad"}, + }, + }}, + Template: &operatorv1.EgressGatewayDeploymentPodTemplateSpec{Metadata: &operatorv1.EgressGatewayMetadata{Labels: map[string]string{"egress-code": "red"}}}, + }, + Status: operatorv1.EgressGatewayStatus{State: operatorv1.TigeraStatusReady}, + } + Expect(c.Create(ctx, egw)).NotTo(HaveOccurred()) + + // r.multusEnabled is false by default in this test. + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).Should(HaveOccurred()) + mockStatus.AssertExpectations(GinkgoT()) + }) + + It("Should reconcile (with a warning) when a referenced NAD is missing", func() { + mockStatus.On("AddDaemonsets", mock.Anything).Return() + mockStatus.On("AddDeployments", mock.Anything).Return() + mockStatus.On("IsAvailable").Return(true) + mockStatus.On("AddStatefulSets", mock.Anything).Return() + mockStatus.On("AddCronJobs", mock.Anything) + mockStatus.On("OnCRNotFound").Return() + mockStatus.On("ClearDegraded") + mockStatus.On("ReadyToMonitor") + multus := operatorv1.MultiInterfaceModeMultus + installation.Spec.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multus} + Expect(c.Create(ctx, installation)).NotTo(HaveOccurred()) + r.multusEnabled = true + + egw := &operatorv1.EgressGateway{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-red", Namespace: "calico-egress"}, + Spec: operatorv1.EgressGatewaySpec{ + Replicas: ptr.To(int32(1)), + LogSeverity: ptr.To(operatorv1.LogSeverityInfo), + IPPools: []operatorv1.EgressGatewayIPPool{{Name: "ippool-1"}}, + AdditionalInterfaces: []operatorv1.AdditionalInterface{{ + Name: "data0", + Attachment: operatorv1.InterfaceAttachment{ + Multus: &operatorv1.MultusAttachment{Name: "absent-nad"}, + }, + }}, + Template: &operatorv1.EgressGatewayDeploymentPodTemplateSpec{Metadata: &operatorv1.EgressGatewayMetadata{Labels: map[string]string{"egress-code": "red"}}}, + }, + Status: operatorv1.EgressGatewayStatus{State: operatorv1.TigeraStatusReady}, + } + Expect(c.Create(ctx, egw)).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + dep := appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "calico-red", Namespace: "calico-egress"}} + Expect(test.GetResource(c, &dep)).To(BeNil()) + Expect(dep.Spec.Template.Annotations).To(HaveKey("k8s.v1.cni.cncf.io/networks")) + }) + It("Should throw an error when externalNetworks are not present", func() { mockStatus.On("SetDegraded", operatorv1.ResourceValidationError, "Error validating egress gateway Name = calico-red, Namespace = calico-egress", "externalnetworks.crd.projectcalico.org \"three\" not found", mock.Anything, mock.Anything).Return() Expect(c.Create(ctx, installation)).NotTo(HaveOccurred()) @@ -812,7 +904,7 @@ var _ = Describe("Egress Gateway controller tests", func() { It("should not watch namespaced resources", func() { m := &mockController{} var mgr manager.Manager - err := add(mgr, m) + err := add(mgr, m, true) Expect(err).ShouldNot(HaveOccurred()) for _, obj := range m.watchedObjects { Expect(len(obj.GetNamespace())).To(Equal(0)) diff --git a/pkg/controller/utils/discovery.go b/pkg/controller/utils/discovery.go index f80638ea5d..4daa4c107b 100644 --- a/pkg/controller/utils/discovery.go +++ b/pkg/controller/utils/discovery.go @@ -63,6 +63,25 @@ func RequiresTigeraSecure(clientset *kubernetes.Clientset) (bool, error) { return false, nil } +// MultusEnabled reports whether the Multus NetworkAttachmentDefinition CRD +// (k8s.cni.cncf.io/v1) is installed in the cluster. Returns false (no error) +// when the API group is not present. +func MultusEnabled(clientset kubernetes.Interface) (bool, error) { + resources, err := clientset.Discovery().ServerResourcesForGroupVersion("k8s.cni.cncf.io/v1") + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + for _, r := range resources.APIResources { + if r.Kind == "NetworkAttachmentDefinition" { + return true, nil + } + } + return false, nil +} + func MultiTenant(ctx context.Context, c kubernetes.Interface) (bool, error) { resources, err := c.Discovery().ServerResourcesForGroupVersion("operator.tigera.io/v1") if err != nil { diff --git a/pkg/imports/crds/operator/operator.tigera.io_egressgateways.yaml b/pkg/imports/crds/operator/operator.tigera.io_egressgateways.yaml index 0f0d1b1926..357406642f 100644 --- a/pkg/imports/crds/operator/operator.tigera.io_egressgateways.yaml +++ b/pkg/imports/crds/operator/operator.tigera.io_egressgateways.yaml @@ -13,7 +13,11 @@ spec: singular: egressgateway scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - jsonPath: .spec.network + name: Network + type: string + name: v1 schema: openAPIV3Schema: description: EgressGateway is the Schema for the egressgateways API @@ -38,6 +42,89 @@ spec: spec: description: EgressGatewaySpec defines the desired state of EgressGateway properties: + additionalInterfaces: + description: |- + AdditionalInterfaces declares secondary NICs to attach to each Egress + Gateway pod, in addition to the primary eth0 interface. Each entry names + an interface (which becomes the device name inside the pod) and + describes how it is plumbed in via a Multus NetworkAttachmentDefinition. + Requires Installation.spec.calicoNetwork.multiInterfaceMode=Multus and + the Multus CNI to be installed in the cluster. + items: + description: |- + AdditionalInterface describes a secondary network interface attached to + each Egress Gateway pod, in addition to the primary eth0 interface. + properties: + attachment: + description: |- + Attachment selects how this interface is plumbed into the pod. + Exactly one attachment mechanism must be set. + maxProperties: 1 + minProperties: 1 + properties: + multus: + description: + Multus attaches the interface via a Multus + NetworkAttachmentDefinition. + properties: + name: + description: Name of the NetworkAttachmentDefinition. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace of the NetworkAttachmentDefinition. Defaults to the namespace + of the EgressGateway. + maxLength: 63 + minLength: 1 + type: string + required: + - name + type: object + type: object + ipPools: + description: |- + IPPools optionally restricts the Calico IP pools the CNI may allocate + from for this interface. If unset, IP allocation follows the + configuration baked into the attachment (e.g. the NetworkAttachmentDefinition's + CNI config). + items: + properties: + cidr: + description: + CIDR is the IPPool CIDR that the Egress Gateways + can use. + type: string + name: + description: + Name is the name of the IPPool that the Egress + Gateways can use. + type: string + type: object + maxItems: 10 + type: array + name: + description: |- + Name is the interface name inside the pod (e.g. "eth1", "data0"). + Must be a valid Linux interface name, must not be "eth0", and must be + unique within additionalInterfaces. + maxLength: 15 + minLength: 1 + pattern: ^[a-z][a-z0-9-]*$ + type: string + x-kubernetes-validations: + - message: name must not be 'eth0'; eth0 is the primary interface + rule: self != 'eth0' + required: + - attachment + - name + type: object + maxItems: 9 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map aws: description: AWS defines the additional configuration options for @@ -152,6 +239,8 @@ spec: ExternalNetworks defines the external network names this Egress Gateway is associated with. ExternalNetworks must match existing external networks. + Deprecated: superseded by Network and will be removed in a future release. + Mutually exclusive with Network. items: type: string type: array @@ -185,6 +274,17 @@ spec: - Debug - Trace type: string + network: + description: |- + Network names a cluster-scoped Calico Network (projectcalico.org/v3) that + the primary pod interface attaches to. When set, EgressGateway pods are + annotated with cni.projectcalico.org/networks so the Calico CNI plumbs + eth0 into that Network. + Mutually exclusive with ExternalNetworks. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string replicas: default: 1 description: @@ -1656,6 +1756,11 @@ spec: required: - ipPools type: object + x-kubernetes-validations: + - message: network and externalNetworks are mutually exclusive; use network + rule: + "!(has(self.network) && has(self.externalNetworks) && size(self.externalNetworks) + > 0)" status: description: EgressGatewayStatus defines the observed state of EgressGateway properties: diff --git a/pkg/render/egressgateway/egressgateway.go b/pkg/render/egressgateway/egressgateway.go index 84c675e75c..5848268836 100644 --- a/pkg/render/egressgateway/egressgateway.go +++ b/pkg/render/egressgateway/egressgateway.go @@ -194,9 +194,55 @@ func (c *component) egwBuildAnnotations() map[string]string { if len(c.config.EgressGW.Spec.ExternalNetworks) > 0 { annotations["egress.projectcalico.org/externalNetworkNames"] = c.getExternalNetworks() } + if c.config.EgressGW.Spec.Network != "" { + // Calico CNI attaches the primary interface to the named Network. + // Currently a single name; expected to extend to a JSON list later. + annotations["cni.projectcalico.org/networks"] = c.config.EgressGW.Spec.Network + } + if len(c.config.EgressGW.Spec.AdditionalInterfaces) > 0 { + annotations["k8s.v1.cni.cncf.io/networks"] = c.getMultusNetworks() + } return annotations } +// multusNetworkRef is one entry in the k8s.v1.cni.cncf.io/networks annotation. +// See https://github.com/k8snetworkplumbingwg/multus-cni for the schema. +type multusNetworkRef struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + Interface string `json:"interface,omitempty"` +} + +// getMultusNetworks renders the k8s.v1.cni.cncf.io/networks annotation value, +// one entry per AdditionalInterface. The pod-side interface name is taken from +// AdditionalInterface.Name so it is deterministic rather than Multus's +// auto-generated net1/net2. +func (c *component) getMultusNetworks() string { + refs := make([]multusNetworkRef, 0, len(c.config.EgressGW.Spec.AdditionalInterfaces)) + for _, iface := range c.config.EgressGW.Spec.AdditionalInterfaces { + if iface.Attachment.Multus == nil { + continue + } + ns := iface.Attachment.Multus.Namespace + if ns == "" { + ns = c.config.EgressGW.Namespace + } + refs = append(refs, multusNetworkRef{ + Name: iface.Attachment.Multus.Name, + Namespace: ns, + Interface: iface.Name, + }) + } + b, err := json.Marshal(refs) + if err != nil { + // json.Marshal on a slice of trivial structs never errors in practice; + // log and fall back to an empty array so we never break rendering. + log.Error(err, "failed to marshal Multus networks annotation; emitting empty array") + return "[]" + } + return string(b) +} + func (c *component) egwInitContainer() *corev1.Container { return &corev1.Container{ Name: "egress-gateway-init", diff --git a/pkg/render/egressgateway/egressgateway_test.go b/pkg/render/egressgateway/egressgateway_test.go index a83877eb83..44c8d92d26 100644 --- a/pkg/render/egressgateway/egressgateway_test.go +++ b/pkg/render/egressgateway/egressgateway_test.go @@ -345,4 +345,65 @@ var _ = Describe("Egress Gateway rendering tests", func() { Effect: corev1.TaintEffectNoSchedule, })) }) + + It("should set cni.projectcalico.org/networks when spec.network is set", func() { + egw.Spec.ExternalNetworks = nil // mutually exclusive with Network + egw.Spec.Network = "finance-vrf" + + component := egressgateway.EgressGateway(&egressgateway.Config{ + Installation: installation, + OSType: rmeta.OSTypeLinux, + EgressGW: egw, + VXLANVNI: 4097, + VXLANPort: 4790, + }) + resources, _ := component.Objects() + dep := rtest.GetResource(resources, "egress-test", "test-ns", "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(dep.Spec.Template.Annotations).To(HaveKeyWithValue("cni.projectcalico.org/networks", "finance-vrf")) + }) + + It("should render k8s.v1.cni.cncf.io/networks for additionalInterfaces", func() { + egw.Spec.AdditionalInterfaces = []operatorv1.AdditionalInterface{ + { + Name: "data0", + Attachment: operatorv1.InterfaceAttachment{ + Multus: &operatorv1.MultusAttachment{Name: "finance-nad"}, + }, + }, + { + Name: "mgmt0", + Attachment: operatorv1.InterfaceAttachment{ + Multus: &operatorv1.MultusAttachment{Name: "mgmt-nad", Namespace: "kube-system"}, + }, + }, + } + + component := egressgateway.EgressGateway(&egressgateway.Config{ + Installation: installation, + OSType: rmeta.OSTypeLinux, + EgressGW: egw, + VXLANVNI: 4097, + VXLANPort: 4790, + }) + resources, _ := component.Objects() + dep := rtest.GetResource(resources, "egress-test", "test-ns", "apps", "v1", "Deployment").(*appsv1.Deployment) + + expected := `[{"name":"finance-nad","namespace":"test-ns","interface":"data0"},` + + `{"name":"mgmt-nad","namespace":"kube-system","interface":"mgmt0"}]` + Expect(dep.Spec.Template.Annotations).To(HaveKeyWithValue("k8s.v1.cni.cncf.io/networks", expected)) + }) + + It("should not emit the multus annotation when additionalInterfaces is empty", func() { + component := egressgateway.EgressGateway(&egressgateway.Config{ + Installation: installation, + OSType: rmeta.OSTypeLinux, + EgressGW: egw, + VXLANVNI: 4097, + VXLANPort: 4790, + }) + resources, _ := component.Objects() + dep := rtest.GetResource(resources, "egress-test", "test-ns", "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(dep.Spec.Template.Annotations).NotTo(HaveKey("k8s.v1.cni.cncf.io/networks")) + Expect(dep.Spec.Template.Annotations).NotTo(HaveKey("cni.projectcalico.org/networks")) + }) })