diff --git a/api/v1/calico_node_types.go b/api/v1/calico_node_types.go index 35d5ba9520..fd3a9b883c 100644 --- a/api/v1/calico_node_types.go +++ b/api/v1/calico_node_types.go @@ -48,8 +48,8 @@ type CalicoNodeDaemonSetContainer struct { // CalicoNodeDaemonSetInitContainer is a calico-node DaemonSet init container. type CalicoNodeDaemonSetInitContainer struct { // Name is an enum which identifies the calico-node DaemonSet init container by name. - // Supported values are: install-cni, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) - // +kubebuilder:validation:Enum=install-cni;hostpath-init;flexvol-driver;ebpf-bootstrap;node-certs-key-cert-provisioner;calico-node-prometheus-server-tls-key-cert-provisioner;mount-bpffs + // Supported values are: install-cni, cni-plugins, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) + // +kubebuilder:validation:Enum=install-cni;cni-plugins;hostpath-init;flexvol-driver;ebpf-bootstrap;node-certs-key-cert-provisioner;calico-node-prometheus-server-tls-key-cert-provisioner;mount-bpffs Name string `json:"name"` // Resources allows customization of limits and requests for compute resources such as cpu and memory. diff --git a/api/v1/installation_types.go b/api/v1/installation_types.go index 5c8ef6b222..98940237bd 100644 --- a/api/v1/installation_types.go +++ b/api/v1/installation_types.go @@ -1020,8 +1020,36 @@ type CNISpec struct { // +optional // +kubebuilder:validation:Type=string ConfDir *string `json:"confDir,omitempty"` + + // InstallMode controls which CNI plugin binaries the operator installs onto each node + // when CNI.Type is Calico. + // * All (default): the operator runs a cni-plugins init container that stages upstream + // CNI plugin binaries (host-local, portmap, loopback, tuning, flannel) into a shared + // volume, and the install-cni init container copies them onto the host alongside + // Calico's own binaries. + // * CalicoOnly: skip the cni-plugins init container. Only Calico's own binaries are + // installed. Use this when the host already provides the upstream plugins (e.g. kind, + // certain managed node images). + // + // Default: All + // +optional + // +kubebuilder:validation:Enum=All;CalicoOnly + InstallMode *CNIInstallMode `json:"installMode,omitempty"` } +// CNIInstallMode controls which CNI plugin binaries the operator installs onto the host. +type CNIInstallMode string + +const ( + // CNIInstallModeAll installs Calico's own CNI binaries plus the upstream plugin set + // (host-local, portmap, loopback, tuning, flannel) via a dedicated init container. + CNIInstallModeAll CNIInstallMode = "All" + + // CNIInstallModeCalicoOnly installs only Calico's own CNI binaries; the host is + // expected to provide any required upstream plugins. + CNIInstallModeCalicoOnly CNIInstallMode = "CalicoOnly" +) + // InstallationStatus defines the observed state of the Calico or Calico Enterprise installation. type InstallationStatus struct { // Variant is the most recently observed installed variant - one of Calico or CalicoEnterprise. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 7a26d77a2f..78c009ac62 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -912,6 +912,11 @@ func (in *CNISpec) DeepCopyInto(out *CNISpec) { *out = new(string) **out = **in } + if in.InstallMode != nil { + in, out := &in.InstallMode, &out.InstallMode + *out = new(CNIInstallMode) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNISpec. diff --git a/config/calico_versions.yml b/config/calico_versions.yml index c18bc8aa4b..a807df8e54 100644 --- a/config/calico_versions.yml +++ b/config/calico_versions.yml @@ -13,6 +13,8 @@ components: version: master cni-windows: version: master + cni-plugins: + version: master kube-controllers: version: master goldmane: diff --git a/config/enterprise_versions.yml b/config/enterprise_versions.yml index e5761a627a..5f2fb3daa2 100644 --- a/config/enterprise_versions.yml +++ b/config/enterprise_versions.yml @@ -51,6 +51,9 @@ components: tigera-cni-windows: image: cni-windows version: master + tigera-cni-plugins: + image: cni-plugins + version: master # coreos-prometheus holds the version of prometheus built for tigera/prometheus, # which prometheus operator uses to validate. coreos-prometheus: diff --git a/hack/gen-versions/calico.go.tpl b/hack/gen-versions/calico.go.tpl index 0f5b062a2f..cffae65b3b 100644 --- a/hack/gen-versions/calico.go.tpl +++ b/hack/gen-versions/calico.go.tpl @@ -19,6 +19,15 @@ package components var ( CalicoRelease string = "{{ .Title }}" +{{ with index .Components "cni-plugins" }} + ComponentCalicoCNIPlugins = Component{ + Version: "{{ .Version }}", + Image: "{{ .Image }}", + Registry: "{{ .Registry }}", + imagePath: "{{ .ImagePath }}", + variant: calicoVariant, + } +{{- end }} {{ with index .Components "cni-windows" }} ComponentCalicoCNIWindows = Component{ Version: "{{ .Version }}", @@ -145,6 +154,7 @@ var ( {{- end }} CalicoImages = []Component{ + ComponentCalicoCNIPlugins, ComponentCalicoCNIWindows, ComponentCalicoNode, ComponentCalicoNodeFIPS, diff --git a/hack/gen-versions/components.go b/hack/gen-versions/components.go index 5715c26eae..e51f73c0b5 100644 --- a/hack/gen-versions/components.go +++ b/hack/gen-versions/components.go @@ -54,6 +54,8 @@ var ( "coreos-alertmanager": "unused-image", "tigera-cni": "cni", "tigera-cni-windows": "cni-windows", + "cni-plugins": "cni-plugins", + "tigera-cni-plugins": "cni-plugins", "linseed": "linseed", "gateway-api-envoy-gateway": "envoy-gateway", "gateway-api-envoy-proxy": "envoy-proxy", diff --git a/hack/gen-versions/enterprise.go.tpl b/hack/gen-versions/enterprise.go.tpl index 7ed9089073..77e7714315 100644 --- a/hack/gen-versions/enterprise.go.tpl +++ b/hack/gen-versions/enterprise.go.tpl @@ -237,6 +237,15 @@ var ( variant: enterpriseVariant, } {{- end }} +{{ with index .Components "tigera-cni-plugins" }} + ComponentTigeraCNIPlugins = Component{ + Version: "{{ .Version }}", + Image: "{{ .Image }}", + Registry: "{{ .Registry }}", + imagePath: "{{ .ImagePath }}", + variant: enterpriseVariant, + } +{{- end }} {{ with index .Components "gateway-api-envoy-gateway" }} ComponentGatewayAPIEnvoyGateway = Component{ Version: "{{ .Version }}", @@ -321,6 +330,7 @@ var ( ComponentTigeraNode, ComponentTigeraNodeWindows, ComponentTigeraCNIWindows, + ComponentTigeraCNIPlugins, ComponentGatewayAPIEnvoyGateway, ComponentGatewayAPIEnvoyProxy, ComponentGatewayAPIEnvoyRatelimit, diff --git a/pkg/components/calico.go b/pkg/components/calico.go index 2858bfb15a..54cc7691cc 100644 --- a/pkg/components/calico.go +++ b/pkg/components/calico.go @@ -20,6 +20,14 @@ package components var ( CalicoRelease string = "master" + ComponentCalicoCNIPlugins = Component{ + Version: "master", + Image: "cni-plugins", + Registry: "", + imagePath: "", + variant: calicoVariant, + } + ComponentCalicoCNIWindows = Component{ Version: "master", Image: "cni-windows", @@ -133,6 +141,7 @@ var ( } CalicoImages = []Component{ + ComponentCalicoCNIPlugins, ComponentCalicoCNIWindows, ComponentCalicoNode, ComponentCalicoNodeFIPS, diff --git a/pkg/components/enterprise.go b/pkg/components/enterprise.go index 3753ca105d..561c4ec57c 100644 --- a/pkg/components/enterprise.go +++ b/pkg/components/enterprise.go @@ -212,6 +212,14 @@ var ( variant: enterpriseVariant, } + ComponentTigeraCNIPlugins = Component{ + Version: "master", + Image: "cni-plugins", + Registry: "", + imagePath: "", + variant: enterpriseVariant, + } + ComponentGatewayAPIEnvoyGateway = Component{ Version: "master", Image: "envoy-gateway", @@ -288,6 +296,7 @@ var ( ComponentTigeraNode, ComponentTigeraNodeWindows, ComponentTigeraCNIWindows, + ComponentTigeraCNIPlugins, ComponentGatewayAPIEnvoyGateway, ComponentGatewayAPIEnvoyProxy, ComponentGatewayAPIEnvoyRatelimit, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index ebce8b40f4..c88fa21704 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -656,6 +656,10 @@ func fillDefaults(instance *operatorv1.Installation, currentPools *v3.IPPoolList if instance.Spec.CNI.BinDir == nil || *instance.Spec.CNI.BinDir == "" { instance.Spec.CNI.BinDir = &defaultCNIBinDir } + if instance.Spec.CNI.InstallMode == nil { + mode := operatorv1.CNIInstallModeAll + instance.Spec.CNI.InstallMode = &mode + } // While a number of the fields in this section are relevant to all CNI plugins, // there are some settings which are currently only applicable if using Calico CNI. diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index aa113b34e3..f5e86c13df 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -452,7 +452,7 @@ var _ = Describe("Testing core-controller installation", func() { components.TigeraImagePath, components.ComponentTigeraNode.Image, components.ComponentTigeraNode.Version))) - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(5)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(6)) fv := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "flexvol-driver") Expect(fv).ToNot(BeNil()) Expect(fv.Image).To(Equal( @@ -467,6 +467,13 @@ var _ = Describe("Testing core-controller installation", func() { components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version))) + cniPlugins := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "cni-plugins") + Expect(cniPlugins).ToNot(BeNil()) + Expect(cniPlugins.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s:%s", + components.TigeraImagePath, + components.ComponentTigeraCNIPlugins.Image, + components.ComponentTigeraCNIPlugins.Version))) csrinit = test.GetContainer(ds.Spec.Template.Spec.InitContainers, fmt.Sprintf("%s-key-cert-provisioner", render.NodeTLSSecretName)) Expect(csrinit).ToNot(BeNil()) Expect(csrinit.Image).To(Equal( @@ -497,6 +504,7 @@ var _ = Describe("Testing core-controller installation", func() { Images: []operator.Image{ {Image: "tigera/calico", Digest: "sha256:tigeracalicohash"}, {Image: "tigera/node", Digest: "sha256:tigeranodehash"}, + {Image: "tigera/cni-plugins", Digest: "sha256:tigeracnipluginshash"}, }, }, } @@ -563,7 +571,7 @@ var _ = Describe("Testing core-controller installation", func() { components.TigeraImagePath, components.ComponentTigeraNode.Image, "sha256:tigeranodehash"))) - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(5)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(6)) fv := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "flexvol-driver") Expect(fv).ToNot(BeNil()) Expect(fv.Image).To(Equal( @@ -578,6 +586,13 @@ var _ = Describe("Testing core-controller installation", func() { components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:tigeracalicohash"))) + cniPlugins := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "cni-plugins") + Expect(cniPlugins).ToNot(BeNil()) + Expect(cniPlugins.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s@%s", + components.TigeraImagePath, + components.ComponentTigeraCNIPlugins.Image, + "sha256:tigeracnipluginshash"))) csrinit = test.GetContainer(ds.Spec.Template.Spec.InitContainers, fmt.Sprintf("%s-key-cert-provisioner", render.NodeTLSSecretName)) Expect(csrinit).ToNot(BeNil()) Expect(csrinit.Image).To(Equal( diff --git a/pkg/controller/installation/defaults_test.go b/pkg/controller/installation/defaults_test.go index 24b9c6a373..d8d948e117 100644 --- a/pkg/controller/installation/defaults_test.go +++ b/pkg/controller/installation/defaults_test.go @@ -115,6 +115,7 @@ var _ = Describe("Defaulting logic tests", func() { var linuxPolicySetupTimeoutSeconds int32 = 1 cniBinDir := "/opt/custom/cni/bin" cniConfDir := "/etc/custom/cni/net.d" + cniInstallMode := operator.CNIInstallModeAll hpEnabled := operator.HostPortsEnabled disabled := operator.BGPDisabled @@ -134,10 +135,11 @@ var _ = Describe("Defaulting logic tests", func() { }, }, CNI: &operator.CNISpec{ - Type: operator.PluginCalico, - IPAM: &operator.IPAMSpec{Type: operator.IPAMPluginCalico}, - BinDir: &cniBinDir, - ConfDir: &cniConfDir, + Type: operator.PluginCalico, + IPAM: &operator.IPAMSpec{Type: operator.IPAMPluginCalico}, + BinDir: &cniBinDir, + ConfDir: &cniConfDir, + InstallMode: &cniInstallMode, }, CalicoNetwork: &operator.CalicoNetworkSpec{ LinuxDataplane: &dpIptables, // Actually the default but BPF would make other values invalid. @@ -210,6 +212,7 @@ var _ = Describe("Defaulting logic tests", func() { logSeverity := operator.LogLevelError cniBinDir := "/opt/custom/cni/bin" cniConfDir := "/etc/custom/cni/net.d" + cniInstallMode := operator.CNIInstallModeAll disabled := operator.BGPDisabled miMode := operator.MultiInterfaceModeNone @@ -229,10 +232,11 @@ var _ = Describe("Defaulting logic tests", func() { }, }, CNI: &operator.CNISpec{ - Type: operator.PluginCalico, - IPAM: &operator.IPAMSpec{Type: operator.IPAMPluginCalico}, - BinDir: &cniBinDir, - ConfDir: &cniConfDir, + Type: operator.PluginCalico, + IPAM: &operator.IPAMSpec{Type: operator.IPAMPluginCalico}, + BinDir: &cniBinDir, + ConfDir: &cniConfDir, + InstallMode: &cniInstallMode, }, CalicoNetwork: &operator.CalicoNetworkSpec{ LinuxDataplane: &dpBPF, // Actually the default but BPF would make other values invalid. diff --git a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml index 1e04c6eb26..a55dee796b 100644 --- a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml +++ b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml @@ -2903,9 +2903,10 @@ spec: name: description: |- Name is an enum which identifies the calico-node DaemonSet init container by name. - Supported values are: install-cni, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) + Supported values are: install-cni, cni-plugins, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) enum: - install-cni + - cni-plugins - hostpath-init - flexvol-driver - ebpf-bootstrap @@ -5709,6 +5710,22 @@ spec: * For KubernetesProvider OpenShift, this field defaults to "/var/run/multus/cni/net.d". * Otherwise, this field defaults to "/etc/cni/net.d". type: string + installMode: + description: |- + InstallMode controls which CNI plugin binaries the operator installs onto each node + when CNI.Type is Calico. + * All (default): the operator runs a cni-plugins init container that stages upstream + CNI plugin binaries (host-local, portmap, loopback, tuning, flannel) into a shared + volume, and the install-cni init container copies them onto the host alongside + Calico's own binaries. + * CalicoOnly: skip the cni-plugins init container. Only Calico's own binaries are + installed. Use this when the host already provides the upstream plugins (e.g. kind, + certain managed node images). + Default: All + enum: + - All + - CalicoOnly + type: string ipam: description: |- IPAM specifies the pod IP address management that will be used in the Calico or @@ -12178,9 +12195,10 @@ spec: name: description: |- Name is an enum which identifies the calico-node DaemonSet init container by name. - Supported values are: install-cni, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) + Supported values are: install-cni, cni-plugins, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) enum: - install-cni + - cni-plugins - hostpath-init - flexvol-driver - ebpf-bootstrap @@ -15030,6 +15048,22 @@ spec: * For KubernetesProvider OpenShift, this field defaults to "/var/run/multus/cni/net.d". * Otherwise, this field defaults to "/etc/cni/net.d". type: string + installMode: + description: |- + InstallMode controls which CNI plugin binaries the operator installs onto each node + when CNI.Type is Calico. + * All (default): the operator runs a cni-plugins init container that stages upstream + CNI plugin binaries (host-local, portmap, loopback, tuning, flannel) into a shared + volume, and the install-cni init container copies them onto the host alongside + Calico's own binaries. + * CalicoOnly: skip the cni-plugins init container. Only Calico's own binaries are + installed. Use this when the host already provides the upstream plugins (e.g. kind, + certain managed node images). + Default: All + enum: + - All + - CalicoOnly + type: string ipam: description: |- IPAM specifies the pod IP address management that will be used in the Calico or diff --git a/pkg/render/node.go b/pkg/render/node.go index c72fe93c79..b5e6c98d8f 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -167,8 +167,9 @@ type nodeComponent struct { cfg *NodeConfiguration // Calculated internal fields based on the given information. - calicoImage string - nodeImage string + calicoImage string + cniPluginsImage string + nodeImage string } func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -184,6 +185,13 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) + if c.installUpstreamPlugins() { + if c.cfg.Installation.Variant.IsEnterprise() { + c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIPlugins, reg, path, prefix, is)) + } else { + c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIPlugins, reg, path, prefix, is)) + } + } switch { case c.cfg.Installation.Variant.IsEnterprise(): c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) @@ -1062,6 +1070,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo } if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.installUpstreamPlugins() { + // cniPluginsContainer must run before cniContainer: it populates the + // staging volume that install-cni reads from at /opt/cni/bin. + ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniPluginsContainer()) + } ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } @@ -1121,6 +1134,11 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) } + if c.installUpstreamPlugins() { + // Staging volume populated by the cni-plugins init container and read + // by install-cni when copying upstream plugins onto the host. + volumes = append(volumes, corev1.Volume{Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + } // Override with Tigera-specific config. if c.cfg.Installation.Variant.IsEnterprise() { @@ -1205,6 +1223,11 @@ func (c *nodeComponent) cniContainer() corev1.Container { {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } + if c.installUpstreamPlugins() { + // Upstream plugin binaries staged by the cni-plugins init container. + // install.go walks /opt/cni/bin to copy them onto the host. + cniVolumeMounts = append(cniVolumeMounts, corev1.VolumeMount{MountPath: "/opt/cni/bin", Name: "cni-plugins-stage"}) + } return corev1.Container{ Name: "install-cni", @@ -1216,6 +1239,35 @@ func (c *nodeComponent) cniContainer() corev1.Container { } } +// installUpstreamPlugins reports whether the operator should stage the upstream +// CNI plugin binaries onto the host. Gated on CNI.Type == Calico and the +// CNI.InstallMode override (defaults to All). +func (c *nodeComponent) installUpstreamPlugins() bool { + if c.cfg.Installation.CNI == nil || c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { + return false + } + if c.cfg.Installation.CNI.InstallMode != nil && *c.cfg.Installation.CNI.InstallMode == operatorv1.CNIInstallModeCalicoOnly { + return false + } + return true +} + +// cniPluginsContainer creates the init container that stages upstream CNI +// plugin binaries (host-local, portmap, loopback, tuning, flannel) into a +// shared volume read by the install-cni init container. The plugins ship as +// a separate image rather than baked into the combined calico image so the +// main image stays small. +func (c *nodeComponent) cniPluginsContainer() corev1.Container { + return corev1.Container{ + Name: "cni-plugins", + Image: c.cniPluginsImage, + SecurityContext: securitycontext.NewRootContext(true), + VolumeMounts: []corev1.VolumeMount{ + {MountPath: "/stage", Name: "cni-plugins-stage"}, + }, + } +} + // flexVolumeContainer creates the node's init container that installs the Unix Domain Socket to allow Dikastes // to communicate with Felix over the Policy Sync API. func (c *nodeComponent) flexVolumeContainer() corev1.Container { diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index e7a4ba37d4..224fb69c1f 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -322,6 +322,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -513,6 +514,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, @@ -912,6 +914,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -1114,6 +1117,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) verifyInitContainers(ds, cfg.Installation) @@ -1314,6 +1318,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -1560,6 +1565,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -2149,6 +2155,34 @@ var _ = Describe("Node rendering tests", func() { verifyInitContainers(ds, defaultInstance) }) + It("should omit the cni-plugins init container when CNI.InstallMode is CalicoOnly", func() { + mode := operatorv1.CNIInstallModeCalicoOnly + defaultInstance.CNI.InstallMode = &mode + component := render.Node(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + ds := dsResource.(*appsv1.DaemonSet) + Expect(ds).ToNot(BeNil()) + + // cni-plugins init container is absent and install-cni does not mount + // the staging volume. + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "cni-plugins")).To(BeNil()) + installCNI := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(installCNI).NotTo(BeNil()) + for _, m := range installCNI.VolumeMounts { + Expect(m.Name).NotTo(Equal("cni-plugins-stage")) + } + // Pod has no cni-plugins-stage volume. + for _, v := range ds.Spec.Template.Spec.Volumes { + Expect(v.Name).NotTo(Equal("cni-plugins-stage")) + } + + verifyInitContainers(ds, defaultInstance) + }) + It("should render MaxUnavailable if a custom value was set", func() { two := intstr.FromInt(2) defaultInstance.NodeUpdateStrategy.RollingUpdate.MaxUnavailable = &two @@ -3375,10 +3409,15 @@ func verifyInitContainers(ds *appsv1.DaemonSet, instance *operatorv1.Installatio // Validate correct number of init containers. numInitContainers := 1 isCalicoCNI := instance.CNI != nil && instance.CNI.Type == operatorv1.PluginCalico - // If using Calico CNI, the CNI install container is present. + // Default to InstallMode=All when unset. + installUpstreamPlugins := isCalicoCNI && + (instance.CNI.InstallMode == nil || *instance.CNI.InstallMode != operatorv1.CNIInstallModeCalicoOnly) if isCalicoCNI { numInitContainers++ } + if installUpstreamPlugins { + numInitContainers++ + } // Certificate management adds an additional key/cert init container. if instance.CertificateManagement != nil { numInitContainers++ @@ -3452,11 +3491,45 @@ func verifyInitContainers(ds *appsv1.DaemonSet, instance *operatorv1.Installatio {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } + if installUpstreamPlugins { + expectedCNIVolumeMounts = append(expectedCNIVolumeMounts, corev1.VolumeMount{MountPath: "/opt/cni/bin", Name: "cni-plugins-stage"}) + } Expect(cniContainer.VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) } else { Expect(cniContainer).To(BeNil()) } + // Verify the cni-plugins init container is present and runs before + // install-cni when using Calico CNI with the default InstallMode. + cniPluginsContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "cni-plugins") + if installUpstreamPlugins { + Expect(cniPluginsContainer).NotTo(BeNil()) + expectedImage := fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIPlugins.Image, components.ComponentCalicoCNIPlugins.Version) + if instance.Variant.IsEnterprise() { + expectedImage = fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIPlugins.Image, components.ComponentTigeraCNIPlugins.Version) + } + Expect(cniPluginsContainer.Image).To(Equal(expectedImage)) + Expect(cniPluginsContainer.VolumeMounts).To(ConsistOf([]corev1.VolumeMount{ + {MountPath: "/stage", Name: "cni-plugins-stage"}, + })) + // cni-plugins must come before install-cni so it populates the staging + // volume before install-cni reads from it. + var pluginsIdx, installIdx = -1, -1 + for i, ic := range ds.Spec.Template.Spec.InitContainers { + switch ic.Name { + case "cni-plugins": + pluginsIdx = i + case "install-cni": + installIdx = i + } + } + Expect(pluginsIdx).To(BeNumerically(">=", 0)) + Expect(installIdx).To(BeNumerically(">=", 0)) + Expect(pluginsIdx).To(BeNumerically("<", installIdx)) + } else { + Expect(cniPluginsContainer).To(BeNil()) + } + // Verify the ebpf-bootstrap container image and security context. ebpfBootstrap := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "ebpf-bootstrap") Expect(ebpfBootstrap).NotTo(BeNil())