diff --git a/.github/workflows/scripts/integration-test-ci b/.github/workflows/scripts/integration-test-ci index 9078084c58..fd4b3544e4 100755 --- a/.github/workflows/scripts/integration-test-ci +++ b/.github/workflows/scripts/integration-test-ci @@ -47,7 +47,7 @@ try --waitmsg "Waiting for CRDs" crds_exist echo "Shutting down core rancher" kubectl scale deploy rancher -n cattle-system --replicas=0 kubectl wait pods -l app=rancher --for=delete -n cattle-system -# Make sure the webhook recreates configurations on startup +# Delete the existing webhook configurations so the helm upgrade recreates them from the chart templates. kubectl delete validatingwebhookconfiguration rancher.cattle.io kubectl delete mutatingwebhookconfiguration rancher.cattle.io @@ -83,6 +83,25 @@ upgrade_rancher_webhook() { kubectl rollout status deployment/rancher-webhook -n cattle-system try --max 90 --waitmsg "Waiting for webhooks to be registered" --failmsg "Webhooks not registered" check_webhooks + + # The chart now owns the WebhookConfigurations with caBundle: "" (filled by needacert at runtime). + # needacert runs inside the rancher pod, which is scaled to 0 in this test. Patch caBundle + # directly from the TLS secret so the kube-apiserver can verify the webhook's serving cert. + populate_ca_bundle() { + local ca_bundle + ca_bundle=$(kubectl get secret cattle-webhook-tls -n cattle-system \ + -o jsonpath='{.data.tls\.crt}' 2>/dev/null) + [[ -z "$ca_bundle" ]] && return 1 + for resource in validatingwebhookconfiguration mutatingwebhookconfiguration; do + local patch + patch=$(kubectl get "$resource" rancher.cattle.io -o json \ + | jq --arg ca "$ca_bundle" \ + '[range(.webhooks | length) | {"op":"replace","path":"/webhooks/\(.)/clientConfig/caBundle","value":$ca}]') + kubectl patch "$resource" rancher.cattle.io --type=json -p="$patch" + done + } + try --max 10 --delay 3 --waitmsg "Populating caBundle from TLS secret" \ + --failmsg "Failed to populate caBundle" populate_ca_bundle } try --max 3 --delay 2 --waitmsg "Upgrading Webhook" --failmsg "Failed to upgrade webhook" upgrade_rancher_webhook diff --git a/charts/rancher-webhook/templates/deployment.yaml b/charts/rancher-webhook/templates/deployment.yaml index 926eef7045..c5f97a7470 100644 --- a/charts/rancher-webhook/templates/deployment.yaml +++ b/charts/rancher-webhook/templates/deployment.yaml @@ -12,8 +12,12 @@ spec: labels: app: rancher-webhook spec: - {{- if $auth.clientCA }} volumes: + - name: tls + secret: + secretName: cattle-webhook-tls + optional: false + {{- if $auth.clientCA }} - name: client-ca secret: secretName: client-ca @@ -65,8 +69,11 @@ spec: port: "https" scheme: "HTTPS" periodSeconds: 5 - {{- if $auth.clientCA }} volumeMounts: + - name: tls + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + {{- if $auth.clientCA }} - name: client-ca mountPath: /tmp/k8s-webhook-server/client-ca readOnly: true @@ -74,11 +81,11 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - {{- if .Values.capNetBindService }} + {{- if .Values.capNetBindService }} capabilities: add: - NET_BIND_SERVICE - {{- end }} + {{- end }} serviceAccountName: rancher-webhook {{- if .Values.priorityClassName }} priorityClassName: "{{.Values.priorityClassName}}" diff --git a/charts/rancher-webhook/templates/rbac.yaml b/charts/rancher-webhook/templates/rbac.yaml index f4364995c0..5433b06d02 100644 --- a/charts/rancher-webhook/templates/rbac.yaml +++ b/charts/rancher-webhook/templates/rbac.yaml @@ -1,12 +1,95 @@ apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRole metadata: name: rancher-webhook +rules: +# --- Read-only: informer caches for validators/mutators --- +- apiGroups: ["management.cattle.io"] + resources: + - clusters + - clusterproxyconfigs + - clusterroletemplatebindings + - features + - globalroles + - globalrolebindings + - nodes + - podsecurityadmissionconfigurationtemplates + - projects + - projectroletemplatebindings + - roletemplates + - settings + - tokens + - userattributes + - users + verbs: [get, list, watch] +- apiGroups: ["provisioning.cattle.io"] + resources: [clusters] + verbs: [get, list, watch, patch] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [roles, rolebindings, clusterroles, clusterrolebindings] + verbs: [get, list, watch] +- apiGroups: [""] + resources: [namespaces, secrets] + verbs: [get, list, watch] +- apiGroups: ["rke.cattle.io"] + resources: [etcdsnapshots] + verbs: [get, list, watch] +- apiGroups: ["apiregistration.k8s.io"] + resources: [apiservices] + verbs: [get, list, watch] +- apiGroups: ["apiextensions.k8s.io"] + resources: [customresourcedefinitions] + verbs: [get, list, watch] +- apiGroups: ["cluster.x-k8s.io"] + resources: [machinedeployments, clusters] + verbs: [get, list, watch] +- apiGroups: ["rke-machine-config.cattle.io"] + resources: ["*"] + verbs: [get, list, watch] +- apiGroups: ["rke-machine.cattle.io"] + resources: + - amazonec2machines + - azuremachines + - digitaloceanmachines + - harvestermachines + - linodemachines + - vmwarevspheremachines + verbs: [get, list, watch] +- apiGroups: ["catalog.cattle.io"] + resources: [clusterrepos] + verbs: [get, list, watch] +- apiGroups: ["auditlog.cattle.io"] + resources: [auditpolicies] + verbs: [get, list, watch] +- apiGroups: ["cluster.cattle.io"] + resources: [clusterauthtokens] + verbs: [get, list, watch] +# --- Write operations --- +- apiGroups: ["authorization.k8s.io"] + resources: [subjectaccessreviews] + verbs: [create] +- apiGroups: [""] + resources: [namespaces] + verbs: [create] +- apiGroups: [""] + resources: [secrets] + verbs: [create, update, delete] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [roles] + verbs: [update] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [rolebindings, clusterroles, clusterrolebindings] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rancher-webhook-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: cluster-admin + name: rancher-webhook subjects: - kind: ServiceAccount name: rancher-webhook - namespace: {{.Release.Namespace}} \ No newline at end of file + namespace: {{.Release.Namespace}} diff --git a/charts/rancher-webhook/templates/service.yaml b/charts/rancher-webhook/templates/service.yaml index 220afebeae..e0848b8bce 100644 --- a/charts/rancher-webhook/templates/service.yaml +++ b/charts/rancher-webhook/templates/service.yaml @@ -3,6 +3,8 @@ apiVersion: v1 metadata: name: rancher-webhook namespace: cattle-system + annotations: + need-a-cert.cattle.io/secret-name: cattle-webhook-tls spec: ports: - port: 443 diff --git a/charts/rancher-webhook/templates/webhook.yaml b/charts/rancher-webhook/templates/webhook.yaml index 53a0687b6f..fe73a1d084 100644 --- a/charts/rancher-webhook/templates/webhook.yaml +++ b/charts/rancher-webhook/templates/webhook.yaml @@ -2,8 +2,1167 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: rancher.cattle.io +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/features.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.features.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + resources: + - features + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusters.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - clusters + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/rke-machine-config.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rke-machine-config.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - rke-machine-config.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - '*' + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.delete-namespace + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - fleet-local + - local + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - DELETE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - UPDATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.create-non-kubesystem + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.create-kubesystem-only + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - kube-system + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterrepos.catalog.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterrepos.catalog.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - catalog.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + - CREATE + resources: + - clusterrepos + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/auditpolicies.auditlog.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.auditpolicies.auditlog.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - auditlog.cattle.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - auditpolicies + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +{{- if .Values.mcm.enabled }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusters.provisioning.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.provisioning.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - provisioning.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - clusters + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterproxyconfigs.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterproxyconfigs.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - clusterproxyconfigs + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/podsecurityadmissionconfigurationtemplates.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.podsecurityadmissionconfigurationtemplates.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - podsecurityadmissionconfigurationtemplates + scope: '*' + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/globalroles.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalroles.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - globalroles + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/globalrolebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalrolebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - globalrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/projectroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projectroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - projectroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - clusterroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/roletemplates.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.roletemplates.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - roletemplates + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/secrets + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.secrets + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - DELETE + resources: + - secrets + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/nodedrivers.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.nodedrivers.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - DELETE + resources: + - nodedrivers + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/projects.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projects.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - projects + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/roles.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.roles.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/gr-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - roles + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/rolebindings.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rolebindings.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/grb-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - rolebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/settings.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.settings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - settings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/tokens.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.tokens.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - tokens + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/userattributes.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.userattributes.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - userattributes + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterroles.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroles.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/gr-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - clusterroles + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterrolebindings.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterrolebindings.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/grb-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - clusterrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/authconfigs.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.authconfigs.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - authconfigs + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/users.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.users.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - users + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/machinedeployments-scale.cluster.x-k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.machinedeployments-scale.cluster.x-k8s.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - machinedeployments/scale + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/proxyendpoints.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.proxyendpoints.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - proxyendpoints + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +{{- end }} --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: rancher.cattle.io +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusters.provisioning.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.provisioning.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - provisioning.cattle.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - clusters + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusters.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - clusters + scope: Cluster + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/fleetworkspaces.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.fleetworkspaces.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - fleetworkspaces + scope: Cluster + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/rke-machine-config.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rke-machine-config.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - rke-machine-config.cattle.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - '*' + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +{{- if .Values.mcm.enabled }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/secrets + port: 443 + failurePolicy: Fail + matchConditions: + - expression: request.operation == 'DELETE' || (object != null && object.type + == "provisioning.cattle.io/cloud-credential" && request.operation == 'CREATE') + || (object != null && object.metadata.namespace == "cattle-local-user-passwords") + name: filter-by-secret-type-cloud-credential + matchPolicy: Equivalent + name: rancher.cattle.io.secrets + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + - DELETE + - UPDATE + resources: + - secrets + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 15 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/projects.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projects.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - projects + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/globalrolebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalrolebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - globalrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/projectroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projectroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - projectroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusterroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - clusterroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/charts/rancher-webhook/values.yaml b/charts/rancher-webhook/values.yaml index ca04ddb914..5a50ba52e7 100644 --- a/charts/rancher-webhook/values.yaml +++ b/charts/rancher-webhook/values.yaml @@ -12,6 +12,11 @@ global: mcm: enabled: true +# caBundle is populated at runtime by needacert (wrangler/pkg/needacert) +# once the cattle-webhook-tls secret has been generated. Override only +# when running this chart standalone without needacert. +caBundle: "" + # tolerations for the webhook deployment. See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ for more info tolerations: [] nodeSelector: {} diff --git a/go.mod b/go.mod index 4839625326..9b31a2ab6d 100644 --- a/go.mod +++ b/go.mod @@ -35,8 +35,6 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/evanphx/json-patch v5.9.11+incompatible github.com/go-ldap/ldap/v3 v3.4.13 - github.com/google/uuid v1.6.0 - github.com/rancher/dynamiclistener v0.9.0-rc.1 github.com/rancher/jsonpath v0.0.0-20260423141252-c4e0c565a09f github.com/rancher/lasso v0.2.9 github.com/rancher/rancher/pkg/apis v0.0.0-20260211194119-d0c9ffaf3cb0 @@ -72,6 +70,7 @@ require ( github.com/go-openapi/swag/stringutils v0.25.4 // indirect github.com/go-openapi/swag/typeutils v0.25.4 // indirect github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/google/uuid v1.6.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect ) diff --git a/go.sum b/go.sum index fb508c5873..190c203848 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,6 @@ github.com/rancher/aks-operator v1.13.0-rc.4 h1:tc7p2gZmRg4c6VBwWTQJYwmh1hlN68kf github.com/rancher/aks-operator v1.13.0-rc.4/go.mod h1:1ZjZB6zGHK+NGchN9KLplq+xPxRRi+q6Uzet5bjFwxo= github.com/rancher/ali-operator v1.13.0-rc.2 h1:a0biHGez+Np9XybJVh3yKN4RGPdaCzfM6D6cAXJac6o= github.com/rancher/ali-operator v1.13.0-rc.2/go.mod h1:s5HznpxsN9LsgtX6u5UoW9dZNKnDLuXcwzQRAEoDcog= -github.com/rancher/dynamiclistener v0.9.0-rc.1 h1:tmV3fgvB5pXSOMrb7G1DSno5C9XTfoyzHaDeVRhtUd0= -github.com/rancher/dynamiclistener v0.9.0-rc.1/go.mod h1:HG2E4ZcZKAc6JymJLy74ROKMTtsKyrA1fyQSR/g/+0w= github.com/rancher/eks-operator v1.13.0-rc.4 h1:XowN8+m3QZTIBOBLzar4frtz0xtREb9kcX6KXhF4eas= github.com/rancher/eks-operator v1.13.0-rc.4/go.mod h1:SbaKX2ttFWCxGOYkrKYeWH/6E4oToq2rRTcrMa2Mmdk= github.com/rancher/fleet/pkg/apis v0.15.0-alpha.6 h1:T9ELFwdKQMgkvSEfxxIGQu0G/ek3hqZb97iwUX6AcuY= diff --git a/pkg/server/server.go b/pkg/server/server.go index c41b3d09c9..30a7c7613e 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,50 +12,28 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" - "time" - "github.com/google/uuid" - "github.com/rancher/dynamiclistener" - "github.com/rancher/dynamiclistener/server" "github.com/rancher/webhook/pkg/admission" "github.com/rancher/webhook/pkg/clients" "github.com/rancher/webhook/pkg/health" - admissionregistration "github.com/rancher/wrangler/v3/pkg/generated/controllers/admissionregistration.k8s.io/v1" "github.com/sirupsen/logrus" - v1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" ) const ( - serviceName = "rancher-webhook" - namespace = "cattle-system" - tlsName = "rancher-webhook.cattle-system.svc" - certName = "cattle-webhook-tls" - caName = "cattle-webhook-ca" validationPath = "/v1/webhook/validation" mutationPath = "/v1/webhook/mutation" - clientPort = int32(443) webhookHTTPPort = 0 // value of 0 indicates we do not want to use http. defaultWebhookHTTPSPort = 9443 webhookPortEnvKey = "CATTLE_PORT" - webhookURLEnvKey = "CATTLE_WEBHOOK_URL" + webhookCertDirEnvKey = "CATTLE_WEBHOOK_CERT_DIR" + defaultCertDir = "/tmp/k8s-webhook-server/serving-certs" allowedCNsEnv = "ALLOWED_CNS" ) var caFile = filepath.Join(os.TempDir(), "k8s-webhook-server", "client-ca", "ca.crt") -// leaderFlag indicates whether this process is the elected leader. -// Gate config mutation work on this to avoid concurrent writers. -var leaderFlag atomic.Bool - -// tlsOpt option function applied to all webhook servers. +// tlsOpt configures the TLS settings shared by the serving listener. var tlsOpt = func(config *tls.Config) { config.MinVersion = tls.VersionTLS12 config.CipherSuites = []uint16{ @@ -76,13 +54,6 @@ func ListenAndServe(ctx context.Context, cfg *rest.Config, mcmEnabled bool) erro return fmt.Errorf("failed to create a new client: %w", err) } - if err = setCertificateExpirationDays(); err != nil { - // If this error occurs, certificate creation will still work. However, our override will likely not have worked. - // This will not affect functionality of the webhook, but users may have to perform the workaround: - // https://github.com/rancher/docs/issues/3637 - logrus.Infof("[ListenAndServe] could not set certificate expiration days via environment variable: %v", err) - } - validators, err := Validation(clients) if err != nil { return err @@ -93,76 +64,13 @@ func ListenAndServe(ctx context.Context, cfg *rest.Config, mcmEnabled bool) erro return err } - k8sClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - return fmt.Errorf("failed to create kubernetes clientset: %w", err) - } - id := uuid.New().String() - - lock := &resourcelock.LeaseLock{ - LeaseMeta: metav1.ObjectMeta{ - Name: "rancher-webhook-leader", - Namespace: namespace, - }, - Client: k8sClient.CoordinationV1(), - LockConfig: resourcelock.ResourceLockConfig{ - Identity: id, - }, - } - - go leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ - Lock: lock, - LeaseDuration: 15 * time.Second, - RenewDeadline: 10 * time.Second, - RetryPeriod: 2 * time.Second, - ReleaseOnCancel: true, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(_ context.Context) { - leaderFlag.Store(true) - clients.Core.Secret().Enqueue(namespace, caName) - logrus.Infof("[%s] elected leader: will manage webhook configurations", id) - }, - OnStoppedLeading: func() { - leaderFlag.Store(false) - logrus.Infof("[%s] lost leadership: will stop managing webhook configurations", id) - }, - OnNewLeader: func(identity string) { - if identity == id { - logrus.Infof("[%s] I am the new leader", id) - } else { - logrus.Infof("[%s] observed new leader: %s", id, identity) - } - }, - }, - }) - - if err = listenAndServe(ctx, clients, validators, mutators); err != nil { - return err - } - - if err = clients.Start(ctx); err != nil { - return fmt.Errorf("failed to start client: %w", err) - } - - return nil -} - -// By default, dynamiclistener sets newly signed certificates to expire after 365 days. Since the -// self-signed certificate for webhook does not need to be rotated, we increase expiration time -// beyond relevance. In this case, that's 3650 days (10 years). -func setCertificateExpirationDays() error { - certExpirationDaysKey := "CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS" - if os.Getenv(certExpirationDaysKey) == "" { - return os.Setenv(certExpirationDaysKey, "3650") - } - return nil + return listenAndServe(ctx, clients, validators, mutators) } func listenAndServe(ctx context.Context, clients *clients.Clients, validators []admission.ValidatingAdmissionHandler, mutators []admission.MutatingAdmissionHandler) (rErr error) { router := http.NewServeMux() errChecker := health.NewErrorChecker("config-applied") health.RegisterHealthCheckers(router, errChecker) - errChecker.Store(errors.New("webhook configuration not yet applied")) logrus.Debug("Creating Webhook routes") for _, webhook := range validators { @@ -178,24 +86,6 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators [] routerHandler := certAuth()(router) - handler := &secretHandler{ - validators: validators, - mutators: mutators, - errChecker: errChecker, - validatingController: clients.Admission.ValidatingWebhookConfiguration(), - mutatingController: clients.Admission.MutatingWebhookConfiguration(), - } - clients.Core.Secret().OnChange(ctx, "secrets", handler.sync) - - defer func() { - if rErr != nil { - return - } - rErr = clients.Start(ctx) - }() - - tlsConfig := &tls.Config{} - tlsOpt(tlsConfig) webhookHTTPSPort := defaultWebhookHTTPSPort if portStr := os.Getenv(webhookPortEnvKey); portStr != "" { var err error @@ -204,142 +94,66 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators [] return fmt.Errorf("failed to decode webhook port value '%s': %w", portStr, err) } } - return server.ListenAndServe(ctx, webhookHTTPSPort, webhookHTTPPort, routerHandler, &server.ListenOpts{ - Secrets: clients.Core.Secret(), - CertNamespace: namespace, - CertName: certName, - CAName: caName, - TLSListenerConfig: dynamiclistener.Config{ - SANs: []string{ - tlsName, - }, - FilterCN: dynamiclistener.OnlyAllow(tlsName), - TLSConfig: tlsConfig, - }, - DisplayServerLogs: true, - IgnoreTLSHandshakeError: true, - }) -} -type secretHandler struct { - validators []admission.ValidatingAdmissionHandler - mutators []admission.MutatingAdmissionHandler - errChecker *health.ErrorChecker - validatingController admissionregistration.ValidatingWebhookConfigurationClient - mutatingController admissionregistration.MutatingWebhookConfigurationClient -} + certDir := defaultCertDir + if dir := os.Getenv(webhookCertDirEnvKey); dir != "" { + certDir = dir + } + certPath := filepath.Join(certDir, "tls.crt") + keyPath := filepath.Join(certDir, "tls.key") -// sync updates the validating admission configuration whenever the TLS cert changes. -// Only the elected leader performs the updates, followers are a no-op. -func (s *secretHandler) sync(_ string, secret *corev1.Secret) (*corev1.Secret, error) { - // The leader is responsible for applying the webhook configuration. - // Follower pods are only responsible for serving traffic and can be marked as healthy once the certificates are generated. - if !leaderFlag.Load() { - s.errChecker.Store(nil) - return nil, nil + tlsConfig := &tls.Config{ + GetCertificate: certReloader(certPath, keyPath), } + tlsOpt(tlsConfig) - if secret == nil || secret.Name != caName || secret.Namespace != namespace || len(secret.Data[corev1.TLSCertKey]) == 0 { - return nil, nil + server := &http.Server{ + Addr: fmt.Sprintf(":%d", webhookHTTPSPort), + Handler: routerHandler, + TLSConfig: tlsConfig, } - logrus.Info("Applying webhook config") + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + logrus.Warnf("webhook server shutdown returned error: %v", err) + } + }() - validationClientConfig := v1.WebhookClientConfig{ - Service: &v1.ServiceReference{ - Namespace: namespace, - Name: serviceName, - Path: admission.Ptr(validationPath), - Port: admission.Ptr(clientPort), - }, - CABundle: secret.Data[corev1.TLSCertKey], + logrus.Infof("listening on :%d serving certs from %s", webhookHTTPSPort, certDir) + if _, err := tls.LoadX509KeyPair(certPath, keyPath); err != nil { + return fmt.Errorf("failed to load serving cert from %s: %w", certDir, err) } + errChecker.Store(nil) - mutationClientConfig := v1.WebhookClientConfig{ - Service: &v1.ServiceReference{ - Namespace: namespace, - Name: serviceName, - Path: admission.Ptr(mutationPath), - Port: admission.Ptr(clientPort), - }, - CABundle: secret.Data[corev1.TLSCertKey], - } - if devURL, ok := os.LookupEnv(webhookURLEnvKey); ok { - validationURL := devURL + validationPath - mutationURL := devURL + mutationPath - validationClientConfig = v1.WebhookClientConfig{ - URL: &validationURL, - } - mutationClientConfig = v1.WebhookClientConfig{ - URL: &mutationURL, + go func() { + if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + logrus.Fatalf("webhook server stopped: %v", err) } - } - validatingWebhooks := make([]v1.ValidatingWebhook, 0, len(s.validators)) - for _, webhook := range s.validators { - validatingWebhooks = append(validatingWebhooks, webhook.ValidatingWebhook(validationClientConfig)...) - } - mutatingWebhooks := make([]v1.MutatingWebhook, 0, len(s.mutators)) - for _, webhook := range s.mutators { - mutatingWebhooks = append(mutatingWebhooks, webhook.MutatingWebhook(mutationClientConfig)...) - } - validatingConfig := &v1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rancher.cattle.io", - }, - Webhooks: validatingWebhooks, - } - mutatingConfig := &v1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rancher.cattle.io", - }, - Webhooks: mutatingWebhooks, - } - err := s.ensureWebhookConfiguration(validatingConfig, mutatingConfig) - if err != nil { - logrus.Errorf("Failed to ensure configuration: %s", err.Error()) - } + }() - s.errChecker.Store(err) - return secret, err + if err := clients.Start(ctx); err != nil { + return fmt.Errorf("failed to start clients: %w", err) + } + <-ctx.Done() + return nil } -// ensureWebhookConfiguration creates or updates the current validating and mutating webhook configuration to have the desired webhook. -func (s *secretHandler) ensureWebhookConfiguration(validatingConfig *v1.ValidatingWebhookConfiguration, mutatingConfig *v1.MutatingWebhookConfiguration) error { - - currValidating, err := s.validatingController.Get(validatingConfig.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = s.validatingController.Create(validatingConfig) +// certReloader returns a GetCertificate that re-reads the cert files on every TLS +// handshake. This is intentionally re-read each time so that needacert can rotate +// the underlying secret (kubelet refreshes the projected files on its own cycle) +// without needing a webhook pod restart. +func certReloader(certPath, keyPath string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) if err != nil { - return fmt.Errorf("failed to create validating configuration: %w", err) - } - } else if err != nil { - return fmt.Errorf("failed to get validating configuration: %w", err) - } else { - currValidating.Webhooks = validatingConfig.Webhooks - _, err = s.validatingController.Update(currValidating) - if err != nil { - return fmt.Errorf("failed to update validating configuration: %w", err) + return nil, err } + return &cert, nil } - - currMutation, err := s.mutatingController.Get(mutatingConfig.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = s.mutatingController.Create(mutatingConfig) - if err != nil { - return fmt.Errorf("failed to create mutating configuration: %w", err) - } - } else if err != nil { - return fmt.Errorf("failed to get mutating configuration: %w", err) - } else { - currMutation.Webhooks = mutatingConfig.Webhooks - _, err = s.mutatingController.Update(currMutation) - if err != nil { - return fmt.Errorf("failed to update mutating configuration: %w", err) - } - } - - return nil } // certAuth returns a middleware for cert-based authentication. diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go deleted file mode 100644 index cfb2e73ce8..0000000000 --- a/pkg/server/server_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package server - -import ( - "errors" - "testing" - - "github.com/rancher/webhook/pkg/health" - "github.com/rancher/wrangler/v3/pkg/generic/fake" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - v1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -func TestSecretHandlerEnsureWebhookConfigurationCreate(t *testing.T) { - configName := "rancher.cattle.io" - - var ( - storedValidatingConfig *v1.ValidatingWebhookConfiguration - storedMutatingConfig *v1.MutatingWebhookConfiguration - ) - - ctrl := gomock.NewController(t) - validatingController := fake.NewMockNonNamespacedClientInterface[*v1.ValidatingWebhookConfiguration, *v1.ValidatingWebhookConfigurationList](ctrl) - validatingController.EXPECT().Get(configName, gomock.Any()).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: v1.GroupName, Resource: "validatingwebhookconfiguration"}, configName)).Times(1) - validatingController.EXPECT().Create(gomock.Any()).DoAndReturn(func(obj *v1.ValidatingWebhookConfiguration) (*v1.ValidatingWebhookConfiguration, error) { - storedValidatingConfig = obj.DeepCopy() - return obj, nil - }).Times(1) - - mutatingController := fake.NewMockNonNamespacedClientInterface[*v1.MutatingWebhookConfiguration, *v1.MutatingWebhookConfigurationList](ctrl) - mutatingController.EXPECT().Get(configName, gomock.Any()).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: v1.GroupName, Resource: "mutatingwebhookconfiguration"}, configName)).Times(1) - mutatingController.EXPECT().Create(gomock.Any()).DoAndReturn(func(obj *v1.MutatingWebhookConfiguration) (*v1.MutatingWebhookConfiguration, error) { - storedMutatingConfig = obj.DeepCopy() - return obj, nil - }).Times(1) - - handler := &secretHandler{ - validatingController: validatingController, - mutatingController: mutatingController, - } - - validatingConfig := &v1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - }, - Webhooks: []v1.ValidatingWebhook{ - { - Name: "rancher.cattle.io.features.management.cattle.io", - }, - }, - } - mutatingConfig := &v1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - }, - Webhooks: []v1.MutatingWebhook{ - { - Name: "rancher.cattle.io.clusters.provisioning.cattle.io", - }, - }, - } - - err := handler.ensureWebhookConfiguration(validatingConfig, mutatingConfig) - require.NoError(t, err) - - require.NotNil(t, storedValidatingConfig) - require.Len(t, storedValidatingConfig.Webhooks, 1) - assert.Equal(t, validatingConfig.Webhooks[0].Name, storedValidatingConfig.Webhooks[0].Name) - - require.NotNil(t, storedMutatingConfig) - require.Len(t, storedMutatingConfig.Webhooks, 1) - assert.Equal(t, mutatingConfig.Webhooks[0].Name, storedMutatingConfig.Webhooks[0].Name) -} - -// TestSyncLeaderLogic tests the logic within the sync function related to leader election. -func TestSyncLeaderLogic(t *testing.T) { - t.Parallel() - configName := "rancher.cattle.io" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: caName, - Namespace: namespace, - }, - Data: map[string][]byte{ - corev1.TLSCertKey: []byte("cert-data"), - }, - } - testErr := errors.New("test error") - - tests := []struct { - name string - isLeader bool - getMutatingErr error - getValidatingErr error - createMutatingErr error - createValidatingErr error - expectedErr error - expectControllersRun bool - }{ - { - name: "follower becomes healthy", - isLeader: false, - expectedErr: nil, - }, - { - name: "leader becomes healthy on create", - isLeader: true, - getMutatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - getValidatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - expectedErr: nil, - expectControllersRun: true, - }, - { - name: "leader becomes unhealthy on create error", - isLeader: true, - getValidatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - createValidatingErr: testErr, - expectedErr: testErr, - expectControllersRun: true, - }, - { - name: "leader becomes healthy on update", - isLeader: true, - expectedErr: nil, - expectControllersRun: true, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - validatingController := fake.NewMockNonNamespacedClientInterface[*v1.ValidatingWebhookConfiguration, *v1.ValidatingWebhookConfigurationList](ctrl) - mutatingController := fake.NewMockNonNamespacedClientInterface[*v1.MutatingWebhookConfiguration, *v1.MutatingWebhookConfigurationList](ctrl) - errChecker := health.NewErrorChecker("test") - errChecker.Store(errors.New("initial error")) - - handler := &secretHandler{ - validatingController: validatingController, - mutatingController: mutatingController, - errChecker: errChecker, - } - - leaderFlag.Store(tt.isLeader) - - if tt.expectControllersRun { - validatingController.EXPECT().Get(configName, gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, tt.getValidatingErr).Times(1) - if apierrors.IsNotFound(tt.getValidatingErr) { - validatingController.EXPECT().Create(gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, tt.createValidatingErr).Times(1) - } else if tt.getValidatingErr == nil { - validatingController.EXPECT().Update(gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, nil).Times(1) - } - - // Only expect calls to the mutating controller if the validating part is expected to succeed. - if (tt.getValidatingErr == nil || apierrors.IsNotFound(tt.getValidatingErr)) && tt.createValidatingErr == nil { - mutatingController.EXPECT().Get(configName, gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, tt.getMutatingErr).Times(1) - if apierrors.IsNotFound(tt.getMutatingErr) { - mutatingController.EXPECT().Create(gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, tt.createMutatingErr).Times(1) - } else if tt.getMutatingErr == nil { - mutatingController.EXPECT().Update(gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, nil).Times(1) - } - } - } - - _, err := handler.sync("test-sync", secret) - - // The only error we might get is a transient one from ensureWebhookConfiguration. - if tt.expectedErr != nil { - assert.ErrorContains(t, err, tt.expectedErr.Error(), "expected an error when ensuring webhook config") - } else { - require.NoError(t, err) - } - - healthErr := errChecker.Check(nil) - if tt.expectedErr == nil { - assert.NoError(t, healthErr, "expected pod to be healthy") - } else { - assert.Error(t, healthErr, "expected pod to be unhealthy") - } - }) - } -}