diff --git a/build-tools/cncluster b/build-tools/cncluster index 0e7a5e0114..6d5439beda 100755 --- a/build-tools/cncluster +++ b/build-tools/cncluster @@ -989,16 +989,23 @@ subcommand_whitelist[cluster_enable_workload_identity]='Enable workload identity function subcmd_cluster_enable_workload_identity() { if gcloud container clusters describe "${GCP_CLUSTER_NAME}" --format="value(workloadIdentityConfig)" | grep -q "workloadPool"; then _info "Workload identity is already enabled for the cluster." - return + else + _info "Enabling workload identity for the cluster." + gcloud container clusters update "${GCP_CLUSTER_NAME}" --workload-pool="${CLOUDSDK_CORE_PROJECT}.svc.id.goog" fi - _info "Enabling workload identity for the cluster." - gcloud container clusters update "${GCP_CLUSTER_NAME}" --workload-pool="${CLOUDSDK_CORE_PROJECT}.svc.id.goog" - - _info "Enabling GKE metadata on the cn-apps node pool." - gcloud container node-pools update cn-apps-pool \ - --cluster "cn-${GCP_CLUSTER_BASENAME}net" \ - --workload-metadata=GKE_METADATA + local pool current_mode + for pool in cn-apps-pool cn-apps-node-pool-hd cn-infra-node-pool gke-node-pool; do + if gcloud container node-pools describe "${pool}" --cluster "${GCP_CLUSTER_NAME}" --format="value(name)" &>/dev/null; then + current_mode=$(gcloud container node-pools describe "${pool}" --cluster "${GCP_CLUSTER_NAME}" --format="value(config.workloadMetadataConfig.mode)" 2>/dev/null || true) + if [[ "${current_mode}" == "GKE_METADATA" ]]; then + _info "Workload metadata already GKE_METADATA for ${pool}." + else + _info "Enabling GKE metadata on node pool ${pool}." + gcloud container node-pools update "${pool}" --cluster "${GCP_CLUSTER_NAME}" --workload-metadata=GKE_METADATA + fi + fi + done } subcommand_whitelist[cluster_enable_http_load_balancing]='Enable http load balancing for the cluster.' diff --git a/cluster/deployment/scratchneta/config.resolved.yaml b/cluster/deployment/scratchneta/config.resolved.yaml index 792f1989e6..c34d402646 100644 --- a/cluster/deployment/scratchneta/config.resolved.yaml +++ b/cluster/deployment/scratchneta/config.resolved.yaml @@ -26,6 +26,7 @@ cluster: minNodes: 1 nodeType: 'n4d-standard-8' infra: + certManagerUseWorkloadIdentity: true gkeGateway: proxyForIstioHttp: false istio: diff --git a/cluster/deployment/scratchneta/config.yaml b/cluster/deployment/scratchneta/config.yaml index f6984aad72..e0a4e48359 100644 --- a/cluster/deployment/scratchneta/config.yaml +++ b/cluster/deployment/scratchneta/config.yaml @@ -1 +1,5 @@ -!include($SPLICE_ROOT/cluster/configs/shared/scratchnet.yaml) {} +!include($SPLICE_ROOT/cluster/configs/shared/scratchnet.yaml) { + infra: { + certManagerUseWorkloadIdentity: true + } +} diff --git a/cluster/pulumi/infra/src/config.ts b/cluster/pulumi/infra/src/config.ts index 3d42081c5d..f1a940c474 100644 --- a/cluster/pulumi/infra/src/config.ts +++ b/cluster/pulumi/infra/src/config.ts @@ -46,6 +46,7 @@ export const InfraConfigSchema = z.object({ }) .optional(), enableGCReaperJob: z.boolean().default(false), + certManagerUseWorkloadIdentity: z.boolean().optional(), gkeGateway: z.object({ proxyForIstioHttp: z.boolean(), }), @@ -73,6 +74,8 @@ export type Config = z.infer; // @ts-ignore const fullConfig = InfraConfigSchema.parse(clusterYamlConfig); export const enableGCReaperJob = fullConfig.infra.enableGCReaperJob; +export const certManagerUseWorkloadIdentity = + fullConfig.infra.certManagerUseWorkloadIdentity ?? false; console.error( `Loaded infra config: ${util.inspect(fullConfig, { depth: null, diff --git a/cluster/pulumi/infra/src/network.ts b/cluster/pulumi/infra/src/network.ts index 3345b8f408..d0275c6134 100644 --- a/cluster/pulumi/infra/src/network.ts +++ b/cluster/pulumi/infra/src/network.ts @@ -16,7 +16,12 @@ import { import { infraAffinityAndTolerations } from '@canton-network/splice-pulumi-common'; import { svConfigsBasic } from '@canton-network/splice-pulumi-common-sv/src/svConfigsBasic'; -import { gcpDnsProject } from './config'; +import { certManagerUseWorkloadIdentity, gcpDnsProject } from './config'; + +const useCertManagerWorkloadIdentity = certManagerUseWorkloadIdentity; +const dns01SaIamAccount = useCertManagerWorkloadIdentity + ? config.requireEnv('DNS01_SA_IAM_ACCOUNT') + : config.optionalEnv('DNS01_SA_IAM_ACCOUNT') || ''; function ipAddress(addressName: string): gcp.compute.Address { return new gcp.compute.Address(addressName, { @@ -78,6 +83,20 @@ function certManager(certManagerNamespaceName: string): certmanager.CertManager }, }); + const workloadIdentityArgs: Partial = useCertManagerWorkloadIdentity + ? { + extraArgs: [ + '--issuer-ambient-credentials=true', + '--cluster-issuer-ambient-credentials=true', + ], + serviceAccount: { + annotations: { + 'iam.gke.io/gcp-service-account': dns01SaIamAccount, + }, + }, + } + : {}; + return new certmanager.CertManager('cert-manager', { installCRDs: true, helmOptions: { @@ -94,6 +113,7 @@ function certManager(certManagerNamespaceName: string): certmanager.CertManager startupapicheck: { ...infraAffinityAndTolerations, }, + ...workloadIdentityArgs, }); } @@ -140,13 +160,17 @@ function clusterCertificate( solvers: [ { dns01: { - cloudDNS: { - project: 'da-gcp-canton-domain', - serviceAccountSecretRef: { - key: 'key.json', - name: 'clouddns-dns01-solver-svc-acct', - }, - }, + cloudDNS: useCertManagerWorkloadIdentity + ? { + project: 'da-gcp-canton-domain', + } + : { + project: 'da-gcp-canton-domain', + serviceAccountSecretRef: { + key: 'key.json', + name: 'clouddns-dns01-solver-svc-acct', + }, + }, }, }, ], @@ -158,30 +182,38 @@ function clusterCertificate( } ); - const gcpSecretName = config.requireEnv('DNS01_SA_KEY_SECRET'); - - gcp.secretmanager.SecretVersion.get( - 'dns01-sa-key-secret', - `projects/${GCP_PROJECT}/secrets/${gcpSecretName}/versions/latest` - ).secretData.apply(dns01SaKeySecret => { - new k8s.core.v1.Secret( - 'clouddns-dns01-solver-svc-acct', - { - metadata: { - name: 'clouddns-dns01-solver-svc-acct', - namespace: ns.metadata.name, - }, - type: 'Opaque', - data: { - // TODO(#973): Handle this correctly in dump-config. Currently it gets here with an undefined value. - 'key.json': btoa(dns01SaKeySecret || 'dns-secret'), + if (useCertManagerWorkloadIdentity) { + new gcp.serviceaccount.IAMMember('dns01-solver-workload-identity-user', { + serviceAccountId: `projects/${gcpDnsProject}/serviceAccounts/${dns01SaIamAccount}`, + role: 'roles/iam.workloadIdentityUser', + member: pulumi.interpolate`serviceAccount:${GCP_PROJECT}.svc.id.goog[cert-manager/cert-manager]`, + }); + } else { + const gcpSecretName = config.requireEnv('DNS01_SA_KEY_SECRET'); + + gcp.secretmanager.SecretVersion.get( + 'dns01-sa-key-secret', + `projects/${GCP_PROJECT}/secrets/${gcpSecretName}/versions/latest` + ).secretData.apply(dns01SaKeySecret => { + new k8s.core.v1.Secret( + 'clouddns-dns01-solver-svc-acct', + { + metadata: { + name: 'clouddns-dns01-solver-svc-acct', + namespace: ns.metadata.name, + }, + type: 'Opaque', + data: { + // TODO(#973): Handle this correctly in dump-config. Currently it gets here with an undefined value. + 'key.json': btoa(dns01SaKeySecret || 'dns-secret'), + }, }, - }, - { - dependsOn: ns, - } - ); - }); + { + dependsOn: ns, + } + ); + }); + } const certDnsNames = dnsNames .map(dnsName =>