fix(gcp): enable the APIs that read through the host project's quota

.github/workflows/terraform-validate.yml

@@ -10,6 +10,7 @@ on:
   pull_request:
     paths:
       - "gcp-integration-setup/**"
+      - "scripts/test_gcp_installer.py"
       - "aws-integration-setup/**"
       - "azure-integration-setup/**"
       - "scripts/**"
@@ -19,6 +20,7 @@ on:
       - main
     paths:
       - "gcp-integration-setup/**"
+      - "scripts/test_gcp_installer.py"
       - "aws-integration-setup/**"
       - "azure-integration-setup/**"
       - "scripts/**"
@@ -106,6 +108,7 @@ jobs:
           - 1.8.5
           - 1.10.5
         directory:
+          - gcp-integration-setup/terraform
           - aws-integration-setup/terraform/modules/eks-managed-scan-access
           - aws-integration-setup/terraform/modules/k8s-resources
           - aws-integration-setup/terraform/modules/nullify-aws-integration
@@ -162,3 +165,6 @@ jobs:
           sudo apt-get install -y shellcheck >/dev/null
           shellcheck gcp-integration-setup/scripts/install.sh
           shellcheck gcp-integration-setup/scripts/uninstall.sh
+
+      - name: Exercise GCP installer with synthetic gcloud responses
+        run: python3 scripts/test_gcp_installer.py

gcp-integration-setup/scripts/install.sh

@@ -41,6 +41,40 @@ gcloud services enable \
   serviceusage.googleapis.com \
   --project="${NULLIFY_HOST_PROJECT}"
 
+enable_scan_api() {
+  local api="$1" attempt
+  for attempt in 1 2 3; do
+    if gcloud services enable "$api" --project="${NULLIFY_HOST_PROJECT}" &&
+      [[ "$(gcloud services describe "$api" --project="${NULLIFY_HOST_PROJECT}" --format='value(state)')" == "ENABLED" ]]; then
+      return 0
+    fi
+    if [[ "$attempt" -lt 3 ]]; then
+      sleep "$attempt"
+    fi
+  done
+  return 1
+}
+
+SCAN_APIS=(
+  cloudkms.googleapis.com
+  sqladmin.googleapis.com
+  spanner.googleapis.com
+  dataflow.googleapis.com
+  bigquery.googleapis.com
+  orgpolicy.googleapis.com
+  accesscontextmanager.googleapis.com
+)
+UNREADY_SCAN_APIS=()
+echo "==> Enabling scan APIs on ${NULLIFY_HOST_PROJECT}"
+for api in "${SCAN_APIS[@]}"; do
+  if ! enable_scan_api "$api"; then
+    UNREADY_SCAN_APIS+=("$api")
+  fi
+done
+if [[ "${#UNREADY_SCAN_APIS[@]}" -gt 0 ]]; then
+  printf 'WARNING: scan APIs are not ready on %s: %s\n' "${NULLIFY_HOST_PROJECT}" "${UNREADY_SCAN_APIS[*]}" >&2
+fi
+
 # API enablement is documented as synchronous but the IAM API's "enabled"
 # state propagates eventually-consistent. On a fresh project the next
 # service-accounts call can still return 403 for 10-30s. Retry the first
@@ -150,6 +184,10 @@ done
 
 echo
 echo "Nullify GCP integration installed successfully."
+printf 'Scan API readiness: %d/%d enabled\n' "$(( ${#SCAN_APIS[@]} - ${#UNREADY_SCAN_APIS[@]} ))" "${#SCAN_APIS[@]}"
+if [[ "${#UNREADY_SCAN_APIS[@]}" -gt 0 ]]; then
+  printf 'Enable these APIs in %s before scanning their services: %s\n' "${NULLIFY_HOST_PROJECT}" "${UNREADY_SCAN_APIS[*]}"
+fi
 echo
 echo "Paste these values into the Nullify console under Settings -> Cloud Integrations -> GCP:"
 echo "  Service Account Email:        ${SA_EMAIL}"

gcp-integration-setup/terraform/README.md

@@ -80,15 +80,42 @@ If your organisation restricts API enablement via Service Usage org
 policies, ensure the following are allowlisted on the host project:
 `iam.googleapis.com`, `iamcredentials.googleapis.com`, `sts.googleapis.com`,
 `cloudresourcemanager.googleapis.com`, `cloudasset.googleapis.com`,
-`serviceusage.googleapis.com`.
+`serviceusage.googleapis.com`, `cloudkms.googleapis.com`,
+`sqladmin.googleapis.com`, `spanner.googleapis.com`,
+`dataflow.googleapis.com`, `bigquery.googleapis.com`,
+`orgpolicy.googleapis.com`, `accesscontextmanager.googleapis.com`.
+
+The last seven count each read against the quota of the service account's
+own project, so they must be enabled in the host project even when the
+scanned projects already enable them. The module manages them as separate Terraform resources
+(`enable_scan_quota_apis`, default `true`). Terraform apply fails if an
+organisation policy or permission blocks an API. Set
+`enable_scan_quota_apis = false` in `terraform.tfvars` to manage these APIs
+separately, then enable each permitted API in the host project. Nullify
+cannot read a service until its API is enabled there.
+The roles, not the APIs, decide what Nullify can read.
+
+Existing installs: re-apply the module, or enable the seven APIs separately.
+The shell installer retries each scanner API up to three times and reports
+which remain disabled without stopping identity setup. Terraform remains
+strict; the `managed_scan_quota_apis` output lists its configured services,
+not their live readiness.
+
+The API additions are compatible with the deployed scanner. Deploy
+Nullify-Platform/nullify#18346 before checking Cloud Build coverage because
+that change reads its configuration through Cloud Asset Inventory. Customer
+re-application and a fresh scan without `CONSUMER_API_DISABLED` are separate
+activation checks after merging this module.
 
 To enable them manually:
 
 ```bash
 gcloud services enable \
   iam.googleapis.com iamcredentials.googleapis.com sts.googleapis.com \
   cloudresourcemanager.googleapis.com cloudasset.googleapis.com \
-  serviceusage.googleapis.com \
+  serviceusage.googleapis.com cloudkms.googleapis.com sqladmin.googleapis.com \
+  spanner.googleapis.com dataflow.googleapis.com bigquery.googleapis.com \
+  orgpolicy.googleapis.com accesscontextmanager.googleapis.com \
   --project=YOUR_HOST_PROJECT
 ```
 

gcp-integration-setup/terraform/examples/folder/main.tf

@@ -9,8 +9,9 @@
 module "nullify" {
   source = "../../"
 
-  customer_name   = "acme-corp"
-  host_project_id = "acme-security"
+  customer_name          = "acme-corp"
+  host_project_id        = "acme-security"
+  enable_scan_quota_apis = true
 
   scope           = "folder"
   organization_id = "123456789012"

gcp-integration-setup/terraform/examples/organization/main.tf

@@ -8,8 +8,9 @@
 module "nullify" {
   source = "../../"
 
-  customer_name   = "acme-corp"
-  host_project_id = "acme-security"
+  customer_name          = "acme-corp"
+  host_project_id        = "acme-security"
+  enable_scan_quota_apis = true
 
   scope           = "organization"
   organization_id = "123456789012"

gcp-integration-setup/terraform/examples/single-project/main.tf

@@ -6,8 +6,9 @@
 module "nullify" {
   source = "../../"
 
-  customer_name   = "acme-corp"
-  host_project_id = "acme-security"
+  customer_name          = "acme-corp"
+  host_project_id        = "acme-security"
+  enable_scan_quota_apis = true
 
   scope       = "projects"
   project_ids = ["acme-security"]

gcp-integration-setup/terraform/main.tf

@@ -12,4 +12,5 @@ module "nullify_gcp_integration" {
   wif_pool_id             = var.wif_pool_id
   wif_provider_id         = var.wif_provider_id
   service_account_name    = var.service_account_name
+  enable_scan_quota_apis  = var.enable_scan_quota_apis
 }

gcp-integration-setup/terraform/modules/nullify-gcp-integration/apis.tf

@@ -21,3 +21,28 @@ resource "google_project_service" "required" {
   service            = each.value
   disable_on_destroy = false
 }
+
+# Some Google APIs count every call against the quota of the calling
+# service account's own project, not the project that is read. The Nullify
+# service account lives in the host project, so these APIs must also be
+# enabled here, or every read of them fails with "API has not been used in
+# project <host> before or it is disabled". The roles in main.tf, not these
+# APIs, decide what Nullify can read. Google can enable dependent APIs with
+# them.
+#
+# Terraform apply fails if any API cannot be enabled.
+# Set `enable_scan_quota_apis = false` to manage these APIs separately.
+resource "google_project_service" "scan_quota" {
+  for_each = var.enable_scan_quota_apis ? toset([
+    "cloudkms.googleapis.com",
+    "sqladmin.googleapis.com",
+    "spanner.googleapis.com",
+    "dataflow.googleapis.com",
+    "bigquery.googleapis.com",
+    "orgpolicy.googleapis.com",
+    "accesscontextmanager.googleapis.com",
+  ]) : toset([])
+  project            = var.host_project_id
+  service            = each.value
+  disable_on_destroy = false
+}

gcp-integration-setup/terraform/modules/nullify-gcp-integration/main.tf

@@ -183,9 +183,8 @@ locals {
     "bigquery.routines.get",
     "bigquery.routines.list",
 
-    # Cloud Build: there is no buildTriggers.* IAM permission family, and the
-    # builds.* permissions expose execution data (logs/artifacts) which we
-    # intentionally avoid. Trigger-config visibility is therefore not granted.
+    # Cloud Build: no permission is added here. Nullify reads trigger and
+    # worker pool configuration through Cloud Asset Inventory.
 
     # Cloud Batch job spec. No task logs or output artifacts.
     "batch.jobs.get",

gcp-integration-setup/terraform/modules/nullify-gcp-integration/outputs.tf

@@ -27,3 +27,8 @@ output "next_steps" {
   description = "Human-readable next steps. Surfaces in the apply output so customers don't have to dig through the Nullify docs."
   value       = "Paste the service_account_email and workload_identity_provider outputs into the Nullify console under Settings -> Cloud Integrations -> GCP, then click Verify."
 }
+
+output "managed_scan_quota_apis" {
+  description = "Scanner APIs managed by this configuration. This is not a live readiness check."
+  value       = sort([for api in google_project_service.scan_quota : api.service])
+}

gcp-integration-setup/terraform/modules/nullify-gcp-integration/variables.tf

@@ -85,3 +85,9 @@ variable "service_account_name" {
   type        = string
   default     = "nullify-cloud-connector"
 }
+
+variable "enable_scan_quota_apis" {
+  description = "Enable in the host project the APIs that count Nullify's reads against the service account's own project (KMS, Cloud SQL, Spanner, Dataflow, BigQuery, Org Policy, Access Context Manager). Terraform apply fails if any API cannot be enabled. Set false to manage these APIs separately."
+  type        = bool
+  default     = true
+}

gcp-integration-setup/terraform/outputs.tf

@@ -24,3 +24,8 @@ output "next_steps" {
     To revoke access at any time, run `terraform destroy` from this directory.
   EOT
 }
+
+output "managed_scan_quota_apis" {
+  description = "Scanner APIs managed by this configuration. This is not a live readiness check."
+  value       = module.nullify_gcp_integration.managed_scan_quota_apis
+}

gcp-integration-setup/terraform/terraform.tfvars.example

@@ -3,6 +3,7 @@
 
 customer_name   = "acme-corp"
 host_project_id = "acme-security"
+enable_scan_quota_apis = true
 
 # Choose ONE of the three scoping modes below.
 

gcp-integration-setup/terraform/tests/scan_quota_apis.tftest.hcl

@@ -0,0 +1,48 @@
+mock_provider "google" {
+  mock_data "google_project" {
+    defaults = {
+      number = "123456789012"
+    }
+  }
+  mock_resource "google_service_account" {
+    defaults = {
+      email = "nullify-cloud-connector@acme-security.iam.gserviceaccount.com"
+    }
+  }
+}
+
+variables {
+  customer_name           = "acme"
+  host_project_id         = "acme-security"
+  scope                   = "projects"
+  project_ids             = ["acme-security"]
+  nullify_oidc_issuer_uri = "https://gcp.nullify.ai"
+  nullify_tenant_id       = "GitHub-12345"
+}
+
+run "manage_scanner_apis_by_default" {
+  command = plan
+  assert {
+    condition = output.managed_scan_quota_apis == tolist([
+      "accesscontextmanager.googleapis.com",
+      "bigquery.googleapis.com",
+      "cloudkms.googleapis.com",
+      "dataflow.googleapis.com",
+      "orgpolicy.googleapis.com",
+      "spanner.googleapis.com",
+      "sqladmin.googleapis.com",
+    ])
+    error_message = "The supplied root configuration must manage all seven scanner APIs by default."
+  }
+}
+
+run "root_opt_out_manages_no_scanner_apis" {
+  command = plan
+  variables {
+    enable_scan_quota_apis = false
+  }
+  assert {
+    condition     = length(output.managed_scan_quota_apis) == 0
+    error_message = "The root opt-out must reach the nested module so restricted projects can manage scanner APIs separately."
+  }
+}

gcp-integration-setup/terraform/variables.tf

@@ -75,3 +75,9 @@ variable "service_account_name" {
   type        = string
   default     = "nullify-cloud-connector"
 }
+
+variable "enable_scan_quota_apis" {
+  description = "Manage the scanner APIs in the host project. Terraform apply fails if an API cannot be enabled. Set false to manage these APIs separately."
+  type        = bool
+  default     = true
+}