diff --git a/bq-workers/parsers.cloudbuild.yaml b/bq-workers/parsers.cloudbuild.yaml index b3471f3f..a57d53ee 100644 --- a/bq-workers/parsers.cloudbuild.yaml +++ b/bq-workers/parsers.cloudbuild.yaml @@ -14,20 +14,15 @@ steps: - # Build parser image - name: gcr.io/cloud-builders/docker:latest + name: gcr.io/cloud-builders/docker dir: ${_SERVICE}-parser - args: ['build', - '--tag=gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}', '.'] + args: ['build', '-t', + '${_REGION}-docker.pkg.dev/$PROJECT_ID/${_SERVICE}-parser/parser:${_TAG}', '.'] id: build -- # Push the container image to Container Registry - name: gcr.io/cloud-builders/docker - args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}'] - waitFor: build - id: push +# Push the container image to Artifact Registry +images: + - '${_REGION}-docker.pkg.dev/$PROJECT_ID/${_SERVICE}-parser/parser:${_TAG}' -images: [ - 'gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}' -] substitutions: _TAG: latest diff --git a/dashboard/cloudbuild.yaml b/dashboard/cloudbuild.yaml index 875e449c..16b5e5f9 100644 --- a/dashboard/cloudbuild.yaml +++ b/dashboard/cloudbuild.yaml @@ -14,15 +14,14 @@ steps: - # Build grafana image - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag', - 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}', '.'] + name: gcr.io/cloud-builders/docker + args: ['build', '-t', + '${_REGION}-docker.pkg.dev/$PROJECT_ID/dashboard/grafana:${_TAG}', '.'] id: build -- # Push the container image to Container Registry - name: gcr.io/cloud-builders/docker - args: ['push', 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}'] - id: push +# Push the container image to Artifact Registry +images: + - '${_REGION}-docker.pkg.dev/$PROJECT_ID/dashboard/grafana:${_TAG}' # Read more about substitutions # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values diff --git a/event-handler/cloudbuild.yaml b/event-handler/cloudbuild.yaml index 36c752ed..9f006878 100644 --- a/event-handler/cloudbuild.yaml +++ b/event-handler/cloudbuild.yaml @@ -15,17 +15,12 @@ steps: - # Build event_handler image name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=gcr.io/$PROJECT_ID/event-handler:${_TAG}', '.'] + args: ['build', '-t', '${_REGION}-docker.pkg.dev/$PROJECT_ID/event-handler/handler:${_TAG}', '.'] id: build -- # Push the container image to Container Registry - name: gcr.io/cloud-builders/docker - args: ['push', 'gcr.io/$PROJECT_ID/event-handler:${_TAG}'] - waitFor: build - id: push +# Push the container image to Artifact Registry +images: +- '${_REGION}-docker.pkg.dev/$PROJECT_ID/event-handler/handler:${_TAG}' -images: [ - 'gcr.io/$PROJECT_ID/event-handler:${_TAG}' -] substitutions: _TAG: latest diff --git a/setup/README.md b/setup/README.md index 642439b2..4a79e860 100644 --- a/setup/README.md +++ b/setup/README.md @@ -25,20 +25,37 @@ To deploy Four Keys with Terraform, you will first need: export PROJECT_ID="YOUR_PROJECT_ID" ``` +1. Set an environment variable indicating the region where you want to deploy the dashboard, event handler and parsers. You can find a list of available regions [here](https://cloud.google.com/artifact-registry/docs/repositories/repo-locations). + ```sh + export GAR_REGION="ARTIFACT_REGISTRY_REGION" + ``` + +1. Set an environment variable indicating the service name you want to use: + ```sh + export SERVICE="SERVICE_NAME" + ``` + +1. Create an artifact registry repository for the dashboard, event-handler and parsers: + ```sh + gcloud artifacts repositories create dashboard --repository-format=docker --location=$GAR_REGION --project $PROJECT_ID && \ + gcloud artifacts repositories create event-handler --repository-format=docker --location=$GAR_REGION --project $PROJECT_ID && \ + gcloud artifacts repositories create ${SERVICE}-parser --repository-format=docker --location=$GAR_REGION --project $PROJECT_ID + ``` + 1. Clone the fourkeys git repository and change into the root directory ``` git clone https://github.com/dora-team/fourkeys.git && cd fourkeys ``` -1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: +1. Use Cloud Build to build and push containers to Google Artifact Registry for the dashboard, event-handler: ``` - gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID && \ - gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID + gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID --substitutions=_REGION=$GAR_REGION && \ + gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID --substitutions=_REGION=$GAR_REGION ``` -1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: +1. Use Cloud Build to build and push containers to Google Artifact Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: ``` - gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github + gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=$SERVICE,_REGION=$GAR_REGION ``` 1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` diff --git a/terraform/example/main.tf b/terraform/example/main.tf index 272d1cb5..900a541b 100644 --- a/terraform/example/main.tf +++ b/terraform/example/main.tf @@ -1,8 +1,8 @@ module "fourkeys" { - source = "../modules/fourkeys" - project_id = var.project_id - enable_apis = var.enable_apis - region = var.region - bigquery_region = var.bigquery_region - parsers = var.parsers + source = "../modules/fourkeys" + project_id = var.project_id + enable_apis = var.enable_apis + region = var.region + bigquery_region = var.bigquery_region + parsers = var.parsers } diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf index 8b8ea2a8..0b3149b1 100644 --- a/terraform/example/variables.tf +++ b/terraform/example/variables.tf @@ -1,6 +1,6 @@ variable "project_id" { - type = string - description = "project to deploy four keys resources to" + type = string + description = "project to deploy four keys resources to" } variable "enable_apis" { @@ -10,9 +10,9 @@ variable "enable_apis" { } variable "region" { - type = string - default = "us-central1" - description = "Region to deploy four keys resources in." + type = string + default = "us-central1" + description = "Region to deploy four keys resources in." } variable "bigquery_region" { diff --git a/terraform/modules/fourkeys-circleci-parser/main.tf b/terraform/modules/fourkeys-circleci-parser/main.tf index 9e2e5951..3f3d2b52 100644 --- a/terraform/modules/fourkeys-circleci-parser/main.tf +++ b/terraform/modules/fourkeys-circleci-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "circleci_parser" { diff --git a/terraform/modules/fourkeys-circleci-parser/variables.tf b/terraform/modules/fourkeys-circleci-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-circleci-parser/variables.tf +++ b/terraform/modules/fourkeys-circleci-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-cloud-build-parser/main.tf b/terraform/modules/fourkeys-cloud-build-parser/main.tf index 8a7515dd..e42a27d5 100644 --- a/terraform/modules/fourkeys-cloud-build-parser/main.tf +++ b/terraform/modules/fourkeys-cloud-build-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "cloudbuild_parser" { diff --git a/terraform/modules/fourkeys-cloud-build-parser/variables.tf b/terraform/modules/fourkeys-cloud-build-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-cloud-build-parser/variables.tf +++ b/terraform/modules/fourkeys-cloud-build-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-github-parser/main.tf b/terraform/modules/fourkeys-github-parser/main.tf index 1aaebe9b..368c95ce 100644 --- a/terraform/modules/fourkeys-github-parser/main.tf +++ b/terraform/modules/fourkeys-github-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "github_parser" { diff --git a/terraform/modules/fourkeys-github-parser/variables.tf b/terraform/modules/fourkeys-github-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-github-parser/variables.tf +++ b/terraform/modules/fourkeys-github-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-gitlab-parser/main.tf b/terraform/modules/fourkeys-gitlab-parser/main.tf index 1fc2931b..28f99531 100644 --- a/terraform/modules/fourkeys-gitlab-parser/main.tf +++ b/terraform/modules/fourkeys-gitlab-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "gitlab_parser" { @@ -72,7 +72,7 @@ resource "google_pubsub_subscription" "gitlab" { project = var.project_id name = "gitlab" topic = google_pubsub_topic.gitlab.id - + expiration_policy { ttl = "" } diff --git a/terraform/modules/fourkeys-gitlab-parser/variables.tf b/terraform/modules/fourkeys-gitlab-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-gitlab-parser/variables.tf +++ b/terraform/modules/fourkeys-gitlab-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-pagerduty-parser/main.tf b/terraform/modules/fourkeys-pagerduty-parser/main.tf index b91df506..521bed56 100644 --- a/terraform/modules/fourkeys-pagerduty-parser/main.tf +++ b/terraform/modules/fourkeys-pagerduty-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "pagerduty_parser" { @@ -37,7 +37,7 @@ resource "google_cloud_run_service" "pagerduty_parser" { percent = 100 latest_revision = true } - + metadata { annotations = { "run.googleapis.com/ingress" = "internal" diff --git a/terraform/modules/fourkeys-pagerduty-parser/variables.tf b/terraform/modules/fourkeys-pagerduty-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-pagerduty-parser/variables.tf +++ b/terraform/modules/fourkeys-pagerduty-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-tekton-parser/main.tf b/terraform/modules/fourkeys-tekton-parser/main.tf index dfcd1bd0..92f5b55b 100644 --- a/terraform/modules/fourkeys-tekton-parser/main.tf +++ b/terraform/modules/fourkeys-tekton-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "tekton_parser" { diff --git a/terraform/modules/fourkeys-tekton-parser/variables.tf b/terraform/modules/fourkeys-tekton-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-tekton-parser/variables.tf +++ b/terraform/modules/fourkeys-tekton-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys/locals.tf b/terraform/modules/fourkeys/locals.tf index a8f92647..b06477ee 100644 --- a/terraform/modules/fourkeys/locals.tf +++ b/terraform/modules/fourkeys/locals.tf @@ -4,14 +4,14 @@ data "google_project" "project" { locals { cloud_build_service_account = "${data.google_project.project.number}@cloudbuild.gserviceaccount.com" - event_handler_container_url = var.event_handler_container_url == "" ? format("gcr.io/%s/event-handler", var.project_id) : var.event_handler_container_url - dashboard_container_url = var.dashboard_container_url == "" ? format("gcr.io/%s/fourkeys-grafana-dashboard", var.project_id) : var.dashboard_container_url - github_parser_url = var.github_parser_url == "" ? format("gcr.io/%s/github-parser", var.project_id) : var.github_parser_url - gitlab_parser_url = var.gitlab_parser_url == "" ? format("gcr.io/%s/gitlab-parser", var.project_id) : var.gitlab_parser_url - cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("gcr.io/%s/cloud-build-parser", var.project_id) : var.cloud_build_parser_url - tekton_parser_url = var.tekton_parser_url == "" ? format("gcr.io/%s/tekton-parser", var.project_id) : var.tekton_parser_url - circleci_parser_url = var.circleci_parser_url == "" ? format("gcr.io/%s/circleci-parser", var.project_id) : var.circleci_parser_url - pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("gcr.io/%s/pagerduty-parser", var.project_id) : var.pagerduty_parser_url + event_handler_container_url = var.event_handler_container_url == "" ? format("%s-docker.pkg.dev/%s/event-handler/handler", var.region, var.project_id) : var.event_handler_container_url + dashboard_container_url = var.dashboard_container_url == "" ? format("%s-docker.pkg.dev/%s/dashboard/grafana", var.region, var.project_id) : var.dashboard_container_url + github_parser_url = var.github_parser_url == "" ? format("%s-docker.pkg.dev/%s/github-parser/parser", var.region, var.project_id) : var.github_parser_url + gitlab_parser_url = var.gitlab_parser_url == "" ? format("%s-docker.pkg.dev/%s/gitlab-parser/parser", var.region, var.project_id) : var.gitlab_parser_url + cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("%s-docker.pkg.dev/%s/cloud-build-parser/parser", var.region, var.project_id) : var.cloud_build_parser_url + tekton_parser_url = var.tekton_parser_url == "" ? format("%s-docker.pkg.dev/%s/tekton-parser/parser", var.region, var.project_id) : var.tekton_parser_url + circleci_parser_url = var.circleci_parser_url == "" ? format("%s-docker.pkg.dev/%s/circleci-parser/parser", var.region, var.project_id) : var.circleci_parser_url + pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("%s-docker.pkg.dev/%s/pagerduty-parser/parser", var.region, var.project_id) : var.pagerduty_parser_url services = var.enable_apis ? [ "bigquery.googleapis.com", "cloudbuild.googleapis.com", diff --git a/terraform/modules/fourkeys/variables.tf b/terraform/modules/fourkeys/variables.tf index df6504e7..e638b01d 100644 --- a/terraform/modules/fourkeys/variables.tf +++ b/terraform/modules/fourkeys/variables.tf @@ -38,48 +38,48 @@ variable "enable_dashboard" { variable "event_handler_container_url" { type = string - description = "The URL for the event_handler container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the event_handler container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "dashboard_container_url" { type = string - description = "The URL for the dashboard container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the dashboard container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "github_parser_url" { type = string - description = "The URL for the Github parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the Github parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "gitlab_parser_url" { type = string - description = "The URL for the Gitlab parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the Gitlab parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "cloud_build_parser_url" { type = string - description = "The URL for the Cloud Build parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the Cloud Build parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "tekton_parser_url" { type = string - description = "The URL for the Tekton parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the Tekton parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "circleci_parser_url" { type = string - description = "The URL for the CircleCI parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the CircleCI parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" } variable "pagerduty_parser_url" { type = string - description = "The URL for the Pager Duty parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + description = "The URL for the Pager Duty parser container image. A default value pointing to the project's artifact registry is defined in under local values of this module." default = "" }