[IDEA] Add 'connect_storage' and 'create_catalog' modules to Azure Terraform code

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/databricks.tf

@@ -72,4 +72,26 @@ resource "databricks_metastore_assignment" "this" {
   provider     = databricks.accounts
   workspace_id = azurerm_databricks_workspace.this.workspace_id
   metastore_id = var.existing_metastore_id == "" ? databricks_metastore.this[0].id : var.existing_metastore_id
+}
+
+# catalog creation
+
+module "connect_storage" {
+  source = "./modules/connect_storage"
+  create_storage_account = true
+  resource_group = azurerm_resource_group.this.name
+  location = azurerm_resource_group.this.location
+  storage_account_name = "gergeljkissa"
+  tags = var.tags
+}
+
+module "create_catalog" {
+  source = "./modules/catalog"
+  storage_account_id = module.connect_storage.storage_account_id
+  storage_account_name = module.connect_storage.storage_account_name
+  storage_credential_id = module.connect_storage.storage_credential_id
+  catalog_name = "demo"
+  storage_container_name = "demo"
+  force_destroy_external_location = true
+  force_destroy_catalog = true
 }

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/main.tf

@@ -0,0 +1,40 @@
+locals {
+  external_location_name = "${var.catalog_name}_external_location"
+}
+
+// Create a container in storage account to be used by unity catalog metastore as root storage
+resource "azurerm_storage_container" "this" {
+  name                  = var.storage_container_name
+  storage_account_id    = var.storage_account_id
+  container_access_type = "private"
+}
+
+resource "databricks_external_location" "this" {
+  name = local.external_location_name
+  url = format("abfss://%s@%s.dfs.core.windows.net",
+    azurerm_storage_container.this.name,
+  var.storage_account_name)
+  credential_name = var.storage_credential_id
+  comment         = "Managed by TF"
+  # Made configurable to prevent accidental data loss in production
+  force_destroy = var.force_destroy_external_location
+}
+
+resource "databricks_catalog" "this" {
+  name         = var.catalog_name
+  storage_root = format(
+    "abfss://%s@%s.dfs.core.windows.net/%s",
+    azurerm_storage_container.this.name,
+    var.storage_account_name,
+    var.catalog_name
+  )
+  depends_on = [
+    databricks_external_location.this
+  ]
+  # Made configurable to prevent accidental data loss in production
+  # This setting controls whether the catalog and all its contents are deleted on terraform destroy
+  force_destroy = var.force_destroy_catalog
+}
+
+
+

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/variables.tf

@@ -0,0 +1,38 @@
+variable "storage_account_id" {
+  type        = string
+  description = "ID of the Azure storage account"
+}
+
+variable "storage_account_name" {
+  type        = string
+  description = "Name of the Azure storage account"
+}
+
+variable "storage_container_name" {
+  type        = string
+  description = "Name of the storage container"
+}
+
+variable "storage_credential_id" {
+  type        = string
+  description = "ID of the storage credential in Unity Catalog"
+}
+
+variable "catalog_name" {
+  type        = string
+  description = "Name of the Unity Catalog catalog to create"
+}
+
+# Making force_destroy configurable to prevent accidental data loss
+variable "force_destroy_external_location" {
+  description = "Whether to force destroy the external location when removing. WARNING: This will delete all data!"
+  type        = bool
+  default     = false  # Default to false for safety in production
+}
+
+# Making catalog force_destroy configurable for production safety
+variable "force_destroy_catalog" {
+  description = "Whether to force destroy the catalog when removing. WARNING: This will delete all catalog data including schemas and tables!"
+  type        = bool
+  default     = false  # Default to false for safety in production environments
+}

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_version = "~> 1.3"
+
+
+  required_providers {
+    azurerm = {
+      source = "hashicorp/azurerm"
+      version = "~> 4.50"
+    }
+    databricks = {
+      source = "databricks/databricks"
+      version = "~> 1.84"
+    }
+  }
+}

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/main.tf

@@ -0,0 +1,49 @@
+locals {
+  access_connector_name = "${var.storage_account_name}_access_connector"
+  storage_credential_name = "${var.storage_account_name}_storage_credential"
+}
+
+// Create a storage account to be used by catalog
+resource "azurerm_storage_account" "this" {
+  count = var.create_storage_account ? 1 : 0
+  name                     = var.storage_account_name
+  resource_group_name      = var.resource_group
+  location                 = var.location
+  tags                     = var.tags
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  is_hns_enabled           = true
+}
+
+data "azurerm_storage_account" "this" {
+  count = var.create_storage_account ? 0 : 1
+  name = var.storage_account_name
+  resource_group_name = var.resource_group
+}
+
+resource "azurerm_databricks_access_connector" "this" {
+  name                = local.access_connector_name
+  resource_group_name = var.resource_group
+  location            = var.location
+  identity {
+    type = "SystemAssigned"
+  }
+}
+
+// Assign the Storage Blob Data Contributor role to managed identity to allow unity catalog to access the storage
+resource "azurerm_role_assignment" "this" {
+  scope                = var.create_storage_account ? azurerm_storage_account.this[0].id : data.azurerm_storage_account.this[0].id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = azurerm_databricks_access_connector.this.identity[0].principal_id
+}
+
+resource "databricks_storage_credential" "this" {
+  name = local.storage_credential_name
+  azure_managed_identity {
+    access_connector_id = azurerm_databricks_access_connector.this.id
+  }
+  comment = "Managed identity credential managed by TF"
+  depends_on = [
+    azurerm_role_assignment.this
+  ]
+}

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/outputs.tf

@@ -0,0 +1,14 @@
+output "storage_account_id" {
+  description = "ID of the Azure storage account"
+  value = var.create_storage_account ? azurerm_storage_account.this[0].id : data.azurerm_storage_account.this[0].id
+}
+
+output "storage_account_name" {
+  description = "Name of the Azure storage account"
+  value = var.create_storage_account ? azurerm_storage_account.this[0].name : data.azurerm_storage_account.this[0].name
+}
+
+output "storage_credential_id" {
+  description = "ID of the storage credential in Unity Catalog"
+  value = databricks_storage_credential.this.id
+}

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/variables.tf

@@ -0,0 +1,25 @@
+variable "resource_group" {
+  type        = string
+  description = "Azure resource group name"
+}
+
+variable "location" {
+  type        = string
+  description = "Azure region/location for resources"
+}
+
+variable "create_storage_account" {
+  type        = bool
+  description = "Whether to create a storage account"
+  default     = true
+}
+
+variable "storage_account_name" {
+  type        = string
+  description = "Name of the Azure storage account"
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "Tags to apply to Azure resources"
+}

workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_version = "~> 1.3"
+
+
+  required_providers {
+    azurerm = {
+      source = "hashicorp/azurerm"
+      version = "~> 4.50"
+    }
+    databricks = {
+      source = "databricks/databricks"
+      version = "~> 1.84"
+    }
+  }
+}