diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/databricks.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/databricks.tf index 15064d9..d88a26d 100644 --- a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/databricks.tf +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/databricks.tf @@ -72,4 +72,26 @@ resource "databricks_metastore_assignment" "this" { provider = databricks.accounts workspace_id = azurerm_databricks_workspace.this.workspace_id metastore_id = var.existing_metastore_id == "" ? databricks_metastore.this[0].id : var.existing_metastore_id +} + +# catalog creation + +module "connect_storage" { + source = "./modules/connect_storage" + create_storage_account = true + resource_group = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + storage_account_name = "gergeljkissa" + tags = var.tags +} + +module "create_catalog" { + source = "./modules/catalog" + storage_account_id = module.connect_storage.storage_account_id + storage_account_name = module.connect_storage.storage_account_name + storage_credential_id = module.connect_storage.storage_credential_id + catalog_name = "demo" + storage_container_name = "demo" + force_destroy_external_location = true + force_destroy_catalog = true } \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/main.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/main.tf new file mode 100644 index 0000000..4df462c --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/main.tf @@ -0,0 +1,40 @@ +locals { + external_location_name = "${var.catalog_name}_external_location" +} + +// Create a container in storage account to be used by unity catalog metastore as root storage +resource "azurerm_storage_container" "this" { + name = var.storage_container_name + storage_account_id = var.storage_account_id + container_access_type = "private" +} + +resource "databricks_external_location" "this" { + name = local.external_location_name + url = format("abfss://%s@%s.dfs.core.windows.net", + azurerm_storage_container.this.name, + var.storage_account_name) + credential_name = var.storage_credential_id + comment = "Managed by TF" + # Made configurable to prevent accidental data loss in production + force_destroy = var.force_destroy_external_location +} + +resource "databricks_catalog" "this" { + name = var.catalog_name + storage_root = format( + "abfss://%s@%s.dfs.core.windows.net/%s", + azurerm_storage_container.this.name, + var.storage_account_name, + var.catalog_name + ) + depends_on = [ + databricks_external_location.this + ] + # Made configurable to prevent accidental data loss in production + # This setting controls whether the catalog and all its contents are deleted on terraform destroy + force_destroy = var.force_destroy_catalog +} + + + diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/variables.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/variables.tf new file mode 100644 index 0000000..14404df --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/variables.tf @@ -0,0 +1,38 @@ +variable "storage_account_id" { + type = string + description = "ID of the Azure storage account" +} + +variable "storage_account_name" { + type = string + description = "Name of the Azure storage account" +} + +variable "storage_container_name" { + type = string + description = "Name of the storage container" +} + +variable "storage_credential_id" { + type = string + description = "ID of the storage credential in Unity Catalog" +} + +variable "catalog_name" { + type = string + description = "Name of the Unity Catalog catalog to create" +} + +# Making force_destroy configurable to prevent accidental data loss +variable "force_destroy_external_location" { + description = "Whether to force destroy the external location when removing. WARNING: This will delete all data!" + type = bool + default = false # Default to false for safety in production +} + +# Making catalog force_destroy configurable for production safety +variable "force_destroy_catalog" { + description = "Whether to force destroy the catalog when removing. WARNING: This will delete all catalog data including schemas and tables!" + type = bool + default = false # Default to false for safety in production environments +} \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/versions.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/versions.tf new file mode 100644 index 0000000..1720928 --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/catalog/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = "~> 1.3" + + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.50" + } + databricks = { + source = "databricks/databricks" + version = "~> 1.84" + } + } +} \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/main.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/main.tf new file mode 100644 index 0000000..cd0f522 --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/main.tf @@ -0,0 +1,49 @@ +locals { + access_connector_name = "${var.storage_account_name}_access_connector" + storage_credential_name = "${var.storage_account_name}_storage_credential" +} + +// Create a storage account to be used by catalog +resource "azurerm_storage_account" "this" { + count = var.create_storage_account ? 1 : 0 + name = var.storage_account_name + resource_group_name = var.resource_group + location = var.location + tags = var.tags + account_tier = "Standard" + account_replication_type = "LRS" + is_hns_enabled = true +} + +data "azurerm_storage_account" "this" { + count = var.create_storage_account ? 0 : 1 + name = var.storage_account_name + resource_group_name = var.resource_group +} + +resource "azurerm_databricks_access_connector" "this" { + name = local.access_connector_name + resource_group_name = var.resource_group + location = var.location + identity { + type = "SystemAssigned" + } +} + +// Assign the Storage Blob Data Contributor role to managed identity to allow unity catalog to access the storage +resource "azurerm_role_assignment" "this" { + scope = var.create_storage_account ? azurerm_storage_account.this[0].id : data.azurerm_storage_account.this[0].id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_databricks_access_connector.this.identity[0].principal_id +} + +resource "databricks_storage_credential" "this" { + name = local.storage_credential_name + azure_managed_identity { + access_connector_id = azurerm_databricks_access_connector.this.id + } + comment = "Managed identity credential managed by TF" + depends_on = [ + azurerm_role_assignment.this + ] +} \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/outputs.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/outputs.tf new file mode 100644 index 0000000..2da61bf --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/outputs.tf @@ -0,0 +1,14 @@ +output "storage_account_id" { + description = "ID of the Azure storage account" + value = var.create_storage_account ? azurerm_storage_account.this[0].id : data.azurerm_storage_account.this[0].id +} + +output "storage_account_name" { + description = "Name of the Azure storage account" + value = var.create_storage_account ? azurerm_storage_account.this[0].name : data.azurerm_storage_account.this[0].name +} + +output "storage_credential_id" { + description = "ID of the storage credential in Unity Catalog" + value = databricks_storage_credential.this.id +} \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/variables.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/variables.tf new file mode 100644 index 0000000..62b06fa --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/variables.tf @@ -0,0 +1,25 @@ +variable "resource_group" { + type = string + description = "Azure resource group name" +} + +variable "location" { + type = string + description = "Azure region/location for resources" +} + +variable "create_storage_account" { + type = bool + description = "Whether to create a storage account" + default = true +} + +variable "storage_account_name" { + type = string + description = "Name of the Azure storage account" +} + +variable "tags" { + type = map(string) + description = "Tags to apply to Azure resources" +} \ No newline at end of file diff --git a/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/versions.tf b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/versions.tf new file mode 100644 index 0000000..1720928 --- /dev/null +++ b/workspace-setup/terraform-examples/azure/azure-vnet-injection/tf/modules/connect_storage/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = "~> 1.3" + + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.50" + } + databricks = { + source = "databricks/databricks" + version = "~> 1.84" + } + } +} \ No newline at end of file