diff --git a/README.md b/README.md index 1c66db1..d473f81 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ The folder `examples` contains the following Terraform implementation examples : | Azure | [adb-uc](examples/adb-uc/) | ADB Unity Catalog Process | | Azure | [adb-unity-catalog-basic-demo](examples/adb-unity-catalog-basic-demo/) | ADB Unity Catalog end-to-end demo including UC metastore setup, Users/groups sync from AAD to databricks account, UC Catalog, External locations, Schemas, & Access Grants | | Azure | [adb-overwatch](examples/adb-overwatch/) | Overwatch multi-workspace deployment on Azure | +| Azure | [adb-uc-external-location-file-events](examples/adb-uc-external-location-file-events/) | UC external location with managed AQS file events, documented Azure RBAC, and grants | | AWS | [aws-workspace-basic](examples/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-workspace-with-firewall](examples/aws-workspace-with-firewall/) | Provisioning AWS Databricks E2 with an AWS Firewall | | AWS | [aws-exfiltration-protection](examples/aws-exfiltration-protection/) | An implementation of [Data Exfiltration Protection on AWS](https://www.databricks.com/blog/2021/02/02/data-exfiltration-protection-with-databricks-on-aws.html) | @@ -82,6 +83,7 @@ The folder `modules` contains the following Terraform modules : | Azure | [adb-overwatch-main-ws](modules/adb-overwatch-main-ws/) | Main Overwatch workspace deployment | | Azure | [adb-overwatch-ws-to-monitor](modules/adb-overwatch-ws-to-monitor/) | Overwatch deployment on the Azure workspace to monitor | | Azure | [adb-overwatch-analysis](modules/adb-overwatch-analysis/) | Overwatch analysis notebooks deployment on Azure | +| Azure | [adb-uc-external-location-file-events](modules/adb-uc-external-location-file-events/) | UC storage credential + external locations with managed AQS file events and documented Azure RBAC | | AWS | [aws-workspace-basic](modules/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-databricks-base-infra](modules/aws-databricks-base-infra/) | Provisioning AWS Infrastructure to be used for the deployment of a Databricks E2 workspace | | AWS | [aws-databricks-unity-catalog](modules/aws-databricks-unity-catalog/) | Provisioning the AWS Infrastructure and setting up the metastore for Databricks Unity Catalog | diff --git a/examples/adb-uc-external-location-file-events/README.md b/examples/adb-uc-external-location-file-events/README.md new file mode 100644 index 0000000..f351a13 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/README.md @@ -0,0 +1,29 @@ +# Azure UC external location with managed file events + +Provisions a Unity Catalog storage credential and external location with +**automatic managed AQS file events**, plus the documented Azure RBAC roles and +optional UC grants. + +See the [module README](../../modules/adb-uc-external-location-file-events/README.md) +for RBAC details and [file arrival triggers](https://docs.databricks.com/aws/en/jobs/file-arrival-triggers). + +## Prerequisites + +- UC-enabled Azure Databricks workspace +- Existing ADLS Gen2 storage account and Access Connector +- `az login` (or Azure env credentials) with permission to assign RBAC +- Databricks CLI auth to the workspace (`databricks auth login --host ...`) + +## Usage + +1. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in values. +2. `terraform init` +3. `terraform plan` +4. `terraform apply` + +## What gets created + +- Storage credential (Azure managed identity via access connector) +- External location with `enable_file_events = true` and `file_event_queue.managed_aqs` +- Azure RBAC: Blob Data Contributor/Reader, Queue Data Contributor, Storage Account Contributor, EventGrid EventSubscription Contributor +- Optional UC grants on credential and location diff --git a/examples/adb-uc-external-location-file-events/main.tf b/examples/adb-uc-external-location-file-events/main.tf new file mode 100644 index 0000000..f206942 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/main.tf @@ -0,0 +1,30 @@ +module "uc_external_location_file_events" { + source = "../../modules/adb-uc-external-location-file-events" + + name_prefix = var.name_prefix + access_connector_id = var.access_connector_id + storage_account_id = var.storage_account_id + resource_group_name = var.resource_group_name + + external_locations = [ + { + name = var.external_location_name + url = var.external_location_url + comment = "Landing zone with managed AQS file events" + } + ] + + credential_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["CREATE_EXTERNAL_LOCATION", "READ_FILES", "WRITE_FILES"] + } + ] + + location_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} diff --git a/examples/adb-uc-external-location-file-events/outputs.tf b/examples/adb-uc-external-location-file-events/outputs.tf new file mode 100644 index 0000000..4f6f0d8 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/outputs.tf @@ -0,0 +1,11 @@ +output "storage_credential_name" { + value = module.uc_external_location_file_events.storage_credential_name +} + +output "external_location_ids" { + value = module.uc_external_location_file_events.external_location_ids +} + +output "azure_rbac_roles" { + value = module.uc_external_location_file_events.azure_rbac_roles +} diff --git a/examples/adb-uc-external-location-file-events/providers.tf b/examples/adb-uc-external-location-file-events/providers.tf new file mode 100644 index 0000000..6989231 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/providers.tf @@ -0,0 +1,10 @@ +provider "azurerm" { + features {} +} + +# Authenticate to an existing UC-enabled workspace. +# Prefer `databricks auth login --host ` and omit explicit credentials, +# or set host / token / Azure auth via environment variables. +provider "databricks" { + host = var.databricks_host +} diff --git a/examples/adb-uc-external-location-file-events/terraform.tfvars.example b/examples/adb-uc-external-location-file-events/terraform.tfvars.example new file mode 100644 index 0000000..2d43369 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/terraform.tfvars.example @@ -0,0 +1,8 @@ +databricks_host = "https://adb-xxxxxxxxxxxx.azuredatabricks.net" +name_prefix = "file-events-demo" +access_connector_id = "/subscriptions//resourceGroups//providers/Microsoft.Databricks/accessConnectors/" +storage_account_id = "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/" +resource_group_name = "" +external_location_name = "file-events-landing" +external_location_url = "abfss://landing@.dfs.core.windows.net/incoming" +grant_principal = "data-engineers" diff --git a/examples/adb-uc-external-location-file-events/variables.tf b/examples/adb-uc-external-location-file-events/variables.tf new file mode 100644 index 0000000..cf250c3 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/variables.tf @@ -0,0 +1,42 @@ +variable "databricks_host" { + type = string + description = "Workspace URL, e.g. https://adb-xxxx.azuredatabricks.net" +} + +variable "name_prefix" { + type = string + description = "Prefix for UC object names" + default = "file-events-demo" +} + +variable "access_connector_id" { + type = string + description = "Azure resource ID of the Access Connector for Azure Databricks" +} + +variable "storage_account_id" { + type = string + description = "Azure resource ID of the ADLS Gen2 storage account" +} + +variable "resource_group_name" { + type = string + description = "Resource group containing the storage account" +} + +variable "external_location_name" { + type = string + description = "Name of the Unity Catalog external location" + default = "file-events-landing" +} + +variable "external_location_url" { + type = string + description = "abfss:// URL for the external location path" +} + +variable "grant_principal" { + type = string + description = "UC group or user to grant on the credential and external location. Leave empty to skip grants." + default = "" +} diff --git a/examples/adb-uc-external-location-file-events/versions.tf b/examples/adb-uc-external-location-file-events/versions.tf new file mode 100644 index 0000000..fe80276 --- /dev/null +++ b/examples/adb-uc-external-location-file-events/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.0.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.50.0" + } + } +} diff --git a/modules/adb-uc-external-location-file-events/Makefile b/modules/adb-uc-external-location-file-events/Makefile new file mode 100644 index 0000000..653039d --- /dev/null +++ b/modules/adb-uc-external-location-file-events/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/modules/adb-uc-external-location-file-events/README.md b/modules/adb-uc-external-location-file-events/README.md new file mode 100644 index 0000000..dccc61a --- /dev/null +++ b/modules/adb-uc-external-location-file-events/README.md @@ -0,0 +1,133 @@ +# adb-uc-external-location-file-events + +Creates Azure Unity Catalog **storage credentials** and **external locations** with +[managed file events](https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/cloud-storage/manage-external-locations) +enabled via Azure Queue Storage (managed AQS). Also assigns the documented Azure RBAC roles +required for data access and automatic file-event setup. + +This is useful for [file arrival triggers](https://docs.databricks.com/aws/en/jobs/file-arrival-triggers) +and other ingestion patterns that benefit from cloud storage change notifications. + +## Prerequisites + +- Unity Catalog–enabled Azure Databricks workspace +- Existing Azure Data Lake Storage Gen2 account + container path +- Existing [Access Connector for Azure Databricks](https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/cloud-storage/azure-managed-identities) +- Permission to assign Azure RBAC on the storage account and its resource group +- Permission to create UC storage credentials and external locations + +## Azure RBAC (automatic managed file events) + +When `assign_azure_rbac = true` (default), the access connector managed identity receives: + +| Scope | Role | Purpose | +| ----- | ---- | ------- | +| Storage account | `Storage Blob Data Contributor` (or `Reader` if all locations are `read_only`) | Data plane access | +| Storage account | `Storage Queue Data Contributor` | Subscribe to file-change notifications | +| Storage account | `Storage Account Contributor` | Let Databricks auto-create the queue / routing | +| Resource group | `EventGrid EventSubscription Contributor` | Let Databricks auto-create Event Grid subscriptions | + +`Storage Account Contributor` and the Event Grid role are only needed for **automatic** file-event +setup. Manual / provided-queue configuration can omit them but is unsupported by Databricks. + +## Example + +```hcl +module "uc_locations" { + source = "../../modules/adb-uc-external-location-file-events" + + name_prefix = "demo" + access_connector_id = "/subscriptions/.../accessConnectors/uc-access-connector" + storage_account_id = "/subscriptions/.../storageAccounts/stdemo" + resource_group_name = "rg-demo" + + external_locations = [ + { + name = "demo-landing" + url = "abfss://landing@stdemo.dfs.core.windows.net/incoming" + comment = "Landing zone with file events" + } + ] + + credential_grants = [ + { + principal = "data-engineers" + privileges = ["CREATE_EXTERNAL_LOCATION", "READ_FILES", "WRITE_FILES"] + } + ] + + location_grants = [ + { + principal = "data-engineers" + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} +``` + +See also [examples/adb-uc-external-location-file-events](../../examples/adb-uc-external-location-file-events). + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [azurerm](#requirement\_azurerm) | >= 3.0.0 | +| [databricks](#requirement\_databricks) | >= 1.50.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [azurerm](#provider\_azurerm) | >= 3.0.0 | +| [databricks](#provider\_databricks) | >= 1.50.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [azurerm_role_assignment.blob_data](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.eventgrid_subscription](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.queue_data](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.storage_account_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [databricks_external_location.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/external_location) | resource | +| [databricks_grants.credential](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_grants.location](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_storage_credential.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) | resource | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_databricks_access_connector.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/databricks_access_connector) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [access\_connector\_id](#input\_access\_connector\_id) | Azure resource ID of the Access Connector for Azure Databricks (managed identity used by the storage credential) | `string` | n/a | yes | +| [external\_locations](#input\_external\_locations) | External locations to create. Each location gets managed AQS file events enabled. |
list(object({
name = string
url = string
comment = optional(string, "Managed by Terraform")
read_only = optional(bool, false)
}))
| n/a | yes | +| [name\_prefix](#input\_name\_prefix) | Prefix used for storage credential and external location names | `string` | n/a | yes | +| [resource\_group\_name](#input\_resource\_group\_name) | Resource group that contains the storage account (used for managed AQS and Event Grid RBAC) | `string` | n/a | yes | +| [storage\_account\_id](#input\_storage\_account\_id) | Azure resource ID of the ADLS Gen2 storage account that backs the external location(s) | `string` | n/a | yes | +| [assign\_azure\_rbac](#input\_assign\_azure\_rbac) | Assign the documented Azure RBAC roles required for data access and automatic managed file events | `bool` | `true` | no | +| [create\_storage\_credential](#input\_create\_storage\_credential) | When true, create a storage credential backed by the access connector. When false, use existing\_credential\_name. | `bool` | `true` | no | +| [credential\_grants](#input\_credential\_grants) | UC grants on the storage credential. Defaults to empty (owner-only). |
list(object({
principal = string
privileges = list(string)
}))
| `[]` | no | +| [existing\_credential\_name](#input\_existing\_credential\_name) | Name of an existing storage credential to reuse when create\_storage\_credential is false | `string` | `""` | no | +| [force\_destroy](#input\_force\_destroy) | Force destroy UC objects even if dependents exist | `bool` | `true` | no | +| [location\_grants](#input\_location\_grants) | UC grants applied to every external location.
Recommended privileges for data engineers: BROWSE, READ\_FILES, WRITE\_FILES,
CREATE\_EXTERNAL\_TABLE, CREATE\_EXTERNAL\_VOLUME. |
list(object({
principal = string
privileges = list(string)
}))
| `[]` | no | +| [storage\_credential\_name](#input\_storage\_credential\_name) | Name for the created storage credential. Defaults to "-storage-credential". | `string` | `""` | no | +| [subscription\_id](#input\_subscription\_id) | Azure subscription ID for managed AQS file-event configuration. Defaults to the current azurerm client subscription when empty. | `string` | `""` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [azure\_rbac\_roles](#output\_azure\_rbac\_roles) | Azure RBAC roles assigned to the access connector managed identity when assign\_azure\_rbac is true | +| [external\_location\_ids](#output\_external\_location\_ids) | Map of external location name to ID | +| [external\_location\_names](#output\_external\_location\_names) | Names of the created external locations | +| [external\_location\_urls](#output\_external\_location\_urls) | Map of external location name to URL | +| [storage\_credential\_id](#output\_storage\_credential\_id) | ID of the created storage credential (null when reusing an existing credential) | +| [storage\_credential\_name](#output\_storage\_credential\_name) | Name of the storage credential used by the external locations | + diff --git a/modules/adb-uc-external-location-file-events/azure_rbac.tf b/modules/adb-uc-external-location-file-events/azure_rbac.tf new file mode 100644 index 0000000..36e7d09 --- /dev/null +++ b/modules/adb-uc-external-location-file-events/azure_rbac.tf @@ -0,0 +1,41 @@ +# Documented Azure RBAC for automatic managed file events on ADLS Gen2. +# See: https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/cloud-storage/manage-external-locations +# +# Storage account: +# - Storage Blob Data Contributor|Reader — data plane access +# - Storage Queue Data Contributor — subscribe to file-change notifications +# - Storage Account Contributor — let Databricks auto-create the queue / routing +# Resource group: +# - EventGrid EventSubscription Contributor — let Databricks auto-create Event Grid subscriptions + +resource "azurerm_role_assignment" "blob_data" { + count = var.assign_azure_rbac ? 1 : 0 + + scope = var.storage_account_id + role_definition_name = local.blob_data_role_name + principal_id = local.access_connector_principal_id +} + +resource "azurerm_role_assignment" "queue_data" { + count = var.assign_azure_rbac ? 1 : 0 + + scope = var.storage_account_id + role_definition_name = "Storage Queue Data Contributor" + principal_id = local.access_connector_principal_id +} + +resource "azurerm_role_assignment" "storage_account_contributor" { + count = var.assign_azure_rbac ? 1 : 0 + + scope = var.storage_account_id + role_definition_name = "Storage Account Contributor" + principal_id = local.access_connector_principal_id +} + +resource "azurerm_role_assignment" "eventgrid_subscription" { + count = var.assign_azure_rbac ? 1 : 0 + + scope = "/subscriptions/${local.subscription_id}/resourceGroups/${var.resource_group_name}" + role_definition_name = "EventGrid EventSubscription Contributor" + principal_id = local.access_connector_principal_id +} diff --git a/modules/adb-uc-external-location-file-events/main.tf b/modules/adb-uc-external-location-file-events/main.tf new file mode 100644 index 0000000..48b9410 --- /dev/null +++ b/modules/adb-uc-external-location-file-events/main.tf @@ -0,0 +1,96 @@ +data "azurerm_client_config" "current" {} + +data "azurerm_databricks_access_connector" "this" { + name = local.access_connector_name + resource_group_name = local.access_connector_rg +} + +locals { + subscription_id = var.subscription_id != "" ? var.subscription_id : data.azurerm_client_config.current.subscription_id + + # Parse /subscriptions/.../resourceGroups//providers/Microsoft.Databricks/accessConnectors/ + access_connector_parts = split("/", var.access_connector_id) + access_connector_name = element(local.access_connector_parts, length(local.access_connector_parts) - 1) + access_connector_rg = element(local.access_connector_parts, index(local.access_connector_parts, "resourceGroups") + 1) + + storage_credential_name = var.storage_credential_name != "" ? var.storage_credential_name : "${var.name_prefix}-storage-credential" + + credential_name = var.create_storage_credential ? databricks_storage_credential.this[0].name : var.existing_credential_name + + # Data-plane role: Reader for read-only locations only when ALL locations are read-only. + all_read_only = alltrue([for loc in var.external_locations : loc.read_only]) + blob_data_role_name = local.all_read_only ? "Storage Blob Data Reader" : "Storage Blob Data Contributor" + access_connector_principal_id = data.azurerm_databricks_access_connector.this.identity[0].principal_id +} + +resource "databricks_storage_credential" "this" { + count = var.create_storage_credential ? 1 : 0 + + name = local.storage_credential_name + + azure_managed_identity { + access_connector_id = var.access_connector_id + } + + comment = "Managed identity credential for external locations with file events. Managed by Terraform." + force_destroy = var.force_destroy + force_update = var.force_destroy +} + +resource "databricks_external_location" "this" { + for_each = { for loc in var.external_locations : loc.name => loc } + + name = each.value.name + url = each.value.url + credential_name = local.credential_name + comment = each.value.comment + read_only = each.value.read_only + force_destroy = var.force_destroy + + # Automatic managed Azure Queue Storage file events. + # Requires the Azure RBAC roles assigned in azure_rbac.tf. + # Docs: https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/cloud-storage/manage-external-locations + enable_file_events = true + file_event_queue { + managed_aqs { + resource_group = var.resource_group_name + subscription_id = local.subscription_id + } + } + + depends_on = [ + databricks_storage_credential.this, + azurerm_role_assignment.blob_data, + azurerm_role_assignment.queue_data, + azurerm_role_assignment.storage_account_contributor, + azurerm_role_assignment.eventgrid_subscription, + ] +} + +resource "databricks_grants" "credential" { + count = var.create_storage_credential && length(var.credential_grants) > 0 ? 1 : 0 + + storage_credential = databricks_storage_credential.this[0].id + + dynamic "grant" { + for_each = var.credential_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} + +resource "databricks_grants" "location" { + for_each = length(var.location_grants) > 0 ? databricks_external_location.this : {} + + external_location = each.value.id + + dynamic "grant" { + for_each = var.location_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} diff --git a/modules/adb-uc-external-location-file-events/outputs.tf b/modules/adb-uc-external-location-file-events/outputs.tf new file mode 100644 index 0000000..57839af --- /dev/null +++ b/modules/adb-uc-external-location-file-events/outputs.tf @@ -0,0 +1,38 @@ +output "storage_credential_name" { + description = "Name of the storage credential used by the external locations" + value = local.credential_name +} + +output "storage_credential_id" { + description = "ID of the created storage credential (null when reusing an existing credential)" + value = try(databricks_storage_credential.this[0].id, null) +} + +output "external_location_names" { + description = "Names of the created external locations" + value = [for loc in databricks_external_location.this : loc.name] +} + +output "external_location_ids" { + description = "Map of external location name to ID" + value = { for k, v in databricks_external_location.this : k => v.id } +} + +output "external_location_urls" { + description = "Map of external location name to URL" + value = { for k, v in databricks_external_location.this : k => v.url } +} + +output "azure_rbac_roles" { + description = "Azure RBAC roles assigned to the access connector managed identity when assign_azure_rbac is true" + value = var.assign_azure_rbac ? { + storage_account = [ + local.blob_data_role_name, + "Storage Queue Data Contributor", + "Storage Account Contributor", + ] + resource_group = [ + "EventGrid EventSubscription Contributor", + ] + } : null +} diff --git a/modules/adb-uc-external-location-file-events/providers.tf b/modules/adb-uc-external-location-file-events/providers.tf new file mode 100644 index 0000000..f5d7f44 --- /dev/null +++ b/modules/adb-uc-external-location-file-events/providers.tf @@ -0,0 +1,13 @@ +# This module expects: +# - a workspace-level Databricks provider (authenticated to a UC-enabled workspace) +# - an AzureRM provider with permission to assign RBAC on the storage account / RG +# +# Example: +# +# provider "databricks" { +# host = "https://adb-xxxx.azuredatabricks.net" +# } +# +# provider "azurerm" { +# features {} +# } diff --git a/modules/adb-uc-external-location-file-events/variables.tf b/modules/adb-uc-external-location-file-events/variables.tf new file mode 100644 index 0000000..8b7cc81 --- /dev/null +++ b/modules/adb-uc-external-location-file-events/variables.tf @@ -0,0 +1,97 @@ +variable "name_prefix" { + type = string + description = "Prefix used for storage credential and external location names" +} + +variable "access_connector_id" { + type = string + description = "Azure resource ID of the Access Connector for Azure Databricks (managed identity used by the storage credential)" +} + +variable "storage_account_id" { + type = string + description = "Azure resource ID of the ADLS Gen2 storage account that backs the external location(s)" +} + +variable "resource_group_name" { + type = string + description = "Resource group that contains the storage account (used for managed AQS and Event Grid RBAC)" +} + +variable "subscription_id" { + type = string + description = "Azure subscription ID for managed AQS file-event configuration. Defaults to the current azurerm client subscription when empty." + default = "" +} + +variable "external_locations" { + type = list(object({ + name = string + url = string + comment = optional(string, "Managed by Terraform") + read_only = optional(bool, false) + })) + description = "External locations to create. Each location gets managed AQS file events enabled." + + validation { + condition = length(var.external_locations) > 0 + error_message = "At least one external location is required." + } +} + +variable "create_storage_credential" { + type = bool + description = "When true, create a storage credential backed by the access connector. When false, use existing_credential_name." + default = true +} + +variable "existing_credential_name" { + type = string + description = "Name of an existing storage credential to reuse when create_storage_credential is false" + default = "" + + validation { + condition = var.create_storage_credential || var.existing_credential_name != "" + error_message = "existing_credential_name must be set when create_storage_credential is false." + } +} + +variable "storage_credential_name" { + type = string + description = "Name for the created storage credential. Defaults to \"-storage-credential\"." + default = "" +} + +variable "assign_azure_rbac" { + type = bool + description = "Assign the documented Azure RBAC roles required for data access and automatic managed file events" + default = true +} + +variable "force_destroy" { + type = bool + description = "Force destroy UC objects even if dependents exist" + default = true +} + +variable "credential_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = "UC grants on the storage credential. Defaults to empty (owner-only)." + default = [] +} + +variable "location_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = <<-EOT + UC grants applied to every external location. + Recommended privileges for data engineers: BROWSE, READ_FILES, WRITE_FILES, + CREATE_EXTERNAL_TABLE, CREATE_EXTERNAL_VOLUME. + EOT + default = [] +} diff --git a/modules/adb-uc-external-location-file-events/versions.tf b/modules/adb-uc-external-location-file-events/versions.tf new file mode 100644 index 0000000..fe80276 --- /dev/null +++ b/modules/adb-uc-external-location-file-events/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.0.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.50.0" + } + } +}