222 lines
6.2 KiB
HCL
222 lines
6.2 KiB
HCL
terraform {
|
|
|
|
required_version = ">= 0.14"
|
|
|
|
required_providers {
|
|
databricks = {
|
|
source = "databricks/databricks"
|
|
version = ">= 1.47.0"
|
|
configuration_aliases = [ databricks.main, databricks.accounts ]
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
|
|
locals {
|
|
cluster_nameUpper = upper(var.cluster_name)
|
|
}
|
|
|
|
locals {
|
|
resource_regex = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)"
|
|
subscription_id = regex(local.resource_regex, var.databricks_resource_id)[0]
|
|
resource_group = regex(local.resource_regex, var.databricks_resource_id)[1]
|
|
databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2]
|
|
tenant_id = data.azurerm_client_config.current.tenant_id
|
|
databricks_workspace_host = data.azurerm_databricks_workspace.this.workspace_url
|
|
databricks_workspace_id = data.azurerm_databricks_workspace.this.workspace_id
|
|
prefix = replace(replace(lower(data.azurerm_resource_group.this.name), "rg", ""), "-", "")
|
|
}
|
|
|
|
data "azurerm_resource_group" "this" {
|
|
name = local.resource_group
|
|
}
|
|
|
|
data "azurerm_client_config" "current" {
|
|
}
|
|
|
|
data "azurerm_databricks_workspace" "this" {
|
|
name = local.databricks_workspace_name
|
|
resource_group_name = local.resource_group
|
|
}
|
|
|
|
resource "databricks_metastore" "this" {
|
|
provider = databricks.accounts
|
|
name = "primary"
|
|
force_destroy = true
|
|
region = data.azurerm_resource_group.this.location
|
|
}
|
|
|
|
resource "databricks_metastore_assignment" "this" {
|
|
provider = databricks.accounts
|
|
workspace_id = local.databricks_workspace_id
|
|
metastore_id = databricks_metastore.this.id
|
|
default_catalog_name = "hive_metastore"
|
|
}
|
|
|
|
/*resource "databricks_workspace_conf" "this" {
|
|
|
|
custom_config = {
|
|
"enableIpAccessLists" : true
|
|
}
|
|
}
|
|
|
|
resource "databricks_ip_access_list" "allowed-list" {
|
|
|
|
label = "allow_in"
|
|
list_type = "ALLOW"
|
|
ip_addresses = [
|
|
"52.249.189.38/32",
|
|
"13.67.39.86/32",
|
|
"20.67.110.207/32",
|
|
"14.98.59.168/29",
|
|
"182.74.33.8/29",
|
|
"188.65.156.32/28",
|
|
"212.73.252.96/29",
|
|
"194.90.96.176/29",
|
|
"213.57.84.160/29",
|
|
"4.4.97.104/29",
|
|
"206.196.17.32/27",
|
|
#QCS
|
|
"18.205.71.36/32",
|
|
"18.232.32.199/32",
|
|
"34.237.68.254/32",
|
|
"34.247.21.179/32",
|
|
"52.31.212.214/32",
|
|
"54.154.95.18/32",
|
|
"13.210.43.241/32",
|
|
"13.236.104.42/32",
|
|
"13.236.206.172/32",
|
|
"18.138.163.172/32",
|
|
"18.142.157.182/32",
|
|
"54.179.13.251/32"
|
|
]
|
|
|
|
}*/
|
|
|
|
|
|
data "databricks_group" "admins" {
|
|
display_name = "admins"
|
|
}
|
|
|
|
|
|
resource "databricks_user" "aor" {
|
|
provider = databricks.main
|
|
count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
|
|
|
|
allow_cluster_create = true
|
|
databricks_sql_access = true
|
|
|
|
user_name = "aor@qlik.com"
|
|
display_name = "Manuel Romero"
|
|
}
|
|
|
|
resource "databricks_user" "me" {
|
|
provider = databricks.main
|
|
allow_cluster_create = true
|
|
databricks_sql_access = true
|
|
|
|
user_name = var.user_email
|
|
display_name = var.user_id
|
|
}
|
|
|
|
|
|
resource "databricks_group_member" "i-am-admin" {
|
|
provider = databricks.main
|
|
group_id = data.databricks_group.admins.id
|
|
member_id = databricks_user.me.id
|
|
}
|
|
|
|
|
|
resource "databricks_group_member" "i-am-admin-aor" {
|
|
provider = databricks.main
|
|
count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
|
|
group_id = data.databricks_group.admins.id
|
|
member_id = databricks_user.aor[0].id
|
|
}
|
|
|
|
resource "databricks_secret_scope" "scopeapp" {
|
|
provider = databricks.main
|
|
count = (var.sku == "premium")? 1 : 0
|
|
|
|
name = "${var.cluster_name}scope"
|
|
}
|
|
resource "databricks_secret" "publishing_api" {
|
|
provider = databricks.main
|
|
count = (var.sku == "premium")? 1 : 0
|
|
|
|
key = "accesskey"
|
|
string_value = var.app_reg_secret
|
|
scope = databricks_secret_scope.scopeapp[0].name
|
|
}
|
|
|
|
resource "databricks_sql_endpoint" "sqlep" {
|
|
provider = databricks.main
|
|
count = (var.sku == "premium")? 1 : 0
|
|
|
|
name = "${local.cluster_nameUpper} SQL warehouse"
|
|
cluster_size = var.cluster_size
|
|
max_num_clusters = 1
|
|
|
|
}
|
|
|
|
resource "databricks_sql_global_config" "this" {
|
|
provider = databricks.main
|
|
count = (var.sku == "premium")? 1 : 0
|
|
|
|
security_policy = "DATA_ACCESS_CONTROL"
|
|
data_access_config = {
|
|
"spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
|
|
"spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
|
|
"spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
|
|
"spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
|
|
"spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
|
|
}
|
|
|
|
enable_serverless_compute = false
|
|
sql_config_params = {}
|
|
}
|
|
|
|
|
|
|
|
resource "databricks_token" "pat" {
|
|
provider = databricks.main
|
|
depends_on = [
|
|
databricks_user.me,
|
|
databricks_user.aor
|
|
]
|
|
comment = "qmi"
|
|
lifetime_seconds = 8640000
|
|
}
|
|
|
|
resource "databricks_cluster" "dbrickscluster" {
|
|
provider = databricks.main
|
|
count = (var.cluster_name != null)? 1 : 0
|
|
|
|
|
|
cluster_name = "cluster-${var.cluster_name}"
|
|
spark_version = var.spark_version
|
|
node_type_id = var.node_type_id
|
|
|
|
spark_conf = {
|
|
"spark.hadoop.hive.server2.enable.doAs": false,
|
|
"spark.databricks.delta.preview.enabled": true,
|
|
"spark.databricks.repl.allowedLanguages": "sql,python,r",
|
|
"spark.databricks.cluster.profile": "serverless",
|
|
"fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
|
|
"spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
|
|
"spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
|
|
}
|
|
|
|
|
|
spark_env_vars = {
|
|
"PYSPARK_PYTHON": "/databricks/python3/bin/python3"
|
|
}
|
|
|
|
autoscale {
|
|
min_workers = 1
|
|
max_workers = 4
|
|
}
|
|
|
|
autotermination_minutes = 45
|
|
} |