53 lines
1.1 KiB
HCL
53 lines
1.1 KiB
HCL
terraform {
|
|
required_providers {
|
|
databricks = {
|
|
source = "databrickslabs/databricks"
|
|
version = ">= 0.2.7"
|
|
}
|
|
}
|
|
}
|
|
|
|
provider "databricks" {
|
|
alias = "mws"
|
|
}
|
|
|
|
resource "databricks_user" "me" {
|
|
|
|
provider = databricks.mws
|
|
|
|
user_name = var.user_email
|
|
display_name = var.user_id
|
|
}
|
|
|
|
resource "databricks_token" "pat" {
|
|
provider = databricks.mws
|
|
|
|
comment = "qmi"
|
|
}
|
|
|
|
resource "databricks_cluster" "dbrickscluster" {
|
|
provider = databricks.mws
|
|
|
|
cluster_name = "cluster-${var.cluster_name}"
|
|
spark_version = "7.3.x-scala2.12"
|
|
node_type_id = "Standard_DS3_v2"
|
|
|
|
spark_conf = {
|
|
"spark.hadoop.hive.server2.enable.doAs": false,
|
|
"spark.databricks.delta.preview.enabled": true,
|
|
"spark.databricks.repl.allowedLanguages": "sql,python,r",
|
|
"spark.databricks.cluster.profile": "serverless",
|
|
"fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
|
|
}
|
|
|
|
spark_env_vars = {
|
|
"PYSPARK_PYTHON": "/databricks/python3/bin/python3"
|
|
}
|
|
|
|
autoscale {
|
|
min_workers = 2
|
|
max_workers = 8
|
|
}
|
|
|
|
autotermination_minutes = 120
|
|
} |