new config for cluster

compose-install/scripts/di-compose-installQMICertificate.ps1

@@ -6,6 +6,11 @@ Param(
 
 try {
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

databases/dbricks/main.tf

@@ -5,11 +5,16 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 1.6.0"
+      version = ">= 1.13.0"
     }
   }
 }
 
+
+locals {
+  cluster_nameUpper = upper(var.cluster_name)
+}
+
 /*resource "databricks_workspace_conf" "this" {
   
   custom_config = {
@@ -88,16 +93,42 @@ resource "databricks_group_member" "i-am-admin-aor" {
   member_id = databricks_user.aor[0].id
 }
 
+resource "databricks_secret_scope" "scopeapp" {
+
+    name = "${var.cluster_name}scope"
+}
+resource "databricks_secret" "publishing_api" {
+
+    key = "accesskey"
+    string_value = var.app_reg_secret
+    scope = databricks_secret_scope.scopeapp.name
+}
+
 resource "databricks_sql_endpoint" "sqlep" {
 
   count = (var.sku == "premium")? 1 : 0
 
-  name             = "qmi-sqlwh-${databricks_user.me.id}"
-  cluster_size     = "Small"
+  name             = "${local.cluster_nameUpper} SQL warehouse"
+  cluster_size     = var.cluster_size
   max_num_clusters = 1
 
 }
 
+resource "databricks_sql_global_config" "this" {
+
+  security_policy      = "DATA_ACCESS_CONTROL"
+  data_access_config = {
+    "spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
+    "spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
+    "spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
+    "spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
+  }
+
+  enable_serverless_compute = false
+  sql_config_params = {}
+}
+
 
 
 resource "databricks_token" "pat" {  
@@ -122,9 +153,12 @@ resource "databricks_cluster" "dbrickscluster" {
     "spark.databricks.delta.preview.enabled": true,
     "spark.databricks.repl.allowedLanguages": "sql,python,r",
     "spark.databricks.cluster.profile": "serverless",
-    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
+    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
+    "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
+    "spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
   } 
 
+
   spark_env_vars = {
       "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
   }
@@ -134,5 +168,5 @@ resource "databricks_cluster" "dbrickscluster" {
     max_workers = 4
   }
 
-  autotermination_minutes = 120
+  autotermination_minutes = 45
 }

databases/dbricks/varaibles.tf

@@ -24,4 +24,16 @@ variable "node_type_id" {
 
 variable "sku" {
     default = "standard"
+}
+
+variable "app_reg_id" {
+  default = null
+}
+
+variable "app_reg_secret" {
+  default = null
+}
+
+variable "cluster_size" {
+  default = "Small"
 }

linux-common/centos/common/extract-certs.sh

@@ -3,6 +3,9 @@
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD

linux-common/centos/common/save-cert.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+
+echo $1 > $BASEDIR/qmicerts/myserver.crt
+
+echo $2 > $BASEDIR/qmicerts/myserver.key
+
+

linux-common/centos/common/update.sh

@@ -0,0 +1,2 @@
+echo 'Updating OS. This will take a around 10 minutes'
+yum -y --quiet update

linux-common/main.tf

@@ -13,15 +13,22 @@ data "azurerm_key_vault_secret" "cert_password" {
   key_vault_id = var.key_vault_id
 }
 
+#data "azurerm_key_vault_certificate_data" "cert-data" {
+#  name         = "star-qmi-qlikpoc-com-cert"
+#  key_vault_id = var.key_vault_id
+#}
+
+
 locals {
   falcon_id     = nonsensitive(data.azurerm_key_vault_secret.cid.value)
   tenable_key   = nonsensitive(data.azurerm_key_vault_secret.tenable-key.value)
   cert_password = nonsensitive(data.azurerm_key_vault_secret.cert_password.value)
-}
+  #cert_pem =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.pem)
+  #cert_key =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.key)
+} 
 
-resource "null_resource" "post-linux-vm" {
-    
-    provisioner "file" {
+resource "null_resource" "files" {
+  provisioner "file" {
         connection {
             type     = "ssh"
             host     = var.private_ip_address
@@ -33,6 +40,13 @@ resource "null_resource" "post-linux-vm" {
         source      = "${path.module}/${var.os_type}/common"
         destination = "/home/${var.admin_username}"
     }
+}
+
+resource "null_resource" "post-linux-vm" {
+    
+    depends_on = [
+      null_resource.files
+    ]
 
     provisioner "remote-exec" {
         connection {
@@ -50,6 +64,34 @@ resource "null_resource" "post-linux-vm" {
             "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
             "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
+            #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
         ]
     }
-}
+}
+
+resource "null_resource" "update" {
+
+  count = var.update? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.post-linux-vm
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "60s"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/update.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -3,6 +3,9 @@
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD

linux-common/ubuntu/common/falcon.sh

@@ -9,7 +9,7 @@ binary="falcon-sensor_5.17.0-8103_amd64.deb"
 echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
 
-sudo apt -qq -y update
+#sudo apt -qq -y update
 sudo apt install libnl-3-200 libnl-genl-3-200 -qq -y
 
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary

linux-common/ubuntu/common/update.sh

@@ -0,0 +1,3 @@
+echo 'Updating OS. This will take a around 10 minutes'
+apt -qq -y update
+apt --fix-broken -qq -y upgrade

linux-common/variables.tf

@@ -16,3 +16,8 @@ variable "key_vault_id" {
   default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
 }
 
+variable "update" {
+  type = bool
+  default = true
+}
+

replicate-install/scripts/di-replicate-installQMICertificate.ps1

@@ -7,6 +7,11 @@ Param(
 try {
 
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+    
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

vm-qdc/main.tf

@@ -123,5 +123,6 @@ module "linux-common" {
   private_ip_address  = module.qmi-nic.private_ip_address
   admin_username      = local.admin_username
   admin_password      = local.admin_password  
+  update              = false
   
 }

vm-qdc/scripts/feb2022/1-qdc-nextgen-xml.sh

@@ -13,6 +13,7 @@ if [[ $2 == *"4.9.0"* ]]; then
  sudo cp /home/qmi/scripts/feb2022/tomcat.service /etc/systemd/system/
 fi
 
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O /home/qmi/scripts/wildcard_qmi_qlik-poc_com.pfx
 sudo cp /home/qmi/scripts/wildcard_qmi_qlik-poc_com.pfx /usr/local/qdc
 
 if [[ $2 == *"4.13.0"* ]]; then

vm-qdc/scripts/feb2022/2-core_env_setup.sh

@@ -7,6 +7,9 @@ TOMCAT_HOME=$BASE/$TOMCAT
 BASEDIR=$(dirname "$0")
 
 echo "Replacing server.xml for Tomcat - Stopping Tomcat"
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O /home/qmi/scripts/wildcard_qmi_qlik-poc_com.pfx
+sudo cp /home/qmi/scripts/wildcard_qmi_qlik-poc_com.pfx /usr/local/qdc
+
 sudo cp /home/qmi/scripts/wildcard_qmi_qlik-poc_com.pfx /usr/local/qdc
 sudo systemctl stop tomcat.service
 mv $TOMCAT_HOME/conf/server.xml $TOMCAT_HOME/conf/server.xml_backup

vm-qdc/scripts/feb2022/nextgen-xml_server.xml

@@ -99,7 +99,7 @@
     clientAuth="false" sslProtocol="TLS"
     keystoreFile="/usr/local/qdc/wildcard_qmi_qlik-poc_com.pfx"
     keystoreType="PKCS12"
-    keystorePass="QJ]5M$>m,X)t=_:BNNt:mwFks7NqocSMbR@/zA" />   
+    keystorePass="8@afBR2AwKMb@4Rj" />   
      
     <!--
     <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"

vm-qs-alerting/scripts/qs-post-cfg.ps1

@@ -117,6 +117,11 @@ While( (Connect-Qlik $($env:COMPUTERNAME) -TrustAllCerts -UseDefaultCredentials
 
 ### Install qlik-poc_com certificate
 Write-Log -Message "Install 'qmi.qlik-poc.com' certificate, set thumbsprint and whitelist domain in QS central virtual proxy"
+
+$ProgressPreference = 'SilentlyContinue'
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "C:\provision\wildcard_qmi_qlik-poc_com.pfx") 
+
 try {
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert=Import-PfxCertificate -FilePath C:/provision/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd

vm-qs/main.tf

@@ -129,7 +129,7 @@ resource "azurerm_windows_virtual_machine" "vm" {
 }
 
 
-/*resource "null_resource" "vm-qs-post" {
+resource "null_resource" "vm-qs-post" {
 
   depends_on = [
     azurerm_windows_virtual_machine.vm
@@ -210,5 +210,4 @@ module "win-common" {
   private_ip_address  = module.qmi-nic.private_ip_address
   admin_username      = local.admin_username
   admin_password      = local.admin_password  
-}
-*/
+}

vm-qs/scripts/qs-post-cfg.ps1

@@ -243,6 +243,13 @@ restartQse
 
 ### Install qlik-poc_com certificate
 Write-Log -Message "Install qmi_qlik-poc_com certificate on Windows"
+
+$ProgressPreference = 'SilentlyContinue'
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "C:\provision\wildcard_qmi_qlik-poc_com.pfx") 
+
+write-log -Message "Cert password is: $CertPwd"
+
 try {
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert=Import-PfxCertificate -FilePath C:/provision/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd