new config for cluster

adls/main.tf

@@ -0,0 +1,67 @@
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 5
+}
+
+resource "azurerm_storage_account" "saccount" {
+
+  name = var.storage_account_name != null? var.storage_account_name : "qmiadlsgen2${random_id.randomMachineId.hex}"
+
+  resource_group_name       = var.resource_group_name
+  location                  = var.location
+  account_kind              = "StorageV2"
+  account_tier              = "Standard"
+  access_tier               = "Hot"
+  account_replication_type  = "RAGRS"
+  is_hns_enabled            = "true"
+
+  tags = var.tags
+}
+
+resource "azurerm_storage_container" "scontainer" {
+
+  name                  = var.container_name != null? var.container_name : "qmicontainer"
+  storage_account_name  = azurerm_storage_account.saccount.name
+  #container_access_type = "container"
+}
+
+resource "azurerm_role_assignment" "data-contributor-role" {
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Contributor"
+  principal_id         = var.tpm_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-role2" {  
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.tpm_app_registration_principal_id
+}
+
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp1" {
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp2" { 
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+
+###### EXTRA ASSIGN ROLE #######
+
+resource "azurerm_role_assignment" "machine_role_assignment" { 
+
+  count = var.principal_id_storage_blob_contributor != null? 1 : 0
+
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.principal_id_storage_blob_contributor
+}

adls/outputs.tf

@@ -0,0 +1,35 @@
+output "StorageAccount-Scope" {
+  value = azurerm_storage_account.saccount.id
+}
+
+output "StorageAccount-AccessKey" {
+  value = nonsensitive(azurerm_storage_account.saccount.primary_access_key)
+}
+
+output "StorageAccount-ConnectionString" {
+  value = nonsensitive(azurerm_storage_account.saccount.primary_connection_string)
+}
+
+output "StorageAccount-Name" {
+  value = azurerm_storage_account.saccount.name
+}
+
+output "StorageAccount-ContainerName" {
+  value = azurerm_storage_container.scontainer.name
+}
+
+output "StorageAccount-ContainerId" {
+  value = azurerm_storage_container.scontainer.id
+}
+
+output "Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}

adls/variable.tf

@@ -0,0 +1,38 @@
+variable "resource_group_name" {
+}
+
+variable "location" {
+  description = "The Azure Region in which the resources in this example should exist"
+  default = "East US"
+}
+
+variable "storage_account_name" {
+  default = null
+}
+
+variable "container_name" {
+  default = null
+}
+
+variable "tags" {
+    default = null
+}
+
+variable "tpm_app_registration_principal_id" {
+    description = "tpm"
+    default = "163a72e3-8ce3-4e33-baae-954383f87e3e"
+}
+
+variable "dbricks_app_registration_principal_id" {
+    description = "databricks-qmi"
+    default = "efeee17c-d2b3-4e7c-a163-9995b7d281e2"
+}
+
+variable "dbricks_app_registration_application_id" {
+    description = "databricks-qmi"
+    default = "9ccb0d99-3bba-4695-aa47-df77bf512084"
+}
+
+variable "principal_id_storage_blob_contributor" {
+  default = null
+}

appgateways/appgw-443-only/main.tf

@@ -151,6 +151,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
   
 
@@ -167,6 +168,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 2
   }
 
 }

appgateways/appgw-443-only/output.tf

@@ -11,5 +11,5 @@ output "appgw_public_ip" {
 }
 
 output "appgw_backend_address_pool_0_id" {
-  value = azurerm_application_gateway.qmi-app-gw.backend_address_pool[0].id
+  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).0.id
 }

appgateways/appgw-e2e/main.tf

@@ -416,6 +416,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -426,6 +427,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 2
   }
 
   # NPrinting routing rules
@@ -435,6 +437,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 3
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -442,6 +445,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 4
   }
 
 
@@ -451,6 +455,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4552
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4552
+    priority                   = 5
   }
 
 
@@ -461,6 +466,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 6
   }
   
 
@@ -477,6 +483,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 7
   }
 
   # Redirect QDC 8080 to 8443
@@ -492,6 +499,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_8080
     redirect_configuration_name = local.redirect_configuration_8080
+    priority                    = 8
   }
 
 }

appgateways/appgw-e2e/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qdc-qs/main.tf

@@ -357,6 +357,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -367,6 +368,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 2
   }
 
   # NPrinting routing rules
@@ -376,6 +378,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 3
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -383,6 +386,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 4
   }
 
   # QIB routing rule
@@ -392,6 +396,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 5
   }
   
 
@@ -408,6 +413,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                   = 6
   }
 
 }

appgateways/appgw-qdc-qs/outputs.tf

@@ -10,10 +10,14 @@ output "appgw_public_ip" {
   value = azurerm_public_ip.appgw-ip.ip_address
 }
 
+locals {
+  backend_pools = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool)
+}
+
 output "appgw_backend_address_pool_0_id" {
-  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).0.id
+  value = local.backend_pools[index(local.backend_pools.*.name, "${var.appgw_hostname}-qs-bp")].id
 }
 
 output "appgw_backend_address_pool_1_id" {
-  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).1.id
+  value = local.backend_pools[index(local.backend_pools.*.name, "${var.appgw_hostname}-qdc-bp")].id
 }

appgateways/appgw-qdc-qs/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qdc/main.tf

@@ -127,6 +127,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 1
   }
 
 }

appgateways/appgw-qdc/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qs/main.tf

@@ -338,6 +338,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -348,6 +349,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 2
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -355,6 +357,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 3
   }
 
   # QIB routing rule (needs QIB using qmi.qlik-poc.com certs)
@@ -364,6 +367,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 4
   }
 
   # Qlik Alerting routing rule (needs Qlik Alerting using qmi.qlik-poc.com certs)
@@ -373,6 +377,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4552
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4552
+    priority                   = 5
   }
   
 
@@ -389,6 +394,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 6
   }
 
 }

appgateways/appgw-qs/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

aws-credentials/main.tf

@@ -1,19 +0,0 @@
-terraform {
-    required_version = ">= 0.13"
-}
-
-provider "azurerm" {    
-    features {}
-    subscription_id = var.subscription_id
-    use_msi = true
-}
-
-data "azurerm_key_vault_secret" "qmi-aws-access-key" {
-  name         = "qmi-aws-access-key"
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "qmi-aws-access-secret" {
-  name         = "qmi-aws-access-secret"
-  key_vault_id = var.key_vault_id
-}

aws-credentials/output.tf

@@ -1,7 +0,0 @@
-output "qmi-aws-access-key" {
-    value = data.azurerm_key_vault_secret.qmi-aws-access-key.value
-}
-
-output "qmi-aws-access-secret" {
-    value = data.azurerm_key_vault_secret.qmi-aws-access-secret.value
-}

aws-credentials/variables.tf

@@ -1,7 +0,0 @@
-variable "subscription_id" {
-    default = "62ebff8f-c40b-41be-9239-252d6c0c8ad9"
-}
-
-variable "key_vault_id" {
-    default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
-}

compose-install/main.tf

@@ -1,20 +1,9 @@
-data "azurerm_key_vault_secret" "compose-license" {
-  name         = "compose-license"
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "c4dw" {
-  name         = "compose-datawarehouse-license"
-  key_vault_id = var.key_vault_id
-}
-
 data "azurerm_key_vault_secret" "cert-password" {
   name         = "star-qmi-qlikpoc-com-password"
   key_vault_id = var.key_vault_id
 }
 
 locals {
-  composeLicense = (var.c_version == "c4dw")? nonsensitive(data.azurerm_key_vault_secret.c4dw.value) : nonsensitive(data.azurerm_key_vault_secret.compose-license.value)
   cert_password = nonsensitive(data.azurerm_key_vault_secret.cert-password.value)
 }
 
@@ -49,8 +38,8 @@ resource "null_resource" "install" {
     inline = [
       "powershell.exe -File C:/provision/compose-install/prep-files.ps1",
       "powershell.exe -File C:/provision/compose-install/di-compose-getBinary.ps1 -url ${var.download_url}",
-      "powershell.exe -File C:/provision/compose-install/di-compose-install.ps1 -url ${var.download_url} -version ${var.c_version}",
+      "powershell.exe -File C:/provision/compose-install/di-compose-install.ps1 -url ${var.download_url}",
-      "powershell.exe -File C:/provision/compose-install/di-compose-setlicense.ps1 -composeLicense \"${local.composeLicense}\" -version ${var.c_version}",
+      "powershell.exe -File C:/provision/compose-install/di-compose-setlicense.ps1 -version ${var.c_version}",
       "powershell.exe -File C:/provision/compose-install/di-compose-installQMICertificate.ps1 -CertPwd \"${local.cert_password}\" -version ${var.c_version}",
     ]
 

compose-install/scripts/Compose_install.iss

@@ -1,25 +1,20 @@
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-DlgOrder]
 Dlg0={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0
 Count=5
-Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicense2Rtf-0
+Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0
 Dlg2={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0
 Dlg3={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0
 Dlg4={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcomeMaint-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-MessageBox-0]
-Result=6
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0]
 Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicense2Rtf-0]
+[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0]
 Result=1
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0]
 szDir=C:\Program Files\Qlik\Compose\
 Result=1
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0]
 Result=1
-
+[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

compose-install/scripts/Compose_install_2021.8.0.iss

@@ -1,20 +0,0 @@
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-DlgOrder]
-Dlg0={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0
-Count=5
-Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0
-Dlg2={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0
-Dlg3={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0
-Dlg4={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0]
-szDir=C:\Program Files\Qlik\Compose\
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0

compose-install/scripts/compose-license.txt

@@ -0,0 +1,21 @@
+{
+  "$type": "ComposeLicense",
+  "product": "QlikCompose",
+  "issued_to": "Attunity Americas",
+  "issued_by": "Attunity US",
+  "license_type": "EVALUATION",
+  "serial_no": "85008595",
+  "expiration_date": "2023-12-31",
+  "hosts": "",
+  "product_version": "2022.5",
+  "notes": "Qlik internal use only ",
+  "host_role": "",
+  "source_db_types": "",
+  "dwh_type": "*",
+  "dl_type": "*",
+  "number_of_dms": "0",
+  "number_of_developers": "0",
+  "managed_dwh_size": "0",
+  "issue_time": "12/15/2022 7:06:16 PM",
+  "signature": "iDk5NujmAcxcI80BSt4JaOEaSj35u0vzoRXvRcYXy4NpWJLFVvCRJ67LfVcA1WuLwHmarnYWYA+RWT0CyqpH1e4n3nZPaJAwBgnU5Z074rIUrHZi+Z5Hmgux2ptNqXJIr7S1JNc6+fBMHuG//hxYppf9Knmk+5cx5rafxNpCYUU="
+}

compose-install/scripts/di-compose-install.ps1

@@ -9,26 +9,36 @@ Intent: Install the selected version of Attunity Replicate
 #>
 
 Param(
-    [string] $url,
+    [string] $url
-    [string] $version = "gen2"
 )
 
 Import-Module $PSScriptRoot\q-helper.psm1 | Out-Null
 
 $fileName = $url.Substring($url.LastIndexOf("/") + 1)
 
+Write-Host "--- Starting di-compose-install.ps1"
+
 $bin = "$PSScriptRoot\binaries\Attunity"
 Write-Host "Binary Path $($bin)"
-Write-Host "Starting di-c4dw-install.ps1"
+
+
 $issFile = "Compose_install.iss"
 
-if ( $version -ne "gen2" ) {
+if ( $url -Match "2021.2" ) {
-    $issFile = "Compose_install_$version.iss" 
+	$issFile="Compose_install_2021.2.0.iss"
+}
+if ( $url -Match "Attunity_Compose_for_Data_Warehouses" ) {
+	$issFile="Compose_install_c4dw.iss"
 }
 
+Write-Host "Using ISS file: $issFile"
 
 Write-Host $bin\$fileName
 
+if (Test-Path "$($bin)\Compose_silent_x64_install.log") {
+	Remove-Item -Path "$($bin)\Compose_silent_x64_install.log" -Force
+}
+
 If ((Test-Path $bin\$fileName))
 {
 	Write-Host "Installing Attunity Compose from $($bin)\$($fileName)"
@@ -40,16 +50,16 @@ If ((Test-Path $bin\$fileName))
 	$exec = Get-ChildItem $bin\$fileNoExtension\*.exe | Select-Object -ExpandProperty Name
     #Write-Host "Exec: $($exec)"
 
-    $C4DWInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
+    $silentInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
 
-	Write-Host "Run Compose silent installation : $($C4DWInstall)"
+	Write-Host "Run Compose silent installation : $($silentInstall)"
-    Invoke-Expression $C4DWInstall
+    Invoke-Expression $silentInstall
     while (!(Test-Path "$($bin)\Compose_silent_x64_install.log")) { 
         Write-Host "[Waiting Compose to be installed] ..."
         Start-Sleep 2
     }
-	$C4DWResults = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
+	$resultLogs = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
-	$testResult = $C4DWResults.ResponseResult.ResultCode
+	$testResult = $resultLogs.ResponseResult.ResultCode
 	Write-Host "Installation return code : $($testResult)"
 
 }

compose-install/scripts/di-compose-installQMICertificate.ps1

@@ -6,6 +6,11 @@ Param(
 
 try {
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

compose-install/scripts/di-compose-setlicense.ps1

@@ -9,14 +9,10 @@ Intent: Setting the DBs
 #>
 
 Param(
-    [string] $composeLicense,
     [string] $version = 'gen2'
 )
 
-Write-Host "ComposeLicenseJSON"
+Write-Host "--- Setting Compose License"
-Write-Host $composeLicense
-
-Set-Content $PSScriptRoot\compose-license.txt $composeLicense
 
 # TODO set license
 $bin = "C:\Program Files\Qlik\Compose\bin\"
@@ -26,16 +22,18 @@ if ( $version -eq "c4dw" ) {
 $bin = $bin -replace ' ','` '
 
 $cmd = "$($bin)ComposeCli.exe connect"
-Write-Host "Connect to Compose: $($cmd)"
+Write-Host "--- Connect to Compose: $($cmd)"
 $cmd = "$($bin)ComposeCli.exe connect"
 Invoke-Expression $cmd
 
 Start-Sleep 5
 
-if ( $version -eq "2021.8.0" ) {
+if ( $version -eq "2022.2.0" -or $version -eq "c4dw") {
-    $cmd = "$($bin)ComposeCli.exe register_license --infile $PSScriptRoot\compose-license.txt"
+    # Old way
-} else {
     $cmd = "$($bin)ComposeCli.exe register_license --req @$PSScriptRoot\compose-license.txt"
+} else {
+    # New way
+    $cmd = "$($bin)ComposeCli.exe register_license --infile $PSScriptRoot\compose-license.txt"
 }
 Write-Host "Apply Compose License: $($cmd)"
 Invoke-Expression $cmd

databases/aws-kinesis/output.tf

@@ -23,5 +23,5 @@ output "iam_access_key" {
 }
 
 output "iam_access_secret" {
-  value = module.iam_user.this_iam_access_key_secret
+  value = nonsensitive(module.iam_user.this_iam_access_key_secret)
 }

databases/aws-rds/main.tf

@@ -1,11 +1,11 @@
 terraform {
 
-  required_version = ">= 0.13"
+  required_version = ">= 0.14"
 
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.49.0"
+      version = "3.74.0"
     }
   }
 }
@@ -18,7 +18,7 @@ locals {
   port                 = (var.engine == "oracle-se2") ? "1521" : (var.engine == "postgres") ? "5432" : (var.engine == "aurora-postgresql") ? "5432" : (var.engine == "sqlserver-ex") ? "1433" : "3306"
   name                 = (var.engine == "sqlserver-ex") ? null : (var.engine == "oracle-se2") ? "ora${local.provid5}" : "qmi${var.provision_id}"
   license              = (local.aurora == true) ? "general-public-license" : (var.engine == "mariadb") ? "general-public-license" : (var.engine == "postgres") ? "postgresql-license" : (var.engine == "mysql") ? "general-public-license" : "license-included"
-  engine_version       = (var.engine == "oracle-se2") ? "19.0.0.0.ru-2021-04.rur-2021-04.r1" : (var.engine == "postgres") ? "13.3" : (var.engine == "mysql") ? "8.0.25" : (var.engine == "aurora-postgresql") ? "12.6" : (var.engine == "aurora-mysql") ? "5.7.mysql_aurora.2.10.0" : (var.engine == "sqlserver-ex") ? "15.00.4073.23.v1" : "10.5"   #mariaDB
+  engine_version       = (var.engine == "oracle-se2") ? "19.0.0.0.ru-2021-04.rur-2021-04.r1" : (var.engine == "postgres") ? "13.3" : (var.engine == "mysql") ? "8.0.25" : (var.engine == "aurora-postgresql") ? "14.5" : (var.engine == "aurora-mysql") ? "5.7.mysql_aurora.2.10.0" : (var.engine == "sqlserver-ex") ? "15.00.4073.23.v1" : "10.5"   #mariaDB
   major_engine_version = (var.engine == "oracle-se2") ? "19" : (var.engine == "postgres") ? "13" : (var.engine == "mysql") ? "8.0" : (var.engine == "aurora-postgresql") ? "12" : (var.engine == "aurora-mysql") ? "5.7" : (var.engine == "sqlserver-ex") ? "15.00" : "10.5"                                                                         #mariaDB
   family               = (var.engine == "oracle-se2") ? "oracle-se2-19" : (var.engine == "postgres") ? "postgres13" : (var.engine == "mysql") ? "mysql8.0" : (var.engine == "aurora-postgresql") ? "aurora-postgresql12" : (var.engine == "aurora-mysql") ? "aurora-mysql5.7" : (var.engine == "sqlserver-ex") ? "sqlserver-ex-15.0" : "mariadb10.5" #mariaDB
   tags = {
@@ -31,6 +31,10 @@ locals {
 }
 
 
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
 module "security_group" {
   
   # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
@@ -44,45 +48,8 @@ module "security_group" {
 
   # ingress
 
-  ingress_cidr_blocks = [
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-    "52.249.189.38/32", 
+  
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
 
   ingress_with_cidr_blocks = [
     {
@@ -96,45 +63,7 @@ module "security_group" {
 
   # egress
 
-  egress_cidr_blocks = [
+  egress_cidr_blocks = module.fw-ips.cidr_blocks
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
 
   egress_with_cidr_blocks = [
     {

databases/aws-redshift/main.tf

@@ -37,6 +37,10 @@ locals {
   }
 }
 
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
 module "security_group" {
   # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
   source  = "terraform-aws-modules/security-group/aws"
@@ -49,45 +53,7 @@ module "security_group" {
 
   # ingress
 
-  ingress_cidr_blocks = [
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
 
   ingress_with_cidr_blocks = [
     {
@@ -101,45 +67,7 @@ module "security_group" {
 
   # egress
 
-  egress_cidr_blocks = [
+  egress_cidr_blocks = module.fw-ips.cidr_blocks
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
 
   
   egress_with_cidr_blocks = [
@@ -185,7 +113,7 @@ module "redshift" {
 
 module "qmi-s3-bucket" {
 
-  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket?ref=dev2"
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
   
   provision_id                  = var.provision_id
   region                        = var.region

databases/azure-eventhub/outputs.tf

@@ -1,9 +1,9 @@
 output "default_primary_connection_string" {
-  value = azurerm_eventhub_namespace.ehbnamespace.default_primary_connection_string
+  value = nonsensitive(azurerm_eventhub_namespace.ehbnamespace.default_primary_connection_string)
 }
 
 output "default_primary_key" {
-  value = azurerm_eventhub_namespace.ehbnamespace.default_primary_key
+  value = nonsensitive(azurerm_eventhub_namespace.ehbnamespace.default_primary_key)
 }
 
 output "namespace_name" {

databases/azure-hdinsight-hadoop/main.tf

@@ -0,0 +1,107 @@
+
+resource "random_password" "password1" {
+  length = 16
+  special = true
+  override_special = "_!@"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+resource "random_password" "password2" {
+  length = 16
+  special = true
+  override_special = "_!@"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 5
+}
+
+resource "azurerm_storage_account" "example" {
+  name                     = "hdinsightstor${random_id.randomMachineId.hex}"
+  resource_group_name      = var.resource_group_name
+  location                 = var.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+
+  tags = var.tags
+}
+
+resource "azurerm_storage_container" "example" {
+  name                  = "hdinsight"
+  storage_account_name  = azurerm_storage_account.example.name
+  container_access_type = "private"
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp1" {
+  scope                = azurerm_storage_account.example.id
+  role_definition_name = "Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp2" { 
+  scope                = azurerm_storage_account.example.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_hdinsight_hadoop_cluster" "example" {
+  name                = "hdicluster-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  location            = var.location
+  cluster_version     = "4.0"
+  tier                = "Standard"
+
+  tags = var.tags
+
+  component_version {
+    hadoop = "3.1"
+  }
+
+  gateway {
+    username = "acctestusrgw"
+    password = random_password.password1.result
+  }
+
+  storage_account {
+    storage_container_id = azurerm_storage_container.example.id
+    storage_account_key  = azurerm_storage_account.example.primary_access_key
+    is_default           = true
+  }
+
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = random_password.password2.result
+    }
+
+    worker_node {
+      vm_size               = "Standard_D3_V2"
+      username              = "acctestusrvm"
+      password              = random_password.password2.result
+      target_instance_count = 2
+    }
+
+    zookeeper_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = random_password.password2.result
+    }
+  }
+}

databases/azure-hdinsight-hadoop/outputs.tf

@@ -0,0 +1,59 @@
+output "gateway-creds" {
+  value = {
+    username = "acctestusrgw"
+    password = nonsensitive(random_password.password1.result)
+  }
+}
+
+output "cluster-creds" {
+  value = {
+    username = "acctestusrvm"
+    password = nonsensitive(random_password.password2.result)
+  }
+}
+
+output "https_endpoint" {
+    value = azurerm_hdinsight_hadoop_cluster.example.https_endpoint
+}
+
+output "ssh_endpoint" {
+  value = azurerm_hdinsight_hadoop_cluster.example.ssh_endpoint
+}
+
+
+output "Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}
+
+
+output "adls_StorageAccount-Name" {
+  value = azurerm_storage_account.example.name
+}
+
+output "adls_StorageAccount-ContainerName" {
+  value = azurerm_storage_container.example.name
+}
+
+output "adls_StorageAccount-AccessKey" {
+  value = nonsensitive(azurerm_storage_account.example.primary_access_key)
+}
+
+output "adls_Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "adls_Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "adls_Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}

databases/azure-hdinsight-hadoop/variables.tf

@@ -0,0 +1,26 @@
+variable "resource_group_name" {
+  type = string
+}
+
+variable "provision_id" {
+  type = string
+}
+
+variable "location" {
+  type = string
+  default = "EAST US"
+}
+
+variable "tags" {
+    default = null
+}
+
+variable "dbricks_app_registration_principal_id" {
+    description = "databricks-qmi"
+    default = "efeee17c-d2b3-4e7c-a163-9995b7d281e2"
+}
+
+variable "dbricks_app_registration_application_id" {
+    description = "databricks-qmi"
+    default = "9ccb0d99-3bba-4695-aa47-df77bf512084"
+}

databases/azure-rds-flexmysql/firewall.tf

@@ -0,0 +1,31 @@
+/*resource "azurerm_mysql_flexible_server_firewall_rule" "all-azure-services" {
+  name                = "AllAzureServices"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_flexible_server.mysql-server.name
+  start_ip_address    = "0.0.0.0"
+  end_ip_address      = "0.0.0.0"
+}*/
+
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule1" {
+  name                = "a1"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_flexible_server.mysql-server.name
+  start_ip_address    = "52.249.189.38"
+  end_ip_address      = "52.249.189.38"
+}
+
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule2" {
+  name                = "a2"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_flexible_server.mysql-server.name
+  start_ip_address    = "13.67.39.86"
+  end_ip_address      = "13.67.39.86"
+}
+
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule3" {
+  name                = "a3"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_flexible_server.mysql-server.name
+  start_ip_address    = "20.67.110.207"
+  end_ip_address      = "20.67.110.207"
+}

databases/azure-rds-flexmysql/main.tf

@@ -49,12 +49,4 @@ resource "azurerm_mysql_flexible_server_configuration" "example" {
   resource_group_name = var.resource_group_name
   server_name         = azurerm_mysql_flexible_server.mysql-server.name
   value               = "full"
-}
-
-resource "azurerm_mysql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
 }

databases/azure-rds-flexpostgres/firewall.tf

@@ -0,0 +1,28 @@
+/*resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
+  name                = "AllAzureServices"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "0.0.0.0"
+  end_ip_address      = "0.0.0.0"
+}*/
+
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule1" {
+  name                = "a1"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "52.249.189.38"
+  end_ip_address      = "52.249.189.38"
+}
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule2" {
+  name                = "a2"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "13.67.39.86"
+  end_ip_address      = "13.67.39.86"
+}
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule3" {
+  name                = "a3"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "20.67.110.207"
+  end_ip_address      = "20.67.110.207"
+}

databases/azure-rds-flexpostgres/main.tf

@@ -51,11 +51,3 @@ resource "azurerm_postgresql_flexible_server_database" "postgresql-db" {
   collation = "en_US.utf8"
   charset   = "utf8"
 }
-
-
-resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}

databases/azure-rds-mariadb/firewall.tf

@@ -0,0 +1,26 @@
+resource "azurerm_mariadb_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  subnet_id           = var.subnet_id
+
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mariadb_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}

databases/azure-rds-mariadb/main.tf

@@ -0,0 +1,47 @@
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!@"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+
+resource "azurerm_mariadb_server" "mariadb-server" {
+  name = "qmi-mariadb-${var.provision_id}"
+  location = var.location
+  resource_group_name = var.resource_group_name
+
+  administrator_login = var.admin_login
+  administrator_login_password = random_password.password.result
+
+  sku_name = var.sku-name
+  version = var.db-version
+ 
+  storage_mb = var.storage
+  auto_grow_enabled = true
+
+  
+  backup_retention_days         = 7
+  geo_redundant_backup_enabled  = false
+  public_network_access_enabled = true
+  ssl_enforcement_enabled       = false
+
+  tags = {
+    Deployment = "QMI PoC"
+    "Cost Center" = "3100"
+    QMI_user = var.user_id
+  }
+}
+
+resource "azurerm_mariadb_database" "mariadb-db" {
+  name                = "QlikDB"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  charset             = "utf8"
+  collation           = "utf8_unicode_ci"
+}

databases/azure-rds-mariadb/output.tf

@@ -0,0 +1,11 @@
+output "db_server_fqdn" {
+  value = azurerm_mariadb_server.mariadb-server.fqdn
+}
+
+output "root_username" {
+  value = "${var.admin_login}@qmi-mariadb-${var.provision_id}"
+}
+
+output "root_username_password" {
+  value = nonsensitive(random_password.password.result)
+}

databases/azure-rds-mariadb/variables.tf

@@ -0,0 +1,43 @@
+variable "resource_group_name" {
+
+}
+
+variable "provision_id" {
+  type = string
+  description = "(optional) describe your variable"
+}
+
+variable "location" {
+  type = string
+  description = "(optional) describe your variable"
+  default = "EAST US"
+}
+
+variable "subnet_id" {
+  default = null
+}
+
+variable "user_id" {
+}
+
+variable "admin_login" {
+  type = string
+  description = "Login to authenticate to MySQL Server"
+  default = "qmi"
+}
+
+variable "db-version" {
+  type = string
+  description = "MariaDB Server version to deploy"
+  default = "10.2"
+}
+variable "sku-name" {
+  type = string
+  description = "MariaDB SKU Name"
+  default = "GP_Gen5_2"
+}
+variable "storage" {
+  type = string
+  description = "MariaDB Storage in MB"
+  default = "5120"
+  }

databases/azure-rds-mssql/firewall.tf

@@ -0,0 +1,24 @@
+
+resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+  
+  name                = "vnet-rule-${local.provision_id}"
+  server_id           = azurerm_mssql_server.sqlserver.id
+  subnet_id           = var.subnet_id
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mssql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  server_id           = azurerm_mssql_server.sqlserver.id
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}

databases/azure-rds-mssql/main.tf

@@ -44,216 +44,6 @@ resource "azurerm_mssql_server" "sqlserver" {
   }
 }
 
-resource "azurerm_mssql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
-
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-ITG-rule1" {
-  name                = "ITG"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-ITG-rule2" {
-  name                = "ITG1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-# QCS
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCS3" {
-  name                = "QCSEU3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
-  name                = "sql-vnet-rule-${local.provision_id}"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  subnet_id           = var.subnet_id
-}
-
-# QAA
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus1" {
-  name                = "QAAus1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus2" {
-  name                = "QAAus2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus3" {
-  name                = "QAAus3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
-}
-
 resource "azurerm_mssql_database" "db" {
   name           = "demoDB"
   server_id      = azurerm_mssql_server.sqlserver.id

databases/azure-rds-mssql/variables.tf

@@ -10,7 +10,7 @@ variable "provision_id" {
 }
 
 variable "subnet_id" {
-
+    default = null
 }
 
 variable "user_id" {

databases/azure-rds-mysql/firewall.tf

@@ -0,0 +1,26 @@
+resource "azurerm_mysql_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  subnet_id           = var.subnet_id
+
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mysql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}

databases/azure-rds-mysql/main.tf

@@ -55,241 +55,4 @@ resource "azurerm_mysql_database" "mysql-db" {
   server_name         = azurerm_mysql_server.mysql-server.name
   charset             = "utf8"
   collation           = "utf8_unicode_ci"
-}
-
-resource "azurerm_mysql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
-
-resource "azurerm_mysql_firewall_rule" "mysql-fw-ITG-rule1" {
-  name                = "ITG"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-ITG-rule2" {
-  name                = "ITG1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-# QCS
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCS3" {
-  name                = "QCSEU3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-# QAA
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus1" {
-  name                = "QAAus1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus2" {
-  name                = "QAAus2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus3" {
-  name                = "QAAus3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
 }

databases/azure-rds-mysql/variables.tf

@@ -13,6 +13,10 @@ variable "location" {
   default = "EAST US"
 }
 
+variable "subnet_id" {
+  default = null
+}
+
 variable "user_id" {
 }
 

databases/azure-rds-postgres/firewall.tf

@@ -0,0 +1,25 @@
+resource "azurerm_postgresql_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  subnet_id           = var.subnet_id
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_postgresql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}

databases/azure-rds-postgres/main.tf

@@ -46,242 +46,4 @@ resource "azurerm_postgresql_database" "postgresql-db" {
   server_name         = azurerm_postgresql_server.postgresql-server.name
   charset             = "utf8"
   collation           = "English_United States.1252"
-}
-
-
-resource "azurerm_postgresql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
-
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-ITG-rule1" {
-  name                = "ITG"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-ITG-rule2" {
-  name                = "ITG1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-#QCS
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCS3" {
-  name                = "QCSEU3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-# QAA
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus1" {
-  name                = "QAAus1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus2" {
-  name                = "QAAus2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus3" {
-  name                = "QAAus3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
 }

databases/azure-rds-postgres/variables.tf

@@ -13,6 +13,10 @@ variable "location" {
   default = "EAST US"
 }
 
+variable "subnet_id" {
+  default = null
+}
+
 
 variable "admin_login" {
   type = string

databases/dbricks/main.tf

@@ -5,65 +5,18 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 0.3.0"
+      version = ">= 1.13.0"
     }
   }
 }
 
+
 locals {
-  myRegex = "/[^[:alpha:]]/"
+  cluster_nameUpper = upper(var.cluster_name)
-  splitLower = split(" ", lower(var.user_id))
-  np0 = replace(element(local.splitLower,0), local.myRegex, "")
-  np1 = replace(element(local.splitLower,1), local.myRegex, "")
-  c_n1 = substr(local.np0, 0, 3)
-  c_n2 = substr(local.np1, 0, 1)
-  c_n3 = substr(strrev(local.np1), 0, 1)
-  
-  cluster_name = var.cluster_name != null? var.cluster_name : "${local.c_n1}${local.c_n2}${local.c_n3}"
-
-}
-
-resource "azurerm_databricks_workspace" "dbricksws" {
-
-  count = var.enabled == true? 1 : 0
-
-  location                      = var.location
-  name                          = "${local.cluster_name}-qlikdbricks-ws"
-  resource_group_name           = var.resource_group_name
-  sku                           = "standard"
-
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-  }
-
-}
-
-resource "azurerm_role_assignment" "contributor-dbricks" {
-
-  count = var.enabled == true? 1 : 0
-  
-  scope                = azurerm_databricks_workspace.dbricksws[0].id
-  role_definition_name = "Contributor"
-  principal_id         = var.app_registration_principal_id
-}
-
-provider "databricks" {
-  alias = "mws"
-
-  azure_workspace_resource_id = var.enabled == true? azurerm_databricks_workspace.dbricksws[0].id : null
-  azure_client_id             = var.enabled == true? "9b4761fd-4823-4f9d-ab3a-a95af38e7c29" : null
-  azure_client_secret         = var.enabled == true? "FWH7Q~ByC.U5zNh0BaIDdK_poyxoy4SxW8hi1" : null
-  azure_tenant_id             = var.enabled == true? "c21eeb5f-f5a6-44e8-a997-124f2f7a497c" : null
 }
 
 /*resource "databricks_workspace_conf" "this" {
-
-  count = var.enabled == true? 1 : 0
   
-  provider = databricks.mws
-
   custom_config = {
     "enableIpAccessLists" : true
   }
@@ -71,10 +24,6 @@ provider "databricks" {
 
 resource "databricks_ip_access_list" "allowed-list" {
 
-  count = var.enabled == true? 1 : 0
-
-  provider = databricks.mws
-
   label     = "allow_in"
   list_type = "ALLOW"
   ip_addresses = [
@@ -107,52 +56,109 @@ resource "databricks_ip_access_list" "allowed-list" {
 }*/
 
 
-resource "databricks_user" "me" {
+data "databricks_group" "admins" {
-  provider = databricks.mws
+  display_name = "admins"
-  count = var.enabled == true? 1 : 0
+}
 
-  depends_on = [
+
-    azurerm_role_assignment.contributor-dbricks
+resource "databricks_user" "aor" {
-  ]
+  count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
+  
+  allow_cluster_create = true
+  databricks_sql_access = true
+
+  user_name = "aor@qlik.com"
+  display_name = "Manuel Romero"
+}
+
+resource "databricks_user" "me" {
+
+  allow_cluster_create = true
+  databricks_sql_access = true
 
   user_name = var.user_email
   display_name = var.user_id
 }
 
 
+resource "databricks_group_member" "i-am-admin" {
+  group_id  = data.databricks_group.admins.id
+  member_id = databricks_user.me.id
+}
 
-resource "databricks_token" "pat" {
+
-  provider = databricks.mws
+resource "databricks_group_member" "i-am-admin-aor" {
-  count = var.enabled == true? 1 : 0
+  count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
-  
+  group_id  = data.databricks_group.admins.id
+  member_id = databricks_user.aor[0].id
+}
+
+resource "databricks_secret_scope" "scopeapp" {
+
+    name = "${var.cluster_name}scope"
+}
+resource "databricks_secret" "publishing_api" {
+
+    key = "accesskey"
+    string_value = var.app_reg_secret
+    scope = databricks_secret_scope.scopeapp.name
+}
+
+resource "databricks_sql_endpoint" "sqlep" {
+
+  count = (var.sku == "premium")? 1 : 0
+
+  name             = "${local.cluster_nameUpper} SQL warehouse"
+  cluster_size     = var.cluster_size
+  max_num_clusters = 1
+
+}
+
+resource "databricks_sql_global_config" "this" {
+
+  security_policy      = "DATA_ACCESS_CONTROL"
+  data_access_config = {
+    "spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
+    "spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
+    "spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
+    "spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
+  }
+
+  enable_serverless_compute = false
+  sql_config_params = {}
+}
+
+
+
+resource "databricks_token" "pat" {  
   depends_on = [
-    azurerm_role_assignment.contributor-dbricks,
+    databricks_user.me,
-    databricks_user.me
+    databricks_user.aor
   ]
   comment  = "qmi"
   lifetime_seconds = 8640000
 }
 
 resource "databricks_cluster" "dbrickscluster" {
-  provider = databricks.mws
+  count = (var.cluster_name != null)? 1 : 0
-  count = (var.enabled == true && var.create_cluster == true)? 1 : 0
 
-  depends_on = [
-    azurerm_role_assignment.contributor-dbricks
-  ]
 
-  cluster_name  = "cluster-${local.cluster_name}"
+  cluster_name  = "cluster-${var.cluster_name}"
-  spark_version = "7.3.x-scala2.12"
+  spark_version = var.spark_version
-  node_type_id  = "Standard_DS3_v2"
+  node_type_id  = var.node_type_id
 
   spark_conf = {
     "spark.hadoop.hive.server2.enable.doAs": false,
     "spark.databricks.delta.preview.enabled": true,
     "spark.databricks.repl.allowedLanguages": "sql,python,r",
     "spark.databricks.cluster.profile": "serverless",
-    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
+    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
+    "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
+    "spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
   } 
 
+
   spark_env_vars = {
       "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
   }
@@ -162,5 +168,5 @@ resource "databricks_cluster" "dbrickscluster" {
     max_workers = 4
   }
 
-  autotermination_minutes = 120
+  autotermination_minutes = 45
 }

databases/dbricks/outputs.tf

@@ -1,11 +1,25 @@
-output "databricks_workspace_url" {
+output "cluster_id" {
-    value = var.enabled? azurerm_databricks_workspace.dbricksws[0].workspace_url : null
+  value = (var.cluster_name != null)? databricks_cluster.dbrickscluster[0].id : null
 }
 
 output "databricks_token" {
-    value = var.enabled? nonsensitive(databricks_token.pat[0].token_value) : null
+    value = nonsensitive(databricks_token.pat.token_value)
 }
 
 output "cluster_name" {
-    value = "cluster-${local.cluster_name}"
+    value = "cluster-${var.cluster_name}"
-}
+}
+
+output "sql_endpoint_jdbc_url" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].jdbc_url : null
+}
+
+output "sql_endpoint_data_source_id" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].data_source_id : null
+}
+
+output "sql_endpoint_odbc_params" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].odbc_params : null
+}
+
+ 

databases/dbricks/varaibles.tf

@@ -1,11 +1,3 @@
-variable "location" {
-    default = "East US"
-}
-
-variable "resource_group_name" {
-
-}
-
 variable "cluster_name" {
     default = null
 }
@@ -22,15 +14,26 @@ variable "user_id" {
 variable "user_email" {
 }
 
-variable "app_registration_principal_id" {
+variable "spark_version" {
-    description = "tpm"
+    default = "10.4.x-scala2.12"
-    default = "163a72e3-8ce3-4e33-baae-954383f87e3e"
 }
 
-variable "enabled" {
+variable "node_type_id" {
-    default = false
+    default = "Standard_DS3_v2"
 }
 
-variable "create_cluster" {
+variable "sku" {
-    default = false
+    default = "standard"
+}
+
+variable "app_reg_id" {
+  default = null
+}
+
+variable "app_reg_secret" {
+  default = null
+}
+
+variable "cluster_size" {
+  default = "Small"
 }

databases/firewall_ips/output.tf

@@ -0,0 +1,118 @@
+locals {
+
+  az = {
+      az1          = ["52.249.189.38","52.249.189.38"]
+      az2          = ["13.67.39.86", "13.67.39.86"]
+      az3          = ["20.67.110.207", "20.67.110.207"]
+  }
+
+  q_routes = {
+    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
+    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
+    EMEA_FULLVPN   = ["193.15.228.246", "193.15.228.246"]
+    ITG         = ["188.65.156.32", "188.65.156.47"]
+    ITG1        = ["212.73.252.96", "212.73.252.103"]
+    kfar_saba   = ["194.90.96.176", "194.90.96.183"]
+    kfar_saba1  = ["213.57.84.160", "213.57.84.167"]
+    peak_10     = ["4.4.97.104", "4.4.97.111"]
+    peak_101    = ["206.196.17.32", "206.196.17.63"]
+    Bangalore   = ["14.98.59.168", "14.98.59.175"]
+    Bangalore1  = ["182.74.33.8", "182.74.33.15"]
+  }
+
+  qcs = {
+      QCS_us1          = ["18.205.71.36", "18.205.71.36"]      
+      QCS_us2          = ["18.232.32.199", "18.232.32.199"]   
+      QCS_us3          = ["34.237.68.254", "34.237.68.254"]   
+      QCS_eu1          = ["34.247.21.179", "34.247.21.179"]  
+      QCS_eu2          = ["52.31.212.214", "52.31.212.214"]  
+      QCS_eu3          = ["54.154.95.18", "54.154.95.18"]  
+      QCS_apac1        = ["13.210.43.241", "13.210.43.241"] 
+      QCS_apac2        = ["13.236.104.42", "13.236.104.42"]  
+      QCS_apac3        = ["13.236.206.172", "13.236.206.172"] 
+      QCS_sing1        = ["18.138.163.172", "18.138.163.172"]
+      QCS_sing2        = ["18.142.157.182", "18.142.157.182"]
+      QCS_sing3        = ["54.179.13.251", "54.179.13.251"]
+  }
+
+  qcs_qaa = {
+      QCS_QAA_eu1     = ["54.216.156.88", "54.216.156.88"]
+      QCS_QAA_eu2     = ["3.248.156.131", "3.248.156.131"]
+      QCS_QAA_eu3     = ["52.213.44.55", "52.213.44.55"]
+      QCS_QAA_us1     = ["18.235.133.252", "18.235.133.252"]
+      QCS_QAA_us2     = ["3.217.244.242", "3.217.244.242"]
+      QCS_QAA_us3     = ["18.214.8.201", "18.214.8.201"]
+      QCS_QAA_apac1     = ["54.206.158.27", "54.206.158.27"]
+      QCS_QAA_apac2     = ["3.104.137.20", "3.104.137.20"]
+      QCS_QAA_apac3     = ["3.24.52.178", "3.24.52.178"]
+      QCS_QAA_sing1     = ["54.169.84.213", "54.169.84.213"]
+      QCS_QAA_sing2     = ["13.213.173.37", "13.213.173.37"]
+      QCS_QAA_sing3     = ["13.213.113.162", "13.213.113.162"]
+  }
+
+ 
+}
+
+output "ips2" {
+  value = merge(local.q_routes, local.qcs)
+}
+
+output "ips" {
+  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes)
+}
+
+output "cidr_blocks" {
+  value = [
+    # EMEA/AMER FULL VPN TUNNEL
+    "144.121.57.103/32",
+    "193.15.228.246/32",
+    "50.239.179.6/32",
+    # Azure QMI machines
+    "52.249.189.38/32", 
+    "13.67.39.86/32", 
+    "20.67.110.207/32", 
+    # Qlik network routers
+    "14.98.59.168/29", 
+    "182.74.33.8/29", 
+    "188.65.156.32/28", 
+    "212.73.252.96/29", 
+    "194.90.96.176/29", 
+    "213.57.84.160/29", 
+    "4.4.97.104/29", 
+    "206.196.17.32/27", 
+    # QCS
+    "18.205.71.36/32", 
+    "18.232.32.199/32", 
+    "34.237.68.254/32", 
+    "34.247.21.179/32", 
+    "52.31.212.214/32", 
+    "54.154.95.18/32", 
+    "13.210.43.241/32", 
+    "13.236.104.42/32", 
+    "13.236.206.172/32",
+    "18.138.163.172/32",
+    "18.142.157.182/32",
+    "54.179.13.251/32",
+    # QCS-QAA
+    "54.216.156.88/32",
+    "3.248.156.131/32",
+    "52.213.44.55/32",
+    "18.235.133.252/32",
+    "3.217.244.242/32",
+    "18.214.8.201/32",
+    "54.206.158.27/32",
+    "3.104.137.20/32",
+    "3.24.52.178/32",
+    "54.169.84.213/32",
+    "13.213.173.37/32",
+    "13.213.113.162/32",
+    # QCS STAGING
+    "18.233.22.130/32",
+    "18.205.135.40/32",
+    "35.169.201.36/32",
+    "18.155.181.46/32",
+    "18.155.181.25/32",
+    "18.155.181.128/32",
+    "18.155.181.76/32"
+  ]
+}

databases/synapse-ws/firewall.tf

@@ -6,151 +6,44 @@ resource "azurerm_synapse_firewall_rule" "azureservices" {
   end_ip_address       = "0.0.0.0"
 }
 
+
 resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
+
+  name                = "az1"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "52.249.189.38"
   end_ip_address      = "52.249.189.38"
+
 }
 
 resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
+
+  name                = "az2"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "13.67.39.86"
   end_ip_address      = "13.67.39.86"
+
 }
 
 resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
+
+  name                = "az3"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "20.67.110.207"
   end_ip_address      = "20.67.110.207"
+
 }
 
-resource "azurerm_synapse_firewall_rule" "fw-ITG-rule1" {
+module "fw-ips" {
-  name                = "ITG"
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_synapse_firewall_rule" "fw-ITG-rule2" {
-  name                = "ITG1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_synapse_firewall_rule" "fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_synapse_firewall_rule" "fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
 }
 
-# QCS US
+resource "azurerm_synapse_firewall_rule" "fw_rule" {
-resource "azurerm_synapse_firewall_rule" "qcs_us_1" {
+  for_each = module.fw-ips.ips2
-  name                = "QCS_US_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_us_2" {
-  name                = "QCS_US_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_us_3" {
-  name                = "QCS_US_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
 
-# QCS EU
+  name                = each.key
-resource "azurerm_synapse_firewall_rule" "qcs_eu_1" {
-  name                = "QCS_EU_1"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "34.247.21.179"
+  start_ip_address    = each.value.0
-  end_ip_address      = "34.247.21.179"
+  end_ip_address      = each.value.1
-}
-resource "azurerm_synapse_firewall_rule" "qcs_eu_2" {
-  name                = "QCS_EU_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_eu_3" {
-  name                = "QCS_EU_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-# QCS AP
-resource "azurerm_synapse_firewall_rule" "qcs_ap_1" {
-  name                = "QCS_AP_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_ap_2" {
-  name                = "QCS_AP_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_ap_3" {
-  name                = "QCS_AP_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
 
-# QCS SINGAPORE
+}
-resource "azurerm_synapse_firewall_rule" "qcs_sp_1" {
-  name                = "QCS_SP_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_sp_2" {
-  name                = "QCS_SP_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_sp_3" {
-  name                = "QCS_SP_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}

databases/synapse-ws/main.tf

@@ -33,6 +33,9 @@ resource "azurerm_synapse_workspace" "synapsews" {
   sql_administrator_login              = var.sql_administrator_login
   sql_administrator_login_password     = var.sql_administrator_login_password
 
+
+  managed_virtual_network_enabled      = true
+
   identity {
     type = "SystemAssigned"
   }
@@ -65,19 +68,11 @@ resource "azurerm_synapse_sql_pool" "db" {
 resource "azurerm_synapse_role_assignment" "sqladministrator_aor" {
   
   depends_on = [
-      azurerm_synapse_firewall_rule.azureservices
+      azurerm_synapse_firewall_rule.azureservices,
-      /*,
+      
       azurerm_synapse_firewall_rule.fw-a-rule1,
       azurerm_synapse_firewall_rule.fw-a-rule2,
-      azurerm_synapse_firewall_rule.fw-a-rule3,
+      azurerm_synapse_firewall_rule.fw-a-rule3
-      azurerm_synapse_firewall_rule.fw-ITG-rule1,
-      azurerm_synapse_firewall_rule.fw-ITG-rule2,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule1,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule2,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule1,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule2,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule1,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule2*/
   ]
 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
@@ -91,18 +86,11 @@ resource "azurerm_synapse_role_assignment" "sqladministrator" {
   count = (var.user_oid != null && var.user_oid != "c583b5aa-2844-4baf-b8bf-b6a1ae0b1520") ? 1 : 0
 
   depends_on = [
-      azurerm_synapse_firewall_rule.azureservices/*,
+      azurerm_synapse_firewall_rule.azureservices,
+      
       azurerm_synapse_firewall_rule.fw-a-rule1,
       azurerm_synapse_firewall_rule.fw-a-rule2,
-      azurerm_synapse_firewall_rule.fw-a-rule3,
+      azurerm_synapse_firewall_rule.fw-a-rule3
-      azurerm_synapse_firewall_rule.fw-ITG-rule1,
-      azurerm_synapse_firewall_rule.fw-ITG-rule2,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule1,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule2,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule1,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule2,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule1,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule2*/
   ]
 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id

linux-common/centos/common/extract-certs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+BASEDIR=$(dirname "$0")
+
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
+PASSWORD=$1
+# Extract the private key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
+# Extract the public key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
+# Extract the CA cert chain
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+
+cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
+
+rm -fr $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+echo $PASSWORD > $BASEDIR/qmicerts/pfx-cert-password.txt

linux-common/centos/common/falcon.sh

@@ -1,20 +1,22 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-
+cVer=`rpm -E %{rhel}`
-centos_ver=`rpm --eval %{centos_ver}`
 binary='falcon-sensor-5.29.0-9403.el7.x86_64.rpm'
-if [ $centos_ver -eq '8' ]; then
+if [ $cVer -eq '8' ]; then
     binary='falcon-sensor-5.27.0-9104.el8.x86_64.rpm'
 fi
-echo $binary
 
-
+echo "--- Installing CrowdStrike falcon agent --> $binary"
-echo "Installing CrowdStrike falcon agent"
 echo "CID=$1"
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
+
 if ! rpm -qa | grep -qw falcon-sensor; then
-    sudo yum install $BASEDIR/$binary -y
+    sudo yum -y --quiet install $BASEDIR/$binary
 fi
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1
 sudo systemctl start falcon-sensor 

linux-common/centos/common/resizedisk.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 echo "Resizing main partition to max available disk"
 (
 echo u # Change the units to sectors

linux-common/centos/common/save-cert.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+
+echo $1 > $BASEDIR/qmicerts/myserver.crt
+
+echo $2 > $BASEDIR/qmicerts/myserver.key
+
+

linux-common/centos/common/tenable.sh

@@ -1,14 +1,23 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-echo "Installing Tenable Nessus Agent..."
 KEY=$1
 echo "KEY=$KEY"
 
-sudo yum install $BASEDIR/NessusAgent-10.2.0-es7.x86_64.rpm -y
+cVer=`rpm -E %{rhel}`
 
-echo "Linking Tenable Nessus Agent..."
+FILE="NessusAgent-10.2.1-es$cVer.x86_64.rpm"
+
+echo "--- Installing Tenable Nessus Agent --> $FILE"
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$FILE -O $BASEDIR/$FILE
+
+sudo /bin/yum -y --quiet install $BASEDIR/$FILE -y
+
+echo "--- Linking Tenable Nessus Agent..."
 sudo /bin/systemctl start nessusagent.service
 sudo /opt/nessus_agent/sbin/nessuscli agent link --key=$KEY --groups="Qlik IT Cloud Agents" --host=cloud.tenable.com --port=443
 

linux-common/centos/common/update.sh

@@ -0,0 +1,2 @@
+echo 'Updating OS. This will take a around 10 minutes'
+yum -y --quiet update

linux-common/main.tf

@@ -8,14 +8,27 @@ data "azurerm_key_vault_secret" "tenable-key" {
   key_vault_id = var.key_vault_id
 }
 
+data "azurerm_key_vault_secret" "cert_password" {
+  name         = "star-qmi-qlikpoc-com-password"
+  key_vault_id = var.key_vault_id
+}
+
+#data "azurerm_key_vault_certificate_data" "cert-data" {
+#  name         = "star-qmi-qlikpoc-com-cert"
+#  key_vault_id = var.key_vault_id
+#}
+
+
 locals {
   falcon_id     = nonsensitive(data.azurerm_key_vault_secret.cid.value)
   tenable_key   = nonsensitive(data.azurerm_key_vault_secret.tenable-key.value)
-}
+  cert_password = nonsensitive(data.azurerm_key_vault_secret.cert_password.value)
+  #cert_pem =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.pem)
+  #cert_key =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.key)
+} 
 
-resource "null_resource" "post-linux-vm" {
+resource "null_resource" "files" {
-    
+  provisioner "file" {
-    provisioner "file" {
         connection {
             type     = "ssh"
             host     = var.private_ip_address
@@ -27,6 +40,13 @@ resource "null_resource" "post-linux-vm" {
         source      = "${path.module}/${var.os_type}/common"
         destination = "/home/${var.admin_username}"
     }
+}
+
+resource "null_resource" "post-linux-vm" {
+    
+    depends_on = [
+      null_resource.files
+    ]
 
     provisioner "remote-exec" {
         connection {
@@ -43,6 +63,35 @@ resource "null_resource" "post-linux-vm" {
             "sudo /home/${var.admin_username}/common/falcon.sh '${local.falcon_id}'",
             "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
+            "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
+            #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
         ]
     }
-}
+}
+
+resource "null_resource" "update" {
+
+  count = var.update? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.post-linux-vm
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "60s"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/update.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+BASEDIR=$(dirname "$0")
+
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
+PASSWORD=$1
+# Extract the private key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
+# Extract the public key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
+# Extract the CA cert chain
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+
+cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
+
+rm -fr $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+echo $PASSWORD > $BASEDIR/qmicerts/pfx-cert-password.txt

linux-common/ubuntu/common/falcon.sh

@@ -1,11 +1,21 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-echo "Installing CrowdStrike falcon agent"
+binary="falcon-sensor_5.17.0-8103_amd64.deb"
+
+echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
+
+#sudo apt -qq -y update
+sudo apt install libnl-3-200 libnl-genl-3-200 -qq -y
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
+
 if ! dpkg -l | grep -qw falcon-sensor; then
-    sudo dpkg -i $BASEDIR/falcon-sensor_5.17.0-8103_amd64.deb
+    sudo dpkg -i $BASEDIR/$binary
 fi
 
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1

linux-common/ubuntu/common/resizedisk.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 echo "Resizing main partition to max available disk"
 (
 echo p # List the partitions details

linux-common/ubuntu/common/tenable.sh

@@ -1,14 +1,21 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-echo "Installing Tenable Nessus Agent..."
 KEY=$1
 echo "KEY=$KEY"
 
-sudo dpkg -i $BASEDIR/NessusAgent-10.2.0-ubuntu1110_amd64.deb
+FILE="NessusAgent-10.2.1-ubuntu1404_amd64.deb"
 
-echo "Linking Tenable Nessus Agent..."
+echo "--- Installing Tenable Nessus Agent --> $FILE"
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$FILE -O $BASEDIR/$FILE
+
+sudo dpkg -i $BASEDIR/$FILE
+
+echo "--- Linking Tenable Nessus Agent..."
 sudo /bin/systemctl start nessusagent.service
 sudo /opt/nessus_agent/sbin/nessuscli agent link --key=$KEY --groups="Qlik IT Cloud Agents" --host=cloud.tenable.com --port=443
 

linux-common/ubuntu/common/update.sh

@@ -0,0 +1,3 @@
+echo 'Updating OS. This will take a around 10 minutes'
+apt -qq -y update
+apt --fix-broken -qq -y upgrade

linux-common/variables.tf

@@ -8,9 +8,16 @@ variable "admin_password" {
 }
 
 variable "os_type" { 
+  description = "Either 'centos' or 'ubuntu'"
+  default = "centos"
 }
 
 variable "key_vault_id" {
   default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
 }
 
+variable "update" {
+  type = bool
+  default = true
+}
+

replicate-install/main.tf

@@ -62,7 +62,7 @@ resource "null_resource" "install" {
     inline = [
       "powershell.exe -File C:/provision/replicate-install/di-replicate-getBinary.ps1 -url ${var.download_url}",
       "powershell.exe -File C:/provision/replicate-install/di-replicate-install.ps1 -url ${var.download_url}",
-      "powershell.exe -File C:/provision/replicate-install/di-replicate-installQMICertificate.ps1 -CertPwd \"${local.cert_password}\"",
+      "powershell.exe -File C:/provision/replicate-install/di-replicate-installQMICertificate.ps1 -CertPwd \"${local.cert_password}\" -url ${var.download_url}",
       "powershell.exe -File C:/provision/replicate-install/di-em-register-replicate.ps1 -user ${var.vm_admin_username} -pass ${var.vm_admin_password} -domain ${local.domain}",
     ]
 

replicate-install/scripts/Replicate_install_2022.iss

@@ -0,0 +1,32 @@
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
+Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
+Count=8
+Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0
+Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-1
+Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
+Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
+Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
+Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
+Dlg7={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-1]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
+szDir=C:\Program Files\Attunity\Replicate\
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
+szDir=C:\Program Files\Attunity\Replicate\data
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
+Result=1
+Sel-0=1
+Sel-1=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

replicate-install/scripts/di-em-register-replicate.ps1

@@ -38,4 +38,4 @@ Write-Host "--- Registering Replicate server in Enterprise Manager with name '$r
 Invoke-RestMethod -Uri "https://di.qmi.qlik-poc.com/attunityenterprisemanager/api/v1/servers/$replsrvName/def" -Method "PUT" -Headers $Headers2 -Body $body_json
 
 Write-Host "--- Setting Replicate server license through EM API"
-Invoke-WebRequest -UseBasicParsing -Uri "https://di.qmi.qlik-poc.com/attunityenterprisemanager/api/v1/servers/$replsrvName/license/def" -Method "PUT" -Infile "$PSScriptRoot\replicate_license_exp2022-12-31_ser60026541.txt" -Headers $Headers2
+Invoke-WebRequest -UseBasicParsing -Uri "https://di.qmi.qlik-poc.com/attunityenterprisemanager/api/v1/servers/$replsrvName/license/def" -Method "PUT" -Infile "$PSScriptRoot\replicate_license.txt" -Headers $Headers2

replicate-install/scripts/di-replicate-install.ps1

@@ -18,12 +18,27 @@ if ( $url -Match "2021" ) {
 	$issFile="Replicate_install_2021.iss"
 }
 
+if ( $url -Match "2022" ) {
+	$issFile="Replicate_install_2022.iss"
+}
+
 Import-Module $PSScriptRoot\q-helper.psm1 | Out-Null
 
 $fileName = $url.Substring($url.LastIndexOf("/") + 1)
 
 $bin = "$PSScriptRoot\binaries\Attunity"
 
+if (Test-Path "$($bin)\Replicate_silent_x64_install.log") {
+	Remove-Item -Path "$($bin)\Replicate_silent_x64_install.log" -Force
+}
+
+if (Test-Path "C:\Program Files\Attunity\Replicate") {
+	Remove-Item -Path "C:\Program Files\Attunity\Replicate" -Recurse -Force
+}
+
+
+Write-Host "Using ISS file: $issFile"
+
 If ((Test-Path $bin\$fileName))
 {
 	Write-Host "Installing Attunity Replicate Server from $($bin)\$($fileName)"

replicate-install/scripts/di-replicate-installQMICertificate.ps1

@@ -1,11 +1,17 @@
  
 Param(
-    [string] $CertPwd
+    [string] $CertPwd,
+    [string] $url
 )
 
 try {
 
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+    
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)
@@ -13,16 +19,25 @@ try {
     Start-Sleep 20
     
     Write-Host "Set SSL qmi_qlik-poc_com for Replicate"
-    
+
     Stop-Service AttunityReplicateConsole
-    Stop-Service AttunityReplicateServer
+    if ( $url -Match "2022.11" ) {  
+        Stop-Service QlikReplicateServer
+    } else {
+        Stop-Service AttunityReplicateServer
+    }
 
     Start-Process -FilePath "C:\Program Files\Attunity\Replicate\bin\RepUiCtl.exe" -ArgumentList "certificate clean" -Wait -NoNewWindow
 
     netsh http add sslcert ipport=0.0.0.0:443 certhash=$thumb appid='{4dc3e181-e14b-4a21-b022-59fc669b0914}'
 
     Start-Service AttunityReplicateConsole 
-    Start-Service AttunityReplicateServer
+    if ( $url -Match "2022.11" ) {  
+        Start-Service QlikReplicateServer
+    } else {
+        Start-Service AttunityReplicateServer
+    }
+    
 
     Start-Sleep 10
     Write-Host "Replicate services restarted!"

replicate-install/scripts/license_exp2021-09-30_ser60020486.txt

@@ -1,18 +0,0 @@
-#
-# Attunity  License
-# Generated on 30-Sep-2020 20:41:19.3216+03:00 
-# License Comment: Qlik Internal use only 
-#
-license_type=TERM_LICENSE
-licensed_to=Qlik Tech
-licensed_by=Attunity US
-serial_no=60020486
-expiration_date=2021-09-30
-hosts=
-source_types=
-target_types=
-features=
-version=6.6
-issue_date=2020-09-30
-checksum=3H2K3-9KQ42-J2938-EF235
-

replicate-install/scripts/license_exp2021-12-31_ser60021240.txt

@@ -1,18 +0,0 @@
-#
-# Attunity  License
-# Generated on 30-Nov-2020 19:36:15.5200+02:00 
-# License Comment: 
-#
-license_type=EVALUATION_LICENSE
-licensed_to=ADA Demo License 2021
-licensed_by=Attunity US
-serial_no=60021240
-expiration_date=2021-12-31
-hosts=
-source_types=
-target_types=
-features=
-version=7.0
-issue_date=2020-11-30
-checksum=5ERRJ-EQ395-79QQ8-C92F2
-

replicate-install/scripts/replicate_license.txt

@@ -0,0 +1,17 @@
+#
+# Qlik  License
+# Generated on 15-Dec-2022 19:01:15.1383+02:00 
+# License Comment: Qlik internal use only 
+#
+license_type=EVALUATION_LICENSE
+licensed_to=Attunity Americas
+licensed_by=Attunity US
+serial_no=60031826
+expiration_date=2023-12-31
+hosts=
+source_types=
+target_types=
+features=
+version=2022.11
+issue_date=2022-12-15
+checksum=F4CK8-H36JR-8H58Q-833KH

replicate-install/scripts/replicate_license_exp2022-12-31_ser60026541.txt

@@ -1,18 +0,0 @@
-#
-# Qlik  License
-# Generated on 07-Dec-2021 18:01:15.5540+02:00 
-# License Comment: For internal Qlik use only 
-#
-license_type=EVALUATION_LICENSE
-licensed_to=Attunity Americas
-licensed_by=Attunity US
-serial_no=60026541
-expiration_date=2022-12-31
-hosts=
-source_types=
-target_types=
-features=
-version=2021.11
-issue_date=2021-12-07
-checksum=5Q9FQ-27467-KRC52-77598
-

replicate-install/variables.tf

@@ -11,8 +11,6 @@ variable "vm_admin_password" {
 }
 
 variable "download_url" {
-   //default = "https://da3hntz84uekx.cloudfront.net/QlikReplicate/2021.5/3/_MSI/AttunityReplicate_2021.5.0.924_X64.zip"
-   default = "https://github.com/qlik-download/replicate/releases/download/v2021.11.1/QlikReplicate_2021.11.0.165_X64.zip"
 }
 
 variable "key_vault_id" {

s3-bucket-sftp-public/main.tf

@@ -0,0 +1,229 @@
+
+terraform {
+
+  required_version = ">= 0.13"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "= 3.74.1"
+    }
+  }
+}
+
+locals {
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "sftp-${var.provision_id}"
+  }
+}
+
+resource "tls_private_key" "sftp-key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+
+}
+
+resource "aws_key_pair" "generated_key" {    
+  key_name   = "key-${var.provision_id}"
+  public_key = tls_private_key.sftp-key.public_key_openssh
+
+  tags = local.tags
+}
+
+## Public_secret
+
+resource "aws_secretsmanager_secret" "public_key" {
+  name = "pub-${var.provision_id}"
+
+  tags = local.tags
+}
+
+resource "aws_secretsmanager_secret_version" "public_key" {
+  secret_id = aws_secretsmanager_secret.public_key.id
+  secret_string = aws_key_pair.generated_key.public_key
+
+}
+
+## Private Secret
+
+resource "aws_secretsmanager_secret" "private_key" {
+  name = "priv-${var.provision_id}"
+
+  tags = local.tags
+}
+
+resource "aws_secretsmanager_secret_version" "private_key" {
+  secret_id = aws_secretsmanager_secret.private_key.id
+  secret_string = nonsensitive(tls_private_key.sftp-key.private_key_pem)
+
+}
+
+
+module "s3_bucket" {
+  source = "terraform-aws-modules/s3-bucket/aws"
+
+  version = "~> 2.1.0"
+
+  bucket = "sftp-${var.provision_id}"
+  acl    = "private"
+
+  versioning = {
+    enabled = false
+  }
+
+  force_destroy = true
+
+  tags = local.tags
+
+}
+
+###
+
+resource "aws_s3_bucket_public_access_block" "sftp-block" {
+  bucket = module.s3_bucket.s3_bucket_id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
+}
+
+###
+
+resource "aws_iam_role" "logging" {
+  name  = "${var.provision_id}-transfer-logging"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "transfer.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "logging" {
+  name  = "${var.provision_id}-transfer-logging"
+  role   = aws_iam_role.logging.id
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogStream",
+        "logs:DescribeLogStreams",
+        "logs:CreateLogGroup",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_transfer_server" "public" {
+  endpoint_type          = var.sftp_type 
+  protocols              = var.protocols 
+  identity_provider_type = var.identity_provider_type 
+  url                    = var.api_gw_url
+  invocation_role        = var.invocation_role
+  logging_role           = aws_iam_role.logging.arn
+  force_destroy          = var.force_destroy 
+  security_policy_name   = var.security_policy_name
+
+  tags = local.tags
+}
+
+
+resource "aws_iam_role" "user" {
+  for_each           = var.sftp_users
+  name               = "${var.provision_id}-sftp-user-${each.key}"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "transfer.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+
+resource "aws_iam_role_policy" "user" {
+  for_each = var.sftp_users
+  name     = "${var.provision_id}-sftp-user-${each.key}"
+  role     = aws_iam_role.user[each.key].id
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AllowListingOfUserFolder",
+      "Action": [
+        "s3:ListBucket",
+        "s3:GetBucketLocation"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+       "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id])}"
+      ]
+    },
+    {
+      "Sid": "HomeDirObjectAccess",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObjectVersion",
+        "s3:DeleteObject",
+        "s3:GetObjectVersion"
+      ],
+      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/", each.value, "/*"])}"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_transfer_user" "this" {
+  for_each       = var.sftp_users
+  server_id      = aws_transfer_server.public.id
+  user_name      = each.key
+  home_directory = "/${module.s3_bucket.s3_bucket_id}/${each.value}"
+  role           = aws_iam_role.user[each.key].arn
+
+  tags = local.tags
+}
+
+
+resource "aws_transfer_ssh_key" "this" {
+  for_each   = { "sftpusr" = aws_secretsmanager_secret_version.public_key.secret_string }
+  server_id  = aws_transfer_server.public.id
+  user_name  = each.key
+  body       = each.value
+  depends_on = [aws_transfer_user.this]
+
+}

s3-bucket-sftp-public/outputs.tf

@@ -0,0 +1,20 @@
+
+output "id" {
+  value       = aws_transfer_server.public.id
+  description = "ID of transfer server"
+}
+
+output "endpoint" {
+  value       = aws_transfer_server.public.endpoint
+  description = "Endpoint of transfer server"
+}
+
+output "username" {
+  value = "sftpusr"
+}
+
+output "sftp-private" {
+  value = nonsensitive(aws_secretsmanager_secret_version.private_key.secret_string)
+}
+
+

s3-bucket-sftp-public/variables.tf

@@ -0,0 +1,56 @@
+variable "region" {
+    default = "us-east-1"
+}
+
+variable "provision_id" { 
+}
+
+variable "user_id" {
+}
+
+variable "sftp_type" {
+  type        = string
+  default     = "PUBLIC"
+  description = "Type of SFTP server. **Valid values:** `PUBLIC`, `VPC` or `VPC_ENDPOINT`"
+}
+
+variable "protocols" {
+  type        = list(string)
+  default     = ["SFTP"]
+  description = "List of file transfer protocol(s) over which your FTP client can connect to your server endpoint. **Possible Values:** FTP, FTPS and SFTP"
+}
+
+variable "identity_provider_type" {
+  type        = string
+  default     = "SERVICE_MANAGED"
+  description = "Mode of authentication to use for accessing the service. **Valid Values:** `SERVICE_MANAGED`, `API_GATEWAY`, `AWS_DIRECTORY_SERVICE` or `AWS_LAMBDA`"
+}
+
+variable "api_gw_url" {
+  type        = string
+  default     = null
+  description = "URL of the service endpoint to authenticate users when `identity_provider_type` is of type `API_GATEWAY`"
+}
+
+variable "invocation_role" {
+  type        = string
+  default     = null
+  description = "ARN of the IAM role to authenticate the user when `identity_provider_type` is set to `API_GATEWAY`"
+}
+
+variable "force_destroy" {
+  type        = bool
+  default     = true
+  description = "Whether to delete all the users associated with server so that server can be deleted successfully. **Note:** Supported only if `identity_provider_type` is set to `SERVICE_MANAGED`"
+}
+
+variable "security_policy_name" {
+  type        = string
+  default     = "TransferSecurityPolicy-2020-06"
+  description = "Specifies the name of the [security policy](https://docs.aws.amazon.com/transfer/latest/userguide/security-policies.html) to associate with the server"
+}
+
+variable "sftp_users" {
+  type    = map(string)
+  default = { sftpusr = "sftpusr" }
+}

s3-bucket-sftp/main.tf

@@ -0,0 +1,351 @@
+terraform {
+
+  required_version = ">= 0.13"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "= 3.74.1"
+    }
+  }
+}
+
+resource "tls_private_key" "sftp-key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "aws_key_pair" "generated_key" {    
+  key_name   = "key-${var.provision_id}"
+  public_key = tls_private_key.sftp-key.public_key_openssh
+}
+
+## Public_secret
+
+resource "aws_secretsmanager_secret" "public_key" {
+  name = "pub-${var.provision_id}"
+}
+
+resource "aws_secretsmanager_secret_version" "public_key" {
+  secret_id = aws_secretsmanager_secret.public_key.id
+  secret_string = aws_key_pair.generated_key.public_key
+}
+
+## Private Secret
+
+resource "aws_secretsmanager_secret" "private_key" {
+  name = "priv-${var.provision_id}"
+}
+
+resource "aws_secretsmanager_secret_version" "private_key" {
+  secret_id = aws_secretsmanager_secret.private_key.id
+  secret_string = nonsensitive(tls_private_key.sftp-key.private_key_pem)
+}
+
+
+module "s3_bucket" {
+  source = "terraform-aws-modules/s3-bucket/aws"
+
+  version = "~> 2.1.0"
+
+  bucket = "sftp-${var.provision_id}"
+  acl    = "private"
+
+  versioning = {
+    enabled = false
+  }
+
+  force_destroy = true
+
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    #QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "sftp-${var.provision_id}"
+  }
+
+}
+
+###
+
+resource "aws_s3_bucket_public_access_block" "sftp-block" {
+  bucket = module.s3_bucket.s3_bucket_id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+###
+
+resource "aws_iam_role" "logging" {
+  name               = "${var.provision_id}-transfer-logging"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "transfer.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "logging" {
+  name   = "${var.provision_id}-transfer-logging"
+  role   = aws_iam_role.logging.id
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogStream",
+        "logs:DescribeLogStreams",
+        "logs:CreateLogGroup",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_transfer_server" "sftp" {
+  endpoint_type          = "VPC"
+  protocols              = ["SFTP"]
+  identity_provider_type = "SERVICE_MANAGED"
+  logging_role           = aws_iam_role.logging.arn
+  force_destroy          = var.force_destroy
+  security_policy_name   = var.security_policy_name
+  endpoint_details {
+    vpc_id             = var.vpc_id
+    subnet_ids         = var.subnet_ids
+    security_group_ids = [module.security_group.security_group_id]
+  }
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "sftp-${var.provision_id}"
+  }
+}
+
+module "security_group" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
+
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG-SFTP"
+  vpc_id      = var.vpc_id
+
+
+  # ingress
+
+  ingress_cidr_blocks = [
+    "52.249.189.38/32", 
+    "13.67.39.86/32", 
+    "20.67.110.207/32", 
+    "14.98.59.168/29", 
+    "182.74.33.8/29", 
+    "188.65.156.32/28", 
+    "212.73.252.96/29", 
+    "194.90.96.176/29", 
+    "213.57.84.160/29", 
+    "4.4.97.104/29", 
+    "206.196.17.32/27", 
+    #QCS
+    "18.205.71.36/32", 
+    "18.232.32.199/32", 
+    "34.237.68.254/32", 
+    "34.247.21.179/32", 
+    "52.31.212.214/32", 
+    "54.154.95.18/32", 
+    "13.210.43.241/32", 
+    "13.236.104.42/32", 
+    "13.236.206.172/32",
+    "18.138.163.172/32",
+    "18.142.157.182/32",
+    "54.179.13.251/32",
+    #QAA
+    "54.216.156.88/32",
+    "3.248.156.131/32",
+    "52.213.44.55/32",
+    "18.235.133.252/32",
+    "3.217.244.242/32",
+    "18.214.8.201/32",
+    "54.206.158.27/32",
+    "3.104.137.20/32",
+    "3.24.52.178/32",
+    "54.169.84.213/32",
+    "13.213.173.37/32",
+    "13.213.113.162/32"
+  ]
+
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = 22
+      to_port     = 22
+      protocol    = "tcp"
+      description = "Allow SFTP Inbound"
+
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = [
+    "52.249.189.38/32", 
+    "13.67.39.86/32", 
+    "20.67.110.207/32", 
+    "14.98.59.168/29", 
+    "182.74.33.8/29", 
+    "188.65.156.32/28", 
+    "212.73.252.96/29", 
+    "194.90.96.176/29", 
+    "213.57.84.160/29", 
+    "4.4.97.104/29", 
+    "206.196.17.32/27", 
+    #QCS
+    "18.205.71.36/32", 
+    "18.232.32.199/32", 
+    "34.237.68.254/32", 
+    "34.247.21.179/32", 
+    "52.31.212.214/32", 
+    "54.154.95.18/32", 
+    "13.210.43.241/32", 
+    "13.236.104.42/32", 
+    "13.236.206.172/32",
+    "18.138.163.172/32",
+    "18.142.157.182/32",
+    "54.179.13.251/32",
+    #QAA
+    "54.216.156.88/32",
+    "3.248.156.131/32",
+    "52.213.44.55/32",
+    "18.235.133.252/32",
+    "3.217.244.242/32",
+    "18.214.8.201/32",
+    "54.206.158.27/32",
+    "3.104.137.20/32",
+    "3.24.52.178/32",
+    "54.169.84.213/32",
+    "13.213.173.37/32",
+    "13.213.113.162/32"
+  ]
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = 22
+      to_port     = 22
+      protocol    = "tcp"
+      description = "Allow SFTP outbound"
+
+    },
+  ]
+
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    #QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "sftp-${var.provision_id}"
+  }
+  
+}
+
+
+resource "aws_iam_role" "user" {
+  for_each           = var.sftp_users
+  name               = "${var.provision_id}-sftp-user-${each.key}"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "transfer.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "user" {
+  for_each = var.sftp_users
+  name     = "${var.provision_id}-sftp-user-${each.key}"
+  role     = aws_iam_role.user[each.key].id
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AllowListingOfUserFolder",
+      "Action": [
+        "s3:ListBucket",
+        "s3:GetBucketLocation"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+       "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id])}"
+      ]
+    },
+    {
+      "Sid": "HomeDirObjectAccess",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObjectVersion",
+        "s3:DeleteObject",
+        "s3:GetObjectVersion"
+      ],
+      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/", each.value, "/*"])}"
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_transfer_user" "this" {
+  for_each       = var.sftp_users
+  server_id      = aws_transfer_server.sftp.id
+  user_name      = each.key
+  home_directory = "/${module.s3_bucket.s3_bucket_id}/${each.value}"
+  role           = aws_iam_role.user[each.key].arn
+}
+
+resource "aws_transfer_ssh_key" "this" {
+  for_each   = { "sftpusr" = aws_secretsmanager_secret_version.public_key.secret_string }
+  server_id  = aws_transfer_server.sftp.id
+  user_name  = each.key
+  body       = each.value
+  depends_on = [aws_transfer_user.this]
+}
+
+/*resource "null_resource" "get-endpoint-dns" {
+  provisioner "local-exec" {
+    command = "aws ec2 describe-vpc-endpoints --vpc-endpoint-ids ${aws_transfer_server.sftp.endpoint_details[0].vpc_endpoint_id} --query 'VpcEndpoints[*].DnsEntries[0].DnsName'> dns.txt"
+  }
+}
+
+data "local_file" "endpoint-dns" {
+  filename   = "dns.txt"
+  depends_on = [null_resource.get-endpoint-dns]
+}*/

s3-bucket-sftp/outputs.tf

@@ -0,0 +1,28 @@
+output "vpc_endpoint" {
+  value       = aws_transfer_server.sftp.endpoint_details[0].vpc_endpoint_id
+  description = "VPC Endpoint of transfer server"
+}
+
+output "id" {
+  value       = aws_transfer_server.sftp.id
+  description = "ID of transfer server"
+}
+
+output "endpoint" {
+  value       = aws_transfer_server.sftp.endpoint
+  description = "Endpoint of transfer server"
+}
+
+output "endpoint-dns" {
+  value = "" #data.local_file.endpoint-dns.content    
+}
+
+
+output "sftp-private" {
+  value = nonsensitive(aws_secretsmanager_secret_version.private_key.secret_string)
+}
+
+output "username" {
+  value = "sftpusr"
+}
+

s3-bucket-sftp/variables.tf

@@ -0,0 +1,38 @@
+variable "region" {
+    default = "us-east-1"
+}
+
+variable "provision_id" { 
+}
+
+variable "user_id" {
+}
+
+variable "vpc_id" {
+  default = "vpc-c079f5bd"
+  description = "VPC For Transfer Server"
+}
+
+variable "sftp_users" {
+  type    = map(string)
+  default = { sftpusr = "sftpusr" }
+}
+
+variable "subnet_ids" {
+  default = ["subnet-4d26552b"]
+  description = "Subnets For Transfer Server"
+}
+
+variable "force_destroy" {
+  type        = bool
+  default     = true
+  description = "Whether to delete all the users associated with server so that server can be deleted successfully."
+}
+
+variable "security_policy_name" {
+  type        = string
+  default     = "TransferSecurityPolicy-2020-06"
+  description = "Specifies the name of the [security policy](https://docs.aws.amazon.com/transfer/latest/userguide/security-policies.html) to associate with the server"
+}
+
+

s3-bucket/main.tf

@@ -47,6 +47,11 @@ module "s3_bucket" {
 
   force_destroy = true
 
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
   tags = {
     Deployment = "QMI"
     "Cost Center" = "3100"

vm-centos/main.tf

@@ -57,8 +57,8 @@ resource "azurerm_linux_virtual_machine" "vm" {
   }
 
   source_image_reference {
-    publisher = "OpenLogic"
+    publisher = var.publisher
-    offer     = "CentOS"
+    offer     = var.offer
     sku       = var.sku
     version   = "latest"
   }
@@ -76,7 +76,7 @@ resource "azurerm_linux_virtual_machine" "vm" {
 
 module "linux-common" {
 
-  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common?ref=dev2"
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common"
 
   depends_on = [
     azurerm_linux_virtual_machine.vm

vm-centos/variables.tf

@@ -35,6 +35,14 @@ variable "sku" {
   default = "7.5"
 }
 
+variable "publisher" {
+  default = "OpenLogic"
+}
+
+variable "offer" {
+  default = "CentOS"
+}
+
 variable "user_id" {
   default = null
 }

vm-fort-az/outputs.tf

@@ -11,11 +11,11 @@ output "admin_username" {
 }
 
 output "admin_password" {
-  value = random_password.password.result
+  value = nonsensitive(random_password.password.result)
 }
 
 output "ssh" {
-  value = "ssh ${var.admin_username}@${module.qmi-nic.private_ip_address}   --> password: ${random_password.password.result}"
+  value = nonsensitive("ssh ${var.admin_username}@${module.qmi-nic.private_ip_address}   --> password: ${random_password.password.result}")
 }
 
 output "nic_id" {

vm-fromsnapshot-linux/main.tf

@@ -96,6 +96,10 @@ resource "azurerm_virtual_machine" "vm" {
     disable_password_authentication = false
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   tags = {
     "Deployment" = "QMI PoC"
     "Cost Center" = "3100"

vm-fromsnapshot-linux/outputs.tf

@@ -24,4 +24,8 @@ output "nic_private_ip_address" {
 
 output "nic_ip_configuration_name" {
   value = module.qmi-nic.ip_configuration_name
+}
+
+output "principal_id" {
+  value = azurerm_virtual_machine.vm.identity.0.principal_id
 }

vm-fromsnapshot-win/main.tf

@@ -129,10 +129,9 @@ resource "azurerm_virtual_machine" "vm" {
 }
 
 module "win-common" {
-
   count = var.wincommon? 1 : 0
   
-  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//win-common?ref=dev2"
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//win-common"
 
   depends_on = [
     azurerm_virtual_machine.vm
@@ -142,6 +141,7 @@ module "win-common" {
   admin_username      = local.admin_username
   admin_password      = var.initial_password  
   carbonblack         = var.carbonblack
+  tenable             = var.tenable
   
 }
 

vm-fromsnapshot-win/variables.tf

@@ -81,6 +81,10 @@ variable "carbonblack" {
   default = true
 }
 
+variable "tenable" {
+  default = true
+}
+
 variable "wincommon" {
   default = true
 }