craeting metasttore

.gitignore

@@ -0,0 +1 @@
+.DS_Store

adls/main.tf

@@ -0,0 +1,70 @@
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 5
+}
+
+
+resource "azurerm_storage_account" "saccount" {
+
+  name = var.storage_account_name != null? var.storage_account_name : "qmiadlsgen2${random_id.randomMachineId.hex}"
+
+  resource_group_name       = var.resource_group_name
+  location                  = var.location
+
+  account_kind              = "StorageV2"
+  account_replication_type  = var.account_replication_type != null? var.account_replication_type : "RAGRS"
+  account_tier              = "Standard"
+  access_tier               = "Hot"
+
+  is_hns_enabled            = "true"
+
+  tags = var.tags
+}
+
+resource "azurerm_storage_container" "scontainer" {
+
+  name                  = var.container_name != null? var.container_name : "qmicontainer"
+  storage_account_name  = azurerm_storage_account.saccount.name
+  #container_access_type = "container"
+}
+
+resource "azurerm_role_assignment" "data-contributor-role" {
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Contributor"
+  principal_id         = var.tpm_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-role2" {  
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.tpm_app_registration_principal_id
+}
+
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp1" {
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp2" { 
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+
+###### EXTRA ASSIGN ROLE #######
+
+resource "azurerm_role_assignment" "machine_role_assignment" { 
+
+  count = var.principal_id_storage_blob_contributor != null? 1 : 0
+
+  scope                = azurerm_storage_account.saccount.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.principal_id_storage_blob_contributor
+}

adls/outputs.tf

@@ -0,0 +1,35 @@
+output "StorageAccount-Scope" {
+  value = azurerm_storage_account.saccount.id
+}
+
+output "StorageAccount-AccessKey" {
+  value = nonsensitive(azurerm_storage_account.saccount.primary_access_key)
+}
+
+output "StorageAccount-ConnectionString" {
+  value = nonsensitive(azurerm_storage_account.saccount.primary_connection_string)
+}
+
+output "StorageAccount-Name" {
+  value = azurerm_storage_account.saccount.name
+}
+
+output "StorageAccount-ContainerName" {
+  value = azurerm_storage_container.scontainer.name
+}
+
+output "StorageAccount-ContainerId" {
+  value = azurerm_storage_container.scontainer.id
+}
+
+output "Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}

adls/variable.tf

@@ -0,0 +1,46 @@
+variable "resource_group_name" {
+}
+
+variable "location" {
+  description = "The Azure Region in which the resources in this example should exist"
+  default = "East US"
+}
+
+variable "storage_account_name" {
+  default = null
+}
+
+variable "container_name" {
+  default = null
+}
+
+variable "tags" {
+    default = null
+}
+
+variable "tpm_app_registration_principal_id" {
+    description = "tpm"
+    default = "163a72e3-8ce3-4e33-baae-954383f87e3e"
+}
+
+variable "dbricks_app_registration_principal_id" {
+    description = "databricks-qmi"
+    default = "efeee17c-d2b3-4e7c-a163-9995b7d281e2"
+}
+
+variable "dbricks_app_registration_application_id" {
+    description = "databricks-qmi"
+    default = "9ccb0d99-3bba-4695-aa47-df77bf512084"
+}
+
+variable "principal_id_storage_blob_contributor" {
+  default = null
+}
+
+variable "sa_config_type" {
+  default = "1"
+}
+
+variable "account_replication_type" {
+  default = null
+}

appgateways/appgw-443-only/logs.tf

@@ -0,0 +1,23 @@
+resource "azurerm_monitor_diagnostic_setting" "example" {
+
+    name               = "${local.appgw_name}-${var.provision_id}-diagsettings"
+    target_resource_id = azurerm_application_gateway.qmi-app-gw.id
+
+    log_analytics_workspace_id = var.log_analytics_workspace_id
+
+    enabled_log {
+        category = "ApplicationGatewayAccessLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+
+    enabled_log {
+        category = "ApplicationGatewayFirewallLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+}

appgateways/appgw-443-only/main.tf

@@ -30,6 +30,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -48,6 +49,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {
@@ -151,6 +153,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
   
 
@@ -167,6 +170,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 2
   }
 
 }

appgateways/appgw-443-only/output.tf

@@ -11,5 +11,5 @@ output "appgw_public_ip" {
 }
 
 output "appgw_backend_address_pool_0_id" {
-  value = azurerm_application_gateway.qmi-app-gw.backend_address_pool[0].id
+  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).0.id
 }

appgateways/appgw-443-only/variables.tf

@@ -38,3 +38,7 @@ variable "user_id" {
 variable "probe_path" {
   default = "/"
 }
+
+variable "log_analytics_workspace_id" {
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
+}

appgateways/appgw-e2e/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-e2e/main.tf

@@ -62,6 +62,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -80,6 +81,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {
@@ -416,6 +418,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -426,6 +429,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 2
   }
 
   # NPrinting routing rules
@@ -435,6 +439,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 3
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -442,6 +447,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 4
   }
 
 
@@ -451,6 +457,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4552
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4552
+    priority                   = 5
   }
 
 
@@ -461,6 +468,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 6
   }
   
 
@@ -477,6 +485,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 7
   }
 
   # Redirect QDC 8080 to 8443
@@ -492,6 +501,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_8080
     redirect_configuration_name = local.redirect_configuration_8080
+    priority                    = 8
   }
 
 }

appgateways/appgw-e2e/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qdc-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc-qs/main.tf

@@ -54,6 +54,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
   
 }
@@ -73,6 +74,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {
@@ -357,6 +359,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -367,6 +370,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 2
   }
 
   # NPrinting routing rules
@@ -376,6 +380,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 3
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -383,6 +388,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 4
   }
 
   # QIB routing rule
@@ -392,6 +398,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 5
   }
   
 
@@ -408,6 +415,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                   = 6
   }
 
 }

appgateways/appgw-qdc-qs/outputs.tf

@@ -10,10 +10,14 @@ output "appgw_public_ip" {
   value = azurerm_public_ip.appgw-ip.ip_address
 }
 
+locals {
+  backend_pools = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool)
+}
+
 output "appgw_backend_address_pool_0_id" {
-  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).0.id
+  value = local.backend_pools[index(local.backend_pools.*.name, "${var.appgw_hostname}-qs-bp")].id
 }
 
 output "appgw_backend_address_pool_1_id" {
-  value = tolist(azurerm_application_gateway.qmi-app-gw.backend_address_pool).1.id
+  value = local.backend_pools[index(local.backend_pools.*.name, "${var.appgw_hostname}-qdc-bp")].id
 }

appgateways/appgw-qdc-qs/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qdc/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc/main.tf

@@ -28,6 +28,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -46,6 +47,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {
@@ -127,6 +129,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_8443
     backend_address_pool_name  = local.backend_address_pool_qdc
     backend_http_settings_name = local.http_setting_name_8443
+    priority                   = 1
   }
 
 }

appgateways/appgw-qdc/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

appgateways/appgw-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qs/main.tf

@@ -52,6 +52,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -70,6 +71,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {
@@ -338,6 +340,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_https
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_443
+    priority                   = 1
   }
 
 
@@ -348,6 +351,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4993
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4993
+    priority                   = 2
   }
   request_routing_rule {
     name                       = local.request_routing_rule_4994
@@ -355,6 +359,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4994
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4994
+    priority                   = 3
   }
 
   # QIB routing rule (needs QIB using qmi.qlik-poc.com certs)
@@ -364,6 +369,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4435
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4435
+    priority                   = 4
   }
 
   # Qlik Alerting routing rule (needs Qlik Alerting using qmi.qlik-poc.com certs)
@@ -373,6 +379,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     http_listener_name         = local.listener_name_4552
     backend_address_pool_name  = local.backend_address_pool_qs
     backend_http_settings_name = local.http_setting_name_4552
+    priority                   = 5
   }
   
 
@@ -389,6 +396,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     rule_type                   = "Basic"
     http_listener_name          = local.listener_name_http
     redirect_configuration_name = local.redirect_configuration_80
+    priority                    = 6
   }
 
 }

appgateways/appgw-qs/variables.tf

@@ -24,7 +24,7 @@ variable "app_gw_subnet" {
 }
 
 variable "log_analytics_workspace_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourcegroups/appgw_rg/providers/microsoft.operationalinsights/workspaces/qmi-log-analytics"
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
 }
 
 variable "cert_name" {

aws-credentials/main.tf

@@ -1,19 +0,0 @@
-terraform {
-    required_version = ">= 0.13"
-}
-
-provider "azurerm" {    
-    features {}
-    subscription_id = var.subscription_id
-    use_msi = true
-}
-
-data "azurerm_key_vault_secret" "qmi-aws-access-key" {
-  name         = "qmi-aws-access-key"
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "qmi-aws-access-secret" {
-  name         = "qmi-aws-access-secret"
-  key_vault_id = var.key_vault_id
-}

aws-credentials/output.tf

@@ -1,7 +0,0 @@
-output "qmi-aws-access-key" {
-    value = data.azurerm_key_vault_secret.qmi-aws-access-key.value
-}
-
-output "qmi-aws-access-secret" {
-    value = data.azurerm_key_vault_secret.qmi-aws-access-secret.value
-}

aws-credentials/variables.tf

@@ -1,7 +0,0 @@
-variable "subscription_id" {
-    default = "62ebff8f-c40b-41be-9239-252d6c0c8ad9"
-}
-
-variable "key_vault_id" {
-    default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
-}

compose-install/main.tf

@@ -1,20 +1,9 @@
-data "azurerm_key_vault_secret" "compose-license" {
-  name         = "compose-license"
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "c4dw" {
-  name         = "compose-datawarehouse-license"
-  key_vault_id = var.key_vault_id
-}
-
 data "azurerm_key_vault_secret" "cert-password" {
   name         = "star-qmi-qlikpoc-com-password"
   key_vault_id = var.key_vault_id
 }
 
 locals {
-  composeLicense = (var.c_version == "c4dw")? nonsensitive(data.azurerm_key_vault_secret.c4dw.value) : nonsensitive(data.azurerm_key_vault_secret.compose-license.value)
   cert_password = nonsensitive(data.azurerm_key_vault_secret.cert-password.value)
 }
 
@@ -49,8 +38,8 @@ resource "null_resource" "install" {
     inline = [
       "powershell.exe -File C:/provision/compose-install/prep-files.ps1",
       "powershell.exe -File C:/provision/compose-install/di-compose-getBinary.ps1 -url ${var.download_url}",
-      "powershell.exe -File C:/provision/compose-install/di-compose-install.ps1 -url ${var.download_url} -version ${var.c_version}",
-      "powershell.exe -File C:/provision/compose-install/di-compose-setlicense.ps1 -composeLicense \"${local.composeLicense}\" -version ${var.c_version}",
+      "powershell.exe -File C:/provision/compose-install/di-compose-install.ps1 -url ${var.download_url}",
+      "powershell.exe -File C:/provision/compose-install/di-compose-setlicense.ps1 -version ${var.c_version}",
       "powershell.exe -File C:/provision/compose-install/di-compose-installQMICertificate.ps1 -CertPwd \"${local.cert_password}\" -version ${var.c_version}",
     ]
 

compose-install/scripts/Compose_install.iss

@@ -1,25 +1,20 @@
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-DlgOrder]
 Dlg0={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0
 Count=5
-Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicense2Rtf-0
+Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0
 Dlg2={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0
 Dlg3={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0
 Dlg4={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcomeMaint-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-MessageBox-0]
-Result=6
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0]
 Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicense2Rtf-0]
+[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0]
 Result=1
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0]
 szDir=C:\Program Files\Qlik\Compose\
 Result=1
 [{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0]
 Result=1
-
+[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

compose-install/scripts/Compose_install_2021.8.0.iss

@@ -1,20 +0,0 @@
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-DlgOrder]
-Dlg0={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0
-Count=5
-Dlg1={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0
-Dlg2={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0
-Dlg3={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0
-Dlg4={CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdWelcome-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdLicenseAgreement-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdAskDestPath-0]
-szDir=C:\Program Files\Qlik\Compose\
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdStartCopy-0]
-Result=1
-[{CC3EB4FA-1694-4961-93E6-D7F0DA630806}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0

compose-install/scripts/compose-license.txt

@@ -0,0 +1,21 @@
+{
+  "$type": "ComposeLicense",
+  "product": "QlikCompose",
+  "issued_to": "Qlik Internal Use only",
+  "issued_by": "Attunity US",
+  "license_type": "EVALUATION",
+  "serial_no": "85010344",
+  "expiration_date": "2024-12-31",
+  "hosts": "",
+  "product_version": "2023.11",
+  "notes": "Qlik Internal Use only",
+  "host_role": "",
+  "source_db_types": "",
+  "dwh_type": "*",
+  "dl_type": "*",
+  "number_of_dms": "0",
+  "number_of_developers": "0",
+  "managed_dwh_size": "0",
+  "issue_time": "12/5/2023 9:36:23 PM",
+  "signature": "LnAeWQPSDsQw0quG6h5HxvAdnzerIbW/KwOCYvFeq6T8Tp8i6hxAgnKpeVeqTKNiA3t5ovkvIjBvH5n0JGjG75r4OohWiNLFaWYrSvgf6d/fPzFK/4tgyhdImmbS8dZm3tInY3Y8ZNM+kEGi8Hze2hPoHZGwh5nZMM0eCo2E4Sk="
+}

compose-install/scripts/di-compose-install.ps1

@@ -9,26 +9,36 @@ Intent: Install the selected version of Attunity Replicate
 #>
 
 Param(
-    [string] $url,
-    [string] $version = "gen2"
+    [string] $url
 )
 
 Import-Module $PSScriptRoot\q-helper.psm1 | Out-Null
 
 $fileName = $url.Substring($url.LastIndexOf("/") + 1)
 
+Write-Host "--- Starting di-compose-install.ps1"
+
 $bin = "$PSScriptRoot\binaries\Attunity"
 Write-Host "Binary Path $($bin)"
-Write-Host "Starting di-c4dw-install.ps1"
+
+
 $issFile = "Compose_install.iss"
 
-if ( $version -ne "gen2" ) {
-    $issFile = "Compose_install_$version.iss" 
+if ( $url -Match "2021.2" ) {
+	$issFile="Compose_install_2021.2.0.iss"
+}
+if ( $url -Match "Attunity_Compose_for_Data_Warehouses" ) {
+	$issFile="Compose_install_c4dw.iss"
 }
 
+Write-Host "Using ISS file: $issFile"
 
 Write-Host $bin\$fileName
 
+if (Test-Path "$($bin)\Compose_silent_x64_install.log") {
+	Remove-Item -Path "$($bin)\Compose_silent_x64_install.log" -Force
+}
+
 If ((Test-Path $bin\$fileName))
 {
 	Write-Host "Installing Attunity Compose from $($bin)\$($fileName)"
@@ -36,20 +46,22 @@ If ((Test-Path $bin\$fileName))
     $fileNoExtension = [IO.Path]::GetFileNameWithoutExtension("$bin\$fileName")
 	Expand-Archive $bin\$fileName -DestinationPath $bin\$fileNoExtension -Force
 
-    #Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
+    Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
 	$exec = Get-ChildItem $bin\$fileNoExtension\*.exe | Select-Object -ExpandProperty Name
     #Write-Host "Exec: $($exec)"
 
-    $C4DWInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
+    $silentInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
 
-	Write-Host "Run Compose silent installation : $($C4DWInstall)"
-    Invoke-Expression $C4DWInstall
+    Start-Sleep 5 
+    
+	Write-Host "Run Compose silent installation : $($silentInstall)"
+    Invoke-Expression $silentInstall
     while (!(Test-Path "$($bin)\Compose_silent_x64_install.log")) { 
         Write-Host "[Waiting Compose to be installed] ..."
-        Start-Sleep 2
+        Start-Sleep 4
     }
-	$C4DWResults = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
-	$testResult = $C4DWResults.ResponseResult.ResultCode
+	$resultLogs = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
+	$testResult = $resultLogs.ResponseResult.ResultCode
 	Write-Host "Installation return code : $($testResult)"
 
 }

compose-install/scripts/di-compose-installQMICertificate.ps1

@@ -6,6 +6,11 @@ Param(
 
 try {
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

compose-install/scripts/di-compose-setlicense.ps1

@@ -9,14 +9,10 @@ Intent: Setting the DBs
 #>
 
 Param(
-    [string] $composeLicense,
     [string] $version = 'gen2'
 )
 
-Write-Host "ComposeLicenseJSON"
-Write-Host $composeLicense
-
-Set-Content $PSScriptRoot\compose-license.txt $composeLicense
+Write-Host "--- Setting Compose License"
 
 # TODO set license
 $bin = "C:\Program Files\Qlik\Compose\bin\"
@@ -26,16 +22,18 @@ if ( $version -eq "c4dw" ) {
 $bin = $bin -replace ' ','` '
 
 $cmd = "$($bin)ComposeCli.exe connect"
-Write-Host "Connect to Compose: $($cmd)"
+Write-Host "--- Connect to Compose: $($cmd)"
 $cmd = "$($bin)ComposeCli.exe connect"
 Invoke-Expression $cmd
 
 Start-Sleep 5
 
-if ( $version -eq "2021.8.0" ) {
-    $cmd = "$($bin)ComposeCli.exe register_license --infile $PSScriptRoot\compose-license.txt"
-} else {
+if ( $version -eq "2022.2.0" -or $version -eq "c4dw") {
+    # Old way
     $cmd = "$($bin)ComposeCli.exe register_license --req @$PSScriptRoot\compose-license.txt"
+} else {
+    # New way
+    $cmd = "$($bin)ComposeCli.exe register_license --infile $PSScriptRoot\compose-license.txt"
 }
 Write-Host "Apply Compose License: $($cmd)"
 Invoke-Expression $cmd

databases/aws-emr/main.tf

@@ -0,0 +1,146 @@
+terraform {
+
+  required_version = ">= 1.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+    }
+  }
+}
+
+
+module "s3_bucket" {
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "~> 3.0"
+
+  bucket_prefix = "qmi-bucket-${var.provision_id}"
+
+  # Allow deletion of non-empty bucket
+  # Example usage only - not recommended for production
+  force_destroy = true
+
+  attach_deny_insecure_transport_policy = true
+  attach_require_latest_tls_policy      = true
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
+  server_side_encryption_configuration = {
+    rule = {
+      apply_server_side_encryption_by_default = {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+
+  tags = {
+    QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+    Owner    = var.user_id
+  }
+}
+
+module "emr" {
+  source = "terraform-aws-modules/emr/aws"
+
+  name = var.provision_id
+
+  release_label = "emr-6.11.0"
+  applications  = ["spark", "hadoop", "hive", "hue"]
+  auto_termination_policy = {
+    idle_timeout = 3600
+  }
+
+  bootstrap_action = {
+    example = {
+      name = "Just an example",
+      path = "file:/bin/echo",
+      args = ["Hello World!"]
+    }
+  }
+
+  configurations_json = jsonencode([
+    {
+      "classification" : "spark-env",
+      "configurations" : [
+        {
+          "classification" : "export",
+          "properties" : {
+            "JAVA_HOME" : "/usr/lib/jvm/java-1.8.0"
+          }
+        }
+      ],
+      "properties" : {}
+    },
+    {
+      "classification": "hive-site",
+      "properties": {
+         "hive.support.concurrency": "true",
+         "hive.exec.dynamic.partition.mode": "nonstrict",
+         "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"
+      }
+    }
+  ])
+
+  master_instance_group = {
+    name           = "master-group"
+    instance_count = 1
+    instance_type  = "m5.xlarge"
+  }
+
+  core_instance_group = {
+    name           = "core-group"
+    instance_count = 1
+    instance_type  = "c4.large"
+  }
+
+  task_instance_group = {
+    name           = "task-group"
+    instance_count = 1
+    instance_type  = "c5.xlarge"
+    bid_price      = "0.1"
+
+    ebs_config = {
+      size                 = 64
+      type                 = "gp3"
+      volumes_per_instance = 1
+    }
+    ebs_optimized = true
+  }
+
+  ebs_root_volume_size = 64
+  ec2_attributes = {
+    # Instance groups only support one Subnet/AZ
+    # Subnets should be private subnets and tagged with
+    # { "for-use-with-amazon-emr-managed-policies" = true }
+    subnet_id = var.subnet_ids_us[0]
+  }
+  vpc_id = var.vpc_id_us
+
+  list_steps_states  = ["PENDING", "RUNNING", "CANCEL_PENDING", "CANCELLED", "FAILED", "INTERRUPTED", "COMPLETED"]
+  log_uri            = "s3://${module.s3_bucket.s3_bucket_id}/"
+
+  scale_down_behavior    = "TERMINATE_AT_TASK_COMPLETION"
+  step_concurrency_level = 3
+  termination_protection = false
+  visible_to_all_users   = true
+  
+  is_private_cluster = false
+  #create_service_iam_role = false
+  #service_iam_role_arn = "arn:aws:iam::192018133564:role/service-role/AmazonEMR-ServiceRole-20230622T122656"
+  #create_iam_instance_profile = false
+  #iam_instance_profile_name = "AmazonEMR-InstanceProfile-20230622T122640"
+
+  tags = {
+    Terraform   = "true"
+    Environment = "dev"
+    QMI_user = var.user_id
+    Owner    = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+  }
+}

databases/aws-emr/outputs.tf

@@ -0,0 +1,15 @@
+output "s3_bucket" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_id
+}
+
+output "s3_iam_name" {
+  value = module.qmi-s3-bucket.iam_name
+}
+
+output "s3_iam_access_key" {
+  value = module.qmi-s3-bucket.iam_access_key
+}
+
+output "s3_iam_access_secret" {
+  value =  module.qmi-s3-bucket.iam_access_secret
+}

databases/aws-emr/variables.tf

@@ -0,0 +1,34 @@
+variable "region" {
+  default = "us-east-1"
+}
+
+variable "provision_id" {
+  
+}
+
+variable "user_id" {
+}
+
+variable "vpc_id_ap" {
+  default = "vpc-22ee2844"
+}
+
+variable "vpc_id_eu" {
+  default = "vpc-73f0500a"
+}
+
+variable "vpc_id_us" {
+  default = "vpc-c079f5bd"
+}
+
+variable "subnet_ids_ap" {
+  default = ["subnet-658aec3c", "subnet-e030eba8", "subnet-94309bf2"]
+}
+
+variable "subnet_ids_eu" {
+  default = ["subnet-4d441b17", "subnet-95c22fde", "subnet-70938116"]
+}
+
+variable "subnet_ids_us" {
+  default = ["subnet-4d26552b", "subnet-0414685b", "subnet-c97f7c84", "subnet-7f695c71", "subnet-96acd2b7", "subnet-88ab2cb9"]
+}

databases/aws-kinesis/main.tf

@@ -26,6 +26,7 @@ locals {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
   }

databases/aws-kinesis/output.tf

@@ -23,5 +23,5 @@ output "iam_access_key" {
 }
 
 output "iam_access_secret" {
-  value = module.iam_user.this_iam_access_key_secret
+  value = nonsensitive(module.iam_user.this_iam_access_key_secret)
 }

databases/aws-rds/main.tf

@@ -1,36 +1,101 @@
 terraform {
 
-  required_version = ">= 0.13"
+  required_version = ">= 0.14"
 
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.49.0"
     }
   }
 }
 
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
 locals {
   provid5              = substr(var.provision_id, 0, 5)
   aurora               = (var.engine == "aurora-mysql") ? true : (var.engine == "aurora-postgresql") ? true : false
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
-  port                 = (var.engine == "oracle-se2") ? "1521" : (var.engine == "postgres") ? "5432" : (var.engine == "aurora-postgresql") ? "5432" : (var.engine == "sqlserver-ex") ? "1433" : "3306"
+  
   name                 = (var.engine == "sqlserver-ex") ? null : (var.engine == "oracle-se2") ? "ora${local.provid5}" : "qmi${var.provision_id}"
   license              = (local.aurora == true) ? "general-public-license" : (var.engine == "mariadb") ? "general-public-license" : (var.engine == "postgres") ? "postgresql-license" : (var.engine == "mysql") ? "general-public-license" : "license-included"
-  engine_version       = (var.engine == "oracle-se2") ? "19.0.0.0.ru-2021-04.rur-2021-04.r1" : (var.engine == "postgres") ? "13.3" : (var.engine == "mysql") ? "8.0.25" : (var.engine == "aurora-postgresql") ? "12.6" : (var.engine == "aurora-mysql") ? "5.7.mysql_aurora.2.10.0" : (var.engine == "sqlserver-ex") ? "15.00.4073.23.v1" : "10.5"   #mariaDB
-  major_engine_version = (var.engine == "oracle-se2") ? "19" : (var.engine == "postgres") ? "13" : (var.engine == "mysql") ? "8.0" : (var.engine == "aurora-postgresql") ? "12" : (var.engine == "aurora-mysql") ? "5.7" : (var.engine == "sqlserver-ex") ? "15.00" : "10.5"                                                                         #mariaDB
-  family               = (var.engine == "oracle-se2") ? "oracle-se2-19" : (var.engine == "postgres") ? "postgres13" : (var.engine == "mysql") ? "mysql8.0" : (var.engine == "aurora-postgresql") ? "aurora-postgresql12" : (var.engine == "aurora-mysql") ? "aurora-mysql5.7" : (var.engine == "sqlserver-ex") ? "sqlserver-ex-15.0" : "mariadb10.5" #mariaDB
+  
+  port                 = var.port[var.engine] 
+  engine_version       = var.engine_version[var.engine]
+  major_engine_version = var.major_engine_version[var.engine]
+  family               = var.family[var.engine]
+
   tags = {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
+
+  is_postgres = length(regexall("postgres", local.family)) > 0 ? true : false
+  is_mysql = length(regexall("mysql", local.family)) > 0 ? true : false
 }
 
 
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+module "security_group_2" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
+
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+
+  # ingress
+
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
+  
+
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "RDS"
+
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "RDS"
+
+    },
+  ]
+
+  tags = local.tags
+}
+
 module "security_group" {
   
   # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
@@ -44,45 +109,8 @@ module "security_group" {
 
   # ingress
 
-  ingress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks
+  
 
   ingress_with_cidr_blocks = [
     {
@@ -96,45 +124,7 @@ module "security_group" {
 
   # egress
 
-  egress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  egress_cidr_blocks = module.fw-ips.cidr_blocks
 
   egress_with_cidr_blocks = [
     {
@@ -151,7 +141,7 @@ module "security_group" {
 
 module "common_rds_instance" {
   source  = "terraform-aws-modules/rds/aws"
-  version = "~> 3.3"
+  version = "= 6.1.1"
 
   count = local.aurora? 0 : 1
 
@@ -163,58 +153,174 @@ module "common_rds_instance" {
   major_engine_version = local.major_engine_version # DB option group
   instance_class       = var.instance_size
   allocated_storage    = var.storage
+  storage_encrypted    = (var.engine == "sqlserver-ex")? false : true
 
   license_model = local.license
 
-  name                   = local.name
+  db_name                = local.name
   username               = "qmirdsuser"
-  create_random_password = true
-  random_password_length = 12
+  password               = random_password.password.result
+  manage_master_user_password = false
   port                   = local.port
 
   multi_az               = false
   subnet_ids             = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+    module.security_group.security_group_id,
+    module.security_group_2.security_group_id
+  ]
   publicly_accessible    = true
 
   maintenance_window = "Mon:00:00-Mon:03:00"
   backup_window      = "03:00-06:00"
 
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
+  parameters = local.is_postgres? [
+    {
+      apply_method = "pending-reboot"
+      name  = "rds.logical_replication"
+      value = 1
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_wal_senders"
+      value = 10
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_replication_slots"
+      value = 10
+    }
+  ] : local.is_mysql? [{
+      name  = "binlog_format"
+      value = "row"
+  }]: []
+
   tags = local.tags
 
 }
 
+
+resource "aws_rds_cluster_parameter_group" "pg-postgres" {
+
+  count =  var.engine == "aurora-postgresql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-postgresql14"
+  description = "RDS aurora-postgresql14 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "rds.logical_replication"
+    value = 1
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_wal_senders"
+    value = 10
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_replication_slots"
+    value = 10
+  }
+
+}
+
+resource "aws_rds_cluster_parameter_group" "pg-mysql" {
+
+  count =  var.engine == "aurora-mysql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-mysql8.0"
+  description = "RDS aurora-mysql8.0 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "binlog_format"
+    value = "row"
+  }
+
+}
+
 module "aurora_rds_instance" {
+
+  depends_on = [  
+    aws_rds_cluster_parameter_group.pg-postgres, 
+    aws_rds_cluster_parameter_group.pg-mysql
+  ]
+
   source  = "terraform-aws-modules/rds-aurora/aws"
-  version = "~> 5.2"
+  version = "~> 8.3.1"
 
   count = local.aurora? 1 : 0
 
   name           = local.name
   engine         = var.engine
   engine_version = local.engine_version
-  instance_type  = var.instance_size
+  instance_class  = var.instance_size
+
+  apply_immediately = true
+
+  database_name = local.name
+
+  instances       = { 1 = {} }
+
+  autoscaling_enabled      = true
+  autoscaling_min_capacity = 1
+  autoscaling_max_capacity = 3
 
   vpc_id                 = local.vpc_id
   subnets                = local.subnet_ids
   create_security_group  = false
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+      module.security_group.security_group_id,
+      module.security_group_2.security_group_id
+  ]
   port                   = local.port
   publicly_accessible    = true
 
-  username               = "qmirdsuser"
-  create_random_password = true
+  master_username         = "qmirdsuser"
+  manage_master_user_password = false
+  master_password = random_password.password.result
+  create_db_subnet_group = true
 
 
-
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
   tags = local.tags
 
+  db_cluster_parameter_group_name = "rds-cluster-pg-${var.provision_id}"
+
+}
+
+locals {
+    
+    type      = (var.engine == "mysql" || var.engine == "mariadb")? "mysql" :  (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : (var.engine == "sqlserver-ex")? "mssql" : null
+    port4dummy  = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
+    host      = (local.aurora)? "${module.aurora_rds_instance[0].cluster_endpoint}:${local.port4dummy}" : module.common_rds_instance[0].db_instance_endpoint
+    username  = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+    password  = nonsensitive(random_password.password.result)
+    database  = (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : local.name
+}
+
+
+module "dummy-data" {
+
+  count = var.dummydata != null && local.type != null? 1 : 0
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = local.type
+  host      = local.host
+  username  = local.username
+  password  = local.password
+  database  = local.database
 }

databases/aws-rds/output.tf

@@ -1,23 +1,31 @@
  output "db_username" {
   description = "The master username for the database"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
 } 
 
  output "db_instance_password" {
   description = "The database password (this password may be old, because Terraform doesn't track it after initial creation)"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_password) : nonsensitive(module.common_rds_instance[0].db_instance_password)
+  value       = nonsensitive(random_password.password.result)
 }
 
  output "db_instance_port" {
   description = "The database port"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_port : module.common_rds_instance[0].db_instance_port
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
 }
 
 output "db_instance_endpoint" {
   description = "The connection endpoint"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+}
+
+output "db_instance_id" {
+  value       = (local.aurora)? null : "${var.engine}${var.provision_id}"
 }
 
 output "db_instance_name" {
   value       = local.name
 }
+
+output "dummy_data_databases_available" {
+   value = var.dummydata != null && local.type != null? module.dummy-data[0].dbs : null
+}

databases/aws-rds/variables.tf

@@ -38,21 +38,6 @@ variable "engine" {
   description = "Accepted values are mysql, sqlserver-ex, postgres, oracle-se2, aurora-mysql, aurora-postgresql, mariadb"
 }
 
-variable "engine_version" {
-  default = "8.0.25"
-  description = "Not currently used"
-}
-
-variable "major_eng" {
-  default = "8.0"
-  description = "Not currently used"
-}
-
-variable "family" {
-  default = "mysql8.0"
-  description = "Not currently used"
-}
-
 variable "instance_size" {
   default = "db.t3.large"
 }
@@ -60,3 +45,68 @@ variable "instance_size" {
 variable "storage" {
   default = "20"
 }
+
+variable "engine_version" {
+  type = map
+
+  default = {
+    "mysql"  = "8.0.32"
+    "postgres"  = "14.9" 
+    "oracle-se2"  = "19.0.0.0.ru-2023-04.rur-2023-04.r1"
+    "sqlserver-ex" = "15.00.4236.7.v1"
+    "mariadb" = "10.5"
+    "aurora-mysql" = "8.0.mysql_aurora.3.05.2" #"5.7.mysql_aurora.2.11.2"
+    "aurora-postgresql"  = "14.9"
+  }
+  
+}
+
+variable "major_engine_version" {
+  type = map
+
+  default = {
+    "mysql"  = "8.0"  
+    "postgres"  = "14"
+    "oracle-se2"  = "19"
+    "sqlserver-ex" = "15.00"
+    "mariadb" = "10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+  
+}
+
+
+variable "family" {
+  type = map
+
+  default = {
+    "mysql"  = "mysql8.0"
+    "postgres"  = "postgres14"
+    "oracle-se2"  = "oracle-se2-19"
+    "sqlserver-ex" = "sqlserver-ex-15.0"
+    "mariadb" = "mariadb10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+}
+
+variable "port" {
+  type = map
+
+  default = {
+    "mysql"  = "3306"
+    "postgres"  = "5432"
+    "oracle-se2"  = "1521"
+    "sqlserver-ex" = "1433"
+    "mariadb" = "3306"
+    "aurora-mysql" = "3306"
+    "aurora-postgresql"  = "5432"
+  }
+}
+
+variable "dummydata" {
+  default = null
+}

databases/aws-redshift/main.tf

@@ -17,13 +17,14 @@ resource "random_password" "password" {
   min_numeric = 1
   upper = true
   lower = true
-  number = true
+  numeric = true
   min_lower = 2
   min_upper = 2
   min_special = 2
 }
 
 locals {
+  s3_prefix = "redshift/qmi-${var.provision_id}"
   provid5              = substr(var.provision_id, 0, 5)
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
@@ -34,9 +35,14 @@ locals {
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
 }
 
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
 module "security_group" {
   # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
   source  = "terraform-aws-modules/security-group/aws"
@@ -49,45 +55,7 @@ module "security_group" {
 
   # ingress
 
-  ingress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks
 
   ingress_with_cidr_blocks = [
     {
@@ -101,45 +69,7 @@ module "security_group" {
 
   # egress
 
-  egress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  egress_cidr_blocks = module.fw-ips.cidr_blocks
 
   
   egress_with_cidr_blocks = [
@@ -155,27 +85,89 @@ module "security_group" {
   tags = local.tags
 }
 
+module "security_group_2" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
+
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+
+  # ingress
+
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
+  
+
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Redshift"
+
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Redshift"
+
+    },
+  ]
+
+  tags = local.tags
+}
+
+module "qmi-s3-bucket" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
+  
+  provision_id                  = var.provision_id
+  region                        = var.region
+  user_id                       = var.user_id
+}
+
 module "redshift" {
 
   source  = "terraform-aws-modules/redshift/aws"
-  version = "~> 3.0"
+  version = "~> 5.0.0"
 
   cluster_identifier      = "qmi-${var.provision_id}"
-  cluster_node_type       = "dc2.large" #"dc1.large"
-  cluster_number_of_nodes = 1
+  node_type       = "dc2.large" #"dc1.large"
+  number_of_nodes = 1
 
-  cluster_database_name   = var.cluster_database_name
-  cluster_master_username = var.cluster_master_username
-  cluster_master_password = random_password.password.result
+  database_name   = var.cluster_database_name
+  master_username = var.cluster_master_username
+  create_random_password = false
+  master_password = random_password.password.result
 
   # Group parameters
   #wlm_json_configuration = "[{\"query_concurrency\": 5}]"
 
   # DB Subnet Group Inputs
-  subnets = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  subnet_ids = local.subnet_ids
+  vpc_security_group_ids = [
+      module.security_group.security_group_id,
+      module.security_group_2.security_group_id
+  ]
   publicly_accessible = true
 
+  /*logging = {
+    enable        = true
+    bucket_name   = module.qmi-s3-bucket.bucket.s3_bucket_id
+    s3_key_prefix = local.s3_prefix
+  }*/
+
   # IAM Roles
   #cluster_iam_roles = ["arn:aws:iam::225367859851:role/developer"]
 
@@ -183,11 +175,3 @@ module "redshift" {
 
 }
 
-module "qmi-s3-bucket" {
-
-  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket?ref=dev2"
-  
-  provision_id                  = var.provision_id
-  region                        = var.region
-  user_id                       = var.user_id
-}

databases/aws-redshift/outputs.tf

@@ -1,25 +1,25 @@
 output "redshift_cluster_id" {
   description = "The availability zone of the RDS instance"
-  value       = module.redshift.redshift_cluster_id
+  value       = module.redshift.cluster_id
 }
 
 output "redshift_cluster_endpoint" {
   description = "Redshift endpoint"
-  value       = module.redshift.redshift_cluster_endpoint
+  value       = module.redshift.cluster_endpoint
 }
 
 output "redshift_cluster_hostname" {
   description = "Redshift hostname"
-  value       = module.redshift.redshift_cluster_hostname
+  value       = module.redshift.cluster_hostname
 }
 
 output "redshift_cluster_port" {
   description = "Redshift port"
-  value       = module.redshift.redshift_cluster_port
+  value       = module.redshift.cluster_port
 }
 
 output "redshift_cluster_database_name" {
-    value = module.redshift.redshift_cluster_database_name
+    value = module.redshift.cluster_database_name
 }
 
 output "redshift_cluster_master_username" {

databases/aws-redshift/variables.tf

@@ -38,5 +38,5 @@ variable "cluster_database_name" {
 }
 
 variable "cluster_master_username" {
-    default = "qmi"
+    default = "qmiuser"
 }

databases/azure-eventhub/main.tf

@@ -25,6 +25,7 @@ resource "azurerm_eventhub_namespace" "ehbnamespace" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   network_rulesets {

databases/azure-eventhub/outputs.tf

@@ -1,9 +1,9 @@
 output "default_primary_connection_string" {
-  value = azurerm_eventhub_namespace.ehbnamespace.default_primary_connection_string
+  value = nonsensitive(azurerm_eventhub_namespace.ehbnamespace.default_primary_connection_string)
 }
 
 output "default_primary_key" {
-  value = azurerm_eventhub_namespace.ehbnamespace.default_primary_key
+  value = nonsensitive(azurerm_eventhub_namespace.ehbnamespace.default_primary_key)
 }
 
 output "namespace_name" {

databases/azure-hdinsight-hadoop/main.tf

@@ -0,0 +1,107 @@
+
+resource "random_password" "password1" {
+  length = 16
+  special = true
+  override_special = "_!@"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+resource "random_password" "password2" {
+  length = 16
+  special = true
+  override_special = "_!@"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 5
+}
+
+resource "azurerm_storage_account" "example" {
+  name                     = "hdinsightstor${random_id.randomMachineId.hex}"
+  resource_group_name      = var.resource_group_name
+  location                 = var.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+
+  tags = var.tags
+}
+
+resource "azurerm_storage_container" "example" {
+  name                  = "hdinsight"
+  storage_account_name  = azurerm_storage_account.example.name
+  container_access_type = "private"
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp1" {
+  scope                = azurerm_storage_account.example.id
+  role_definition_name = "Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_role_assignment" "data-contributor-dbricksapp2" { 
+  scope                = azurerm_storage_account.example.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.dbricks_app_registration_principal_id
+}
+
+resource "azurerm_hdinsight_hadoop_cluster" "example" {
+  name                = "hdicluster-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  location            = var.location
+  cluster_version     = "4.0"
+  tier                = "Standard"
+
+  tags = var.tags
+
+  component_version {
+    hadoop = "3.1"
+  }
+
+  gateway {
+    username = "acctestusrgw"
+    password = random_password.password1.result
+  }
+
+  storage_account {
+    storage_container_id = azurerm_storage_container.example.id
+    storage_account_key  = azurerm_storage_account.example.primary_access_key
+    is_default           = true
+  }
+
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = random_password.password2.result
+    }
+
+    worker_node {
+      vm_size               = "Standard_D3_V2"
+      username              = "acctestusrvm"
+      password              = random_password.password2.result
+      target_instance_count = 2
+    }
+
+    zookeeper_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = random_password.password2.result
+    }
+  }
+}

databases/azure-hdinsight-hadoop/outputs.tf

@@ -0,0 +1,59 @@
+output "gateway-creds" {
+  value = {
+    username = "acctestusrgw"
+    password = nonsensitive(random_password.password1.result)
+  }
+}
+
+output "cluster-creds" {
+  value = {
+    username = "acctestusrvm"
+    password = nonsensitive(random_password.password2.result)
+  }
+}
+
+output "https_endpoint" {
+    value = azurerm_hdinsight_hadoop_cluster.example.https_endpoint
+}
+
+output "ssh_endpoint" {
+  value = azurerm_hdinsight_hadoop_cluster.example.ssh_endpoint
+}
+
+
+output "Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}
+
+
+output "adls_StorageAccount-Name" {
+  value = azurerm_storage_account.example.name
+}
+
+output "adls_StorageAccount-ContainerName" {
+  value = azurerm_storage_container.example.name
+}
+
+output "adls_StorageAccount-AccessKey" {
+  value = nonsensitive(azurerm_storage_account.example.primary_access_key)
+}
+
+output "adls_Azure_Active_Directory_Tenant_ID" {
+  value = "c21eeb5f-f5a6-44e8-a997-124f2f7a497c"
+}
+
+output "adls_Azure_Application_Registration_Client_ID" {
+  value = var.dbricks_app_registration_application_id
+}
+
+output "adls_Azure_Application_Registration_Secret" {
+  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+}

databases/azure-hdinsight-hadoop/variables.tf

@@ -0,0 +1,26 @@
+variable "resource_group_name" {
+  type = string
+}
+
+variable "provision_id" {
+  type = string
+}
+
+variable "location" {
+  type = string
+  default = "EAST US"
+}
+
+variable "tags" {
+    default = null
+}
+
+variable "dbricks_app_registration_principal_id" {
+    description = "databricks-qmi"
+    default = "efeee17c-d2b3-4e7c-a163-9995b7d281e2"
+}
+
+variable "dbricks_app_registration_application_id" {
+    description = "databricks-qmi"
+    default = "9ccb0d99-3bba-4695-aa47-df77bf512084"
+}

databases/azure-rds-flexmysql/firewall.tf

@@ -0,0 +1,29 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips_az_qcs
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_flexible_server.mysql-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mysql_flexible_server_firewall_rule.fw_rule ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_flexible_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
+}

databases/azure-rds-flexmysql/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_numeric = 2
@@ -33,6 +33,7 @@ resource "azurerm_mysql_flexible_server" "mysql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -49,12 +50,4 @@ resource "azurerm_mysql_flexible_server_configuration" "example" {
   resource_group_name = var.resource_group_name
   server_name         = azurerm_mysql_flexible_server.mysql-server.name
   value               = "full"
-}
-
-resource "azurerm_mysql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
 }

databases/azure-rds-flexmysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-flexmysql/variables.tf

@@ -36,4 +36,8 @@ variable "mysql-storage" {
   type = string
   description = "MySQL Storage in MB"
   default = "20"
-  }
+}
+
+variable "dummydata" {
+  default = null
+}

databases/azure-rds-flexpostgres/firewall.tf

@@ -0,0 +1,28 @@
+/*resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
+  name                = "AllAzureServices"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "0.0.0.0"
+  end_ip_address      = "0.0.0.0"
+}*/
+
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule1" {
+  name                = "a1"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "52.249.189.38"
+  end_ip_address      = "52.249.189.38"
+}
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule2" {
+  name                = "a2"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "13.67.39.86"
+  end_ip_address      = "13.67.39.86"
+}
+
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule3" {
+  name                = "a3"
+  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
+  start_ip_address    = "20.67.110.207"
+  end_ip_address      = "20.67.110.207"
+}

databases/azure-rds-flexpostgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!)"
   upper = true
   lower = true
   min_numeric = 2
@@ -29,6 +29,7 @@ resource "azurerm_postgresql_flexible_server" "postgresql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
 }
@@ -51,11 +52,3 @@ resource "azurerm_postgresql_flexible_server_database" "postgresql-db" {
   collation = "en_US.utf8"
   charset   = "utf8"
 }
-
-
-resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}

databases/azure-rds-mariadb/firewall.tf

@@ -0,0 +1,26 @@
+resource "azurerm_mariadb_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  subnet_id           = var.subnet_id
+
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mariadb_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}

databases/azure-rds-mariadb/main.tf

@@ -0,0 +1,61 @@
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
+
+resource "azurerm_mariadb_server" "mariadb-server" {
+  name = "qmi-mariadb-${var.provision_id}"
+  location = var.location
+  resource_group_name = var.resource_group_name
+
+  administrator_login = var.admin_login
+  administrator_login_password = random_password.password.result
+
+  sku_name = var.sku-name
+  version = var.db-version
+ 
+  storage_mb = var.storage
+  auto_grow_enabled = true
+
+  
+  backup_retention_days         = 7
+  geo_redundant_backup_enabled  = false
+  public_network_access_enabled = true
+  ssl_enforcement_enabled       = false
+
+  tags = {
+    Deployment = "QMI PoC"
+    "Cost Center" = "3100"
+    QMI_user = var.user_id
+    Owner = var.user_id
+  }
+}
+
+resource "azurerm_mariadb_database" "mariadb-db" {
+  name                = "QlikDB"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mariadb_server.mariadb-server.name
+  charset             = "utf8"
+  collation           = "utf8_unicode_ci"
+}
+
+/*
+module "dummy-data" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mariadb_server.mariadb-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
+}
+*/

databases/azure-rds-mariadb/output.tf

@@ -0,0 +1,11 @@
+output "db_server_fqdn" {
+  value = azurerm_mariadb_server.mariadb-server.fqdn
+}
+
+output "root_username" {
+  value = "${var.admin_login}@qmi-mariadb-${var.provision_id}"
+}
+
+output "root_username_password" {
+  value = nonsensitive(random_password.password.result)
+}

databases/azure-rds-mariadb/variables.tf

@@ -0,0 +1,43 @@
+variable "resource_group_name" {
+
+}
+
+variable "provision_id" {
+  type = string
+  description = "(optional) describe your variable"
+}
+
+variable "location" {
+  type = string
+  description = "(optional) describe your variable"
+  default = "EAST US"
+}
+
+variable "subnet_id" {
+  default = null
+}
+
+variable "user_id" {
+}
+
+variable "admin_login" {
+  type = string
+  description = "Login to authenticate to MySQL Server"
+  default = "qmi"
+}
+
+variable "db-version" {
+  type = string
+  description = "MariaDB Server version to deploy"
+  default = "10.2"
+}
+variable "sku-name" {
+  type = string
+  description = "MariaDB SKU Name"
+  default = "GP_Gen5_2"
+}
+variable "storage" {
+  type = string
+  description = "MariaDB Storage in MB"
+  default = "5120"
+  }

databases/azure-rds-mssql/firewall.tf

@@ -0,0 +1,49 @@
+
+
+resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+  
+  name                = "vnet-rule-${local.provision_id}"
+  server_id           = azurerm_mssql_server.sqlserver.id
+  subnet_id           = var.subnet_id
+}
+
+resource "azurerm_mssql_virtual_network_rule" "sqlvnet_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                = "vnet-qmiinfra-${local.provision_id}"
+  server_id           = azurerm_mssql_server.sqlserver.id
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mssql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  server_id           = azurerm_mssql_server.sqlserver.id
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mssql_firewall_rule.fw_rule ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mssql"
+  host      = azurerm_mssql_server.sqlserver.fully_qualified_domain_name
+  username  = var.sql_administrator_login
+  password  = nonsensitive(local.sql_administrator_login_password)
+  database  = ""
+}

databases/azure-rds-mssql/main.tf

@@ -10,7 +10,7 @@ resource "random_id" "randomServerId" {
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -37,6 +37,7 @@ resource "azurerm_mssql_server" "sqlserver" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
   identity {
@@ -44,216 +45,6 @@ resource "azurerm_mssql_server" "sqlserver" {
   }
 }
 
-resource "azurerm_mssql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
-
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-ITG-rule1" {
-  name                = "ITG"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-ITG-rule2" {
-  name                = "ITG1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-# QCS
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCS3" {
-  name                = "QCSEU3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
-  name                = "sql-vnet-rule-${local.provision_id}"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  subnet_id           = var.subnet_id
-}
-
-# QAA
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus1" {
-  name                = "QAAus1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus2" {
-  name                = "QAAus2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAus3" {
-  name                = "QAAus3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_mssql_firewall_rule" "azuresql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  server_id           = azurerm_mssql_server.sqlserver.id
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
-}
-
 resource "azurerm_mssql_database" "db" {
   name           = "demoDB"
   server_id      = azurerm_mssql_server.sqlserver.id
@@ -263,6 +54,7 @@ resource "azurerm_mssql_database" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -276,6 +68,7 @@ resource "azurerm_mssql_database" "dbname_source" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -288,5 +81,6 @@ resource "azurerm_mssql_database" "dbname_target" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }

databases/azure-rds-mssql/outputs.tf

@@ -26,4 +26,8 @@ output "principal_id" {
   value = azurerm_mssql_server.sqlserver.identity.0.principal_id
 }
 
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}
+
 

databases/azure-rds-mssql/variables.tf

@@ -10,7 +10,7 @@ variable "provision_id" {
 }
 
 variable "subnet_id" {
-
+    default = null
 }
 
 variable "user_id" {
@@ -23,4 +23,12 @@ variable "sql_administrator_login" {
 
 variable "sql_administrator_login_password" {
     default = null
-}
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-mysql/firewall.tf

@@ -0,0 +1,55 @@
+resource "azurerm_mysql_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  subnet_id           = var.subnet_id
+
+}
+
+resource "azurerm_mysql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                  = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name   = var.resource_group_name
+  server_name           = azurerm_mysql_server.mysql-server.name
+  subnet_id             = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_mysql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_mysql_virtual_network_rule.vnetrule,
+    azurerm_mysql_virtual_network_rule.vnetrule_qmiinfra,
+    azurerm_mysql_firewall_rule.fw_rule 
+  ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
+}

databases/azure-rds-mysql/main.tf

@@ -1,18 +1,7 @@
-
-/*resource "azurerm_resource_group" "mysql-rg" {
-  name     = "mysql-${var.provision_id}"
-  location = var.location
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-  }
-}*/
-
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -57,239 +46,10 @@ resource "azurerm_mysql_database" "mysql-db" {
   collation           = "utf8_unicode_ci"
 }
 
-resource "azurerm_mysql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
 
-resource "azurerm_mysql_firewall_rule" "mysql-fw-ITG-rule1" {
-  name                = "ITG"
+resource "azurerm_mysql_configuration" "example" {
+  name                = "binlog_row_image"
   resource_group_name = var.resource_group_name
   server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
+  value               = "full"
 }
-resource "azurerm_mysql_firewall_rule" "mysql-fw-ITG-rule2" {
-  name                = "ITG1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-# QCS
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCS3" {
-  name                = "QCSEU3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-# QAA
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus1" {
-  name                = "QAAus1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus2" {
-  name                = "QAAus2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAus3" {
-  name                = "QAAus3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_mysql_firewall_rule" "mysql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_server.mysql-server.name
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
-}

databases/azure-rds-mysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-mysql/variables.tf

@@ -13,6 +13,10 @@ variable "location" {
   default = "EAST US"
 }
 
+variable "subnet_id" {
+  default = null
+}
+
 variable "user_id" {
 }
 
@@ -25,7 +29,7 @@ variable "admin_login" {
 variable "mysql-version" {
   type = string
   description = "MySQL Server version to deploy"
-  default = "5.7"
+  default = "8.0"
 }
 variable "mysql-sku-name" {
   type = string
@@ -37,3 +41,11 @@ variable "mysql-storage" {
   description = "MySQL Storage in MB"
   default = "5120"
   }
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-postgres/firewall.tf

@@ -0,0 +1,55 @@
+resource "azurerm_postgresql_virtual_network_rule" "vnetrule" {
+
+  count = var.subnet_id != null? 1 : 0
+
+  name                = "vnet-rule-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  subnet_id           = var.subnet_id
+}
+
+resource "azurerm_postgresql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+
+  name                = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+
+}
+
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+
+resource "azurerm_postgresql_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips
+
+  name                = each.key
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_postgresql_firewall_rule.fw_rule, 
+    azurerm_postgresql_virtual_network_rule.vnetrule,
+    azurerm_postgresql_virtual_network_rule.vnetrule_qmiinfra 
+  ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "postgres"
+  host      = azurerm_postgresql_server.postgresql-server.fqdn
+  username  = "${var.admin_login}@qmi-postgresql-${var.provision_id}"
+  password  = nonsensitive(random_password.password.result)
+  database  = "postgres"
+}

databases/azure-rds-postgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -46,242 +46,4 @@ resource "azurerm_postgresql_database" "postgresql-db" {
   server_name         = azurerm_postgresql_server.postgresql-server.name
   charset             = "utf8"
   collation           = "English_United States.1252"
-}
-
-
-resource "azurerm_postgresql_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}
-
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-ITG-rule1" {
-  name                = "ITG"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-ITG-rule2" {
-  name                = "ITG1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
-}
-
-#QCS
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS1" {
-  name                = "QCSUS1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS2" {
-  name                = "QCSUS2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSUS3" {
-  name                = "QCSUS3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSEU1" {
-  name                = "QCSEU1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSEU2" {
-  name                = "QCSEU2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCS3" {
-  name                = "QCSEU3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC1" {
-  name                = "QCSAPAC1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC2" {
-  name                = "QCSAPAC2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSAPAC3" {
-  name                = "QCSAPAC3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore1" {
-  name                = "QCSSingapore1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore2" {
-  name                = "QCSSingapore2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QCSSingapore3" {
-  name                = "QCSSingapore3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
-
-# QAA
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu1" {
-  name                = "QAAeu1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.216.156.88"
-  end_ip_address      = "54.216.156.88"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu2" {
-  name                = "QAAeu2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.248.156.131"
-  end_ip_address      = "3.248.156.131"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAeu3" {
-  name                = "QAAeu3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "52.213.44.55"
-  end_ip_address      = "52.213.44.55"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus1" {
-  name                = "QAAus1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.235.133.252"
-  end_ip_address      = "18.235.133.252"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus2" {
-  name                = "QAAus2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.217.244.242"
-  end_ip_address      = "3.217.244.242"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAus3" {
-  name                = "QAAus3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "18.214.8.201"
-  end_ip_address      = "18.214.8.201"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac1" {
-  name                = "QAAapac1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.206.158.27"
-  end_ip_address      = "54.206.158.27"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac2" {
-  name                = "QAAapac2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.104.137.20"
-  end_ip_address      = "3.104.137.20"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAapac3" {
-  name                = "QAAapac3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "3.24.52.178"
-  end_ip_address      = "3.24.52.178"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing1" {
-  name                = "QAAsing1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "54.169.84.213"
-  end_ip_address      = "54.169.84.213"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing2" {
-  name                = "QAAsing2"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.213.173.37"
-  end_ip_address      = "13.213.173.37"
-}
-resource "azurerm_postgresql_firewall_rule" "postgresql-fw-QAAsing3" {
-  name                = "QAAsing3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_postgresql_server.postgresql-server.name
-  start_ip_address    = "13.213.113.162"
-  end_ip_address      = "13.213.113.162"
 }

databases/azure-rds-postgres/output.tf

@@ -13,3 +13,7 @@ output "root_username" {
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
 }
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}

databases/azure-rds-postgres/variables.tf

@@ -13,11 +13,15 @@ variable "location" {
   default = "EAST US"
 }
 
+variable "subnet_id" {
+  default = null
+}
+
 
 variable "admin_login" {
   type = string
   description = "Login to authenticate to PostgreSQL Server"
-  default = "qmi"
+  default = "qmirdsuser"
 }
 
 variable "postgresql-version" {
@@ -38,4 +42,12 @@ variable "postgresql-storage" {
 
 variable "user_id" {
 
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
 }

databases/dbricks/main.tf

@@ -5,65 +5,57 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 0.3.0"
+      version = ">= 1.47.0"
+      configuration_aliases = [ databricks.main, databricks.accounts ]
     }
+    
   }
 }
 
+
+locals {
+  cluster_nameUpper = upper(var.cluster_name)
+}
+
 locals {
-  myRegex = "/[^[:alpha:]]/"
-  splitLower = split(" ", lower(var.user_id))
-  np0 = replace(element(local.splitLower,0), local.myRegex, "")
-  np1 = replace(element(local.splitLower,1), local.myRegex, "")
-  c_n1 = substr(local.np0, 0, 3)
-  c_n2 = substr(local.np1, 0, 1)
-  c_n3 = substr(strrev(local.np1), 0, 1)
-  
-  cluster_name = var.cluster_name != null? var.cluster_name : "${local.c_n1}${local.c_n2}${local.c_n3}"
-
+  resource_regex            = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)"
+  subscription_id           = regex(local.resource_regex, var.databricks_resource_id)[0]
+  resource_group            = regex(local.resource_regex, var.databricks_resource_id)[1]
+  databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2]
+  tenant_id                 = data.azurerm_client_config.current.tenant_id
+  databricks_workspace_host = data.azurerm_databricks_workspace.this.workspace_url
+  databricks_workspace_id   = data.azurerm_databricks_workspace.this.workspace_id
+  prefix                    = replace(replace(lower(data.azurerm_resource_group.this.name), "rg", ""), "-", "")
 }
 
-resource "azurerm_databricks_workspace" "dbricksws" {
-
-  count = var.enabled == true? 1 : 0
-
-  location                      = var.location
-  name                          = "${local.cluster_name}-qlikdbricks-ws"
-  resource_group_name           = var.resource_group_name
-  sku                           = "standard"
-
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-  }
-
+data "azurerm_resource_group" "this" {
+  name = local.resource_group
 }
 
-resource "azurerm_role_assignment" "contributor-dbricks" {
-
-  count = var.enabled == true? 1 : 0
-  
-  scope                = azurerm_databricks_workspace.dbricksws[0].id
-  role_definition_name = "Contributor"
-  principal_id         = var.app_registration_principal_id
+data "azurerm_client_config" "current" {
 }
 
-provider "databricks" {
-  alias = "mws"
+data "azurerm_databricks_workspace" "this" {
+  name                = local.databricks_workspace_name
+  resource_group_name = local.resource_group
+}
 
-  azure_workspace_resource_id = var.enabled == true? azurerm_databricks_workspace.dbricksws[0].id : null
-  azure_client_id             = var.enabled == true? "9b4761fd-4823-4f9d-ab3a-a95af38e7c29" : null
-  azure_client_secret         = var.enabled == true? "FWH7Q~ByC.U5zNh0BaIDdK_poyxoy4SxW8hi1" : null
-  azure_tenant_id             = var.enabled == true? "c21eeb5f-f5a6-44e8-a997-124f2f7a497c" : null
+resource "databricks_metastore" "this" {
+  provider      = databricks.accounts
+  name          = "primary"
+  force_destroy = true
+  region        = data.azurerm_resource_group.this.location
+}
+
+resource "databricks_metastore_assignment" "this" {
+  provider             = databricks.accounts
+  workspace_id         = local.databricks_workspace_id
+  metastore_id         = databricks_metastore.this.id
+  default_catalog_name = "hive_metastore"
 }
 
 /*resource "databricks_workspace_conf" "this" {
-
-  count = var.enabled == true? 1 : 0
   
-  provider = databricks.mws
-
   custom_config = {
     "enableIpAccessLists" : true
   }
@@ -71,10 +63,6 @@ provider "databricks" {
 
 resource "databricks_ip_access_list" "allowed-list" {
 
-  count = var.enabled == true? 1 : 0
-
-  provider = databricks.mws
-
   label     = "allow_in"
   list_type = "ALLOW"
   ip_addresses = [
@@ -107,52 +95,120 @@ resource "databricks_ip_access_list" "allowed-list" {
 }*/
 
 
-resource "databricks_user" "me" {
-  provider = databricks.mws
-  count = var.enabled == true? 1 : 0
+data "databricks_group" "admins" {
+  display_name = "admins"
+}
 
-  depends_on = [
-    azurerm_role_assignment.contributor-dbricks
-  ]
+
+resource "databricks_user" "aor" {
+  provider      = databricks.main
+  count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
+  
+  allow_cluster_create = true
+  databricks_sql_access = true
+
+  user_name = "aor@qlik.com"
+  display_name = "Manuel Romero"
+}
+
+resource "databricks_user" "me" {
+  provider      = databricks.main
+  allow_cluster_create = true
+  databricks_sql_access = true
 
   user_name = var.user_email
   display_name = var.user_id
 }
 
 
+resource "databricks_group_member" "i-am-admin" {
+  provider      = databricks.main
+  group_id  = data.databricks_group.admins.id
+  member_id = databricks_user.me.id
+}
 
-resource "databricks_token" "pat" {
-  provider = databricks.mws
-  count = var.enabled == true? 1 : 0
-  
+
+resource "databricks_group_member" "i-am-admin-aor" {
+  provider      = databricks.main
+  count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
+  group_id  = data.databricks_group.admins.id
+  member_id = databricks_user.aor[0].id
+}
+
+resource "databricks_secret_scope" "scopeapp" {
+    provider      = databricks.main
+    count = (var.sku == "premium")? 1 : 0
+
+    name = "${var.cluster_name}scope"
+}
+resource "databricks_secret" "publishing_api" {
+    provider      = databricks.main
+    count = (var.sku == "premium")? 1 : 0
+
+    key = "accesskey"
+    string_value = var.app_reg_secret
+    scope = databricks_secret_scope.scopeapp[0].name
+}
+
+resource "databricks_sql_endpoint" "sqlep" {
+  provider      = databricks.main
+  count = (var.sku == "premium")? 1 : 0
+
+  name             = "${local.cluster_nameUpper} SQL warehouse"
+  cluster_size     = var.cluster_size
+  max_num_clusters = 1
+
+}
+
+resource "databricks_sql_global_config" "this" {
+  provider      = databricks.main
+  count = (var.sku == "premium")? 1 : 0
+
+  security_policy      = "DATA_ACCESS_CONTROL"
+  data_access_config = {
+    "spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
+    "spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
+    "spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
+    "spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
+  }
+
+  enable_serverless_compute = false
+  sql_config_params = {}
+}
+
+
+
+resource "databricks_token" "pat" {  
+  provider      = databricks.main
   depends_on = [
-    azurerm_role_assignment.contributor-dbricks,
-    databricks_user.me
+    databricks_user.me,
+    databricks_user.aor
   ]
   comment  = "qmi"
   lifetime_seconds = 8640000
 }
 
 resource "databricks_cluster" "dbrickscluster" {
-  provider = databricks.mws
-  count = (var.enabled == true && var.create_cluster == true)? 1 : 0
+  provider      = databricks.main
+  count = (var.cluster_name != null)? 1 : 0
 
-  depends_on = [
-    azurerm_role_assignment.contributor-dbricks
-  ]
 
-  cluster_name  = "cluster-${local.cluster_name}"
-  spark_version = "7.3.x-scala2.12"
-  node_type_id  = "Standard_DS3_v2"
+  cluster_name  = "cluster-${var.cluster_name}"
+  spark_version = var.spark_version
+  node_type_id  = var.node_type_id
 
   spark_conf = {
     "spark.hadoop.hive.server2.enable.doAs": false,
     "spark.databricks.delta.preview.enabled": true,
     "spark.databricks.repl.allowedLanguages": "sql,python,r",
     "spark.databricks.cluster.profile": "serverless",
-    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
+    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
+    "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
+    "spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
   } 
 
+
   spark_env_vars = {
       "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
   }
@@ -162,5 +218,5 @@ resource "databricks_cluster" "dbrickscluster" {
     max_workers = 4
   }
 
-  autotermination_minutes = 120
+  autotermination_minutes = 45
 }

databases/dbricks/outputs.tf

@@ -1,11 +1,25 @@
-output "databricks_workspace_url" {
-    value = var.enabled? azurerm_databricks_workspace.dbricksws[0].workspace_url : null
+output "cluster_id" {
+  value = (var.cluster_name != null)? databricks_cluster.dbrickscluster[0].id : null
 }
 
 output "databricks_token" {
-    value = var.enabled? nonsensitive(databricks_token.pat[0].token_value) : null
+    value = nonsensitive(databricks_token.pat.token_value)
 }
 
 output "cluster_name" {
-    value = "cluster-${local.cluster_name}"
-}
+    value = "cluster-${var.cluster_name}"
+}
+
+output "sql_endpoint_jdbc_url" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].jdbc_url : null
+}
+
+output "sql_endpoint_data_source_id" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].data_source_id : null
+}
+
+output "sql_endpoint_odbc_params" {
+    value = var.sku == "premium"? databricks_sql_endpoint.sqlep[0].odbc_params : null
+}
+
+ 

databases/dbricks/varaibles.tf

@@ -1,11 +1,3 @@
-variable "location" {
-    default = "East US"
-}
-
-variable "resource_group_name" {
-
-}
-
 variable "cluster_name" {
     default = null
 }
@@ -22,15 +14,29 @@ variable "user_id" {
 variable "user_email" {
 }
 
-variable "app_registration_principal_id" {
-    description = "tpm"
-    default = "163a72e3-8ce3-4e33-baae-954383f87e3e"
+variable "spark_version" {
+    default = "10.4.x-scala2.12"
 }
 
-variable "enabled" {
-    default = false
+variable "node_type_id" {
+    default = "Standard_DS3_v2"
 }
 
-variable "create_cluster" {
-    default = false
+variable "sku" {
+    default = "standard"
+}
+
+variable "app_reg_id" {
+  default = null
+}
+
+variable "app_reg_secret" {
+  default = null
+}
+
+variable "cluster_size" {
+  default = "Small"
+}
+
+variable "databricks_resource_id" {
 }

databases/dummy/main.tf

@@ -0,0 +1,21 @@
+resource "null_resource" "dummy-data" {
+    
+  provisioner "local-exec" {
+      command = "chmod +x ${path.module}/scripts/*.sh"
+      interpreter = ["/bin/bash", "-c"]
+  }
+
+  provisioner "local-exec" {
+      command = "${path.module}/scripts/dummy-${var.type}.sh \"$username\" \"$password\" \"$host\" \"$db\""
+      interpreter = ["/bin/bash", "-c"]
+
+      environment = {
+          
+          username  = var.username
+          password  = var.password
+          host      = var.host 
+          db        = var.database
+      }
+  }
+
+}

databases/dummy/outputs.tf

@@ -0,0 +1,3 @@
+output "dbs" {
+  value       = var.type == "postgres"? "'AdventureWorks' and 'dvdrental'": (var.type == "mysql")? "'sakila', 'employees' and 'classicmodels'" : (var.type == "mssql")? "'BikeStores' and 'Northwind'" : null
+}

databases/dummy/scripts/dummy-mssql.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}`
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+
+
+echo "---- Loading BikeStore sample database -----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+
+unzip qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE BikeStores"
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/createbike.sql
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/Bikeloaddata.sql > /tmp/bikestore_load.logs
+
+echo "---- Loading Northwind sample database -----"
+cd /tmp
+curl https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/northwind-pubs/instnwnd.sql -o instnwnd.sql
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE Northwind"
+sqlcmd -C -S "$host" -d "Northwind" -U "$1" -P "$2" -i instnwnd.sql > /tmp/northwind_load.logs
+
+rm -fr /tmp/qmi-cloud-tf-modules-*

databases/dummy/scripts/dummy-mysql.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4" 
+echo "--------------------------"    
+
+
+cd /tmp
+git clone https://github.com/datacharmer/test_db.git
+cd test_db
+echo "-------- Loading Mysql Employees sample database -----------"  
+mysql -h $host -u$1 -p$2 --ssl=TRUE < employees.sql
+cd ..
+wget -q https://downloads.mysql.com/docs/sakila-db.tar.gz
+tar xvfz sakila-db.tar.gz
+cd sakila-db
+
+echo "-------- Loading Mysql Sakila sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-schema.sql 
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-data.sql 
+
+cd /tmp
+
+curl https://www.mysqltutorial.org/wp-content/uploads/2018/03/mysqlsampledatabase.zip -o mysqlsampledatabase.zip
+unzip mysqlsampledatabase.zip
+echo "-------- Loading Mysql ClassicModels sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < mysqlsampledatabase.sql
+
+
+rm -fr /tmp/*
+

databases/dummy/scripts/dummy-postgres.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+echo "---- Install Adventureworks for Postgres ----"
+cd /tmp
+wget -q https://github.com/lorint/AdventureWorks-for-Postgres/archive/master.zip
+unzip master.zip
+cd AdventureWorks-for-Postgres-master/
+wget -q https://github.com/microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks-oltp-install-script.zip
+unzip AdventureWorks-oltp-install-script.zip
+
+ruby update_csvs.rb
+
+export PGPASSWORD=$2
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"AdventureWorks\";" 
+psql -h $host -U $1 -d "AdventureWorks" < ./install.sql > /tmp/AdventureWorks_load.log
+
+
+echo "---- Install DvdRental sample database for Postgres ----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+unzip -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"dvdrental\";" 
+psql -h $host -U $1 -d "dvdrental" < ./postgres/dump_dvdrental.sql > /tmp/dvdrental_load.log
+
+
+rm -fr /tmp/*
+
+

databases/dummy/scripts/mssql/createbike.sql

@@ -0,0 +1,109 @@
+/*
+--------------------------------------------------------------------
+© 2017 sqlservertutorial.net All Rights Reserved
+--------------------------------------------------------------------
+Name   : BikeStores
+Link   : http://www.sqlservertutorial.net/load-sample-database/
+Version: 1.0
+--------------------------------------------------------------------
+*/
+-- create schemas
+CREATE SCHEMA production;
+go
+
+CREATE SCHEMA sales;
+go
+
+-- create tables
+CREATE TABLE production.categories (
+	category_id INT IDENTITY (1, 1) PRIMARY KEY,
+	category_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.brands (
+	brand_id INT IDENTITY (1, 1) PRIMARY KEY,
+	brand_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.products (
+	product_id INT IDENTITY (1, 1) PRIMARY KEY,
+	product_name VARCHAR (255) NOT NULL,
+	brand_id INT NOT NULL,
+	category_id INT NOT NULL,
+	model_year SMALLINT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	FOREIGN KEY (category_id) REFERENCES production.categories (category_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (brand_id) REFERENCES production.brands (brand_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE sales.customers (
+	customer_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (255) NOT NULL,
+	last_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255) NOT NULL,
+	street VARCHAR (255),
+	city VARCHAR (50),
+	state VARCHAR (25),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.stores (
+	store_id INT IDENTITY (1, 1) PRIMARY KEY,
+	store_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255),
+	street VARCHAR (255),
+	city VARCHAR (255),
+	state VARCHAR (10),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.staffs (
+	staff_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (50) NOT NULL,
+	last_name VARCHAR (50) NOT NULL,
+	email VARCHAR (255) NOT NULL UNIQUE,
+	phone VARCHAR (25),
+	active tinyint NOT NULL,
+	store_id INT NOT NULL,
+	manager_id INT,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (manager_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.orders (
+	order_id INT IDENTITY (1, 1) PRIMARY KEY,
+	customer_id INT,
+	order_status tinyint NOT NULL,
+	-- Order status: 1 = Pending; 2 = Processing; 3 = Rejected; 4 = Completed
+	order_date DATE NOT NULL,
+	required_date DATE NOT NULL,
+	shipped_date DATE,
+	store_id INT NOT NULL,
+	staff_id INT NOT NULL,
+	FOREIGN KEY (customer_id) REFERENCES sales.customers (customer_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (staff_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.order_items (
+	order_id INT,
+	item_id INT,
+	product_id INT NOT NULL,
+	quantity INT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	discount DECIMAL (4, 2) NOT NULL DEFAULT 0,
+	PRIMARY KEY (order_id, item_id),
+	FOREIGN KEY (order_id) REFERENCES sales.orders (order_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE production.stocks (
+	store_id INT,
+	product_id INT,
+	quantity INT,
+	PRIMARY KEY (store_id, product_id),
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);

databases/dummy/variables.tf

@@ -0,0 +1,15 @@
+variable "host" { 
+}
+
+variable "username" {
+}
+
+variable "password" {
+}
+
+variable "database" {
+}
+
+variable "type" {
+    description = "mysql | postgres | mssql"
+}

databases/firewall_ips/output.tf

@@ -0,0 +1,224 @@
+locals {
+
+  az = {
+      az1          = ["52.249.189.38","52.249.189.38"]
+      az2          = ["13.67.39.86", "13.67.39.86"]
+      az3          = ["20.67.110.207", "20.67.110.207"]
+  }
+
+  q_routes = {
+    ITG         = ["188.65.156.32", "188.65.156.47"]
+    ITG1        = ["212.73.252.96", "212.73.252.103"]
+    kfar_saba   = ["194.90.96.176", "194.90.96.183"]
+    kfar_saba1  = ["213.57.84.160", "213.57.84.167"]
+    peak_10     = ["4.4.97.104", "4.4.97.111"]
+    peak_101    = ["206.196.17.32", "206.196.17.63"]
+    Bangalore   = ["14.98.59.168", "14.98.59.175"]
+    Bangalore1  = ["182.74.33.8", "182.74.33.15"]
+    T_Beijing   = ["60.247.114.145","60.247.114.158"]
+    T_Nantes    = ["81.255.115.33","81.255.115.38"]
+    T_PA7_1     = ["217.117.156.85","217.117.156.86"]
+    T_PA7_2     = ["217.117.156.141","217.117.156.142"]
+    T_PA7_3     = ["217.117.156.161","217.117.156.190"]
+    T_Sanmateo  = ["12.202.47.153","12.202.47.158"]
+    T_Suresnes  = ["62.23.50.121","62.23.50.126"]
+    T_SV2_1     = ["4.31.193.69","4.31.193.70"]
+    T_SV2_2     = ["4.79.217.1","4.79.217.126"]
+    T_SV2_3     = ["149.97.160.217","149.97.160.218"]
+    T_SV2_4     = ["149.97.160.221","149.97.160.222"]
+    T_SV2_5     = ["149.97.185.1","149.97.185.30"]
+    T_Blr_1     = ["14.97.166.193","14.97.166.198"]
+    T_Blr_2     = ["14.97.167.21","14.97.167.22"]
+    T_Blr_3     = ["122.185.123.61","122.185.123.62"]
+    T_Blr_4     = ["182.79.10.25","182.79.10.26"]
+  }
+
+  fullvpn = {
+    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
+    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
+    EMEA_FULLVPN    = ["193.15.228.246", "193.15.228.246"]
+  }
+
+  qcs = {
+      QCS_us1          = ["18.205.71.36", "18.205.71.36"]      
+      QCS_us2          = ["18.232.32.199", "18.232.32.199"]   
+      QCS_us3          = ["34.237.68.254", "34.237.68.254"]   
+      QCS_eu1          = ["34.247.21.179", "34.247.21.179"]  
+      QCS_eu2          = ["52.31.212.214", "52.31.212.214"]  
+      QCS_eu3          = ["54.154.95.18", "54.154.95.18"]  
+      QCS_ap1          = ["13.210.43.241", "13.210.43.241"] 
+      QCS_ap2          = ["13.236.104.42", "13.236.104.42"]  
+      QCS_ap3          = ["13.236.206.172", "13.236.206.172"] 
+      QCS_sg1          = ["18.138.163.172", "18.138.163.172"]
+      QCS_sg2          = ["18.142.157.182", "18.142.157.182"]
+      QCS_sg3          = ["54.179.13.251", "54.179.13.251"]
+      QCS_de1          = ["3.76.162.169", "3.76.162.169"]
+      QCS_de2          = ["3.77.162.68", "3.77.162.68"]
+      QCS_de3          = ["3.122.137.91", "3.122.137.91"]
+      QCS_uk1          = ["13.42.141.246", "13.42.141.246"]
+      QCS_uk2          = ["18.135.245.97", "18.135.245.97"]
+      QCS_uk3          = ["35.179.0.171", "35.179.0.171"]
+  }
+
+  qcs_qaa = {
+      QCS_QAA_eu1     = ["54.216.156.88", "54.216.156.88"]
+      QCS_QAA_eu2     = ["3.248.156.131", "3.248.156.131"]
+      QCS_QAA_eu3     = ["52.213.44.55", "52.213.44.55"]
+      QCS_QAA_us1     = ["18.235.133.252", "18.235.133.252"]
+      QCS_QAA_us2     = ["3.217.244.242", "3.217.244.242"]
+      QCS_QAA_us3     = ["18.214.8.201", "18.214.8.201"]
+      QCS_QAA_ap1     = ["54.206.158.27", "54.206.158.27"]
+      QCS_QAA_ap2     = ["3.104.137.20", "3.104.137.20"]
+      QCS_QAA_ap3     = ["3.24.52.178", "3.24.52.178"]
+      QCS_QAA_sg1     = ["54.169.84.213", "54.169.84.213"]
+      QCS_QAA_sg2     = ["13.213.173.37", "13.213.173.37"]
+      QCS_QAA_sg3     = ["13.213.113.162", "13.213.113.162"]
+      QCS_QAA_de1     = ["3.69.132.138", "3.69.132.138"]
+      QCS_QAA_de2     = ["3.69.222.86", "3.69.222.86"]
+      QCS_QAA_de3     = ["3.72.136.160", "3.72.136.160"]
+      QCS_QAA_uk1     = ["18.168.120.199", "18.168.120.199"]
+      QCS_QAA_uk2     = ["18.169.91.38", "18.169.91.38"]
+      QCS_QAA_uk3     = ["13.42.135.168", "13.42.135.168"]
+  }
+
+  stitch = {
+      STITCH_1 = ["52.23.137.21", "52.23.137.21"]
+      STITCH_2 = ["52.204.223.208", "52.204.223.208"]
+      STITCH_3 = ["52.204.228.32", "52.204.228.32"]
+      STITCH_4 = ["52.204.230.227", "52.204.230.227"]
+      STITCH_5 = ["3.126.102.29", "3.126.102.29"]
+      STITCH_6 = ["18.158.16.164", "18.158.16.164"]
+      STITCH_7 = ["18.158.251.55", "18.158.251.55"]
+      STITCH_8 = ["52.57.235.168", "52.57.235.168"]
+  }
+
+ 
+}
+
+output "ips2" {
+  value = merge(local.q_routes, local.fullvpn, local.qcs, local.stitch)
+}
+
+output "ips" {
+  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes, local.fullvpn, local.stitch)
+}
+
+output "ips_az_qcs" {
+  value = merge(local.az, local.qcs, local.fullvpn, local.stitch)
+}
+
+output "cidr_blocks" {
+  value = [
+    # EMEA/AMER FULL VPN TUNNEL
+    "144.121.57.103/32",
+    "193.15.228.246/32",
+    "50.239.179.6/32",
+    # Azure QMI machines
+    "52.249.189.38/32", 
+    "13.67.39.86/32", 
+    "20.67.110.207/32", 
+    # Qlik network routers
+    "14.98.59.168/29", 
+    "182.74.33.8/29", 
+    "188.65.156.32/28", 
+    "212.73.252.96/29", 
+    "194.90.96.176/29", 
+    "213.57.84.160/29", 
+    "4.4.97.104/29", 
+    "206.196.17.32/27", 
+    # T Beijing
+    "60.247.114.144/28",
+    # T Nantes
+    "81.255.115.32/29",
+    "84.14.92.152/29",
+    # T PA7
+    "217.117.156.84/30",
+    "217.117.156.140/30",
+    "217.117.156.160/27",
+    # T San Mateo
+    "12.202.47.152/29",
+    # T Suresnes
+    "62.23.50.120/29",
+    # T SV2
+    "4.31.193.68/30",
+    "4.79.217.0/25",
+    "149.97.160.216/30",
+    "149.97.160.220/30",
+    "149.97.185.0/27",
+    # T Blr Office
+    "14.97.166.192/29",
+    "14.97.167.20/30",
+    "122.185.123.60/30",
+    "182.79.10.24/29",
+    # QCS Amercias (us)
+    "18.205.71.36/32", 
+    "18.232.32.199/32", 
+    "34.237.68.254/32",
+    # QCS Ireland (eu) 
+    "34.247.21.179/32", 
+    "52.31.212.214/32", 
+    "54.154.95.18/32",
+    # QCS Australia (ap)  
+    "13.210.43.241/32", 
+    "13.236.104.42/32", 
+    "13.236.206.172/32",
+    # QCS Singapore (sg)
+    "18.138.163.172/32",
+    "18.142.157.182/32",
+    "54.179.13.251/32",
+    # QCS Frankfurt (de)
+    "3.76.162.169/32",
+    "3.77.162.68/32",
+    "3.122.137.91/32",
+    # QCS London (uk)
+    "13.42.141.246/32",
+    "18.135.245.97/32",
+    "35.179.0.171/32",
+    # QCS-QAA Ireland (eu)
+    "54.216.156.88/32",
+    "3.248.156.131/32",
+    "52.213.44.55/32",
+    # QCS-QAA Americas (us)
+    "18.235.133.252/32",
+    "3.217.244.242/32",
+    "18.214.8.201/32",
+    # QCS-QAA Australia (ap)
+    "54.206.158.27/32",
+    "3.104.137.20/32",
+    "3.24.52.178/32",
+    # QCS-QAA Singapore (sg)
+    "54.169.84.213/32",
+    "13.213.173.37/32",
+    "13.213.113.162/32",
+    # QCS-QAA Frankfurt (de)
+    "3.69.132.138/32",
+    "3.69.222.86/32",
+    "3.72.136.160/32",
+    # QCS-QAA London (uk)
+    "18.168.120.199/32",
+    "18.169.91.38/32",
+    "13.42.135.168/32",
+    # QCS STAGING
+    "18.233.22.130/32",
+    "18.205.135.40/32",
+    "35.169.201.36/32",
+    "18.155.181.46/32",
+    "18.155.181.25/32",
+    "18.155.181.128/32",
+    "18.155.181.76/32",
+  ]
+}
+
+output "cidr_blocks_others" {
+  value = [
+    # Stitch
+    "52.23.137.21/32",
+    "52.204.223.208/32",
+    "52.204.228.32/32",
+    "52.204.230.227/32",
+    "3.126.102.29/32",
+    "18.158.16.164/32",
+    "18.158.251.55/32",
+    "52.57.235.168/32",
+  ]
+}

databases/synapse-ws/firewall.tf

@@ -6,151 +6,44 @@ resource "azurerm_synapse_firewall_rule" "azureservices" {
   end_ip_address       = "0.0.0.0"
 }
 
+
 resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
+
+  name                = "az1"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "52.249.189.38"
   end_ip_address      = "52.249.189.38"
+
 }
 
 resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
+
+  name                = "az2"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "13.67.39.86"
   end_ip_address      = "13.67.39.86"
+
 }
 
 resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
+
+  name                = "az3"
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "20.67.110.207"
   end_ip_address      = "20.67.110.207"
+
 }
 
-resource "azurerm_synapse_firewall_rule" "fw-ITG-rule1" {
-  name                = "ITG"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "188.65.156.32"
-  end_ip_address      = "188.65.156.47"
-}
-resource "azurerm_synapse_firewall_rule" "fw-ITG-rule2" {
-  name                = "ITG1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "212.73.252.96"
-  end_ip_address      = "212.73.252.103"
-}
-resource "azurerm_synapse_firewall_rule" "fw-kfar_saba-rule1" {
-  name                = "Kfar_Saba"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "194.90.96.176"
-  end_ip_address      = "194.90.96.183"
-}
-resource "azurerm_synapse_firewall_rule" "fw-kfar_saba-rule2" {
-  name                = "Kfar_Saba1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "213.57.84.160"
-  end_ip_address      = "213.57.84.167"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Peak_10-rule1" {
-  name                = "Peak_10"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "4.4.97.104"
-  end_ip_address      = "4.4.97.111"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Peak_10-rule2" {
-  name                = "Peak_101"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "206.196.17.32"
-  end_ip_address      = "206.196.17.63"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Bangalore-rule1" {
-  name                = "Bangalore"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "14.98.59.168"
-  end_ip_address      = "14.98.59.175"
-}
-resource "azurerm_synapse_firewall_rule" "fw-Bangalore-rule2" {
-  name                = "Bangalore1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "182.74.33.8"
-  end_ip_address      = "182.74.33.15"
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
-# QCS US
-resource "azurerm_synapse_firewall_rule" "qcs_us_1" {
-  name                = "QCS_US_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.205.71.36"
-  end_ip_address      = "18.205.71.36"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_us_2" {
-  name                = "QCS_US_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.232.32.199"
-  end_ip_address      = "18.232.32.199"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_us_3" {
-  name                = "QCS_US_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "34.237.68.254"
-  end_ip_address      = "34.237.68.254"
-}
+resource "azurerm_synapse_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips2
 
-# QCS EU
-resource "azurerm_synapse_firewall_rule" "qcs_eu_1" {
-  name                = "QCS_EU_1"
+  name                = each.key
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "34.247.21.179"
-  end_ip_address      = "34.247.21.179"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_eu_2" {
-  name                = "QCS_EU_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "52.31.212.214"
-  end_ip_address      = "52.31.212.214"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_eu_3" {
-  name                = "QCS_EU_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "54.154.95.18"
-  end_ip_address      = "54.154.95.18"
-}
-# QCS AP
-resource "azurerm_synapse_firewall_rule" "qcs_ap_1" {
-  name                = "QCS_AP_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.210.43.241"
-  end_ip_address      = "13.210.43.241"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_ap_2" {
-  name                = "QCS_AP_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.236.104.42"
-  end_ip_address      = "13.236.104.42"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_ap_3" {
-  name                = "QCS_AP_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "13.236.206.172"
-  end_ip_address      = "13.236.206.172"
-}
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
 
-# QCS SINGAPORE
-resource "azurerm_synapse_firewall_rule" "qcs_sp_1" {
-  name                = "QCS_SP_1"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.138.163.172"
-  end_ip_address      = "18.138.163.172"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_sp_2" {
-  name                = "QCS_SP_2"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "18.142.157.182"
-  end_ip_address      = "18.142.157.182"
-}
-resource "azurerm_synapse_firewall_rule" "qcs_sp_3" {
-  name                = "QCS_SP_3"
-  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
-  start_ip_address    = "54.179.13.251"
-  end_ip_address      = "54.179.13.251"
-}
+}

databases/synapse-ws/main.tf

@@ -33,6 +33,9 @@ resource "azurerm_synapse_workspace" "synapsews" {
   sql_administrator_login              = var.sql_administrator_login
   sql_administrator_login_password     = var.sql_administrator_login_password
 
+
+  managed_virtual_network_enabled      = true
+
   identity {
     type = "SystemAssigned"
   }
@@ -41,6 +44,7 @@ resource "azurerm_synapse_workspace" "synapsews" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner    = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 
@@ -57,6 +61,7 @@ resource "azurerm_synapse_sql_pool" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 
@@ -65,19 +70,11 @@ resource "azurerm_synapse_sql_pool" "db" {
 resource "azurerm_synapse_role_assignment" "sqladministrator_aor" {
   
   depends_on = [
-      azurerm_synapse_firewall_rule.azureservices
-      /*,
+      azurerm_synapse_firewall_rule.azureservices,
+      
       azurerm_synapse_firewall_rule.fw-a-rule1,
       azurerm_synapse_firewall_rule.fw-a-rule2,
-      azurerm_synapse_firewall_rule.fw-a-rule3,
-      azurerm_synapse_firewall_rule.fw-ITG-rule1,
-      azurerm_synapse_firewall_rule.fw-ITG-rule2,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule1,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule2,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule1,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule2,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule1,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule2*/
+      azurerm_synapse_firewall_rule.fw-a-rule3
   ]
 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
@@ -91,18 +88,11 @@ resource "azurerm_synapse_role_assignment" "sqladministrator" {
   count = (var.user_oid != null && var.user_oid != "c583b5aa-2844-4baf-b8bf-b6a1ae0b1520") ? 1 : 0
 
   depends_on = [
-      azurerm_synapse_firewall_rule.azureservices/*,
+      azurerm_synapse_firewall_rule.azureservices,
+      
       azurerm_synapse_firewall_rule.fw-a-rule1,
       azurerm_synapse_firewall_rule.fw-a-rule2,
-      azurerm_synapse_firewall_rule.fw-a-rule3,
-      azurerm_synapse_firewall_rule.fw-ITG-rule1,
-      azurerm_synapse_firewall_rule.fw-ITG-rule2,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule1,
-      azurerm_synapse_firewall_rule.fw-kfar_saba-rule2,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule1,
-      azurerm_synapse_firewall_rule.fw-Peak_10-rule2,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule1,
-      azurerm_synapse_firewall_rule.fw-Bangalore-rule2*/
+      azurerm_synapse_firewall_rule.fw-a-rule3
   ]
 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id

linux-common/centos/common/extract-certs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+BASEDIR=$(dirname "$0")
+
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+
+PASSWORD=$1
+# Extract the private key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
+# Extract the public key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
+# Extract the CA cert chain
+#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+
+cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
+
+rm -fr $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+echo $PASSWORD > $BASEDIR/qmicerts/pfx-cert-password.txt

linux-common/centos/common/falcon.sh

@@ -1,20 +1,22 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-
-centos_ver=`rpm --eval %{centos_ver}`
+cVer=`rpm -E %{rhel}`
 binary='falcon-sensor-5.29.0-9403.el7.x86_64.rpm'
-if [ $centos_ver -eq '8' ]; then
+if [ $cVer -eq '8' ]; then
     binary='falcon-sensor-5.27.0-9104.el8.x86_64.rpm'
 fi
-echo $binary
 
-
-echo "Installing CrowdStrike falcon agent"
+echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
+
 if ! rpm -qa | grep -qw falcon-sensor; then
-    sudo yum install $BASEDIR/$binary -y
+    sudo yum -y --quiet install $BASEDIR/$binary
 fi
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1
 sudo systemctl start falcon-sensor 

linux-common/centos/common/resizedisk.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 echo "Resizing main partition to max available disk"
 (
 echo u # Change the units to sectors

linux-common/centos/common/save-cert.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+
+echo $1 > $BASEDIR/qmicerts/myserver.crt
+
+echo $2 > $BASEDIR/qmicerts/myserver.key
+
+

linux-common/centos/common/tenable.sh

@@ -1,14 +1,23 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-echo "Installing Tenable Nessus Agent..."
 KEY=$1
 echo "KEY=$KEY"
 
-sudo yum install $BASEDIR/NessusAgent-10.2.0-es7.x86_64.rpm -y
+cVer=`rpm -E %{rhel}`
 
-echo "Linking Tenable Nessus Agent..."
+FILE="NessusAgent-10.2.1-es$cVer.x86_64.rpm"
+
+echo "--- Installing Tenable Nessus Agent --> $FILE"
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$FILE -O $BASEDIR/$FILE
+
+sudo /bin/yum -y --quiet install $BASEDIR/$FILE -y
+
+echo "--- Linking Tenable Nessus Agent..."
 sudo /bin/systemctl start nessusagent.service
 sudo /opt/nessus_agent/sbin/nessuscli agent link --key=$KEY --groups="Qlik IT Cloud Agents" --host=cloud.tenable.com --port=443
 

linux-common/centos/common/update.sh

@@ -0,0 +1,2 @@
+echo 'Updating OS. This will take a around 10 minutes'
+yum -y --quiet update

linux-common/main.tf

@@ -8,14 +8,27 @@ data "azurerm_key_vault_secret" "tenable-key" {
   key_vault_id = var.key_vault_id
 }
 
+data "azurerm_key_vault_secret" "cert_password" {
+  name         = "star-qmi-qlikpoc-com-password"
+  key_vault_id = var.key_vault_id
+}
+
+#data "azurerm_key_vault_certificate_data" "cert-data" {
+#  name         = "star-qmi-qlikpoc-com-cert"
+#  key_vault_id = var.key_vault_id
+#}
+
+
 locals {
   falcon_id     = nonsensitive(data.azurerm_key_vault_secret.cid.value)
   tenable_key   = nonsensitive(data.azurerm_key_vault_secret.tenable-key.value)
-}
+  cert_password = nonsensitive(data.azurerm_key_vault_secret.cert_password.value)
+  #cert_pem =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.pem)
+  #cert_key =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.key)
+} 
 
-resource "null_resource" "post-linux-vm" {
-    
-    provisioner "file" {
+resource "null_resource" "files" {
+  provisioner "file" {
         connection {
             type     = "ssh"
             host     = var.private_ip_address
@@ -27,6 +40,13 @@ resource "null_resource" "post-linux-vm" {
         source      = "${path.module}/${var.os_type}/common"
         destination = "/home/${var.admin_username}"
     }
+}
+
+resource "null_resource" "post-linux-vm" {
+    
+    depends_on = [
+      null_resource.files
+    ]
 
     provisioner "remote-exec" {
         connection {
@@ -43,6 +63,35 @@ resource "null_resource" "post-linux-vm" {
             "sudo /home/${var.admin_username}/common/falcon.sh '${local.falcon_id}'",
             "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
+            "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
+            #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
         ]
     }
-}
+}
+
+resource "null_resource" "update" {
+
+  count = var.update? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.post-linux-vm
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "60s"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/update.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+BASEDIR=$(dirname "$0")
+
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-full.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/star_qmi_qlik-poc_com.pem -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pem
+
+PASSWORD=$1
+# Extract the private key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
+# Extract the public key
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
+# Extract the CA cert chain
+#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+
+cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
+
+rm -fr $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+echo $PASSWORD > $BASEDIR/qmicerts/pfx-cert-password.txt

linux-common/ubuntu/common/falcon.sh

@@ -1,11 +1,21 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
-echo "Installing CrowdStrike falcon agent"
+binary="falcon-sensor_5.17.0-8103_amd64.deb"
+
+echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
+
+#sudo apt -qq -y update
+sudo apt install libnl-3-200 libnl-genl-3-200 -qq -y
+
+wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
+
 if ! dpkg -l | grep -qw falcon-sensor; then
-    sudo dpkg -i $BASEDIR/falcon-sensor_5.17.0-8103_amd64.deb
+    sudo dpkg -i $BASEDIR/$binary
 fi
 
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1

linux-common/ubuntu/common/resizedisk.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 echo "Resizing main partition to max available disk"
 (
 echo p # List the partitions details