craeting metasttore

.gitignore

@@ -0,0 +1 @@
+.DS_Store

adls/main.tf

@@ -7,16 +7,19 @@ resource "random_id" "randomMachineId" {
     byte_length = 5
 }
 
+
 resource "azurerm_storage_account" "saccount" {
 
   name = var.storage_account_name != null? var.storage_account_name : "qmiadlsgen2${random_id.randomMachineId.hex}"
 
   resource_group_name       = var.resource_group_name
   location                  = var.location
+
   account_kind              = "StorageV2"
+  account_replication_type  = var.account_replication_type != null? var.account_replication_type : "RAGRS"
   account_tier              = "Standard"
   access_tier               = "Hot"
-  account_replication_type  = "RAGRS"
+
   is_hns_enabled            = "true"
 
   tags = var.tags

adls/variable.tf

@@ -35,4 +35,12 @@ variable "dbricks_app_registration_application_id" {
 
 variable "principal_id_storage_blob_contributor" {
   default = null
+}
+
+variable "sa_config_type" {
+  default = "1"
+}
+
+variable "account_replication_type" {
+  default = null
 }

appgateways/appgw-443-only/logs.tf

@@ -0,0 +1,23 @@
+resource "azurerm_monitor_diagnostic_setting" "example" {
+
+    name               = "${local.appgw_name}-${var.provision_id}-diagsettings"
+    target_resource_id = azurerm_application_gateway.qmi-app-gw.id
+
+    log_analytics_workspace_id = var.log_analytics_workspace_id
+
+    enabled_log {
+        category = "ApplicationGatewayAccessLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+
+    enabled_log {
+        category = "ApplicationGatewayFirewallLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+}

appgateways/appgw-443-only/main.tf

@@ -30,6 +30,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -48,6 +49,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-443-only/variables.tf

@@ -38,3 +38,7 @@ variable "user_id" {
 variable "probe_path" {
   default = "/"
 }
+
+variable "log_analytics_workspace_id" {
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
+}

appgateways/appgw-e2e/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-e2e/main.tf

@@ -62,6 +62,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -80,6 +81,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qdc-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc-qs/main.tf

@@ -54,6 +54,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
   
 }
@@ -73,6 +74,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qdc/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc/main.tf

@@ -28,6 +28,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -46,6 +47,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qs/main.tf

@@ -52,6 +52,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -70,6 +71,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

compose-install/scripts/compose-license.txt

@@ -1,14 +1,14 @@
 {
   "$type": "ComposeLicense",
   "product": "QlikCompose",
-  "issued_to": "Attunity Americas",
+  "issued_to": "Qlik Internal Use only",
   "issued_by": "Attunity US",
   "license_type": "EVALUATION",
-  "serial_no": "85008595",
-  "expiration_date": "2023-12-31",
+  "serial_no": "85010344",
+  "expiration_date": "2024-12-31",
   "hosts": "",
-  "product_version": "2022.5",
-  "notes": "Qlik internal use only ",
+  "product_version": "2023.11",
+  "notes": "Qlik Internal Use only",
   "host_role": "",
   "source_db_types": "",
   "dwh_type": "*",
@@ -16,6 +16,6 @@
   "number_of_dms": "0",
   "number_of_developers": "0",
   "managed_dwh_size": "0",
-  "issue_time": "12/15/2022 7:06:16 PM",
-  "signature": "iDk5NujmAcxcI80BSt4JaOEaSj35u0vzoRXvRcYXy4NpWJLFVvCRJ67LfVcA1WuLwHmarnYWYA+RWT0CyqpH1e4n3nZPaJAwBgnU5Z074rIUrHZi+Z5Hmgux2ptNqXJIr7S1JNc6+fBMHuG//hxYppf9Knmk+5cx5rafxNpCYUU="
+  "issue_time": "12/5/2023 9:36:23 PM",
+  "signature": "LnAeWQPSDsQw0quG6h5HxvAdnzerIbW/KwOCYvFeq6T8Tp8i6hxAgnKpeVeqTKNiA3t5ovkvIjBvH5n0JGjG75r4OohWiNLFaWYrSvgf6d/fPzFK/4tgyhdImmbS8dZm3tInY3Y8ZNM+kEGi8Hze2hPoHZGwh5nZMM0eCo2E4Sk="
 }

compose-install/scripts/di-compose-install.ps1

@@ -46,17 +46,19 @@ If ((Test-Path $bin\$fileName))
     $fileNoExtension = [IO.Path]::GetFileNameWithoutExtension("$bin\$fileName")
 	Expand-Archive $bin\$fileName -DestinationPath $bin\$fileNoExtension -Force
 
-    #Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
+    Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
 	$exec = Get-ChildItem $bin\$fileNoExtension\*.exe | Select-Object -ExpandProperty Name
     #Write-Host "Exec: $($exec)"
 
     $silentInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
 
+    Start-Sleep 5 
+    
 	Write-Host "Run Compose silent installation : $($silentInstall)"
     Invoke-Expression $silentInstall
     while (!(Test-Path "$($bin)\Compose_silent_x64_install.log")) { 
         Write-Host "[Waiting Compose to be installed] ..."
-        Start-Sleep 2
+        Start-Sleep 4
     }
 	$resultLogs = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
 	$testResult = $resultLogs.ResponseResult.ResultCode

compose-install/scripts/di-compose-installQMICertificate.ps1

@@ -6,6 +6,11 @@ Param(
 
 try {
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

databases/aws-emr/main.tf

@@ -0,0 +1,146 @@
+terraform {
+
+  required_version = ">= 1.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+    }
+  }
+}
+
+
+module "s3_bucket" {
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "~> 3.0"
+
+  bucket_prefix = "qmi-bucket-${var.provision_id}"
+
+  # Allow deletion of non-empty bucket
+  # Example usage only - not recommended for production
+  force_destroy = true
+
+  attach_deny_insecure_transport_policy = true
+  attach_require_latest_tls_policy      = true
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
+  server_side_encryption_configuration = {
+    rule = {
+      apply_server_side_encryption_by_default = {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+
+  tags = {
+    QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+    Owner    = var.user_id
+  }
+}
+
+module "emr" {
+  source = "terraform-aws-modules/emr/aws"
+
+  name = var.provision_id
+
+  release_label = "emr-6.11.0"
+  applications  = ["spark", "hadoop", "hive", "hue"]
+  auto_termination_policy = {
+    idle_timeout = 3600
+  }
+
+  bootstrap_action = {
+    example = {
+      name = "Just an example",
+      path = "file:/bin/echo",
+      args = ["Hello World!"]
+    }
+  }
+
+  configurations_json = jsonencode([
+    {
+      "classification" : "spark-env",
+      "configurations" : [
+        {
+          "classification" : "export",
+          "properties" : {
+            "JAVA_HOME" : "/usr/lib/jvm/java-1.8.0"
+          }
+        }
+      ],
+      "properties" : {}
+    },
+    {
+      "classification": "hive-site",
+      "properties": {
+         "hive.support.concurrency": "true",
+         "hive.exec.dynamic.partition.mode": "nonstrict",
+         "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"
+      }
+    }
+  ])
+
+  master_instance_group = {
+    name           = "master-group"
+    instance_count = 1
+    instance_type  = "m5.xlarge"
+  }
+
+  core_instance_group = {
+    name           = "core-group"
+    instance_count = 1
+    instance_type  = "c4.large"
+  }
+
+  task_instance_group = {
+    name           = "task-group"
+    instance_count = 1
+    instance_type  = "c5.xlarge"
+    bid_price      = "0.1"
+
+    ebs_config = {
+      size                 = 64
+      type                 = "gp3"
+      volumes_per_instance = 1
+    }
+    ebs_optimized = true
+  }
+
+  ebs_root_volume_size = 64
+  ec2_attributes = {
+    # Instance groups only support one Subnet/AZ
+    # Subnets should be private subnets and tagged with
+    # { "for-use-with-amazon-emr-managed-policies" = true }
+    subnet_id = var.subnet_ids_us[0]
+  }
+  vpc_id = var.vpc_id_us
+
+  list_steps_states  = ["PENDING", "RUNNING", "CANCEL_PENDING", "CANCELLED", "FAILED", "INTERRUPTED", "COMPLETED"]
+  log_uri            = "s3://${module.s3_bucket.s3_bucket_id}/"
+
+  scale_down_behavior    = "TERMINATE_AT_TASK_COMPLETION"
+  step_concurrency_level = 3
+  termination_protection = false
+  visible_to_all_users   = true
+  
+  is_private_cluster = false
+  #create_service_iam_role = false
+  #service_iam_role_arn = "arn:aws:iam::192018133564:role/service-role/AmazonEMR-ServiceRole-20230622T122656"
+  #create_iam_instance_profile = false
+  #iam_instance_profile_name = "AmazonEMR-InstanceProfile-20230622T122640"
+
+  tags = {
+    Terraform   = "true"
+    Environment = "dev"
+    QMI_user = var.user_id
+    Owner    = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+  }
+}

databases/aws-emr/outputs.tf

@@ -0,0 +1,15 @@
+output "s3_bucket" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_id
+}
+
+output "s3_iam_name" {
+  value = module.qmi-s3-bucket.iam_name
+}
+
+output "s3_iam_access_key" {
+  value = module.qmi-s3-bucket.iam_access_key
+}
+
+output "s3_iam_access_secret" {
+  value =  module.qmi-s3-bucket.iam_access_secret
+}

databases/aws-emr/variables.tf

@@ -0,0 +1,34 @@
+variable "region" {
+  default = "us-east-1"
+}
+
+variable "provision_id" {
+  
+}
+
+variable "user_id" {
+}
+
+variable "vpc_id_ap" {
+  default = "vpc-22ee2844"
+}
+
+variable "vpc_id_eu" {
+  default = "vpc-73f0500a"
+}
+
+variable "vpc_id_us" {
+  default = "vpc-c079f5bd"
+}
+
+variable "subnet_ids_ap" {
+  default = ["subnet-658aec3c", "subnet-e030eba8", "subnet-94309bf2"]
+}
+
+variable "subnet_ids_eu" {
+  default = ["subnet-4d441b17", "subnet-95c22fde", "subnet-70938116"]
+}
+
+variable "subnet_ids_us" {
+  default = ["subnet-4d26552b", "subnet-0414685b", "subnet-c97f7c84", "subnet-7f695c71", "subnet-96acd2b7", "subnet-88ab2cb9"]
+}

databases/aws-kinesis/main.tf

@@ -26,6 +26,7 @@ locals {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
   }

databases/aws-rds/main.tf

@@ -5,29 +5,47 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.74.0"
     }
   }
 }
 
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
 locals {
   provid5              = substr(var.provision_id, 0, 5)
   aurora               = (var.engine == "aurora-mysql") ? true : (var.engine == "aurora-postgresql") ? true : false
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
-  port                 = (var.engine == "oracle-se2") ? "1521" : (var.engine == "postgres") ? "5432" : (var.engine == "aurora-postgresql") ? "5432" : (var.engine == "sqlserver-ex") ? "1433" : "3306"
+  
   name                 = (var.engine == "sqlserver-ex") ? null : (var.engine == "oracle-se2") ? "ora${local.provid5}" : "qmi${var.provision_id}"
   license              = (local.aurora == true) ? "general-public-license" : (var.engine == "mariadb") ? "general-public-license" : (var.engine == "postgres") ? "postgresql-license" : (var.engine == "mysql") ? "general-public-license" : "license-included"
-  engine_version       = (var.engine == "oracle-se2") ? "19.0.0.0.ru-2021-04.rur-2021-04.r1" : (var.engine == "postgres") ? "13.3" : (var.engine == "mysql") ? "8.0.25" : (var.engine == "aurora-postgresql") ? "14.5" : (var.engine == "aurora-mysql") ? "5.7.mysql_aurora.2.10.0" : (var.engine == "sqlserver-ex") ? "15.00.4073.23.v1" : "10.5"   #mariaDB
-  major_engine_version = (var.engine == "oracle-se2") ? "19" : (var.engine == "postgres") ? "13" : (var.engine == "mysql") ? "8.0" : (var.engine == "aurora-postgresql") ? "12" : (var.engine == "aurora-mysql") ? "5.7" : (var.engine == "sqlserver-ex") ? "15.00" : "10.5"                                                                         #mariaDB
-  family               = (var.engine == "oracle-se2") ? "oracle-se2-19" : (var.engine == "postgres") ? "postgres13" : (var.engine == "mysql") ? "mysql8.0" : (var.engine == "aurora-postgresql") ? "aurora-postgresql12" : (var.engine == "aurora-mysql") ? "aurora-mysql5.7" : (var.engine == "sqlserver-ex") ? "sqlserver-ex-15.0" : "mariadb10.5" #mariaDB
+  
+  port                 = var.port[var.engine] 
+  engine_version       = var.engine_version[var.engine]
+  major_engine_version = var.major_engine_version[var.engine]
+  family               = var.family[var.engine]
+
   tags = {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
+
+  is_postgres = length(regexall("postgres", local.family)) > 0 ? true : false
+  is_mysql = length(regexall("mysql", local.family)) > 0 ? true : false
 }
 
 
@@ -35,6 +53,49 @@ module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
+module "security_group_2" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
+
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+
+  # ingress
+
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
+  
+
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "RDS"
+
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "RDS"
+
+    },
+  ]
+
+  tags = local.tags
+}
+
 module "security_group" {
   
   # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
@@ -80,7 +141,7 @@ module "security_group" {
 
 module "common_rds_instance" {
   source  = "terraform-aws-modules/rds/aws"
-  version = "~> 3.3"
+  version = "= 6.1.1"
 
   count = local.aurora? 0 : 1
 
@@ -92,58 +153,174 @@ module "common_rds_instance" {
   major_engine_version = local.major_engine_version # DB option group
   instance_class       = var.instance_size
   allocated_storage    = var.storage
+  storage_encrypted    = (var.engine == "sqlserver-ex")? false : true
 
   license_model = local.license
 
-  name                   = local.name
+  db_name                = local.name
   username               = "qmirdsuser"
-  create_random_password = true
-  random_password_length = 12
+  password               = random_password.password.result
+  manage_master_user_password = false
   port                   = local.port
 
   multi_az               = false
   subnet_ids             = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+    module.security_group.security_group_id,
+    module.security_group_2.security_group_id
+  ]
   publicly_accessible    = true
 
   maintenance_window = "Mon:00:00-Mon:03:00"
   backup_window      = "03:00-06:00"
 
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
+  parameters = local.is_postgres? [
+    {
+      apply_method = "pending-reboot"
+      name  = "rds.logical_replication"
+      value = 1
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_wal_senders"
+      value = 10
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_replication_slots"
+      value = 10
+    }
+  ] : local.is_mysql? [{
+      name  = "binlog_format"
+      value = "row"
+  }]: []
+
   tags = local.tags
 
 }
 
+
+resource "aws_rds_cluster_parameter_group" "pg-postgres" {
+
+  count =  var.engine == "aurora-postgresql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-postgresql14"
+  description = "RDS aurora-postgresql14 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "rds.logical_replication"
+    value = 1
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_wal_senders"
+    value = 10
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_replication_slots"
+    value = 10
+  }
+
+}
+
+resource "aws_rds_cluster_parameter_group" "pg-mysql" {
+
+  count =  var.engine == "aurora-mysql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-mysql8.0"
+  description = "RDS aurora-mysql8.0 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "binlog_format"
+    value = "row"
+  }
+
+}
+
 module "aurora_rds_instance" {
+
+  depends_on = [  
+    aws_rds_cluster_parameter_group.pg-postgres, 
+    aws_rds_cluster_parameter_group.pg-mysql
+  ]
+
   source  = "terraform-aws-modules/rds-aurora/aws"
-  version = "~> 5.2"
+  version = "~> 8.3.1"
 
   count = local.aurora? 1 : 0
 
   name           = local.name
   engine         = var.engine
   engine_version = local.engine_version
-  instance_type  = var.instance_size
+  instance_class  = var.instance_size
+
+  apply_immediately = true
+
+  database_name = local.name
+
+  instances       = { 1 = {} }
+
+  autoscaling_enabled      = true
+  autoscaling_min_capacity = 1
+  autoscaling_max_capacity = 3
 
   vpc_id                 = local.vpc_id
   subnets                = local.subnet_ids
   create_security_group  = false
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+      module.security_group.security_group_id,
+      module.security_group_2.security_group_id
+  ]
   port                   = local.port
   publicly_accessible    = true
 
-  username               = "qmirdsuser"
-  create_random_password = true
+  master_username         = "qmirdsuser"
+  manage_master_user_password = false
+  master_password = random_password.password.result
+  create_db_subnet_group = true
 
 
-
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
   tags = local.tags
 
+  db_cluster_parameter_group_name = "rds-cluster-pg-${var.provision_id}"
+
+}
+
+locals {
+    
+    type      = (var.engine == "mysql" || var.engine == "mariadb")? "mysql" :  (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : (var.engine == "sqlserver-ex")? "mssql" : null
+    port4dummy  = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
+    host      = (local.aurora)? "${module.aurora_rds_instance[0].cluster_endpoint}:${local.port4dummy}" : module.common_rds_instance[0].db_instance_endpoint
+    username  = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+    password  = nonsensitive(random_password.password.result)
+    database  = (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : local.name
+}
+
+
+module "dummy-data" {
+
+  count = var.dummydata != null && local.type != null? 1 : 0
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = local.type
+  host      = local.host
+  username  = local.username
+  password  = local.password
+  database  = local.database
 }

databases/aws-rds/output.tf

@@ -1,23 +1,31 @@
  output "db_username" {
   description = "The master username for the database"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
 } 
 
  output "db_instance_password" {
   description = "The database password (this password may be old, because Terraform doesn't track it after initial creation)"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_password) : nonsensitive(module.common_rds_instance[0].db_instance_password)
+  value       = nonsensitive(random_password.password.result)
 }
 
  output "db_instance_port" {
   description = "The database port"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_port : module.common_rds_instance[0].db_instance_port
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
 }
 
 output "db_instance_endpoint" {
   description = "The connection endpoint"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+}
+
+output "db_instance_id" {
+  value       = (local.aurora)? null : "${var.engine}${var.provision_id}"
 }
 
 output "db_instance_name" {
   value       = local.name
 }
+
+output "dummy_data_databases_available" {
+   value = var.dummydata != null && local.type != null? module.dummy-data[0].dbs : null
+}

databases/aws-rds/variables.tf

@@ -38,21 +38,6 @@ variable "engine" {
   description = "Accepted values are mysql, sqlserver-ex, postgres, oracle-se2, aurora-mysql, aurora-postgresql, mariadb"
 }
 
-variable "engine_version" {
-  default = "8.0.25"
-  description = "Not currently used"
-}
-
-variable "major_eng" {
-  default = "8.0"
-  description = "Not currently used"
-}
-
-variable "family" {
-  default = "mysql8.0"
-  description = "Not currently used"
-}
-
 variable "instance_size" {
   default = "db.t3.large"
 }
@@ -60,3 +45,68 @@ variable "instance_size" {
 variable "storage" {
   default = "20"
 }
+
+variable "engine_version" {
+  type = map
+
+  default = {
+    "mysql"  = "8.0.32"
+    "postgres"  = "14.9" 
+    "oracle-se2"  = "19.0.0.0.ru-2023-04.rur-2023-04.r1"
+    "sqlserver-ex" = "15.00.4236.7.v1"
+    "mariadb" = "10.5"
+    "aurora-mysql" = "8.0.mysql_aurora.3.05.2" #"5.7.mysql_aurora.2.11.2"
+    "aurora-postgresql"  = "14.9"
+  }
+  
+}
+
+variable "major_engine_version" {
+  type = map
+
+  default = {
+    "mysql"  = "8.0"  
+    "postgres"  = "14"
+    "oracle-se2"  = "19"
+    "sqlserver-ex" = "15.00"
+    "mariadb" = "10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+  
+}
+
+
+variable "family" {
+  type = map
+
+  default = {
+    "mysql"  = "mysql8.0"
+    "postgres"  = "postgres14"
+    "oracle-se2"  = "oracle-se2-19"
+    "sqlserver-ex" = "sqlserver-ex-15.0"
+    "mariadb" = "mariadb10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+}
+
+variable "port" {
+  type = map
+
+  default = {
+    "mysql"  = "3306"
+    "postgres"  = "5432"
+    "oracle-se2"  = "1521"
+    "sqlserver-ex" = "1433"
+    "mariadb" = "3306"
+    "aurora-mysql" = "3306"
+    "aurora-postgresql"  = "5432"
+  }
+}
+
+variable "dummydata" {
+  default = null
+}

databases/aws-redshift/main.tf

@@ -17,13 +17,14 @@ resource "random_password" "password" {
   min_numeric = 1
   upper = true
   lower = true
-  number = true
+  numeric = true
   min_lower = 2
   min_upper = 2
   min_special = 2
 }
 
 locals {
+  s3_prefix = "redshift/qmi-${var.provision_id}"
   provid5              = substr(var.provision_id, 0, 5)
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
@@ -34,6 +35,7 @@ locals {
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
 }
 
@@ -83,32 +85,47 @@ module "security_group" {
   tags = local.tags
 }
 
-module "redshift" {
+module "security_group_2" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
 
-  source  = "terraform-aws-modules/redshift/aws"
-  version = "~> 3.0"
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
 
-  cluster_identifier      = "qmi-${var.provision_id}"
-  cluster_node_type       = "dc2.large" #"dc1.large"
-  cluster_number_of_nodes = 1
 
-  cluster_database_name   = var.cluster_database_name
-  cluster_master_username = var.cluster_master_username
-  cluster_master_password = random_password.password.result
+  # ingress
 
-  # Group parameters
-  #wlm_json_configuration = "[{\"query_concurrency\": 5}]"
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
+  
 
-  # DB Subnet Group Inputs
-  subnets = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
-  publicly_accessible = true
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Redshift"
 
-  # IAM Roles
-  #cluster_iam_roles = ["arn:aws:iam::225367859851:role/developer"]
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Redshift"
+
+    },
+  ]
 
   tags = local.tags
-
 }
 
 module "qmi-s3-bucket" {
@@ -118,4 +135,43 @@ module "qmi-s3-bucket" {
   provision_id                  = var.provision_id
   region                        = var.region
   user_id                       = var.user_id
-}
+}
+
+module "redshift" {
+
+  source  = "terraform-aws-modules/redshift/aws"
+  version = "~> 5.0.0"
+
+  cluster_identifier      = "qmi-${var.provision_id}"
+  node_type       = "dc2.large" #"dc1.large"
+  number_of_nodes = 1
+
+  database_name   = var.cluster_database_name
+  master_username = var.cluster_master_username
+  create_random_password = false
+  master_password = random_password.password.result
+
+  # Group parameters
+  #wlm_json_configuration = "[{\"query_concurrency\": 5}]"
+
+  # DB Subnet Group Inputs
+  subnet_ids = local.subnet_ids
+  vpc_security_group_ids = [
+      module.security_group.security_group_id,
+      module.security_group_2.security_group_id
+  ]
+  publicly_accessible = true
+
+  /*logging = {
+    enable        = true
+    bucket_name   = module.qmi-s3-bucket.bucket.s3_bucket_id
+    s3_key_prefix = local.s3_prefix
+  }*/
+
+  # IAM Roles
+  #cluster_iam_roles = ["arn:aws:iam::225367859851:role/developer"]
+
+  tags = local.tags
+
+}
+

databases/aws-redshift/outputs.tf

@@ -1,25 +1,25 @@
 output "redshift_cluster_id" {
   description = "The availability zone of the RDS instance"
-  value       = module.redshift.redshift_cluster_id
+  value       = module.redshift.cluster_id
 }
 
 output "redshift_cluster_endpoint" {
   description = "Redshift endpoint"
-  value       = module.redshift.redshift_cluster_endpoint
+  value       = module.redshift.cluster_endpoint
 }
 
 output "redshift_cluster_hostname" {
   description = "Redshift hostname"
-  value       = module.redshift.redshift_cluster_hostname
+  value       = module.redshift.cluster_hostname
 }
 
 output "redshift_cluster_port" {
   description = "Redshift port"
-  value       = module.redshift.redshift_cluster_port
+  value       = module.redshift.cluster_port
 }
 
 output "redshift_cluster_database_name" {
-    value = module.redshift.redshift_cluster_database_name
+    value = module.redshift.cluster_database_name
 }
 
 output "redshift_cluster_master_username" {

databases/aws-redshift/variables.tf

@@ -38,5 +38,5 @@ variable "cluster_database_name" {
 }
 
 variable "cluster_master_username" {
-    default = "qmi"
+    default = "qmiuser"
 }

databases/azure-eventhub/main.tf

@@ -25,6 +25,7 @@ resource "azurerm_eventhub_namespace" "ehbnamespace" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   network_rulesets {

databases/azure-rds-flexmysql/firewall.tf

@@ -1,31 +1,29 @@
-/*resource "azurerm_mysql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}*/
-
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "52.249.189.38"
-  end_ip_address      = "52.249.189.38"
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips_az_qcs
+
+  name                = each.key
   resource_group_name = var.resource_group_name
   server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "13.67.39.86"
-  end_ip_address      = "13.67.39.86"
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
 }
 
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "20.67.110.207"
-  end_ip_address      = "20.67.110.207"
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mysql_flexible_server_firewall_rule.fw_rule ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_flexible_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
 }

databases/azure-rds-flexmysql/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_numeric = 2
@@ -33,6 +33,7 @@ resource "azurerm_mysql_flexible_server" "mysql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 

databases/azure-rds-flexmysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-flexmysql/variables.tf

@@ -36,4 +36,8 @@ variable "mysql-storage" {
   type = string
   description = "MySQL Storage in MB"
   default = "20"
-  }
+}
+
+variable "dummydata" {
+  default = null
+}

databases/azure-rds-flexpostgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!)"
   upper = true
   lower = true
   min_numeric = 2
@@ -29,6 +29,7 @@ resource "azurerm_postgresql_flexible_server" "postgresql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
 }

databases/azure-rds-mariadb/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_numeric = 2
@@ -35,6 +35,7 @@ resource "azurerm_mariadb_server" "mariadb-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -44,4 +45,17 @@ resource "azurerm_mariadb_database" "mariadb-db" {
   server_name         = azurerm_mariadb_server.mariadb-server.name
   charset             = "utf8"
   collation           = "utf8_unicode_ci"
-}
+}
+
+/*
+module "dummy-data" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mariadb_server.mariadb-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
+}
+*/

databases/azure-rds-mssql/firewall.tf

@@ -1,4 +1,5 @@
 
+
 resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
 
   count = var.subnet_id != null? 1 : 0
@@ -8,6 +9,15 @@ resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
   subnet_id           = var.subnet_id
 }
 
+resource "azurerm_mssql_virtual_network_rule" "sqlvnet_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                = "vnet-qmiinfra-${local.provision_id}"
+  server_id           = azurerm_mssql_server.sqlserver.id
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -21,4 +31,19 @@ resource "azurerm_mssql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mssql_firewall_rule.fw_rule ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mssql"
+  host      = azurerm_mssql_server.sqlserver.fully_qualified_domain_name
+  username  = var.sql_administrator_login
+  password  = nonsensitive(local.sql_administrator_login_password)
+  database  = ""
 }

databases/azure-rds-mssql/main.tf

@@ -10,7 +10,7 @@ resource "random_id" "randomServerId" {
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -37,6 +37,7 @@ resource "azurerm_mssql_server" "sqlserver" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
   identity {
@@ -53,6 +54,7 @@ resource "azurerm_mssql_database" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -66,6 +68,7 @@ resource "azurerm_mssql_database" "dbname_source" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -78,5 +81,6 @@ resource "azurerm_mssql_database" "dbname_target" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }

databases/azure-rds-mssql/outputs.tf

@@ -26,4 +26,8 @@ output "principal_id" {
   value = azurerm_mssql_server.sqlserver.identity.0.principal_id
 }
 
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}
+
 

databases/azure-rds-mssql/variables.tf

@@ -23,4 +23,12 @@ variable "sql_administrator_login" {
 
 variable "sql_administrator_login_password" {
     default = null
-}
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-mysql/firewall.tf

@@ -9,6 +9,16 @@ resource "azurerm_mysql_virtual_network_rule" "vnetrule" {
 
 }
 
+resource "azurerm_mysql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                  = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name   = var.resource_group_name
+  server_name           = azurerm_mysql_server.mysql-server.name
+  subnet_id             = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -23,4 +33,23 @@ resource "azurerm_mysql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_mysql_virtual_network_rule.vnetrule,
+    azurerm_mysql_virtual_network_rule.vnetrule_qmiinfra,
+    azurerm_mysql_firewall_rule.fw_rule 
+  ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
 }

databases/azure-rds-mysql/main.tf

@@ -1,18 +1,7 @@
-
-/*resource "azurerm_resource_group" "mysql-rg" {
-  name     = "mysql-${var.provision_id}"
-  location = var.location
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-  }
-}*/
-
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -55,4 +44,12 @@ resource "azurerm_mysql_database" "mysql-db" {
   server_name         = azurerm_mysql_server.mysql-server.name
   charset             = "utf8"
   collation           = "utf8_unicode_ci"
-}
+}
+
+
+resource "azurerm_mysql_configuration" "example" {
+  name                = "binlog_row_image"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  value               = "full"
+}

databases/azure-rds-mysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-mysql/variables.tf

@@ -29,7 +29,7 @@ variable "admin_login" {
 variable "mysql-version" {
   type = string
   description = "MySQL Server version to deploy"
-  default = "5.7"
+  default = "8.0"
 }
 variable "mysql-sku-name" {
   type = string
@@ -41,3 +41,11 @@ variable "mysql-storage" {
   description = "MySQL Storage in MB"
   default = "5120"
   }
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-postgres/firewall.tf

@@ -8,6 +8,17 @@ resource "azurerm_postgresql_virtual_network_rule" "vnetrule" {
   subnet_id           = var.subnet_id
 }
 
+resource "azurerm_postgresql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+
+  name                = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -22,4 +33,23 @@ resource "azurerm_postgresql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_postgresql_firewall_rule.fw_rule, 
+    azurerm_postgresql_virtual_network_rule.vnetrule,
+    azurerm_postgresql_virtual_network_rule.vnetrule_qmiinfra 
+  ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "postgres"
+  host      = azurerm_postgresql_server.postgresql-server.fqdn
+  username  = "${var.admin_login}@qmi-postgresql-${var.provision_id}"
+  password  = nonsensitive(random_password.password.result)
+  database  = "postgres"
 }

databases/azure-rds-postgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2

databases/azure-rds-postgres/output.tf

@@ -13,3 +13,7 @@ output "root_username" {
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
 }
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}

databases/azure-rds-postgres/variables.tf

@@ -21,7 +21,7 @@ variable "subnet_id" {
 variable "admin_login" {
   type = string
   description = "Login to authenticate to PostgreSQL Server"
-  default = "qmi"
+  default = "qmirdsuser"
 }
 
 variable "postgresql-version" {
@@ -42,4 +42,12 @@ variable "postgresql-storage" {
 
 variable "user_id" {
 
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
 }

databases/dbricks/main.tf

@@ -5,11 +5,55 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 1.6.0"
+      version = ">= 1.47.0"
+      configuration_aliases = [ databricks.main, databricks.accounts ]
     }
+    
   }
 }
 
+
+locals {
+  cluster_nameUpper = upper(var.cluster_name)
+}
+
+locals {
+  resource_regex            = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)"
+  subscription_id           = regex(local.resource_regex, var.databricks_resource_id)[0]
+  resource_group            = regex(local.resource_regex, var.databricks_resource_id)[1]
+  databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2]
+  tenant_id                 = data.azurerm_client_config.current.tenant_id
+  databricks_workspace_host = data.azurerm_databricks_workspace.this.workspace_url
+  databricks_workspace_id   = data.azurerm_databricks_workspace.this.workspace_id
+  prefix                    = replace(replace(lower(data.azurerm_resource_group.this.name), "rg", ""), "-", "")
+}
+
+data "azurerm_resource_group" "this" {
+  name = local.resource_group
+}
+
+data "azurerm_client_config" "current" {
+}
+
+data "azurerm_databricks_workspace" "this" {
+  name                = local.databricks_workspace_name
+  resource_group_name = local.resource_group
+}
+
+resource "databricks_metastore" "this" {
+  provider      = databricks.accounts
+  name          = "primary"
+  force_destroy = true
+  region        = data.azurerm_resource_group.this.location
+}
+
+resource "databricks_metastore_assignment" "this" {
+  provider             = databricks.accounts
+  workspace_id         = local.databricks_workspace_id
+  metastore_id         = databricks_metastore.this.id
+  default_catalog_name = "hive_metastore"
+}
+
 /*resource "databricks_workspace_conf" "this" {
   
   custom_config = {
@@ -57,6 +101,7 @@ data "databricks_group" "admins" {
 
 
 resource "databricks_user" "aor" {
+  provider      = databricks.main
   count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
   
   allow_cluster_create = true
@@ -67,7 +112,7 @@ resource "databricks_user" "aor" {
 }
 
 resource "databricks_user" "me" {
-
+  provider      = databricks.main
   allow_cluster_create = true
   databricks_sql_access = true
 
@@ -77,30 +122,65 @@ resource "databricks_user" "me" {
 
 
 resource "databricks_group_member" "i-am-admin" {
+  provider      = databricks.main
   group_id  = data.databricks_group.admins.id
   member_id = databricks_user.me.id
 }
 
 
 resource "databricks_group_member" "i-am-admin-aor" {
+  provider      = databricks.main
   count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
   group_id  = data.databricks_group.admins.id
   member_id = databricks_user.aor[0].id
 }
 
-resource "databricks_sql_endpoint" "sqlep" {
+resource "databricks_secret_scope" "scopeapp" {
+    provider      = databricks.main
+    count = (var.sku == "premium")? 1 : 0
 
+    name = "${var.cluster_name}scope"
+}
+resource "databricks_secret" "publishing_api" {
+    provider      = databricks.main
+    count = (var.sku == "premium")? 1 : 0
+
+    key = "accesskey"
+    string_value = var.app_reg_secret
+    scope = databricks_secret_scope.scopeapp[0].name
+}
+
+resource "databricks_sql_endpoint" "sqlep" {
+  provider      = databricks.main
   count = (var.sku == "premium")? 1 : 0
 
-  name             = "qmi-sqlwh-${databricks_user.me.id}"
-  cluster_size     = "Small"
+  name             = "${local.cluster_nameUpper} SQL warehouse"
+  cluster_size     = var.cluster_size
   max_num_clusters = 1
 
 }
 
+resource "databricks_sql_global_config" "this" {
+  provider      = databricks.main
+  count = (var.sku == "premium")? 1 : 0
+
+  security_policy      = "DATA_ACCESS_CONTROL"
+  data_access_config = {
+    "spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
+    "spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
+    "spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
+    "spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
+  }
+
+  enable_serverless_compute = false
+  sql_config_params = {}
+}
+
 
 
 resource "databricks_token" "pat" {  
+  provider      = databricks.main
   depends_on = [
     databricks_user.me,
     databricks_user.aor
@@ -110,6 +190,7 @@ resource "databricks_token" "pat" {
 }
 
 resource "databricks_cluster" "dbrickscluster" {
+  provider      = databricks.main
   count = (var.cluster_name != null)? 1 : 0
 
 
@@ -122,9 +203,12 @@ resource "databricks_cluster" "dbrickscluster" {
     "spark.databricks.delta.preview.enabled": true,
     "spark.databricks.repl.allowedLanguages": "sql,python,r",
     "spark.databricks.cluster.profile": "serverless",
-    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
+    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
+    "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
+    "spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
   } 
 
+
   spark_env_vars = {
       "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
   }
@@ -134,5 +218,5 @@ resource "databricks_cluster" "dbrickscluster" {
     max_workers = 4
   }
 
-  autotermination_minutes = 120
+  autotermination_minutes = 45
 }

databases/dbricks/varaibles.tf

@@ -24,4 +24,19 @@ variable "node_type_id" {
 
 variable "sku" {
     default = "standard"
+}
+
+variable "app_reg_id" {
+  default = null
+}
+
+variable "app_reg_secret" {
+  default = null
+}
+
+variable "cluster_size" {
+  default = "Small"
+}
+
+variable "databricks_resource_id" {
 }

databases/dummy/main.tf

@@ -0,0 +1,21 @@
+resource "null_resource" "dummy-data" {
+    
+  provisioner "local-exec" {
+      command = "chmod +x ${path.module}/scripts/*.sh"
+      interpreter = ["/bin/bash", "-c"]
+  }
+
+  provisioner "local-exec" {
+      command = "${path.module}/scripts/dummy-${var.type}.sh \"$username\" \"$password\" \"$host\" \"$db\""
+      interpreter = ["/bin/bash", "-c"]
+
+      environment = {
+          
+          username  = var.username
+          password  = var.password
+          host      = var.host 
+          db        = var.database
+      }
+  }
+
+}

databases/dummy/outputs.tf

@@ -0,0 +1,3 @@
+output "dbs" {
+  value       = var.type == "postgres"? "'AdventureWorks' and 'dvdrental'": (var.type == "mysql")? "'sakila', 'employees' and 'classicmodels'" : (var.type == "mssql")? "'BikeStores' and 'Northwind'" : null
+}

databases/dummy/scripts/dummy-mssql.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}`
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+
+
+echo "---- Loading BikeStore sample database -----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+
+unzip qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE BikeStores"
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/createbike.sql
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/Bikeloaddata.sql > /tmp/bikestore_load.logs
+
+echo "---- Loading Northwind sample database -----"
+cd /tmp
+curl https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/northwind-pubs/instnwnd.sql -o instnwnd.sql
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE Northwind"
+sqlcmd -C -S "$host" -d "Northwind" -U "$1" -P "$2" -i instnwnd.sql > /tmp/northwind_load.logs
+
+rm -fr /tmp/qmi-cloud-tf-modules-*

databases/dummy/scripts/dummy-mysql.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4" 
+echo "--------------------------"    
+
+
+cd /tmp
+git clone https://github.com/datacharmer/test_db.git
+cd test_db
+echo "-------- Loading Mysql Employees sample database -----------"  
+mysql -h $host -u$1 -p$2 --ssl=TRUE < employees.sql
+cd ..
+wget -q https://downloads.mysql.com/docs/sakila-db.tar.gz
+tar xvfz sakila-db.tar.gz
+cd sakila-db
+
+echo "-------- Loading Mysql Sakila sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-schema.sql 
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-data.sql 
+
+cd /tmp
+
+curl https://www.mysqltutorial.org/wp-content/uploads/2018/03/mysqlsampledatabase.zip -o mysqlsampledatabase.zip
+unzip mysqlsampledatabase.zip
+echo "-------- Loading Mysql ClassicModels sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < mysqlsampledatabase.sql
+
+
+rm -fr /tmp/*
+

databases/dummy/scripts/dummy-postgres.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+echo "---- Install Adventureworks for Postgres ----"
+cd /tmp
+wget -q https://github.com/lorint/AdventureWorks-for-Postgres/archive/master.zip
+unzip master.zip
+cd AdventureWorks-for-Postgres-master/
+wget -q https://github.com/microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks-oltp-install-script.zip
+unzip AdventureWorks-oltp-install-script.zip
+
+ruby update_csvs.rb
+
+export PGPASSWORD=$2
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"AdventureWorks\";" 
+psql -h $host -U $1 -d "AdventureWorks" < ./install.sql > /tmp/AdventureWorks_load.log
+
+
+echo "---- Install DvdRental sample database for Postgres ----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+unzip -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"dvdrental\";" 
+psql -h $host -U $1 -d "dvdrental" < ./postgres/dump_dvdrental.sql > /tmp/dvdrental_load.log
+
+
+rm -fr /tmp/*
+
+

databases/dummy/scripts/mssql/createbike.sql

@@ -0,0 +1,109 @@
+/*
+--------------------------------------------------------------------
+© 2017 sqlservertutorial.net All Rights Reserved
+--------------------------------------------------------------------
+Name   : BikeStores
+Link   : http://www.sqlservertutorial.net/load-sample-database/
+Version: 1.0
+--------------------------------------------------------------------
+*/
+-- create schemas
+CREATE SCHEMA production;
+go
+
+CREATE SCHEMA sales;
+go
+
+-- create tables
+CREATE TABLE production.categories (
+	category_id INT IDENTITY (1, 1) PRIMARY KEY,
+	category_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.brands (
+	brand_id INT IDENTITY (1, 1) PRIMARY KEY,
+	brand_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.products (
+	product_id INT IDENTITY (1, 1) PRIMARY KEY,
+	product_name VARCHAR (255) NOT NULL,
+	brand_id INT NOT NULL,
+	category_id INT NOT NULL,
+	model_year SMALLINT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	FOREIGN KEY (category_id) REFERENCES production.categories (category_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (brand_id) REFERENCES production.brands (brand_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE sales.customers (
+	customer_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (255) NOT NULL,
+	last_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255) NOT NULL,
+	street VARCHAR (255),
+	city VARCHAR (50),
+	state VARCHAR (25),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.stores (
+	store_id INT IDENTITY (1, 1) PRIMARY KEY,
+	store_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255),
+	street VARCHAR (255),
+	city VARCHAR (255),
+	state VARCHAR (10),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.staffs (
+	staff_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (50) NOT NULL,
+	last_name VARCHAR (50) NOT NULL,
+	email VARCHAR (255) NOT NULL UNIQUE,
+	phone VARCHAR (25),
+	active tinyint NOT NULL,
+	store_id INT NOT NULL,
+	manager_id INT,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (manager_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.orders (
+	order_id INT IDENTITY (1, 1) PRIMARY KEY,
+	customer_id INT,
+	order_status tinyint NOT NULL,
+	-- Order status: 1 = Pending; 2 = Processing; 3 = Rejected; 4 = Completed
+	order_date DATE NOT NULL,
+	required_date DATE NOT NULL,
+	shipped_date DATE,
+	store_id INT NOT NULL,
+	staff_id INT NOT NULL,
+	FOREIGN KEY (customer_id) REFERENCES sales.customers (customer_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (staff_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.order_items (
+	order_id INT,
+	item_id INT,
+	product_id INT NOT NULL,
+	quantity INT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	discount DECIMAL (4, 2) NOT NULL DEFAULT 0,
+	PRIMARY KEY (order_id, item_id),
+	FOREIGN KEY (order_id) REFERENCES sales.orders (order_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE production.stocks (
+	store_id INT,
+	product_id INT,
+	quantity INT,
+	PRIMARY KEY (store_id, product_id),
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);

databases/dummy/variables.tf

@@ -0,0 +1,15 @@
+variable "host" { 
+}
+
+variable "username" {
+}
+
+variable "password" {
+}
+
+variable "database" {
+}
+
+variable "type" {
+    description = "mysql | postgres | mssql"
+}

databases/firewall_ips/output.tf

@@ -7,9 +7,6 @@ locals {
   }
 
   q_routes = {
-    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
-    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
-    EMEA_FULLVPN   = ["193.15.228.246", "193.15.228.246"]
     ITG         = ["188.65.156.32", "188.65.156.47"]
     ITG1        = ["212.73.252.96", "212.73.252.103"]
     kfar_saba   = ["194.90.96.176", "194.90.96.183"]
@@ -18,6 +15,28 @@ locals {
     peak_101    = ["206.196.17.32", "206.196.17.63"]
     Bangalore   = ["14.98.59.168", "14.98.59.175"]
     Bangalore1  = ["182.74.33.8", "182.74.33.15"]
+    T_Beijing   = ["60.247.114.145","60.247.114.158"]
+    T_Nantes    = ["81.255.115.33","81.255.115.38"]
+    T_PA7_1     = ["217.117.156.85","217.117.156.86"]
+    T_PA7_2     = ["217.117.156.141","217.117.156.142"]
+    T_PA7_3     = ["217.117.156.161","217.117.156.190"]
+    T_Sanmateo  = ["12.202.47.153","12.202.47.158"]
+    T_Suresnes  = ["62.23.50.121","62.23.50.126"]
+    T_SV2_1     = ["4.31.193.69","4.31.193.70"]
+    T_SV2_2     = ["4.79.217.1","4.79.217.126"]
+    T_SV2_3     = ["149.97.160.217","149.97.160.218"]
+    T_SV2_4     = ["149.97.160.221","149.97.160.222"]
+    T_SV2_5     = ["149.97.185.1","149.97.185.30"]
+    T_Blr_1     = ["14.97.166.193","14.97.166.198"]
+    T_Blr_2     = ["14.97.167.21","14.97.167.22"]
+    T_Blr_3     = ["122.185.123.61","122.185.123.62"]
+    T_Blr_4     = ["182.79.10.25","182.79.10.26"]
+  }
+
+  fullvpn = {
+    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
+    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
+    EMEA_FULLVPN    = ["193.15.228.246", "193.15.228.246"]
   }
 
   qcs = {
@@ -27,12 +46,18 @@ locals {
       QCS_eu1          = ["34.247.21.179", "34.247.21.179"]  
       QCS_eu2          = ["52.31.212.214", "52.31.212.214"]  
       QCS_eu3          = ["54.154.95.18", "54.154.95.18"]  
-      QCS_apac1        = ["13.210.43.241", "13.210.43.241"] 
-      QCS_apac2        = ["13.236.104.42", "13.236.104.42"]  
-      QCS_apac3        = ["13.236.206.172", "13.236.206.172"] 
-      QCS_sing1        = ["18.138.163.172", "18.138.163.172"]
-      QCS_sing2        = ["18.142.157.182", "18.142.157.182"]
-      QCS_sing3        = ["54.179.13.251", "54.179.13.251"]
+      QCS_ap1          = ["13.210.43.241", "13.210.43.241"] 
+      QCS_ap2          = ["13.236.104.42", "13.236.104.42"]  
+      QCS_ap3          = ["13.236.206.172", "13.236.206.172"] 
+      QCS_sg1          = ["18.138.163.172", "18.138.163.172"]
+      QCS_sg2          = ["18.142.157.182", "18.142.157.182"]
+      QCS_sg3          = ["54.179.13.251", "54.179.13.251"]
+      QCS_de1          = ["3.76.162.169", "3.76.162.169"]
+      QCS_de2          = ["3.77.162.68", "3.77.162.68"]
+      QCS_de3          = ["3.122.137.91", "3.122.137.91"]
+      QCS_uk1          = ["13.42.141.246", "13.42.141.246"]
+      QCS_uk2          = ["18.135.245.97", "18.135.245.97"]
+      QCS_uk3          = ["35.179.0.171", "35.179.0.171"]
   }
 
   qcs_qaa = {
@@ -42,23 +67,44 @@ locals {
       QCS_QAA_us1     = ["18.235.133.252", "18.235.133.252"]
       QCS_QAA_us2     = ["3.217.244.242", "3.217.244.242"]
       QCS_QAA_us3     = ["18.214.8.201", "18.214.8.201"]
-      QCS_QAA_apac1     = ["54.206.158.27", "54.206.158.27"]
-      QCS_QAA_apac2     = ["3.104.137.20", "3.104.137.20"]
-      QCS_QAA_apac3     = ["3.24.52.178", "3.24.52.178"]
-      QCS_QAA_sing1     = ["54.169.84.213", "54.169.84.213"]
-      QCS_QAA_sing2     = ["13.213.173.37", "13.213.173.37"]
-      QCS_QAA_sing3     = ["13.213.113.162", "13.213.113.162"]
+      QCS_QAA_ap1     = ["54.206.158.27", "54.206.158.27"]
+      QCS_QAA_ap2     = ["3.104.137.20", "3.104.137.20"]
+      QCS_QAA_ap3     = ["3.24.52.178", "3.24.52.178"]
+      QCS_QAA_sg1     = ["54.169.84.213", "54.169.84.213"]
+      QCS_QAA_sg2     = ["13.213.173.37", "13.213.173.37"]
+      QCS_QAA_sg3     = ["13.213.113.162", "13.213.113.162"]
+      QCS_QAA_de1     = ["3.69.132.138", "3.69.132.138"]
+      QCS_QAA_de2     = ["3.69.222.86", "3.69.222.86"]
+      QCS_QAA_de3     = ["3.72.136.160", "3.72.136.160"]
+      QCS_QAA_uk1     = ["18.168.120.199", "18.168.120.199"]
+      QCS_QAA_uk2     = ["18.169.91.38", "18.169.91.38"]
+      QCS_QAA_uk3     = ["13.42.135.168", "13.42.135.168"]
+  }
+
+  stitch = {
+      STITCH_1 = ["52.23.137.21", "52.23.137.21"]
+      STITCH_2 = ["52.204.223.208", "52.204.223.208"]
+      STITCH_3 = ["52.204.228.32", "52.204.228.32"]
+      STITCH_4 = ["52.204.230.227", "52.204.230.227"]
+      STITCH_5 = ["3.126.102.29", "3.126.102.29"]
+      STITCH_6 = ["18.158.16.164", "18.158.16.164"]
+      STITCH_7 = ["18.158.251.55", "18.158.251.55"]
+      STITCH_8 = ["52.57.235.168", "52.57.235.168"]
   }
 
  
 }
 
 output "ips2" {
-  value = merge(local.q_routes, local.qcs)
+  value = merge(local.q_routes, local.fullvpn, local.qcs, local.stitch)
 }
 
 output "ips" {
-  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes)
+  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes, local.fullvpn, local.stitch)
+}
+
+output "ips_az_qcs" {
+  value = merge(local.az, local.qcs, local.fullvpn, local.stitch)
 }
 
 output "cidr_blocks" {
@@ -80,32 +126,78 @@ output "cidr_blocks" {
     "213.57.84.160/29", 
     "4.4.97.104/29", 
     "206.196.17.32/27", 
-    # QCS
+    # T Beijing
+    "60.247.114.144/28",
+    # T Nantes
+    "81.255.115.32/29",
+    "84.14.92.152/29",
+    # T PA7
+    "217.117.156.84/30",
+    "217.117.156.140/30",
+    "217.117.156.160/27",
+    # T San Mateo
+    "12.202.47.152/29",
+    # T Suresnes
+    "62.23.50.120/29",
+    # T SV2
+    "4.31.193.68/30",
+    "4.79.217.0/25",
+    "149.97.160.216/30",
+    "149.97.160.220/30",
+    "149.97.185.0/27",
+    # T Blr Office
+    "14.97.166.192/29",
+    "14.97.167.20/30",
+    "122.185.123.60/30",
+    "182.79.10.24/29",
+    # QCS Amercias (us)
     "18.205.71.36/32", 
     "18.232.32.199/32", 
-    "34.237.68.254/32", 
+    "34.237.68.254/32",
+    # QCS Ireland (eu) 
     "34.247.21.179/32", 
     "52.31.212.214/32", 
-    "54.154.95.18/32", 
+    "54.154.95.18/32",
+    # QCS Australia (ap)  
     "13.210.43.241/32", 
     "13.236.104.42/32", 
     "13.236.206.172/32",
+    # QCS Singapore (sg)
     "18.138.163.172/32",
     "18.142.157.182/32",
     "54.179.13.251/32",
-    # QCS-QAA
+    # QCS Frankfurt (de)
+    "3.76.162.169/32",
+    "3.77.162.68/32",
+    "3.122.137.91/32",
+    # QCS London (uk)
+    "13.42.141.246/32",
+    "18.135.245.97/32",
+    "35.179.0.171/32",
+    # QCS-QAA Ireland (eu)
     "54.216.156.88/32",
     "3.248.156.131/32",
     "52.213.44.55/32",
+    # QCS-QAA Americas (us)
     "18.235.133.252/32",
     "3.217.244.242/32",
     "18.214.8.201/32",
+    # QCS-QAA Australia (ap)
     "54.206.158.27/32",
     "3.104.137.20/32",
     "3.24.52.178/32",
+    # QCS-QAA Singapore (sg)
     "54.169.84.213/32",
     "13.213.173.37/32",
     "13.213.113.162/32",
+    # QCS-QAA Frankfurt (de)
+    "3.69.132.138/32",
+    "3.69.222.86/32",
+    "3.72.136.160/32",
+    # QCS-QAA London (uk)
+    "18.168.120.199/32",
+    "18.169.91.38/32",
+    "13.42.135.168/32",
     # QCS STAGING
     "18.233.22.130/32",
     "18.205.135.40/32",
@@ -113,6 +205,20 @@ output "cidr_blocks" {
     "18.155.181.46/32",
     "18.155.181.25/32",
     "18.155.181.128/32",
-    "18.155.181.76/32"
+    "18.155.181.76/32",
   ]
-}
+}
+
+output "cidr_blocks_others" {
+  value = [
+    # Stitch
+    "52.23.137.21/32",
+    "52.204.223.208/32",
+    "52.204.228.32/32",
+    "52.204.230.227/32",
+    "3.126.102.29/32",
+    "18.158.16.164/32",
+    "18.158.251.55/32",
+    "52.57.235.168/32",
+  ]
+}

databases/synapse-ws/main.tf

@@ -44,6 +44,7 @@ resource "azurerm_synapse_workspace" "synapsews" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner    = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 
@@ -60,6 +61,7 @@ resource "azurerm_synapse_sql_pool" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 

linux-common/centos/common/extract-certs.sh

@@ -3,13 +3,18 @@
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
 # Extract the public key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
 # Extract the CA cert chain
-openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
 
 cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
 

linux-common/centos/common/save-cert.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+
+echo $1 > $BASEDIR/qmicerts/myserver.crt
+
+echo $2 > $BASEDIR/qmicerts/myserver.key
+
+

linux-common/centos/common/update.sh

@@ -0,0 +1,2 @@
+echo 'Updating OS. This will take a around 10 minutes'
+yum -y --quiet update

linux-common/main.tf

@@ -13,15 +13,22 @@ data "azurerm_key_vault_secret" "cert_password" {
   key_vault_id = var.key_vault_id
 }
 
+#data "azurerm_key_vault_certificate_data" "cert-data" {
+#  name         = "star-qmi-qlikpoc-com-cert"
+#  key_vault_id = var.key_vault_id
+#}
+
+
 locals {
   falcon_id     = nonsensitive(data.azurerm_key_vault_secret.cid.value)
   tenable_key   = nonsensitive(data.azurerm_key_vault_secret.tenable-key.value)
   cert_password = nonsensitive(data.azurerm_key_vault_secret.cert_password.value)
-}
+  #cert_pem =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.pem)
+  #cert_key =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.key)
+} 
 
-resource "null_resource" "post-linux-vm" {
-    
-    provisioner "file" {
+resource "null_resource" "files" {
+  provisioner "file" {
         connection {
             type     = "ssh"
             host     = var.private_ip_address
@@ -33,6 +40,13 @@ resource "null_resource" "post-linux-vm" {
         source      = "${path.module}/${var.os_type}/common"
         destination = "/home/${var.admin_username}"
     }
+}
+
+resource "null_resource" "post-linux-vm" {
+    
+    depends_on = [
+      null_resource.files
+    ]
 
     provisioner "remote-exec" {
         connection {
@@ -50,6 +64,34 @@ resource "null_resource" "post-linux-vm" {
             "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
             "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
+            #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
         ]
     }
-}
+}
+
+resource "null_resource" "update" {
+
+  count = var.update? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.post-linux-vm
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "60s"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/update.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -3,13 +3,18 @@
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-full.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/star_qmi_qlik-poc_com.pem -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pem
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD
 # Extract the public key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
 # Extract the CA cert chain
-openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
 
 cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
 

linux-common/ubuntu/common/falcon.sh

@@ -9,7 +9,7 @@ binary="falcon-sensor_5.17.0-8103_amd64.deb"
 echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
 
-sudo apt -qq -y update
+#sudo apt -qq -y update
 sudo apt install libnl-3-200 libnl-genl-3-200 -qq -y
 
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary

linux-common/ubuntu/common/update.sh

@@ -0,0 +1,3 @@
+echo 'Updating OS. This will take a around 10 minutes'
+apt -qq -y update
+apt --fix-broken -qq -y upgrade

linux-common/variables.tf

@@ -16,3 +16,8 @@ variable "key_vault_id" {
   default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
 }
 
+variable "update" {
+  type = bool
+  default = true
+}
+

qmi-nic/mainf.tf

@@ -71,7 +71,7 @@
   address_prefix       = "10.0.2.0/24"
 }*/
 
-resource "azurerm_public_ip" "publicip" {
+/*resource "azurerm_public_ip" "publicip" {
 
   count = var.isExternal? 1 : 0
   
@@ -80,7 +80,7 @@ resource "azurerm_public_ip" "publicip" {
   location            = var.location
   allocation_method   = "Dynamic"
   domain_name_label   = lower(var.prefix)
-}
+}*/
 
 resource "azurerm_network_interface" "nic" {
   name                = "${var.prefix}-nic"
@@ -92,13 +92,13 @@ resource "azurerm_network_interface" "nic" {
     name                          = "${var.prefix}-ip-cfg"
     subnet_id                     = var.subnet_id
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = var.isExternal? azurerm_public_ip.publicip[0].id : null
   }
 
   tags = {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 

qmi-nic/outputs.tf

@@ -8,12 +8,4 @@ output "private_ip_address" {
 
 output "ip_configuration_name" {
   value = "${var.prefix}-ip-cfg"
-}
-
-output "public_ip_address" {
-  value = var.isExternal? azurerm_public_ip.publicip[0].ip_address : null
-}
-
-output "fqdn" {
-  value = var.isExternal? azurerm_public_ip.publicip[0].fqdn : null
 }

qmi-storage-account/main.tf

@@ -11,12 +11,16 @@ resource "azurerm_storage_account" "qmi-storage-account" {
     name                = "diag${random_id.randomId.hex}"
     resource_group_name = var.resource_group_name
     location            = var.location
-    account_replication_type = "LRS"
-    account_tier = "Standard"
+    
+    account_kind              = "StorageV2"
+    account_replication_type  = "LRS"
+    account_tier              = "Standard"
+    access_tier               = "Hot"
 
     tags = {
         Deployment = "QMI PoC"
         "Cost Center" = "3100"
         QMI_user = var.user_id
+        Owner    = var.user_id
     }
 }

qmicerts/star_qmi_qlik-poc_com.pem

@@ -0,0 +1,89 @@
+-----BEGIN CERTIFICATE-----
+MIIG4DCCBcigAwIBAgIQAxl66Ob3+2PBk9UFiseUtDANBgkqhkiG9w0BAQsFADBZ
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTMwMQYDVQQDEypE
+aWdpQ2VydCBHbG9iYWwgRzIgVExTIFJTQSBTSEEyNTYgMjAyMCBDQTEwHhcNMjQw
+MzI5MDAwMDAwWhcNMjUwNDAzMjM1OTU5WjBdMQswCQYDVQQGEwJTRTENMAsGA1UE
+BxMETHVuZDEiMCAGA1UEChMZUWxpa1RlY2ggSW50ZXJuYXRpb25hbCBBQjEbMBkG
+A1UEAwwSKi5xbWkucWxpay1wb2MuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAy6c9dY5juLMMjCMrKxITmTRFDwAS8igaRo0DZQl3ga8sy2BT0+pv
+CfisX95utappgaOH5EbvjN7yok6YvXQKL9rPz3jFrtv8eAkeUr8zr8DZjIl2OvAw
+5n/gCEhVfJUyKwWjowVa0K2m0Dc07NSt7ZuSmP2ubAC+qP8sirg6JjacEB1PNjRt
+xUSns9EIO8wRfPgRYPeZzjpqGTFMhdM2xQfOrGlB8PTnyGG5UfiLVrD6xN+rxF5u
+jCOIHlsbn3lXLMGOqNwmwO58leXJm6CVnG1O/94KqNnc8YxELYs0d2HcJS1V03rV
+tYWyuepT3xd2pLy6H7GqDgQsomEyGarxbQIDAQABo4IDnjCCA5owHwYDVR0jBBgw
+FoAUdIWAwGbH3zfez70pN6oDHb7tzRcwHQYDVR0OBBYEFA9gOoFIR0J2G2yxCySa
+bP0fiWPAMC8GA1UdEQQoMCaCEioucW1pLnFsaWstcG9jLmNvbYIQcW1pLnFsaWst
+cG9jLmNvbTA+BgNVHSAENzA1MDMGBmeBDAECAjApMCcGCCsGAQUFBwIBFhtodHRw
+Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW
+MBQGCCsGAQUFBwMBBggrBgEFBQcDAjCBnwYDVR0fBIGXMIGUMEigRqBEhkJodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxHMlRMU1JTQVNIQTI1
+NjIwMjBDQTEtMS5jcmwwSKBGoESGQmh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9E
+aWdpQ2VydEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNybDCBhwYIKwYB
+BQUHAQEEezB5MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20w
+UQYIKwYBBQUHMAKGRWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2Vy
+dEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNydDAMBgNVHRMBAf8EAjAA
+MIIBfAYKKwYBBAHWeQIEAgSCAWwEggFoAWYAdgDPEVbu1S58r/OHW9lpLpvpGnFn
+SrAX7KwB0lt3zsw7CAAAAY6LXgYuAAAEAwBHMEUCIQD4iDMCfzcat85RUyRrU535
+2bhUWcjhj8N7k74Z0gMNzwIgCjWhfTgY5d8sOcGb/TharJfYzmQrlS8kMp7SEw1o
+aO0AdQB9WR4S4XgqexxhZ3xe/fjQh1wUoE6VnrkDL9kOjC55uAAAAY6LXgX2AAAE
+AwBGMEQCIDIzZ2lvtqDASkCmNWGNuucy7cxZyPHutwR+W3ONu2M7AiBpk4k2E8Xh
+eaBDsp8SDPxgRIqvfVxg7Gc861A/HKJk6AB1AObSMWNAd4zBEEEG13G5zsHSQPaW
+hIb7uocyHf0eN45QAAABjoteBh8AAAQDAEYwRAIgRIsJ9bny4B86Wi2bgfsFDSQ0
+XnZl0dowBMUQN+06C/8CIGFrEWk0x6GDbjPkFjhy2C0OuaxqxhJ+UFhnMaanXU6b
+MA0GCSqGSIb3DQEBCwUAA4IBAQAydZmIYplxaTFpJl4yXXRd5+SaW5PtM6Y8l+dF
+x8HuENuvLNB4e2Ebl3vE8HqcXF10v2vQ8cwldW85Qov5SSNgXLM4NN4VcLCDub0e
+ArQXzs7OK/uVx+Qd0tIBMD1Ruv/Y8qTOWRlrvfoIduPgS+zjkc1Q0ACex9mCdm3l
+yC0lqB+wtOeCwkZU9yRP4sMZsRiGAtVm2G+fTlQ5/GDxaBI/fOgUHHQZSDICRWvI
+FPqTAgyk+xqPRliTlM9XM22bIxTCtBJceKD3PNwMC2KAhvLV1gP3hrPdMWH0TooD
+uW6bllK/309bMIB5YNlycgBDebddeA4aXWDLP6ldkv72kBo7
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEyDCCA7CgAwIBAgIQDPW9BitWAvR6uFAsI8zwZjANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMTAzMzAwMDAwMDBaFw0zMTAzMjkyMzU5NTlaMFkxCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxMzAxBgNVBAMTKkRpZ2lDZXJ0IEdsb2Jh
+bCBHMiBUTFMgUlNBIFNIQTI1NiAyMDIwIENBMTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMz3EGJPprtjb+2QUlbFbSd7ehJWivH0+dbn4Y+9lavyYEEV
+cNsSAPonCrVXOFt9slGTcZUOakGUWzUb+nv6u8W+JDD+Vu/E832X4xT1FE3LpxDy
+FuqrIvAxIhFhaZAmunjZlx/jfWardUSVc8is/+9dCopZQ+GssjoP80j812s3wWPc
+3kbW20X+fSP9kOhRBx5Ro1/tSUZUfyyIxfQTnJcVPAPooTncaQwywa8WV0yUR0J8
+osicfebUTVSvQpmowQTCd5zWSOTOEeAqgJnwQ3DPP3Zr0UxJqyRewg2C/Uaoq2yT
+zGJSQnWS+Jr6Xl6ysGHlHx+5fwmY6D36g39HaaECAwEAAaOCAYIwggF+MBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHSFgMBmx9833s+9KTeqAx2+7c0XMB8G
+A1UdIwQYMBaAFE4iVCAYlebjbuYP+vq5Eu0GF485MA4GA1UdDwEB/wQEAwIBhjAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdgYIKwYBBQUHAQEEajBoMCQG
+CCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQAYIKwYBBQUHMAKG
+NGh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RH
+Mi5jcnQwQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNybDA9BgNVHSAENjA0MAsGCWCGSAGG/WwC
+ATAHBgVngQwBATAIBgZngQwBAgEwCAYGZ4EMAQICMAgGBmeBDAECAzANBgkqhkiG
+9w0BAQsFAAOCAQEAkPFwyyiXaZd8dP3A+iZ7U6utzWX9upwGnIrXWkOH7U1MVl+t
+wcW1BSAuWdH/SvWgKtiwla3JLko716f2b4gp/DA/JIS7w7d7kwcsr4drdjPtAFVS
+slme5LnQ89/nD/7d+MS5EHKBCQRfz5eeLjJ1js+aWNJXMX43AYGyZm0pGrFmCW3R
+bpD0ufovARTFXFZkAdl9h6g4U5+LXUZtXMYnhIHUfoyMo5tS58aI7Dd8KvvwVVo4
+chDYABPPTHPbqjc1qCmBaZx2vN4Ye5DUys/vZwP9BFohFrH/6j/f3IL16/RZkiMN
+JCqVJUzKoZHm1Lesh3Sz8W2jmdv51b2EQJ8HmA==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
+2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
+1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
+q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
+tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
+vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
+5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
+1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
+NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
+Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
+8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
+pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
+MrY=
+-----END CERTIFICATE-----

qmicerts/wildcard_qmi_qlik-poc_com-ca.crt

@@ -0,0 +1,57 @@
+Bag Attributes
+    friendlyName: DigiCert Global Root G2
+subject=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
+issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
+-----BEGIN CERTIFICATE-----
+MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
+2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
+1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
+q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
+tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
+vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
+5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
+1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
+NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
+Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
+8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
+pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
+MrY=
+-----END CERTIFICATE-----
+Bag Attributes: <Empty Attributes>
+subject=/C=US/O=DigiCert Inc/CN=DigiCert Global G2 TLS RSA SHA256 2020 CA1
+issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
+-----BEGIN CERTIFICATE-----
+MIIEyDCCA7CgAwIBAgIQDPW9BitWAvR6uFAsI8zwZjANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMTAzMzAwMDAwMDBaFw0zMTAzMjkyMzU5NTlaMFkxCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxMzAxBgNVBAMTKkRpZ2lDZXJ0IEdsb2Jh
+bCBHMiBUTFMgUlNBIFNIQTI1NiAyMDIwIENBMTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMz3EGJPprtjb+2QUlbFbSd7ehJWivH0+dbn4Y+9lavyYEEV
+cNsSAPonCrVXOFt9slGTcZUOakGUWzUb+nv6u8W+JDD+Vu/E832X4xT1FE3LpxDy
+FuqrIvAxIhFhaZAmunjZlx/jfWardUSVc8is/+9dCopZQ+GssjoP80j812s3wWPc
+3kbW20X+fSP9kOhRBx5Ro1/tSUZUfyyIxfQTnJcVPAPooTncaQwywa8WV0yUR0J8
+osicfebUTVSvQpmowQTCd5zWSOTOEeAqgJnwQ3DPP3Zr0UxJqyRewg2C/Uaoq2yT
+zGJSQnWS+Jr6Xl6ysGHlHx+5fwmY6D36g39HaaECAwEAAaOCAYIwggF+MBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHSFgMBmx9833s+9KTeqAx2+7c0XMB8G
+A1UdIwQYMBaAFE4iVCAYlebjbuYP+vq5Eu0GF485MA4GA1UdDwEB/wQEAwIBhjAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdgYIKwYBBQUHAQEEajBoMCQG
+CCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQAYIKwYBBQUHMAKG
+NGh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RH
+Mi5jcnQwQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNybDA9BgNVHSAENjA0MAsGCWCGSAGG/WwC
+ATAHBgVngQwBATAIBgZngQwBAgEwCAYGZ4EMAQICMAgGBmeBDAECAzANBgkqhkiG
+9w0BAQsFAAOCAQEAkPFwyyiXaZd8dP3A+iZ7U6utzWX9upwGnIrXWkOH7U1MVl+t
+wcW1BSAuWdH/SvWgKtiwla3JLko716f2b4gp/DA/JIS7w7d7kwcsr4drdjPtAFVS
+slme5LnQ89/nD/7d+MS5EHKBCQRfz5eeLjJ1js+aWNJXMX43AYGyZm0pGrFmCW3R
+bpD0ufovARTFXFZkAdl9h6g4U5+LXUZtXMYnhIHUfoyMo5tS58aI7Dd8KvvwVVo4
+chDYABPPTHPbqjc1qCmBaZx2vN4Ye5DUys/vZwP9BFohFrH/6j/f3IL16/RZkiMN
+JCqVJUzKoZHm1Lesh3Sz8W2jmdv51b2EQJ8HmA==
+-----END CERTIFICATE-----

replicate-install/scripts/Replicate_install.iss

@@ -1,29 +1,29 @@
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
-Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
-Count=7
-Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0
-Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
-Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
-Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
-Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
-Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
-szDir=C:\Program Files\Attunity\Replicate\
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
-szDir=C:\Program Files\Attunity\Replicate\data
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
-Result=1
-Sel-0=1
-Sel-1=0
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
+Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
+Count=7
+Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0
+Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
+Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
+Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
+Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
+Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
+szDir=C:\Program Files\Attunity\Replicate\
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
+szDir=C:\Program Files\Attunity\Replicate\data
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
+Result=1
+Sel-0=1
+Sel-1=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

replicate-install/scripts/Replicate_install_2022.11.0.208.iss

@@ -0,0 +1,29 @@
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
+Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
+Count=7
+Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0
+Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
+Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
+Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
+Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
+Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
+szDir=C:\Program Files\Attunity\Replicate\
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
+szDir=C:\Program Files\Attunity\Replicate\data
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
+Result=1
+Sel-0=1
+Sel-1=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

replicate-install/scripts/di-replicate-installQMICertificate.ps1

@@ -7,6 +7,11 @@ Param(
 try {
 
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+    
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)
@@ -16,7 +21,7 @@ try {
     Write-Host "Set SSL qmi_qlik-poc_com for Replicate"
 
     Stop-Service AttunityReplicateConsole
-    if ( $url -Match "2022.11" ) {  
+    if ( $url -Match "2022.11" -or $url -Match "2023.5" ) {  
         Stop-Service QlikReplicateServer
     } else {
         Stop-Service AttunityReplicateServer
@@ -27,7 +32,7 @@ try {
     netsh http add sslcert ipport=0.0.0.0:443 certhash=$thumb appid='{4dc3e181-e14b-4a21-b022-59fc669b0914}'
 
     Start-Service AttunityReplicateConsole 
-    if ( $url -Match "2022.11" ) {  
+    if ( $url -Match "2022.11" -or $url -Match "2023.5" ) {  
         Start-Service QlikReplicateServer
     } else {
         Start-Service AttunityReplicateServer

replicate-install/scripts/replicate_license.txt

@@ -1,17 +1,17 @@
 #
 # Qlik  License
-# Generated on 15-Dec-2022 19:01:15.1383+02:00 
-# License Comment: Qlik internal use only 
+# Generated on 05-Dec-2023 21:36:16.5775+02:00 
+# License Comment: Qlik Internal Use only
 #
 license_type=EVALUATION_LICENSE
-licensed_to=Attunity Americas
+licensed_to=Qlik Internal Use only
 licensed_by=Attunity US
-serial_no=60031826
-expiration_date=2023-12-31
+serial_no=60036369
+expiration_date=2024-12-31
 hosts=
 source_types=
 target_types=
 features=
-version=2022.11
-issue_date=2022-12-15
-checksum=F4CK8-H36JR-8H58Q-833KH
+version=2023.11
+issue_date=2023-12-05
+checksum=3QHR6-5F3JE-6JCF4-9R63R

s3-bucket-sftp-public/main.tf

@@ -191,6 +191,19 @@ resource "aws_iam_role_policy" "user" {
        "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id])}"
       ]
     },
+    {
+      "Sid": "VisualEditor1",
+      "Effect": "Allow",
+      "Action": [
+          "s3:ListStorageLensConfigurations",
+          "s3:ListAccessPointsForObjectLambda",
+          "s3:ListAllMyBuckets",
+          "s3:ListAccessPoints",
+          "s3:ListJobs",
+          "s3:ListMultiRegionAccessPoints"
+      ],
+      "Resource": "*"
+    },
     {
       "Sid": "HomeDirObjectAccess",
       "Effect": "Allow",
@@ -201,7 +214,7 @@ resource "aws_iam_role_policy" "user" {
         "s3:DeleteObject",
         "s3:GetObjectVersion"
       ],
-      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/", each.value, "/*"])}"
+      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/*"])}"
     }
   ]
 }

s3-bucket-sftp-public/outputs.tf

@@ -17,4 +17,16 @@ output "sftp-private" {
   value = nonsensitive(aws_secretsmanager_secret_version.private_key.secret_string)
 }
 
+output "sftp-public" {
+  value = aws_key_pair.generated_key.public_key
+}
+
+output "host-key-fingerprint" {
+  value = aws_transfer_server.public.host_key_fingerprint
+}
+
+output "s3-bucket-name" {
+  value = "sftp-${var.provision_id}"
+}
+
 

s3-bucket-sftp/main.tf

@@ -42,6 +42,22 @@ resource "aws_secretsmanager_secret_version" "private_key" {
   secret_string = nonsensitive(tls_private_key.sftp-key.private_key_pem)
 }
 
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+locals {
+
+  port                 = "22"
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    #QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "sftp-${var.provision_id}"
+  }
+}
+
 
 module "s3_bucket" {
   source = "terraform-aws-modules/s3-bucket/aws"
@@ -57,13 +73,7 @@ module "s3_bucket" {
 
   force_destroy = true
 
-  tags = {
-    Deployment = "QMI"
-    "Cost Center" = "3100"
-    #QMI_user = var.user_id
-    ProvID   = var.provision_id
-    Name     = "sftp-${var.provision_id}"
-  }
+  tags = local.tags
 
 }
 
@@ -130,12 +140,16 @@ resource "aws_transfer_server" "sftp" {
   endpoint_details {
     vpc_id             = var.vpc_id
     subnet_ids         = var.subnet_ids
-    security_group_ids = [module.security_group.security_group_id]
+    security_group_ids = [
+      module.security_group.security_group_id,
+      module.security_group_2.security_group_id
+    ]
   }
   tags = {
     Deployment = "QMI"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "sftp-${var.provision_id}"
   }
@@ -154,50 +168,12 @@ module "security_group" {
 
   # ingress
 
-  ingress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  ingress_cidr_blocks =  module.fw-ips.cidr_blocks
 
   ingress_with_cidr_blocks = [
     {
-      from_port   = 22
-      to_port     = 22
+      from_port   = local.port
+      to_port     = local.port
       protocol    = "tcp"
       description = "Allow SFTP Inbound"
 
@@ -206,50 +182,12 @@ module "security_group" {
 
   # egress
 
-  egress_cidr_blocks = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32",
-    #QAA
-    "54.216.156.88/32",
-    "3.248.156.131/32",
-    "52.213.44.55/32",
-    "18.235.133.252/32",
-    "3.217.244.242/32",
-    "18.214.8.201/32",
-    "54.206.158.27/32",
-    "3.104.137.20/32",
-    "3.24.52.178/32",
-    "54.169.84.213/32",
-    "13.213.173.37/32",
-    "13.213.113.162/32"
-  ]
+  egress_cidr_blocks =  module.fw-ips.cidr_blocks
 
   egress_with_cidr_blocks = [
     {
-      from_port   = 22
-      to_port     = 22
+      from_port   = local.port
+      to_port     = local.port
       protocol    = "tcp"
       description = "Allow SFTP outbound"
 
@@ -266,6 +204,49 @@ module "security_group" {
   
 }
 
+module "security_group_2" {
+  
+  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "~> 4.3"
+
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2-SFTP"
+  vpc_id      = var.vpc_id
+
+
+  # ingress
+
+  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
+  
+
+  ingress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Allow SFTP Inbound"
+
+    },
+  ]
+
+  # egress
+
+  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
+
+  egress_with_cidr_blocks = [
+    {
+      from_port   = local.port
+      to_port     = local.port
+      protocol    = "tcp"
+      description = "Allow SFTP Inbound"
+
+    },
+  ]
+
+  tags = local.tags
+}
+
 
 resource "aws_iam_role" "user" {
   for_each           = var.sftp_users

s3-bucket/main.tf

@@ -5,7 +5,6 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "= 3.74.1"
     }
   }
 }
@@ -27,6 +26,7 @@ module "iam_user" {
     Deployment = "QMI"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
   }
@@ -36,10 +36,7 @@ module "iam_user" {
 module "s3_bucket" {
   source = "terraform-aws-modules/s3-bucket/aws"
 
-  version = "~> 2.1.0"
-
   bucket = "qmi-bucket-${var.provision_id}"
-  acl    = "private"
 
   versioning = {
     enabled = false
@@ -56,12 +53,14 @@ module "s3_bucket" {
     Deployment = "QMI"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
   }
 
 }
 
+
 resource "aws_iam_user_policy" "lb_ro" {
     
   name = "s3only_policy_${module.iam_user.this_iam_user_name}"

vm-centos/main.tf

@@ -68,6 +68,7 @@ resource "azurerm_linux_virtual_machine" "vm" {
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner"    = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null

vm-centos/outputs.tf

@@ -24,12 +24,4 @@ output "nic_private_ip_address" {
 
 output "nic_ip_configuration_name" {
   value = module.qmi-nic.ip_configuration_name
-}
-
-output "nic_fqdn" {
-  value = var.isExternal? module.qmi-nic.fqdn : null
-}
-
-output "nic_public_ip_address" {
-  value = var.isExternal? module.qmi-nic.public_ip_address : null
 }

vm-fort-az/main.tf

@@ -85,6 +85,7 @@ resource "azurerm_linux_virtual_machine" "vm" {
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null

vm-fromsnapshot-linux/main.tf

@@ -10,7 +10,7 @@ resource "random_id" "randomMachineId" {
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_lower = 2
@@ -43,6 +43,7 @@ resource "azurerm_managed_disk" "md-import" {
     "Deployment" = "QMI PoC"
     "Cost Center" = "3100"
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
   }
 }
 
@@ -62,6 +63,7 @@ resource "azurerm_managed_disk" "md-copy" {
     "Deployment" = "QMI PoC"
     "Cost Center" = "3100"
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
   }
 }
 
@@ -105,6 +107,7 @@ resource "azurerm_virtual_machine" "vm" {
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null

vm-fromsnapshot-win/main.tf

@@ -48,6 +48,7 @@ resource "azurerm_managed_disk" "md-import" {
     "Deployment" = "QMI PoC"
     "Cost Center" = "3100"
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
   }
 }
 
@@ -67,6 +68,7 @@ resource "azurerm_managed_disk" "md-copy" {
     "Deployment" = "QMI PoC"
     "Cost Center" = "3100"
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
   }
 }
 
@@ -121,6 +123,7 @@ resource "azurerm_virtual_machine" "vm" {
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null
@@ -162,7 +165,7 @@ resource "null_resource" "post-vm-fromsnapshot-win" {
       password = var.initial_password
       port     = 5985
       https    = false
-      timeout  = "10m"
+      timeout  = "30m"
     }
     source      = "${path.module}/main"
     destination = "C:/tmp/provision"
@@ -177,7 +180,7 @@ resource "null_resource" "post-vm-fromsnapshot-win" {
       password = var.initial_password
       port     = 5985
       https    = false
-      timeout  = "10m"
+      timeout  = "30m"
     }
 
     inline = [

vm-oraclelinux/main.tf

@@ -1,86 +0,0 @@
-resource "random_id" "randomMachineId" {
-    keepers = {
-        # Generate a new ID only when a new resource group is defined
-        resource_group = var.resource_group_name
-    }
-    
-    byte_length = 3
-}
-
-resource "random_password" "password" {
-  length = 16
-  special = false
-  override_special = "_"
-  upper = true
-  lower = true
-  min_lower = 2
-  min_upper = 2
-  min_special = 0
-  min_numeric = 2
-}
-
-locals {
-  virtual_machine_name = "${var.prefix}-${random_id.randomMachineId.hex}"
-  admin_username = var.admin_username
-  admin_password = random_password.password.result
-}
-
-module "qmi-nic" {
-  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//qmi-nic"
-  
-  prefix = local.virtual_machine_name
-  location = var.location
-  subnet_id = var.subnet_id
-  
-  resource_group_name = var.resource_group_name
-  user_id = var.user_id
-  
-}
-
-resource "azurerm_virtual_machine" "vm" {
-  name                  = local.virtual_machine_name
-  location              = var.location
-  resource_group_name   = var.resource_group_name
-  network_interface_ids = [module.qmi-nic.id]
-  vm_size               = var.vm_type
-
-  delete_os_disk_on_termination = true
-  delete_data_disks_on_termination = true
-
-  storage_image_reference {
-    # this is the Oracle linux image I found.
-    # az vm image list --all --publisher Oracle  -o table
-    publisher = "Oracle"
-    offer     = "Oracle-Linux"
-    sku       = "77"
-    version   = "latest"
-  }
-
-  storage_os_disk {
-    name              = "${local.virtual_machine_name}-osdisk"
-    caching           = "ReadWrite"
-    create_option     = "FromImage"
-    managed_disk_type = var.managed_disk_type
-    disk_size_gb      = var.disk_size_gb
-  }
-
-  os_profile_linux_config {
-      disable_password_authentication = false
-  }
-
-  os_profile {
-    computer_name  = local.virtual_machine_name
-    admin_username = local.admin_username
-    admin_password = local.admin_password
-  }
-
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    "ProvId"  = var.provId != null? var.provId : null
-    "QMI_user" = var.user_id != null? var.user_id : null
-    "24x7" = var.is_24x7 == true? "" : null
-    "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
-    "StartupTime": var.is_24x7 == false? var.startupTime : null
-  }
-}

vm-oraclelinux/outputs.tf

@@ -1,27 +0,0 @@
-output "virtual_machine_id" {
-  value = azurerm_virtual_machine.vm.id
-}
-
-output "virtual_machine_name" {
-    value = "${var.prefix}-${random_id.randomMachineId.hex}"
-}
-
-output "admin_username" {
-  value = var.admin_username
-}
-
-output "admin_password" {
-  value = random_password.password.result
-}
-
-output "nic_id" {
-  value = module.qmi-nic.id
-}
-
-output "nic_private_ip_address" {
-  value = module.qmi-nic.private_ip_address
-}
-
-output "nic_ip_configuration_name" {
-  value = module.qmi-nic.ip_configuration_name
-}

vm-oraclelinux/variables.tf

@@ -1,56 +0,0 @@
-variable "subnet_id" {
-  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-VM-Deployments"
-}
-
-variable "prefix" {
-  description = "The Prefix used for all resources in this example"
-  default = "QMI-ORACLELINUX"
-}
-
-variable "location" {
-  default = "East US"
-}
-
-variable "resource_group_name" {
-}
-
-variable "vm_type" {
-  default = "Standard_DS3_v2"
-}
-
-variable "managed_disk_type" {
-  default = "Premium_LRS"
-}
-
-variable "disk_size_gb" {
-  default = "128"
-}
-
-variable "admin_username" {
-  default = "qmi"
-}
-
-variable "user_id" {
-  default = null
-}
-
-variable "provId" {
-  default = null
-}
-
-variable "is_24x7"{
-  type = bool
-  default = null
-}
-
-variable "shutdownTime"{
-  default = null
-}
-
-variable "startupTime"{
-  default = null
-}
-
-# variable "subnet_id" {
-#   default = "/subscriptions/1f3d4c1d-6509-4c52-8dee-c15fb83f2920/resourceGroups/lkn-rg/providers/Microsoft.Network/virtualNetworks/lkn-vn/subnets/default"
-# }

vm-oraclelinux81/main.tf

@@ -37,52 +37,43 @@ module "qmi-nic" {
   
 }
 
-resource "azurerm_virtual_machine" "vm" {
-  name                  = local.virtual_machine_name
-  location              = var.location
-  resource_group_name   = var.resource_group_name
-  network_interface_ids = [module.qmi-nic.id]
-  vm_size               = var.vm_type
+resource "azurerm_linux_virtual_machine" "vm" {
+  name                = local.virtual_machine_name
+  resource_group_name = var.resource_group_name
+  location            = var.location
+  size                = var.vm_type
+  admin_username      = local.admin_username
+  admin_password      = local.admin_password
+  disable_password_authentication = false
+  
+  network_interface_ids = [
+    module.qmi-nic.id,
+  ]
 
-  delete_os_disk_on_termination = true
-  delete_data_disks_on_termination = true
+  os_disk {
+    name                 = "${local.virtual_machine_name}-osdisk"
+    caching              = "ReadWrite"
+    storage_account_type = var.managed_disk_type
+    disk_size_gb         = var.disk_size_gb
+  }
 
   identity {
     type = "SystemAssigned"
   }
 
-  storage_image_reference {
-    # this is the Oracle linux image I found.
-    # az vm image list --all --publisher Oracle  -o table
+  source_image_reference {
     publisher = "Oracle"
     offer     = "Oracle-Linux"
     sku       = "81"
     version   = "latest"
   }
 
-  storage_os_disk {
-    name              = "${local.virtual_machine_name}-osdisk"
-    caching           = "ReadWrite"
-    create_option     = "FromImage"
-    managed_disk_type = var.managed_disk_type
-    disk_size_gb      = var.disk_size_gb
-  }
-
-  os_profile_linux_config {
-      disable_password_authentication = false
-  }
-
-  os_profile {
-    computer_name  = local.virtual_machine_name
-    admin_username = local.admin_username
-    admin_password = local.admin_password
-  }
-
   tags = {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null
@@ -94,7 +85,7 @@ module "linux-common" {
   source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common"
 
   depends_on = [
-    azurerm_virtual_machine.vm
+    azurerm_linux_virtual_machine.vm
   ]
   
   os_type             = "centos"

vm-oraclelinux81/outputs.tf

@@ -1,5 +1,5 @@
 output "virtual_machine_id" {
-  value = azurerm_virtual_machine.vm.id
+  value = azurerm_linux_virtual_machine.vm.id
 }
 
 output "virtual_machine_name" {
@@ -27,5 +27,5 @@ output "nic_ip_configuration_name" {
 }
 
 output "principal_id" {
-  value = azurerm_virtual_machine.vm.identity.0.principal_id
+  value = azurerm_linux_virtual_machine.vm.identity.0.principal_id
 }

vm-qdc-secure/main.tf

@@ -60,6 +60,7 @@ resource "azurerm_linux_virtual_machine" "vm" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = ""
   }
 

vm-qdc/main.tf

@@ -70,6 +70,7 @@ resource "azurerm_linux_virtual_machine" "vm" {
     "Cost Center" = "3100"
     "ProvId"  = var.provId != null? var.provId : null
     "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
     "24x7" = var.is_24x7 == true? "" : null
     "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
     "StartupTime": var.is_24x7 == false? var.startupTime : null
@@ -123,5 +124,6 @@ module "linux-common" {
   private_ip_address  = module.qmi-nic.private_ip_address
   admin_username      = local.admin_username
   admin_password      = local.admin_password  
+  update              = false
   
 }

vm-qdc/outputs.tf

@@ -38,12 +38,4 @@ output "mysql_demo_password" {
   value = "Qlik1234!"
 }
 
-output "nic_fqdn" {
-  value = var.isExternal? module.qmi-nic.fqdn : null
-}
-
-output "nic_public_ip_address" {
-  value = var.isExternal? module.qmi-nic.public_ip_address : null
-}
-