Merge branch 'master' into dev

adls/main.tf

@@ -7,16 +7,19 @@ resource "random_id" "randomMachineId" {
     byte_length = 5
 }
 
+
 resource "azurerm_storage_account" "saccount" {
 
   name = var.storage_account_name != null? var.storage_account_name : "qmiadlsgen2${random_id.randomMachineId.hex}"
 
   resource_group_name       = var.resource_group_name
   location                  = var.location
+
   account_kind              = "StorageV2"
+  account_replication_type  = var.account_replication_type != null? var.account_replication_type : "RAGRS"
   account_tier              = "Standard"
   access_tier               = "Hot"
-  account_replication_type  = "RAGRS"
+
   is_hns_enabled            = "true"
 
   tags = var.tags

adls/outputs.tf

@@ -31,5 +31,5 @@ output "Azure_Application_Registration_Client_ID" {
 }
 
 output "Azure_Application_Registration_Secret" {
-  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
 }

adls/variable.tf

@@ -35,4 +35,12 @@ variable "dbricks_app_registration_application_id" {
 
 variable "principal_id_storage_blob_contributor" {
   default = null
+}
+
+variable "sa_config_type" {
+  default = "1"
+}
+
+variable "account_replication_type" {
+  default = null
 }

appgateways/appgw-443-only/logs.tf

@@ -0,0 +1,23 @@
+resource "azurerm_monitor_diagnostic_setting" "example" {
+
+    name               = "${local.appgw_name}-${var.provision_id}-diagsettings"
+    target_resource_id = azurerm_application_gateway.qmi-app-gw.id
+
+    log_analytics_workspace_id = var.log_analytics_workspace_id
+
+    enabled_log {
+        category = "ApplicationGatewayAccessLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+
+    enabled_log {
+        category = "ApplicationGatewayFirewallLog"
+
+        retention_policy {
+            enabled = true
+        }
+    }
+}

appgateways/appgw-443-only/main.tf

@@ -30,6 +30,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -48,6 +49,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-443-only/variables.tf

@@ -38,3 +38,7 @@ variable "user_id" {
 variable "probe_path" {
   default = "/"
 }
+
+variable "log_analytics_workspace_id" {
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/appgw_rg/providers/Microsoft.OperationalInsights/workspaces/qmi-log-analytics"
+}

appgateways/appgw-e2e/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-e2e/main.tf

@@ -62,6 +62,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -80,6 +81,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qdc-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc-qs/main.tf

@@ -54,6 +54,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
   
 }
@@ -73,6 +74,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qdc/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qdc/main.tf

@@ -28,6 +28,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -46,6 +47,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

appgateways/appgw-qs/logs.tf

@@ -5,23 +5,19 @@ resource "azurerm_monitor_diagnostic_setting" "example" {
 
     log_analytics_workspace_id = var.log_analytics_workspace_id
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayAccessLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 
-    log {
+    enabled_log {
         category = "ApplicationGatewayFirewallLog"
-        enabled  = true
 
         retention_policy {
             enabled = true
-            days = 7
         }
     }
 }

appgateways/appgw-qs/main.tf

@@ -52,6 +52,7 @@ resource "azurerm_public_ip" "appgw-ip" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 
@@ -70,6 +71,7 @@ resource "azurerm_application_gateway" "qmi-app-gw" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   sku {

aws-olh-prereqs/kinesis.tf

@@ -0,0 +1,9 @@
+resource "aws_kinesis_stream" "qlik_stream" {
+  name             = "qlik-data-stream-${var.provision_id}"  # Change the name as needed
+  shard_count      = 1
+  stream_mode_details {
+    stream_mode = "PROVISIONED"
+  }
+
+  retention_period = 24  # Optional: Retain records for 24 hours (default)
+}

aws-olh-prereqs/kms.tf

@@ -0,0 +1,14 @@
+resource "aws_kms_key" "qlik_key" {
+  description             = "QMIOpenLakehouse - Customer managed key for Qlik resources"
+  deletion_window_in_days = 10
+  enable_key_rotation     = true
+
+  tags = {
+    Alias = "kms-for-qmi-olh-${var.provision_id}"
+  }
+}
+
+resource "aws_kms_alias" "qlik_key_alias" {
+  name          = "alias/kms-for-qmi-olh-${var.provision_id}"
+  target_key_id = aws_kms_key.qlik_key.key_id
+}

aws-olh-prereqs/main.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 6.0.0"
+    }
+  }
+}

aws-olh-prereqs/output.tf

@@ -0,0 +1,71 @@
+output "AWS_ACCOUNT_ID" {
+  value = var.aws_account_id
+}
+
+output "VPC_ID" {
+  value = aws_vpc.main.id
+  
+}
+
+output "VPC_CIDR" {
+  value = var.vpc_cidr
+}
+
+output "PUBLIC_SUBNET_CIDRS" {
+  value = var.public_subnet_cidrs
+}
+
+output "SUBNETS_IDS" {
+  value = aws_subnet.public[*].id
+  
+}
+
+output "PUBLIC_SUBNET_AZS" {
+  value = [
+    for subnet in aws_subnet.public : subnet.availability_zone
+  ]
+}
+
+output "KMS_KEY_ARN" {
+  value = aws_kms_key.qlik_key.arn
+}
+
+output "KINESIS_STREAM_NAME" {
+  value = aws_kinesis_stream.qlik_stream.name
+}
+
+output "SECURITY_GROUP_ID" {
+  value = aws_security_group.allow_tls.id
+}
+
+output "MANAGEMENT_ROLE_ARN" {
+  value = aws_iam_role.olh_mgmt_role.arn
+}
+
+output "EC2_INTANCE_ROLE_ARN" {
+  value = aws_iam_role.olh_ec2_role.arn
+}
+
+output "EC2_INTANCE_PROFILE_ARN" {
+  value = aws_iam_instance_profile.ec2_instance_profile.arn
+}
+
+output "S3_BUCKET_NAME" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_id    
+}
+
+output "S3_BUCKET_REGION" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_region 
+}
+
+output "S3_BUCKET_ACCESS_KEY" {
+  value = module.qmi-s3-bucket.iam_access_key   
+}
+
+output "S3_BUCKET_ACCESS_SECRET" {
+  value = module.qmi-s3-bucket.iam_access_secret
+}
+
+output "TAGS" {
+  value = "Environment = QMI-${var.provision_id}"
+}

aws-olh-prereqs/role-ec2.tf

@@ -0,0 +1,156 @@
+resource "aws_iam_role" "olh_ec2_role" {  
+    name = "qmi_olh_ec2_${var.provision_id}"
+
+    assume_role_policy = jsonencode({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": "sts:AssumeRole",
+                "Principal": {
+                    "Service" : "ec2.amazonaws.com"
+                }
+            }
+        ]
+    })
+
+}
+
+
+resource "aws_iam_role_policy" "server_policy" {
+
+  name = "qmi-olh-${var.provision_id}_s3_policy"
+  role = aws_iam_role.olh_ec2_role.id
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:AttachVolume",
+                "ec2:DetachVolume",
+                "autoscaling:SetInstanceHealth"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:PutMetricData"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:kinesis:*:*:stream/${aws_kinesis_stream.qlik_stream.name}"
+            ],
+            "Action": [
+                "kinesis:PutRecord",
+                "kinesis:DescribeStreamSummary",
+                "kinesis:ListShards",
+                "kinesis:PutRecords",
+                "kinesis:GetShardIterator",
+                "kinesis:GetRecords",
+                "kinesis:DescribeStream",
+                "kinesis:ListStreamConsumers"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}"
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}",
+                "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}/*"
+            ],
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:AbortMultipartUpload",
+                "s3:DeleteObjectVersion",
+                "s3:ListBucket",
+                "s3:DeleteObject",
+                "s3:GetObjectVersion"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ssm:*:*:parameter/qlik/*"
+            ],
+            "Action": [
+                "ssm:GetParameter",
+                "ssm:PutParameter"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "${aws_kms_key.qlik_key.arn}"
+            ],
+            "Action": [
+                "kms:*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "sts:AssumeRole"
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "sts:ExternalId": "6b69f9c3c8f502f2ddfc8434d443f172ebe4c032f4ef1a3c51215d27a58ca799"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "glue:CreateTable",
+                "glue:UpdateTable",
+                "glue:DeleteTable",
+                "glue:BatchDeleteTable",
+                "glue:GetTable",
+                "glue:GetTables",
+                "glue:CreateDatabase",
+                "glue:UpdateDatabase",
+                "glue:GetDatabase",
+                "glue:GetDatabases",
+                "glue:GetUserDefinedFunction"
+            ],
+            "Resource": [
+                "arn:aws:glue:us-east-1:*:catalog",
+                "arn:aws:glue:us-east-1:*:database/*",
+                "arn:aws:glue:us-east-1:*:table/*/*"
+            ]
+        }
+    ]
+}
+  )
+}
+
+
+resource "aws_iam_instance_profile" "ec2_instance_profile" {
+  name = "qmi-olh-instance-profile-${var.provision_id}"
+  role = aws_iam_role.olh_ec2_role.name
+}

aws-olh-prereqs/role-mgmt.tf

@@ -0,0 +1,233 @@
+resource "aws_iam_role" "olh_mgmt_role" {  
+    name = "qmi_olh_mgmt_${var.provision_id}"
+
+    assume_role_policy = jsonencode({
+        "Version": "2012-10-17",
+        "Statement": [
+              {
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": "arn:aws:iam::072398622971:role/upsolver-proxy-assume-role"
+                },
+                "Action": [
+                    "sts:AssumeRole"
+                ],
+                "Condition": {
+                    "StringEquals": {
+                        "sts:ExternalId": "6b69f9c3c8f502f2ddfc8434d443f172ebe4c032f4ef1a3c51215d27a58ca799"
+                    }
+                }
+            },
+            {
+                "Effect": "Allow",
+                "Action": "sts:AssumeRole",
+                "Principal": {
+                    "Service" : "ec2.amazonaws.com"
+                }
+            }
+        ]
+    })
+
+}
+
+resource "aws_iam_role_policy" "inline_policy" {
+
+  name = "qmi-olh-role-policy-${var.provision_id}"
+  role = aws_iam_role.olh_mgmt_role.id
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:CancelSpotInstanceRequests",
+                "ec2:CreateLaunchTemplate",
+                "ec2:CreateTags",
+                "ec2:CreateVolume",
+                "ec2:DescribeAddresses",
+                "ec2:DescribeImageAttribute",
+                "ec2:DescribeImages",
+                "ec2:DescribeInstanceStatus",
+                "ec2:DescribeInstanceTypeOfferings",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
+                "ec2:DescribeLaunchTemplates",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSpotInstanceRequests",
+                "ec2:DescribeSpotPriceHistory",
+                "ec2:DescribeTags",
+                "ec2:DescribeRegions",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups",
+                "ec2:RequestSpotInstances",
+                "ec2:DescribeVolumes"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ec2:*::image/*",
+                "arn:aws:ec2:*:*:instance/*",
+                "arn:aws:ec2:*:*:key-pair/*",
+                "arn:aws:ec2:*:*:launch-template/*",
+                "arn:aws:ec2:*:*:network-interface/*",
+                "arn:aws:ec2:*:*:security-group/*",
+                "arn:aws:ec2:*:*:subnet/*",
+                "arn:aws:ec2:*:*:volume/*"
+            ],
+            "Action": [
+                "ec2:RunInstances"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:AttachVolume",
+                "ec2:DeleteVolume",
+                "ec2:DetachVolume",
+                "ec2:DeleteLaunchTemplate",
+                "ec2:TerminateInstances",
+                "ec2:StartInstances",
+                "ec2:ModifyLaunchTemplate",
+                "ec2:DeleteLaunchTemplateVersions",
+                "ec2:CreateLaunchTemplateVersion"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "autoscaling:DeleteAutoScalingGroup",
+                "autoscaling:DeletePolicy",
+                "autoscaling:PutScalingPolicy",
+                "autoscaling:StartInstanceRefresh",
+                "autoscaling:TerminateInstanceInAutoScalingGroup",
+                "autoscaling:UpdateAutoScalingGroup"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "autoscaling:CreateAutoScalingGroup",
+                "autoscaling:CreateOrUpdateTags",
+                "autoscaling:DescribeAutoScalingGroups",
+                "autoscaling:DescribeInstanceRefreshes",
+                "autoscaling:DescribePolicies",
+                "autoscaling:DescribeScalingActivities"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:DescribeRegions",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:DescribeAlarmHistory",
+                "cloudwatch:DescribeAlarms",
+                "cloudwatch:DescribeAlarmsForMetric",
+                "cloudwatch:GetMetricStatistics",
+                "cloudwatch:ListMetrics",
+                "cloudwatch:PutMetricData"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:PutMetricAlarm",
+                "cloudwatch:DeleteAlarms"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "iam:AddRoleToInstanceProfile",
+                "iam:CreateServiceLinkedRole",
+                "iam:GetPolicy",
+                "iam:GetPolicyVersion",
+                "iam:ListAccountAliases",
+                "iam:ListAttachedRolePolicies",
+                "iam:ListInstanceProfiles",
+                "iam:ListInstanceProfilesForRole",
+                "iam:ListPolicies",
+                "iam:ListRoles",
+                "iam:PassRole"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "sts:DecodeAuthorizationMessage"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ssm:*:${var.aws_account_id}:parameter/qlik/*"
+            ],
+            "Action": [
+                "ssm:PutParameter"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "${aws_kms_key.qlik_key.arn}"
+            ],
+            "Action": [
+                "kms:GenerateDataKeyPairWithoutPlaintext",
+                "kms:Encrypt"
+            ]
+        }
+    ]
+  })
+}

aws-olh-prereqs/s3.tf

@@ -0,0 +1,10 @@
+module "qmi-s3-bucket" {          
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
+  
+  bucket_name                   = "qmi-olh-bucket-${var.provision_id}"
+  provision_id                  = var.provision_id
+  user_id                       = var.user_id
+  forced_destroy                = var.forced_destroy
+  
+}

aws-olh-prereqs/variables.tf

@@ -0,0 +1,30 @@
+
+variable "aws_account_id" {
+    description = "AWS account ID"
+}
+
+variable "region" {
+    description = "AWS region"
+    default     = "us-east-1"
+}
+
+variable "provision_id" {
+}
+
+variable "user_id" {
+}
+
+variable "vpc_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "public_subnet_cidrs" {
+  default = ["10.0.1.0/24", "10.0.2.0/24"]
+}
+
+variable "forced_destroy" {
+  default = null
+}
+
+
+

aws-olh-prereqs/vpc.tf

@@ -0,0 +1,74 @@
+resource "aws_vpc" "main" {
+  cidr_block           = var.vpc_cidr
+  enable_dns_support   = true
+  enable_dns_hostnames = true
+
+  tags = {
+    Name = "qmi-olh-vpc-${var.provision_id}"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "qmi-olh-igw-${var.provision_id}"
+  }
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+resource "aws_subnet" "public" {
+  count                   = length(var.public_subnet_cidrs)
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = var.public_subnet_cidrs[count.index]
+  availability_zone       = data.aws_availability_zones.available.names[count.index]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "qmi-olh-subnet-${var.provision_id}-${count.index + 1}"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+}
+
+resource "aws_route_table_association" "public_assoc" {
+  count          = length(var.public_subnet_cidrs)
+  subnet_id      = aws_subnet.public[count.index].id
+  route_table_id = aws_route_table.public.id
+}
+
+
+resource "aws_security_group" "allow_tls" {
+  name        = "olh-${var.provision_id}-SG"
+  description = "olh-${var.provision_id}-SG"
+  vpc_id      = aws_vpc.main.id
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {  
+    security_group_id = aws_security_group.allow_tls.id
+
+    cidr_ipv4         = var.vpc_cidr
+    ip_protocol           = "tcp"
+    from_port          = 0
+    to_port            = 65535
+
+    description = "olh-${var.provision_id}-ingress-rule"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  security_group_id = aws_security_group.allow_tls.id
+  cidr_ipv4          = "0.0.0.0/0"
+  ip_protocol       = "-1"
+  description = "olh-${var.provision_id}-egress-rule"
+}

compose-install/scripts/compose-license.txt

@@ -1,14 +1,14 @@
 {
   "$type": "ComposeLicense",
   "product": "QlikCompose",
-  "issued_to": "Attunity Americas",
+  "issued_to": "Qlik Internal",
   "issued_by": "Attunity US",
   "license_type": "EVALUATION",
-  "serial_no": "85008595",
-  "expiration_date": "2023-12-31",
+  "serial_no": "85011700",
+  "expiration_date": "2025-12-31",
   "hosts": "",
-  "product_version": "2022.5",
-  "notes": "Qlik internal use only ",
+  "product_version": "2024.12",
+  "notes": "",
   "host_role": "",
   "source_db_types": "",
   "dwh_type": "*",
@@ -16,6 +16,6 @@
   "number_of_dms": "0",
   "number_of_developers": "0",
   "managed_dwh_size": "0",
-  "issue_time": "12/15/2022 7:06:16 PM",
-  "signature": "iDk5NujmAcxcI80BSt4JaOEaSj35u0vzoRXvRcYXy4NpWJLFVvCRJ67LfVcA1WuLwHmarnYWYA+RWT0CyqpH1e4n3nZPaJAwBgnU5Z074rIUrHZi+Z5Hmgux2ptNqXJIr7S1JNc6+fBMHuG//hxYppf9Knmk+5cx5rafxNpCYUU="
+  "issue_time": "12/18/2024 8:31:25 PM",
+  "signature": "bMVg+X1Saxvcux5I96eU+oHndPWHv4E1QzxYwgo/cGpfVltneqGgBrVPmjotMGxp705aj29jGQLHNSx/0bBlwJd9DcfLAKaS/mH9mfph50KWRV1+ZLgPVQmmDHv/vFYLoIQLoyV2RqhMGN2rbVEXrQLzXRa+FWQEbulVa05Uvog="
 }

compose-install/scripts/di-compose-install.ps1

@@ -46,17 +46,19 @@ If ((Test-Path $bin\$fileName))
     $fileNoExtension = [IO.Path]::GetFileNameWithoutExtension("$bin\$fileName")
 	Expand-Archive $bin\$fileName -DestinationPath $bin\$fileNoExtension -Force
 
-    #Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
+    Write-Host "Binary decompressed in folder $($bin)\$($fileNoExtension)"	
 	$exec = Get-ChildItem $bin\$fileNoExtension\*.exe | Select-Object -ExpandProperty Name
     #Write-Host "Exec: $($exec)"
 
     $silentInstall = "$($bin)\$($fileNoExtension)\$($exec) /s /f1$($PSScriptRoot)\$($issFile) /f2$($bin)\Compose_silent_x64_install.log"
 
+    Start-Sleep 5 
+    
 	Write-Host "Run Compose silent installation : $($silentInstall)"
     Invoke-Expression $silentInstall
     while (!(Test-Path "$($bin)\Compose_silent_x64_install.log")) { 
         Write-Host "[Waiting Compose to be installed] ..."
-        Start-Sleep 2
+        Start-Sleep 4
     }
 	$resultLogs = Get-IniFile "$($bin)\Compose_silent_x64_install.log"
 	$testResult = $resultLogs.ResponseResult.ResultCode

compose-install/scripts/di-compose-installQMICertificate.ps1

@@ -6,6 +6,11 @@ Param(
 
 try {
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)

databases/aws-emr/main.tf

@@ -0,0 +1,148 @@
+terraform {
+
+  required_version = ">= 1.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+    }
+  }
+}
+
+
+module "s3_bucket" {
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "~> 3.0"
+
+  bucket_prefix = "qmi-bucket-${var.provision_id}"
+
+  # Allow deletion of non-empty bucket
+  # Example usage only - not recommended for production
+  force_destroy = true
+
+  attach_deny_insecure_transport_policy = true
+  attach_require_latest_tls_policy      = true
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
+  server_side_encryption_configuration = {
+    rule = {
+      apply_server_side_encryption_by_default = {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+
+  tags = {
+    QMI_user = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+    Owner    = var.user_id
+  }
+}
+
+module "emr" {
+  source = "terraform-aws-modules/emr/aws"
+
+  name = var.provision_id
+
+  release_label = "emr-6.11.0"
+  applications  = ["spark", "hadoop", "hive", "hue"]
+  auto_termination_policy = {
+    idle_timeout = 3600
+  }
+
+  bootstrap_action = {
+    example = {
+      name = "Just an example",
+      path = "file:/bin/echo",
+      args = ["Hello World!"]
+    }
+  }
+
+  configurations_json = jsonencode([
+    {
+      "classification" : "spark-env",
+      "configurations" : [
+        {
+          "classification" : "export",
+          "properties" : {
+            "JAVA_HOME" : "/usr/lib/jvm/java-1.8.0"
+          }
+        }
+      ],
+      "properties" : {}
+    },
+    {
+      "classification": "hive-site",
+      "properties": {
+         "hive.support.concurrency": "true",
+         "hive.exec.dynamic.partition.mode": "nonstrict",
+         "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"
+      }
+    }
+  ])
+
+  master_instance_group = {
+    name           = "master-group"
+    instance_count = 1
+    instance_type  = "m5.xlarge"
+  }
+
+  core_instance_group = {
+    name           = "core-group"
+    instance_count = 1
+    instance_type  = "c4.large"
+  }
+
+  task_instance_group = {
+    name           = "task-group"
+    instance_count = 1
+    instance_type  = "c5.xlarge"
+    bid_price      = "0.1"
+
+    ebs_config = {
+      size                 = 64
+      type                 = "gp3"
+      volumes_per_instance = 1
+    }
+    ebs_optimized = true
+  }
+
+  ebs_root_volume_size = 64
+  ec2_attributes = {
+    # Instance groups only support one Subnet/AZ
+    # Subnets should be private subnets and tagged with
+    # { "for-use-with-amazon-emr-managed-policies" = true }
+    subnet_id = var.subnet_ids_us[0]
+  }
+  vpc_id = var.vpc_id_us
+
+  list_steps_states  = ["PENDING", "RUNNING", "CANCEL_PENDING", "CANCELLED", "FAILED", "INTERRUPTED", "COMPLETED"]
+  log_uri            = "s3://${module.s3_bucket.s3_bucket_id}/"
+
+  scale_down_behavior    = "TERMINATE_AT_TASK_COMPLETION"
+  step_concurrency_level = 3
+  termination_protection = false
+  visible_to_all_users   = true
+  
+  is_private_cluster = false
+  #create_service_iam_role = false
+  #service_iam_role_arn = "arn:aws:iam::192018133564:role/service-role/AmazonEMR-ServiceRole-20230622T122656"
+  #create_iam_instance_profile = false
+  #iam_instance_profile_name = "AmazonEMR-InstanceProfile-20230622T122640"
+
+  tags = {
+    Environment = "QMI-${var.provision_id}"
+    Deployment = "QMI-${var.provision_id}"
+    Terraform   = "true"
+    Environment = "dev"
+    QMI_user = var.user_id
+    Owner    = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-emr-${var.provision_id}"
+  }
+}

databases/aws-emr/outputs.tf

@@ -0,0 +1,15 @@
+output "s3_bucket" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_id
+}
+
+output "s3_iam_name" {
+  value = module.qmi-s3-bucket.iam_name
+}
+
+output "s3_iam_access_key" {
+  value = module.qmi-s3-bucket.iam_access_key
+}
+
+output "s3_iam_access_secret" {
+  value =  module.qmi-s3-bucket.iam_access_secret
+}

databases/aws-emr/variables.tf

@@ -0,0 +1,34 @@
+variable "region" {
+  default = "us-east-1"
+}
+
+variable "provision_id" {
+  
+}
+
+variable "user_id" {
+}
+
+variable "vpc_id_ap" {
+  default = "vpc-22ee2844"
+}
+
+variable "vpc_id_eu" {
+  default = "vpc-73f0500a"
+}
+
+variable "vpc_id_us" {
+  default = "vpc-c079f5bd"
+}
+
+variable "subnet_ids_ap" {
+  default = ["subnet-658aec3c", "subnet-e030eba8", "subnet-94309bf2"]
+}
+
+variable "subnet_ids_eu" {
+  default = ["subnet-4d441b17", "subnet-95c22fde", "subnet-70938116"]
+}
+
+variable "subnet_ids_us" {
+  default = ["subnet-4d26552b", "subnet-0414685b", "subnet-c97f7c84", "subnet-7f695c71", "subnet-96acd2b7", "subnet-88ab2cb9"]
+}

databases/aws-kinesis/main.tf

@@ -23,9 +23,11 @@ locals {
   scnamelower = "${local.container_n1}${local.container_n2}${local.container_n3}"
 
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
   }

databases/aws-rds/main.tf

@@ -5,82 +5,54 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.74.0"
     }
   }
 }
 
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!"
+  upper = true
+  lower = true
+  min_numeric = 2
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+}
+
 locals {
   provid5              = substr(var.provision_id, 0, 5)
   aurora               = (var.engine == "aurora-mysql") ? true : (var.engine == "aurora-postgresql") ? true : false
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
-  port                 = (var.engine == "oracle-se2") ? "1521" : (var.engine == "postgres") ? "5432" : (var.engine == "aurora-postgresql") ? "5432" : (var.engine == "sqlserver-ex") ? "1433" : "3306"
+  
   name                 = (var.engine == "sqlserver-ex") ? null : (var.engine == "oracle-se2") ? "ora${local.provid5}" : "qmi${var.provision_id}"
   license              = (local.aurora == true) ? "general-public-license" : (var.engine == "mariadb") ? "general-public-license" : (var.engine == "postgres") ? "postgresql-license" : (var.engine == "mysql") ? "general-public-license" : "license-included"
-  engine_version       = (var.engine == "oracle-se2") ? "19.0.0.0.ru-2021-04.rur-2021-04.r1" : (var.engine == "postgres") ? "13.3" : (var.engine == "mysql") ? "8.0.25" : (var.engine == "aurora-postgresql") ? "14.5" : (var.engine == "aurora-mysql") ? "5.7.mysql_aurora.2.10.0" : (var.engine == "sqlserver-ex") ? "15.00.4073.23.v1" : "10.5"   #mariaDB
-  major_engine_version = (var.engine == "oracle-se2") ? "19" : (var.engine == "postgres") ? "13" : (var.engine == "mysql") ? "8.0" : (var.engine == "aurora-postgresql") ? "12" : (var.engine == "aurora-mysql") ? "5.7" : (var.engine == "sqlserver-ex") ? "15.00" : "10.5"                                                                         #mariaDB
-  family               = (var.engine == "oracle-se2") ? "oracle-se2-19" : (var.engine == "postgres") ? "postgres13" : (var.engine == "mysql") ? "mysql8.0" : (var.engine == "aurora-postgresql") ? "aurora-postgresql12" : (var.engine == "aurora-mysql") ? "aurora-mysql5.7" : (var.engine == "sqlserver-ex") ? "sqlserver-ex-15.0" : "mariadb10.5" #mariaDB
+  
+  port                 = var.port[var.engine] 
+  engine_version       = var.engine_version[var.engine]
+  major_engine_version = var.major_engine_version[var.engine]
+  family               = var.family[var.engine]
+
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
+
+  is_postgres = length(regexall("postgres", local.family)) > 0 ? true : false
+  is_mysql = length(regexall("mysql", local.family)) > 0 ? true : false
 }
 
 
-module "fw-ips" {
-    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-}
-
-module "security_group" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG"
-  description = "${var.provision_id}-SG"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-  
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
 module "common_rds_instance" {
   source  = "terraform-aws-modules/rds/aws"
-  version = "~> 3.3"
+  version = "= 6.1.1"
 
   count = local.aurora? 0 : 1
 
@@ -92,58 +64,172 @@ module "common_rds_instance" {
   major_engine_version = local.major_engine_version # DB option group
   instance_class       = var.instance_size
   allocated_storage    = var.storage
+  storage_encrypted    = (var.engine == "sqlserver-ex")? false : true
 
   license_model = local.license
 
-  name                   = local.name
+  db_name                = local.name
   username               = "qmirdsuser"
-  create_random_password = true
-  random_password_length = 12
+  password               = random_password.password.result
+  manage_master_user_password = false
   port                   = local.port
 
   multi_az               = false
   subnet_ids             = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+    aws_security_group.allow_tls.id,
+    aws_security_group.allow_tls_2.id
+  ]
   publicly_accessible    = true
 
   maintenance_window = "Mon:00:00-Mon:03:00"
   backup_window      = "03:00-06:00"
 
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
+  parameters = local.is_postgres? [
+    {
+      apply_method = "pending-reboot"
+      name  = "rds.logical_replication"
+      value = 1
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_wal_senders"
+      value = 10
+    },
+    {
+      apply_method = "pending-reboot"
+      name  = "max_replication_slots"
+      value = 10
+    }
+  ] : local.is_mysql? [{
+      name  = "binlog_format"
+      value = "row"
+  }]: []
+
   tags = local.tags
 
 }
 
+
+resource "aws_rds_cluster_parameter_group" "pg-postgres" {
+
+  count =  var.engine == "aurora-postgresql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-postgresql14"
+  description = "RDS aurora-postgresql14 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "rds.logical_replication"
+    value = 1
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_wal_senders"
+    value = 10
+  }
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "max_replication_slots"
+    value = 10
+  }
+
+}
+
+resource "aws_rds_cluster_parameter_group" "pg-mysql" {
+
+  count =  var.engine == "aurora-mysql"? 1 : 0
+
+  name        = "rds-cluster-pg-${var.provision_id}"
+  family      = "aurora-mysql8.0"
+  description = "RDS aurora-mysql8.0 cluster parameter group"
+
+  parameter {
+    apply_method = "pending-reboot"
+    name  = "binlog_format"
+    value = "row"
+  }
+
+}
+
 module "aurora_rds_instance" {
+
+  depends_on = [  
+    aws_rds_cluster_parameter_group.pg-postgres, 
+    aws_rds_cluster_parameter_group.pg-mysql
+  ]
+
   source  = "terraform-aws-modules/rds-aurora/aws"
-  version = "~> 5.2"
+  version = "~> 8.3.1"
 
   count = local.aurora? 1 : 0
 
   name           = local.name
   engine         = var.engine
   engine_version = local.engine_version
-  instance_type  = var.instance_size
+  instance_class  = var.instance_size
+
+  database_name = local.name
+
+  instances       = { 1 = {} }
+
+  autoscaling_enabled      = true
+  autoscaling_min_capacity = 1
+  autoscaling_max_capacity = 3
 
   vpc_id                 = local.vpc_id
   subnets                = local.subnet_ids
   create_security_group  = false
-  vpc_security_group_ids = [module.security_group.security_group_id]
+  vpc_security_group_ids = [
+      aws_security_group.allow_tls.id,
+      aws_security_group.allow_tls_2.id
+  ]
   port                   = local.port
   publicly_accessible    = true
 
-  username               = "qmirdsuser"
-  create_random_password = true
+  master_username         = "qmirdsuser"
+  manage_master_user_password = false
+  master_password = random_password.password.result
+  create_db_subnet_group = true
 
 
-
-  backup_retention_period = 0
+  backup_retention_period = 1
   skip_final_snapshot     = true
   deletion_protection     = false
 
   tags = local.tags
 
+  db_cluster_parameter_group_name = "rds-cluster-pg-${var.provision_id}"
+
+}
+
+locals {
+    
+    type      = (var.engine == "mysql" || var.engine == "mariadb")? "mysql" :  (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : (var.engine == "sqlserver-ex")? "mssql" : null
+    port4dummy  = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
+    host      = (local.aurora)? "${module.aurora_rds_instance[0].cluster_endpoint}:${local.port4dummy}" : module.common_rds_instance[0].db_instance_endpoint
+    username  = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+    password  = nonsensitive(random_password.password.result)
+    database  = (var.engine == "postgres" || var.engine == "aurora-postgres")? "postgres" : local.name
+}
+
+
+module "dummy-data" {
+
+  count = var.dummydata != null && local.type != null? 1 : 0
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = local.type
+  host      = local.host
+  username  = local.username
+  password  = local.password
+  database  = local.database
 }

databases/aws-rds/output.tf

@@ -1,23 +1,31 @@
- output "db_username" {
+output "db_username" {
   description = "The master username for the database"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
+  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
 } 
 
  output "db_instance_password" {
   description = "The database password (this password may be old, because Terraform doesn't track it after initial creation)"
-  value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].rds_cluster_master_password) : nonsensitive(module.common_rds_instance[0].db_instance_password)
+  value       = nonsensitive(random_password.password.result)
 }
 
  output "db_instance_port" {
   description = "The database port"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_port : module.common_rds_instance[0].db_instance_port
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_port : module.common_rds_instance[0].db_instance_port
 }
 
 output "db_instance_endpoint" {
   description = "The connection endpoint"
-  value       = (local.aurora)? module.aurora_rds_instance[0].rds_cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+  value       = (local.aurora)? module.aurora_rds_instance[0].cluster_endpoint : module.common_rds_instance[0].db_instance_endpoint
+}
+
+output "db_instance_id" {
+  value       = (local.aurora)? null : "${var.engine}${var.provision_id}"
 }
 
 output "db_instance_name" {
   value       = local.name
 }
+
+output "dummy_data_databases_available" {
+   value = var.dummydata != null && local.type != null? module.dummy-data[0].dbs : null
+}

databases/aws-rds/sec_groups.tf

@@ -0,0 +1,69 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "aws_security_group" "allow_tls" {
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {
+  for_each = toset(module.fw-ips.cidr_blocks)
+  
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  for_each = toset(module.fw-ips.cidr_blocks)
+
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "dbport"
+}
+
+resource "aws_security_group" "allow_tls_2" {
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4_2" {
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+  
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "Others - dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv_2" {
+
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "Others - dbport"
+}

databases/aws-rds/variables.tf

@@ -38,25 +38,79 @@ variable "engine" {
   description = "Accepted values are mysql, sqlserver-ex, postgres, oracle-se2, aurora-mysql, aurora-postgresql, mariadb"
 }
 
-variable "engine_version" {
-  default = "8.0.25"
-  description = "Not currently used"
-}
-
-variable "major_eng" {
-  default = "8.0"
-  description = "Not currently used"
-}
-
-variable "family" {
-  default = "mysql8.0"
-  description = "Not currently used"
-}
-
 variable "instance_size" {
   default = "db.t3.large"
 }
 
 variable "storage" {
-  default = "20"
+  default = "100"
+}
+
+variable "engine_version" {
+  type = map
+
+  default = {
+<<<<<<< HEAD
+    "mysql"  = "8.0.32"
+=======
+    "mysql"  = "8.0.36"
+>>>>>>> master
+    "postgres"  = "14.12" 
+    "oracle-se2"  = "19.0.0.0.ru-2023-04.rur-2023-04.r1"
+    "sqlserver-ex" = "15.00.4236.7.v1"
+    "mariadb" = "10.5"
+    "aurora-mysql" = "8.0.mysql_aurora.3.02.3" #"5.7.mysql_aurora.2.11.2"
+    "aurora-postgresql"  = "14.5"
+  }
+  
+}
+
+variable "major_engine_version" {
+  type = map
+
+  default = {
+    "mysql"  = "8.0"  
+    "postgres"  = "14"
+    "oracle-se2"  = "19"
+    "sqlserver-ex" = "15.00"
+    "mariadb" = "10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+  
+}
+
+
+variable "family" {
+  type = map
+
+  default = {
+    "mysql"  = "mysql8.0"
+    "postgres"  = "postgres14"
+    "oracle-se2"  = "oracle-se2-19"
+    "sqlserver-ex" = "sqlserver-ex-15.0"
+    "mariadb" = "mariadb10.5"
+    # for script compatibility only
+    "aurora-mysql" = "not_used"
+    "aurora-postgresql"  = "not_used"
+  }
+}
+
+variable "port" {
+  type = map
+
+  default = {
+    "mysql"  = "3306"
+    "postgres"  = "5432"
+    "oracle-se2"  = "1521"
+    "sqlserver-ex" = "1433"
+    "mariadb" = "3306"
+    "aurora-mysql" = "3306"
+    "aurora-postgresql"  = "5432"
+  }
+}
+
+variable "dummydata" {
+  default = null
 }

databases/aws-redshift/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.49.0"
+      version = ">= 6.0.0"
     }
   }
 }
@@ -17,105 +17,53 @@ resource "random_password" "password" {
   min_numeric = 1
   upper = true
   lower = true
-  number = true
+  numeric = true
   min_lower = 2
   min_upper = 2
   min_special = 2
 }
 
 locals {
+  s3_prefix = "redshift/qmi-${var.provision_id}"
   provid5              = substr(var.provision_id, 0, 5)
   vpc_id               = (var.region == "eu-west-1") ? var.vpc_id_eu : (var.region == "us-east-1") ? var.vpc_id_us : var.vpc_id_ap
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
   port                 = "5439"
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    Owner    = var.user_id
   }
 }
 
-module "fw-ips" {
-    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-}
-
-module "security_group" {
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG"
-  description = "${var.provision_id}-SG"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
-module "redshift" {
-
-  source  = "terraform-aws-modules/redshift/aws"
-  version = "~> 3.0"
-
-  cluster_identifier      = "qmi-${var.provision_id}"
-  cluster_node_type       = "dc2.large" #"dc1.large"
-  cluster_number_of_nodes = 1
-
-  cluster_database_name   = var.cluster_database_name
-  cluster_master_username = var.cluster_master_username
-  cluster_master_password = random_password.password.result
-
-  # Group parameters
-  #wlm_json_configuration = "[{\"query_concurrency\": 5}]"
-
-  # DB Subnet Group Inputs
-  subnets = local.subnet_ids
-  vpc_security_group_ids = [module.security_group.security_group_id]
-  publicly_accessible = true
-
-  # IAM Roles
-  #cluster_iam_roles = ["arn:aws:iam::225367859851:role/developer"]
-
-  tags = local.tags
-
-}
-
 module "qmi-s3-bucket" {
 
   source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
   
   provision_id                  = var.provision_id
-  region                        = var.region
   user_id                       = var.user_id
-}
+}
+
+
+resource "aws_redshift_cluster" "qmi" {
+  cluster_identifier = "qmi-${var.provision_id}"
+  database_name      = var.cluster_database_name
+  master_username    = var.cluster_master_username
+  master_password    = random_password.password.result
+  node_type          = "ra3.large"
+  cluster_type       = "single-node"
+
+  skip_final_snapshot = true
+
+  vpc_security_group_ids = [aws_security_group.allow_tls.id, aws_security_group.allow_tls_2.id]
+
+  publicly_accessible = true
+
+  tags = local.tags
+}
+
+

databases/aws-redshift/outputs.tf

@@ -1,25 +1,20 @@
 output "redshift_cluster_id" {
   description = "The availability zone of the RDS instance"
-  value       = module.redshift.redshift_cluster_id
+  value       = aws_redshift_cluster.qmi.id
 }
 
 output "redshift_cluster_endpoint" {
   description = "Redshift endpoint"
-  value       = module.redshift.redshift_cluster_endpoint
-}
-
-output "redshift_cluster_hostname" {
-  description = "Redshift hostname"
-  value       = module.redshift.redshift_cluster_hostname
+  value       = aws_redshift_cluster.qmi.endpoint
 }
 
 output "redshift_cluster_port" {
   description = "Redshift port"
-  value       = module.redshift.redshift_cluster_port
+  value       = aws_redshift_cluster.qmi.port
 }
 
 output "redshift_cluster_database_name" {
-    value = module.redshift.redshift_cluster_database_name
+    value = aws_redshift_cluster.qmi.database_name
 }
 
 output "redshift_cluster_master_username" {

databases/aws-redshift/sec_groups.tf

@@ -0,0 +1,69 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "aws_security_group" "allow_tls" {
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {
+  for_each = toset(module.fw-ips.cidr_blocks)
+  
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  for_each = toset(module.fw-ips.cidr_blocks)
+
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "dbport"
+}
+
+resource "aws_security_group" "allow_tls_2" {
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4_2" {
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+  
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "Others - dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv_2" {
+
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "Others - dbport"
+}

databases/aws-redshift/variables.tf

@@ -38,5 +38,5 @@ variable "cluster_database_name" {
 }
 
 variable "cluster_master_username" {
-    default = "qmi"
+    default = "qmiuser"
 }

databases/azure-eventhub/main.tf

@@ -22,9 +22,11 @@ resource "azurerm_eventhub_namespace" "ehbnamespace" {
   maximum_throughput_units = 2
 
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 
   network_rulesets {

databases/azure-hdinsight-hadoop/outputs.tf

@@ -30,7 +30,11 @@ output "Azure_Application_Registration_Client_ID" {
 }
 
 output "Azure_Application_Registration_Secret" {
+<<<<<<< HEAD
   value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+=======
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
+>>>>>>> master
 }
 
 
@@ -55,5 +59,9 @@ output "adls_Azure_Application_Registration_Client_ID" {
 }
 
 output "adls_Azure_Application_Registration_Secret" {
+<<<<<<< HEAD
   value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+=======
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
+>>>>>>> master
 }

databases/azure-rds-flexmysql/firewall.tf

@@ -1,31 +1,29 @@
-/*resource "azurerm_mysql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}*/
-
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "52.249.189.38"
-  end_ip_address      = "52.249.189.38"
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
+resource "azurerm_mysql_flexible_server_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips_az_qcs
+
+  name                = each.key
   resource_group_name = var.resource_group_name
   server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "13.67.39.86"
-  end_ip_address      = "13.67.39.86"
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
 }
 
-resource "azurerm_mysql_flexible_server_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
-  resource_group_name = var.resource_group_name
-  server_name         = azurerm_mysql_flexible_server.mysql-server.name
-  start_ip_address    = "20.67.110.207"
-  end_ip_address      = "20.67.110.207"
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mysql_flexible_server_firewall_rule.fw_rule ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_flexible_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
 }

databases/azure-rds-flexmysql/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_numeric = 2
@@ -33,6 +33,7 @@ resource "azurerm_mysql_flexible_server" "mysql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 

databases/azure-rds-flexmysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-flexmysql/variables.tf

@@ -25,15 +25,23 @@ variable "admin_login" {
 variable "mysql-version" {
   type = string
   description = "MySQL Server version to deploy"
-  default = "5.7"
+  default = "8.0.21"
 }
 variable "mysql-sku-name" {
   type = string
   description = "MySQL SKU Name"
-  default = "GP_Standard_D2ds_v4"
+<<<<<<< HEAD
+  default = "GP_Standard_D4ds_v4"
+=======
+  default = "B_Standard_B2s"
+>>>>>>> master
 }
 variable "mysql-storage" {
   type = string
   description = "MySQL Storage in MB"
   default = "20"
-  }
+}
+
+variable "dummydata" {
+  default = null
+}

databases/azure-rds-flexpostgres/firewall.tf

@@ -1,28 +1,13 @@
-/*resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}*/
-
-
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "52.249.189.38"
-  end_ip_address      = "52.249.189.38"
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "13.67.39.86"
-  end_ip_address      = "13.67.39.86"
-}
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips_az_qcs
 
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
+  name                = each.key
   server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "20.67.110.207"
-  end_ip_address      = "20.67.110.207"
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
 }

databases/azure-rds-flexpostgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!)"
   upper = true
   lower = true
   min_numeric = 2
@@ -29,6 +29,7 @@ resource "azurerm_postgresql_flexible_server" "postgresql-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
 }

databases/azure-rds-flexpostgres/variables.tf

@@ -23,12 +23,12 @@ variable "admin_login" {
 variable "postgresql-version" {
   type        = string
   description = "PostgreSQL Server version to deploy"
-  default     = "12"
+  default     = "14"
 }
 variable "postgresql-sku-name" {
   type        = string
   description = "PostgreSQL SKU Name"
-  default     = "GP_Standard_D2s_v3"
+  default     = "B_Standard_B2s"
 }
 variable "postgresql-storage" {
   type        = string

databases/azure-rds-mariadb/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_!"
   upper = true
   lower = true
   min_numeric = 2
@@ -35,6 +35,7 @@ resource "azurerm_mariadb_server" "mariadb-server" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -44,4 +45,17 @@ resource "azurerm_mariadb_database" "mariadb-db" {
   server_name         = azurerm_mariadb_server.mariadb-server.name
   charset             = "utf8"
   collation           = "utf8_unicode_ci"
-}
+}
+
+/*
+module "dummy-data" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mariadb_server.mariadb-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
+}
+*/

databases/azure-rds-mssql/firewall.tf

@@ -1,4 +1,5 @@
 
+
 resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
 
   count = var.subnet_id != null? 1 : 0
@@ -8,6 +9,15 @@ resource "azurerm_mssql_virtual_network_rule" "sqlvnetrule" {
   subnet_id           = var.subnet_id
 }
 
+resource "azurerm_mssql_virtual_network_rule" "sqlvnet_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                = "vnet-qmiinfra-${local.provision_id}"
+  server_id           = azurerm_mssql_server.sqlserver.id
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -21,4 +31,19 @@ resource "azurerm_mssql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ azurerm_mssql_firewall_rule.fw_rule ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mssql"
+  host      = azurerm_mssql_server.sqlserver.fully_qualified_domain_name
+  username  = var.sql_administrator_login
+  password  = nonsensitive(local.sql_administrator_login_password)
+  database  = ""
 }

databases/azure-rds-mssql/main.tf

@@ -10,7 +10,7 @@ resource "random_id" "randomServerId" {
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -37,6 +37,7 @@ resource "azurerm_mssql_server" "sqlserver" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 
   identity {
@@ -53,6 +54,7 @@ resource "azurerm_mssql_database" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -66,6 +68,7 @@ resource "azurerm_mssql_database" "dbname_source" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }
 
@@ -78,5 +81,6 @@ resource "azurerm_mssql_database" "dbname_target" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner = var.user_id
   }
 }

databases/azure-rds-mssql/outputs.tf

@@ -26,4 +26,8 @@ output "principal_id" {
   value = azurerm_mssql_server.sqlserver.identity.0.principal_id
 }
 
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}
+
 

databases/azure-rds-mssql/variables.tf

@@ -23,4 +23,12 @@ variable "sql_administrator_login" {
 
 variable "sql_administrator_login_password" {
     default = null
-}
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-mysql/firewall.tf

@@ -9,6 +9,16 @@ resource "azurerm_mysql_virtual_network_rule" "vnetrule" {
 
 }
 
+resource "azurerm_mysql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+  
+  name                  = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name   = var.resource_group_name
+  server_name           = azurerm_mysql_server.mysql-server.name
+  subnet_id             = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -23,4 +33,23 @@ resource "azurerm_mysql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_mysql_virtual_network_rule.vnetrule,
+    azurerm_mysql_virtual_network_rule.vnetrule_qmiinfra,
+    azurerm_mysql_firewall_rule.fw_rule 
+  ]
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "mysql"
+  host      = "${azurerm_mysql_server.mysql-server.fqdn}:3306"
+  username  = var.admin_login
+  password  = nonsensitive(random_password.password.result)
+  database  = ""
 }

databases/azure-rds-mysql/main.tf

@@ -1,18 +1,7 @@
-
-/*resource "azurerm_resource_group" "mysql-rg" {
-  name     = "mysql-${var.provision_id}"
-  location = var.location
-  tags = {
-    Deployment = "QMI PoC"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-  }
-}*/
-
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2
@@ -55,4 +44,12 @@ resource "azurerm_mysql_database" "mysql-db" {
   server_name         = azurerm_mysql_server.mysql-server.name
   charset             = "utf8"
   collation           = "utf8_unicode_ci"
-}
+}
+
+
+resource "azurerm_mysql_configuration" "example" {
+  name                = "binlog_row_image"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_mysql_server.mysql-server.name
+  value               = "full"
+}

databases/azure-rds-mysql/output.tf

@@ -8,4 +8,8 @@ output "root_username" {
 
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
+}
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
 }

databases/azure-rds-mysql/variables.tf

@@ -29,7 +29,7 @@ variable "admin_login" {
 variable "mysql-version" {
   type = string
   description = "MySQL Server version to deploy"
-  default = "5.7"
+  default = "8.0"
 }
 variable "mysql-sku-name" {
   type = string
@@ -41,3 +41,11 @@ variable "mysql-storage" {
   description = "MySQL Storage in MB"
   default = "5120"
   }
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
+}

databases/azure-rds-postgres/firewall.tf

@@ -8,6 +8,17 @@ resource "azurerm_postgresql_virtual_network_rule" "vnetrule" {
   subnet_id           = var.subnet_id
 }
 
+resource "azurerm_postgresql_virtual_network_rule" "vnetrule_qmiinfra" {
+
+  count = var.envbranch == "master" && var.location == "eastus" ? 1 : 0
+
+  name                = "vnet-qmiinfra-${var.provision_id}"
+  resource_group_name = var.resource_group_name
+  server_name         = azurerm_postgresql_server.postgresql-server.name
+  subnet_id           = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-Infrastructure"
+
+}
+
 module "fw-ips" {
     source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
@@ -22,4 +33,23 @@ resource "azurerm_postgresql_firewall_rule" "fw_rule" {
   start_ip_address    = each.value.0
   end_ip_address      = each.value.1
 
+}
+
+module "dummy-data" {
+
+  count = var.dummydata != null? 1 : 0
+
+  depends_on = [ 
+    azurerm_postgresql_firewall_rule.fw_rule, 
+    azurerm_postgresql_virtual_network_rule.vnetrule,
+    azurerm_postgresql_virtual_network_rule.vnetrule_qmiinfra 
+  ]
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/dummy"
+
+  type      = "postgres"
+  host      = azurerm_postgresql_server.postgresql-server.fqdn
+  username  = "${var.admin_login}@qmi-postgresql-${var.provision_id}"
+  password  = nonsensitive(random_password.password.result)
+  database  = "postgres"
 }

databases/azure-rds-postgres/main.tf

@@ -1,7 +1,7 @@
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!@"
+  override_special = "_)"
   upper = true
   lower = true
   min_numeric = 2

databases/azure-rds-postgres/output.tf

@@ -13,3 +13,7 @@ output "root_username" {
 output "root_username_password" {
   value = nonsensitive(random_password.password.result)
 }
+
+output "dummy_data_databases_available" {
+  value = var.dummydata != null? module.dummy-data[0].dbs : null
+}

databases/azure-rds-postgres/variables.tf

@@ -21,7 +21,7 @@ variable "subnet_id" {
 variable "admin_login" {
   type = string
   description = "Login to authenticate to PostgreSQL Server"
-  default = "qmi"
+  default = "qmirdsuser"
 }
 
 variable "postgresql-version" {
@@ -42,4 +42,12 @@ variable "postgresql-storage" {
 
 variable "user_id" {
 
+}
+
+variable "dummydata" {
+  default = null
+}
+
+variable "envbranch" {
+  default = "master"
 }

databases/dbricks/main.tf

@@ -5,50 +5,23 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 1.6.0"
+      version = ">= 1.13.0"
     }
   }
 }
 
+
+locals {
+  cluster_nameUpper = upper(var.cluster_name)
+}
+
 /*resource "databricks_workspace_conf" "this" {
   
   custom_config = {
     "enableIpAccessLists" : true
   }
 }
-
-resource "databricks_ip_access_list" "allowed-list" {
-
-  label     = "allow_in"
-  list_type = "ALLOW"
-  ip_addresses = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32"
-  ]
-
-}*/
+*/
 
 
 data "databricks_group" "admins" {
@@ -88,16 +61,48 @@ resource "databricks_group_member" "i-am-admin-aor" {
   member_id = databricks_user.aor[0].id
 }
 
+resource "databricks_secret_scope" "scopeapp" {
+
+    count = (var.sku == "premium")? 1 : 0
+
+    name = "${var.cluster_name}scope"
+}
+resource "databricks_secret" "publishing_api" {
+
+    count = (var.sku == "premium")? 1 : 0
+
+    key = "accesskey"
+    string_value = var.app_reg_secret
+    scope = databricks_secret_scope.scopeapp[0].name
+}
+
 resource "databricks_sql_endpoint" "sqlep" {
 
   count = (var.sku == "premium")? 1 : 0
 
-  name             = "qmi-sqlwh-${databricks_user.me.id}"
-  cluster_size     = "Small"
+  name             = "${local.cluster_nameUpper} SQL warehouse"
+  cluster_size     = var.cluster_size
   max_num_clusters = 1
 
 }
 
+resource "databricks_sql_global_config" "this" {
+
+  count = (var.sku == "premium")? 1 : 0
+
+  security_policy      = "DATA_ACCESS_CONTROL"
+  data_access_config = {
+    "spark.hadoop.fs.azure.account.auth.type.${var.storage_account_name}.dfs.core.windows.net" : "OAuth",
+    "spark.hadoop.fs.azure.account.oauth.provider.type.${var.storage_account_name}.dfs.core.windows.net": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "spark.hadoop.fs.azure.account.oauth2.client.id.${var.storage_account_name}.dfs.core.windows.net" : var.app_reg_id,
+    "spark.hadoop.fs.azure.account.oauth2.client.secret.${var.storage_account_name}.dfs.core.windows.net" : "{{secrets/${var.cluster_name}scope/accesskey}}",
+    "spark.hadoop.fs.azure.account.oauth2.client.endpoint.${var.storage_account_name}.dfs.core.windows.net": "https://login.microsoftonline.com/c21eeb5f-f5a6-44e8-a997-124f2f7a497c/oauth2/token"
+  }
+
+  enable_serverless_compute = false
+  sql_config_params = {}
+}
+
 
 
 resource "databricks_token" "pat" {  
@@ -122,9 +127,12 @@ resource "databricks_cluster" "dbrickscluster" {
     "spark.databricks.delta.preview.enabled": true,
     "spark.databricks.repl.allowedLanguages": "sql,python,r",
     "spark.databricks.cluster.profile": "serverless",
-    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey
+    "fs.azure.account.key.${var.storage_account_name}.dfs.core.windows.net": var.storage_account_accesskey,
+    "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact": true,
+    "spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite": true
   } 
 
+
   spark_env_vars = {
       "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
   }
@@ -134,5 +142,5 @@ resource "databricks_cluster" "dbrickscluster" {
     max_workers = 4
   }
 
-  autotermination_minutes = 120
+  autotermination_minutes = 45
 }

databases/dbricks/varaibles.tf

@@ -24,4 +24,16 @@ variable "node_type_id" {
 
 variable "sku" {
     default = "standard"
+}
+
+variable "app_reg_id" {
+  default = null
+}
+
+variable "app_reg_secret" {
+  default = null
+}
+
+variable "cluster_size" {
+  default = "Small"
 }

databases/dummy/main.tf

@@ -0,0 +1,21 @@
+resource "null_resource" "dummy-data" {
+    
+  provisioner "local-exec" {
+      command = "chmod +x ${path.module}/scripts/*.sh"
+      interpreter = ["/bin/bash", "-c"]
+  }
+
+  provisioner "local-exec" {
+      command = "${path.module}/scripts/dummy-${var.type}.sh \"$username\" \"$password\" \"$host\" \"$db\""
+      interpreter = ["/bin/bash", "-c"]
+
+      environment = {
+          
+          username  = var.username
+          password  = var.password
+          host      = var.host 
+          db        = var.database
+      }
+  }
+
+}

databases/dummy/outputs.tf

@@ -0,0 +1,3 @@
+output "dbs" {
+  value       = var.type == "postgres"? "'AdventureWorks' and 'dvdrental'": (var.type == "mysql")? "'sakila', 'employees' and 'classicmodels'" : (var.type == "mssql")? "'BikeStores' and 'Northwind'" : null
+}

databases/dummy/scripts/dummy-mssql.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}`
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+
+
+echo "---- Loading BikeStore sample database -----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+
+unzip qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE BikeStores"
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/createbike.sql
+sqlcmd -C -S "$host" -d "BikeStores" -U "$1" -P "$2" -i ./mssql/Bikeloaddata.sql > /tmp/bikestore_load.logs
+
+echo "---- Loading Northwind sample database -----"
+cd /tmp
+curl https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/northwind-pubs/instnwnd.sql -o instnwnd.sql
+sqlcmd -C -S "$host" -U "$1" -P "$2" -Q "CREATE DATABASE Northwind"
+sqlcmd -C -S "$host" -d "Northwind" -U "$1" -P "$2" -i instnwnd.sql > /tmp/northwind_load.logs
+
+rm -fr /tmp/qmi-cloud-tf-modules-*

databases/dummy/scripts/dummy-mysql.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4" 
+echo "--------------------------"    
+
+
+cd /tmp
+git clone https://github.com/datacharmer/test_db.git
+cd test_db
+echo "-------- Loading Mysql Employees sample database -----------"  
+mysql -h $host -u$1 -p$2 --ssl=TRUE < employees.sql
+cd ..
+wget -q https://downloads.mysql.com/docs/sakila-db.tar.gz
+tar xvfz sakila-db.tar.gz
+cd sakila-db
+
+echo "-------- Loading Mysql Sakila sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-schema.sql 
+mysql -h $host -u$1 -p$2 --ssl=TRUE < sakila-data.sql 
+
+cd /tmp
+
+curl https://www.mysqltutorial.org/wp-content/uploads/2018/03/mysqlsampledatabase.zip -o mysqlsampledatabase.zip
+unzip mysqlsampledatabase.zip
+echo "-------- Loading Mysql ClassicModels sample database -----------"  
+
+mysql -h $host -u$1 -p$2 --ssl=TRUE < mysqlsampledatabase.sql
+
+
+rm -fr /tmp/*
+

databases/dummy/scripts/dummy-postgres.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+IN="$3"
+arrIN=(${IN//:/ })
+host=`echo ${arrIN[0]}` 
+
+echo "---- Load Dummy Data -----"
+echo "Username: $1"
+echo "Password: $2"
+echo "Host: $host"
+echo "DB: $4"
+echo "--------------------------"
+
+echo "---- Install Adventureworks for Postgres ----"
+cd /tmp
+wget -q https://github.com/lorint/AdventureWorks-for-Postgres/archive/master.zip
+unzip master.zip
+cd AdventureWorks-for-Postgres-master/
+wget -q https://github.com/microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks-oltp-install-script.zip
+unzip AdventureWorks-oltp-install-script.zip
+
+ruby update_csvs.rb
+
+export PGPASSWORD=$2
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"AdventureWorks\";" 
+psql -h $host -U $1 -d "AdventureWorks" < ./install.sql > /tmp/AdventureWorks_load.log
+
+
+echo "---- Install DvdRental sample database for Postgres ----"
+cd /tmp
+curl https://gitlab.com/qmi/qmi-cloud-tf-modules/-/archive/master/qmi-cloud-tf-modules-master.zip?path=databases/dummy/scripts -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+unzip -o qmi-cloud-tf-modules-master-databases-dummy-scripts.zip
+cd qmi-cloud-tf-modules-master-databases-dummy-scripts/databases/dummy/scripts
+
+psql -h $host -U $1 -d $4 -c "CREATE DATABASE \"dvdrental\";" 
+psql -h $host -U $1 -d "dvdrental" < ./postgres/dump_dvdrental.sql > /tmp/dvdrental_load.log
+
+
+rm -fr /tmp/*
+
+

databases/dummy/scripts/mssql/createbike.sql

@@ -0,0 +1,109 @@
+/*
+--------------------------------------------------------------------
+© 2017 sqlservertutorial.net All Rights Reserved
+--------------------------------------------------------------------
+Name   : BikeStores
+Link   : http://www.sqlservertutorial.net/load-sample-database/
+Version: 1.0
+--------------------------------------------------------------------
+*/
+-- create schemas
+CREATE SCHEMA production;
+go
+
+CREATE SCHEMA sales;
+go
+
+-- create tables
+CREATE TABLE production.categories (
+	category_id INT IDENTITY (1, 1) PRIMARY KEY,
+	category_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.brands (
+	brand_id INT IDENTITY (1, 1) PRIMARY KEY,
+	brand_name VARCHAR (255) NOT NULL
+);
+
+CREATE TABLE production.products (
+	product_id INT IDENTITY (1, 1) PRIMARY KEY,
+	product_name VARCHAR (255) NOT NULL,
+	brand_id INT NOT NULL,
+	category_id INT NOT NULL,
+	model_year SMALLINT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	FOREIGN KEY (category_id) REFERENCES production.categories (category_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (brand_id) REFERENCES production.brands (brand_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE sales.customers (
+	customer_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (255) NOT NULL,
+	last_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255) NOT NULL,
+	street VARCHAR (255),
+	city VARCHAR (50),
+	state VARCHAR (25),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.stores (
+	store_id INT IDENTITY (1, 1) PRIMARY KEY,
+	store_name VARCHAR (255) NOT NULL,
+	phone VARCHAR (25),
+	email VARCHAR (255),
+	street VARCHAR (255),
+	city VARCHAR (255),
+	state VARCHAR (10),
+	zip_code VARCHAR (5)
+);
+
+CREATE TABLE sales.staffs (
+	staff_id INT IDENTITY (1, 1) PRIMARY KEY,
+	first_name VARCHAR (50) NOT NULL,
+	last_name VARCHAR (50) NOT NULL,
+	email VARCHAR (255) NOT NULL UNIQUE,
+	phone VARCHAR (25),
+	active tinyint NOT NULL,
+	store_id INT NOT NULL,
+	manager_id INT,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (manager_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.orders (
+	order_id INT IDENTITY (1, 1) PRIMARY KEY,
+	customer_id INT,
+	order_status tinyint NOT NULL,
+	-- Order status: 1 = Pending; 2 = Processing; 3 = Rejected; 4 = Completed
+	order_date DATE NOT NULL,
+	required_date DATE NOT NULL,
+	shipped_date DATE,
+	store_id INT NOT NULL,
+	staff_id INT NOT NULL,
+	FOREIGN KEY (customer_id) REFERENCES sales.customers (customer_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (staff_id) REFERENCES sales.staffs (staff_id) ON DELETE NO ACTION ON UPDATE NO ACTION
+);
+
+CREATE TABLE sales.order_items (
+	order_id INT,
+	item_id INT,
+	product_id INT NOT NULL,
+	quantity INT NOT NULL,
+	list_price DECIMAL (10, 2) NOT NULL,
+	discount DECIMAL (4, 2) NOT NULL DEFAULT 0,
+	PRIMARY KEY (order_id, item_id),
+	FOREIGN KEY (order_id) REFERENCES sales.orders (order_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE production.stocks (
+	store_id INT,
+	product_id INT,
+	quantity INT,
+	PRIMARY KEY (store_id, product_id),
+	FOREIGN KEY (store_id) REFERENCES sales.stores (store_id) ON DELETE CASCADE ON UPDATE CASCADE,
+	FOREIGN KEY (product_id) REFERENCES production.products (product_id) ON DELETE CASCADE ON UPDATE CASCADE
+);

databases/dummy/variables.tf

@@ -0,0 +1,15 @@
+variable "host" { 
+}
+
+variable "username" {
+}
+
+variable "password" {
+}
+
+variable "database" {
+}
+
+variable "type" {
+    description = "mysql | postgres | mssql"
+}

databases/firewall_ips/output.tf

@@ -1,23 +1,42 @@
 locals {
 
   az = {
-      az1          = ["52.249.189.38","52.249.189.38"]
-      az2          = ["13.67.39.86", "13.67.39.86"]
-      az3          = ["20.67.110.207", "20.67.110.207"]
+      #az1          = ["52.249.189.38","52.249.189.38"] # OLD USA
+      #az2          = ["13.67.39.86", "13.67.39.86"]    # OLD APAC
+      #az3          = ["20.67.110.207", "20.67.110.207"] # OLD EUROPE
+      azeurope          = ["20.160.170.99", "20.160.170.99"] # NEW EUROPE
+      azusa          = ["20.169.241.157", "20.169.241.157"] # NEW USA
+      azapac          = ["52.163.112.12", "52.163.112.12"] # NEW APAC
   }
 
   q_routes = {
-    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
-    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
-    EMEA_FULLVPN   = ["193.15.228.246", "193.15.228.246"]
     ITG         = ["188.65.156.32", "188.65.156.47"]
     ITG1        = ["212.73.252.96", "212.73.252.103"]
+    USDC4       = ["155.204.23.130","155.204.23.130"]
     kfar_saba   = ["194.90.96.176", "194.90.96.183"]
     kfar_saba1  = ["213.57.84.160", "213.57.84.167"]
     peak_10     = ["4.4.97.104", "4.4.97.111"]
     peak_101    = ["206.196.17.32", "206.196.17.63"]
     Bangalore   = ["14.98.59.168", "14.98.59.175"]
     Bangalore1  = ["182.74.33.8", "182.74.33.15"]
+    T_Beijing   = ["60.247.114.145","60.247.114.158"]
+    T_Nantes    = ["81.255.115.33","81.255.115.38"]
+    T_PA7_1     = ["217.117.156.85","217.117.156.86"]
+    T_PA7_2     = ["217.117.156.141","217.117.156.142"]
+    T_PA7_3     = ["217.117.156.161","217.117.156.190"]
+    T_Sanmateo  = ["12.202.47.153","12.202.47.158"]
+    T_Suresnes  = ["62.23.50.121","62.23.50.126"]
+    T_SV2_1     = ["4.31.193.69","4.31.193.70"]
+    T_SV2_2     = ["4.79.217.1","4.79.217.126"]
+    T_SV2_3     = ["149.97.160.217","149.97.160.218"]
+    T_SV2_4     = ["149.97.160.221","149.97.160.222"]
+    T_SV2_5     = ["149.97.185.1","149.97.185.30"]
+  }
+
+  fullvpn = {
+    AMER_FULLVPN1   = ["144.121.57.103", "144.121.57.103"]
+    AMER_FULLVPN2   = ["50.239.179.6", "50.239.179.6"]
+    EMEA_FULLVPN    = ["193.15.228.246", "193.15.228.246"]
   }
 
   qcs = {
@@ -27,12 +46,25 @@ locals {
       QCS_eu1          = ["34.247.21.179", "34.247.21.179"]  
       QCS_eu2          = ["52.31.212.214", "52.31.212.214"]  
       QCS_eu3          = ["54.154.95.18", "54.154.95.18"]  
-      QCS_apac1        = ["13.210.43.241", "13.210.43.241"] 
-      QCS_apac2        = ["13.236.104.42", "13.236.104.42"]  
-      QCS_apac3        = ["13.236.206.172", "13.236.206.172"] 
-      QCS_sing1        = ["18.138.163.172", "18.138.163.172"]
-      QCS_sing2        = ["18.142.157.182", "18.142.157.182"]
-      QCS_sing3        = ["54.179.13.251", "54.179.13.251"]
+      QCS_ap1          = ["13.210.43.241", "13.210.43.241"] 
+      QCS_ap2          = ["13.236.104.42", "13.236.104.42"]  
+      QCS_ap3          = ["13.236.206.172", "13.236.206.172"] 
+      QCS_sg1          = ["18.138.163.172", "18.138.163.172"]
+      QCS_sg2          = ["18.142.157.182", "18.142.157.182"]
+      QCS_sg3          = ["54.179.13.251", "54.179.13.251"]
+      QCS_de1          = ["3.76.162.169", "3.76.162.169"]
+      QCS_de2          = ["3.77.162.68", "3.77.162.68"]
+      QCS_de3          = ["3.122.137.91", "3.122.137.91"]
+      QCS_uk1          = ["13.42.141.246", "13.42.141.246"]
+      QCS_uk2          = ["18.135.245.97", "18.135.245.97"]
+      QCS_uk3          = ["35.179.0.171", "35.179.0.171"]
+      QCS_jp1          = ["54.238.168.131", "54.238.168.131"]
+      QCS_jp2          = ["3.113.68.20", "3.113.68.20"]
+      QCS_jp3          = ["35.73.207.58", "35.73.207.58"]
+      QCS_in1          = ["52.66.18.8", "52.66.18.8"]
+      QCS_in2          = ["13.201.250.59", "13.201.250.59"]
+      QCS_in3          = ["15.207.252.34", "15.207.252.34"]
+
   }
 
   qcs_qaa = {
@@ -42,23 +74,51 @@ locals {
       QCS_QAA_us1     = ["18.235.133.252", "18.235.133.252"]
       QCS_QAA_us2     = ["3.217.244.242", "3.217.244.242"]
       QCS_QAA_us3     = ["18.214.8.201", "18.214.8.201"]
-      QCS_QAA_apac1     = ["54.206.158.27", "54.206.158.27"]
-      QCS_QAA_apac2     = ["3.104.137.20", "3.104.137.20"]
-      QCS_QAA_apac3     = ["3.24.52.178", "3.24.52.178"]
-      QCS_QAA_sing1     = ["54.169.84.213", "54.169.84.213"]
-      QCS_QAA_sing2     = ["13.213.173.37", "13.213.173.37"]
-      QCS_QAA_sing3     = ["13.213.113.162", "13.213.113.162"]
+      QCS_QAA_ap1     = ["54.206.158.27", "54.206.158.27"]
+      QCS_QAA_ap2     = ["3.104.137.20", "3.104.137.20"]
+      QCS_QAA_ap3     = ["3.24.52.178", "3.24.52.178"]
+      QCS_QAA_sg1     = ["54.169.84.213", "54.169.84.213"]
+      QCS_QAA_sg2     = ["13.213.173.37", "13.213.173.37"]
+      QCS_QAA_sg3     = ["13.213.113.162", "13.213.113.162"]
+      QCS_QAA_de1     = ["3.69.132.138", "3.69.132.138"]
+      QCS_QAA_de2     = ["3.69.222.86", "3.69.222.86"]
+      QCS_QAA_de3     = ["3.72.136.160", "3.72.136.160"]
+      QCS_QAA_uk1     = ["18.168.120.199", "18.168.120.199"]
+      QCS_QAA_uk2     = ["18.169.91.38", "18.169.91.38"]
+      QCS_QAA_uk3     = ["13.42.135.168", "13.42.135.168"]
+      QCS_QAA_jp1     = ["35.77.238.13", "35.77.238.13"]
+      QCS_QAA_jp2     = ["13.115.58.233", "13.115.58.233"]
+      QCS_QAA_jp3     = ["35.74.220.230", "35.74.220.230"]
+      QCS_QAA_in1     = ["3.109.34.226", "3.109.34.226"]
+      QCS_QAA_in2     = ["15.206.64.196", "15.206.64.196"]
+      QCS_QAA_in3     = ["3.6.11.209", "3.6.11.209"]
+    
+  }
+
+  stitch = {
+      STITCH_1 = ["52.23.137.21", "52.23.137.21"]
+      STITCH_2 = ["52.204.223.208", "52.204.223.208"]
+      STITCH_3 = ["52.204.228.32", "52.204.228.32"]
+      STITCH_4 = ["52.204.230.227", "52.204.230.227"]
+      STITCH_5 = ["3.126.102.29", "3.126.102.29"]
+      STITCH_6 = ["18.158.16.164", "18.158.16.164"]
+      STITCH_7 = ["18.158.251.55", "18.158.251.55"]
+      STITCH_8 = ["52.57.235.168", "52.57.235.168"]
   }
 
  
 }
 
 output "ips2" {
-  value = merge(local.q_routes, local.qcs)
+  value = merge(local.q_routes, local.fullvpn, local.qcs, local.stitch)
 }
 
 output "ips" {
-  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes)
+  value = merge(local.az, local.qcs, local.qcs_qaa, local.q_routes, local.fullvpn, local.stitch)
+}
+
+output "ips_az_qcs" {
+  value = merge(local.az, local.qcs, local.fullvpn, local.stitch)
 }
 
 output "cidr_blocks" {
@@ -68,10 +128,14 @@ output "cidr_blocks" {
     "193.15.228.246/32",
     "50.239.179.6/32",
     # Azure QMI machines
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
+    #"52.249.189.38/32", # OLD USA
+    #"13.67.39.86/32",   # OLD APAC
+    #"20.67.110.207/32", # OLD EUROPE
+    "20.160.170.99/32", # NEW EUROPE
+    "20.169.241.157/32", # NEW USA
+    "52.163.112.12/32", # NEW APAC
     # Qlik network routers
+    "155.204.23.130/32",
     "14.98.59.168/29", 
     "182.74.33.8/29", 
     "188.65.156.32/28", 
@@ -80,32 +144,97 @@ output "cidr_blocks" {
     "213.57.84.160/29", 
     "4.4.97.104/29", 
     "206.196.17.32/27", 
-    # QCS
+    # T Beijing
+    "60.247.114.144/28",
+    # T Nantes
+    "81.255.115.32/29",
+    "84.14.92.152/29",
+    # T PA7
+    "217.117.156.84/30",
+    "217.117.156.140/30",
+    "217.117.156.160/27",
+    # T San Mateo
+    "12.202.47.152/29",
+    # T Suresnes
+    "62.23.50.120/29",
+    # T SV2
+    "4.31.193.68/30",
+    "4.79.217.0/25",
+    "149.97.160.216/30",
+    "149.97.160.220/30",
+    "149.97.185.0/27",
+    # QCS Amercias (us)
     "18.205.71.36/32", 
     "18.232.32.199/32", 
-    "34.237.68.254/32", 
+    "34.237.68.254/32",
+    # QCS Ireland (eu) 
     "34.247.21.179/32", 
     "52.31.212.214/32", 
-    "54.154.95.18/32", 
+    "54.154.95.18/32",
+    # QCS Australia (ap)  
     "13.210.43.241/32", 
     "13.236.104.42/32", 
     "13.236.206.172/32",
+    # QCS Singapore (sg)
     "18.138.163.172/32",
     "18.142.157.182/32",
     "54.179.13.251/32",
-    # QCS-QAA
+    # QCS Frankfurt (de)
+    "3.76.162.169/32",
+    "3.77.162.68/32",
+    "3.122.137.91/32",
+    # QCS Sweden (se)
+    "13.51.129.105/32",
+    "16.170.33.251/32",
+    "16.170.27.83/32",
+    # QCS Japan (jp)
+    "54.238.168.131/32",
+    "3.113.68.20/32",
+    "35.73.207.58/32",
+    # QCS India (in)
+    "52.66.18.8/32", 
+    "13.201.250.59/32",
+    "15.207.252.34/32",
+    # QCS London (uk)
+    "13.42.141.246/32",
+    "18.135.245.97/32",
+    "35.179.0.171/32",
+    # QCS-QAA Ireland (eu)
     "54.216.156.88/32",
     "3.248.156.131/32",
     "52.213.44.55/32",
+    # QCS-QAA Americas (us)
     "18.235.133.252/32",
     "3.217.244.242/32",
     "18.214.8.201/32",
+    # QCS-QAA Australia (ap)
     "54.206.158.27/32",
     "3.104.137.20/32",
     "3.24.52.178/32",
+    # QCS-QAA Singapore (sg)
     "54.169.84.213/32",
     "13.213.173.37/32",
     "13.213.113.162/32",
+    # QCS-QAA Frankfurt (de)
+    "3.69.132.138/32",
+    "3.69.222.86/32",
+    "3.72.136.160/32",
+    # QCS-QAA London (uk)
+    "18.168.120.199/32",
+    "18.169.91.38/32",
+    "13.42.135.168/32",
+    # QCS-QAA Japan (jp)
+    "35.77.238.13/32",
+    "13.115.58.233/32",
+    "35.74.220.230/32",
+    # QCS-QAA India (in)
+    "3.109.34.226/32",
+    "15.206.64.196/32",
+    "3.6.11.209/32",
+    # QCS-QAA Sweden (se)
+    "13.53.211.145/32",
+    "13.50.63.235/32",
+    "13.60.101.12/32",
     # QCS STAGING
     "18.233.22.130/32",
     "18.205.135.40/32",
@@ -113,6 +242,22 @@ output "cidr_blocks" {
     "18.155.181.46/32",
     "18.155.181.25/32",
     "18.155.181.128/32",
-    "18.155.181.76/32"
+    "18.155.181.76/32",
+    "52.16.133.167/32",
+    "34.248.105.199/32", # europe
+  ]
+}
+
+output "cidr_blocks_others" {
+  value = [
+    # Stitch
+    "52.23.137.21/32",
+    "52.204.223.208/32",
+    "52.204.228.32/32",
+    "52.204.230.227/32",
+    "3.126.102.29/32",
+    "18.158.16.164/32",
+    "18.158.251.55/32",
+    "52.57.235.168/32",
   ]
 }

databases/synapse-ws/firewall.tf

@@ -6,16 +6,18 @@ resource "azurerm_synapse_firewall_rule" "azureservices" {
   end_ip_address       = "0.0.0.0"
 }
 
-
+/*
+# OLD USA
 resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
 
-  name                = "az1"
+  name                = "az1" 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "52.249.189.38"
   end_ip_address      = "52.249.189.38"
 
 }
 
+# OLD APAC
 resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
 
   name                = "az2"
@@ -25,6 +27,7 @@ resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
 
 }
 
+# OLD EUROPE
 resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
 
   name                = "az3"
@@ -32,6 +35,35 @@ resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
   start_ip_address    = "20.67.110.207"
   end_ip_address      = "20.67.110.207"
 
+}
+*/
+
+# NEW EUROPE
+resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
+
+  name                = "azeurope"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "20.160.170.99"
+  end_ip_address      = "20.160.170.99"
+
+}
+
+# NEW USA
+resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
+  name                = "azusa"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "20.169.241.157"
+  end_ip_address      = "20.169.241.157"
+
+}
+
+# NEW USA
+resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
+  name                = "azapac"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "52.163.112.12"
+  end_ip_address      = "52.163.112.12"
+
 }
 
 module "fw-ips" {

databases/synapse-ws/main.tf

@@ -44,6 +44,7 @@ resource "azurerm_synapse_workspace" "synapsews" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner    = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 
@@ -60,6 +61,7 @@ resource "azurerm_synapse_sql_pool" "db" {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id != null? var.user_id : null
+    Owner = var.user_id != null? var.user_id : null
     ADAAutomation = "SQLDWSuspend"
   }
 

linux-common/centos/common/extract-certs.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD

linux-common/centos/common/falcon.sh

@@ -16,7 +16,7 @@ echo "CID=$1"
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
 
 if ! rpm -qa | grep -qw falcon-sensor; then
-    sudo yum -y --quiet install $BASEDIR/$binary
+    sudo dnf -y --quiet install $BASEDIR/$binary
 fi
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1
 sudo systemctl start falcon-sensor 

linux-common/centos/common/resizedisk.sh

@@ -2,6 +2,7 @@
 
 echo "--- Executing: $0 $@"
 
+<<<<<<< HEAD
 echo "Resizing main partition to max available disk"
 (
 echo u # Change the units to sectors
@@ -18,6 +19,34 @@ echo w # Write changes
 
 sudo partprobe
 sudo xfs_growfs -d /
+=======
+if  grep -Pq '/dev/(mapper/|disk/by-id/dm)' /etc/fstab  ||  mount | grep -q /dev/mapper/
+then
+    echo "LVM is in use (Oracle Linux)"
+    #work out what disk we need.  About 1 in 5 times we get sdb instead on sda
+    rootdisk=$(df --type=xfs|grep \/dev\/sd | sed -e's/[0-9].*//')
+    sudo gdisk -l $rootdisk
+    sudo growpart $rootdisk 2
+    sudo pvresize $rootdisk"2"
+    sudo lvextend -l +49%FREE /dev/rootvg/rootlv
+    sudo lvextend -l +100%FREE /dev/mapper/rootvg-crashlv
+    sudo xfs_growfs /dev/rootvg/rootlv
+    sudo xfs_growfs /dev/mapper/rootvg-crashlv
+else
+    echo "LVM not in use.  Resizing main partition to max available disk"
+    (
+    echo u # Change the units to sectors
+    echo p # List the partitions details
+    echo d # Delete  partition
+    echo 2 # Partition number
+    echo n # Add a new partition
+    echo p # Primary partition
+    echo 2 # Partition number
+    echo   # First sector (Accept default)
+    echo   # Last sector (Accept default: varies)
+    echo w # Write changes
+    ) | sudo fdisk /dev/sda
+>>>>>>> master
 
 echo "Done resize!"
 echo ""

linux-common/centos/common/save-cert.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "--- Executing: $0 $@"
+
+BASEDIR=$(dirname "$0")
+
+echo $1 > $BASEDIR/qmicerts/myserver.crt
+
+echo $2 > $BASEDIR/qmicerts/myserver.key
+
+

linux-common/centos/common/tenable.sh

@@ -9,13 +9,18 @@ echo "KEY=$KEY"
 
 cVer=`rpm -E %{rhel}`
 
-FILE="NessusAgent-10.2.1-es$cVer.x86_64.rpm"
+FILE="NessusAgent-10.8.2-el$cVer.x86_64.rpm"
 
 echo "--- Installing Tenable Nessus Agent --> $FILE"
 
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$FILE -O $BASEDIR/$FILE
 
-sudo /bin/yum -y --quiet install $BASEDIR/$FILE -y
+
+sudo yum-config-manager --disable mysql80-community > /dev/null 2>&1
+sudo yum-config-manager --disable mysql-connectors-community > /dev/null 2>&1
+sudo yum-config-manager --disable mysql-tools-community > /dev/null 2>&1
+
+sudo dnf -y --quiet install $BASEDIR/$FILE -y
 
 echo "--- Linking Tenable Nessus Agent..."
 sudo /bin/systemctl start nessusagent.service

linux-common/centos/common/update.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+
+cVer=`rpm -E %{rhel}`
+if [ -z "$cVer" ]; then
+    echo "Error: Unable to determine CentOS version."
+    exit 0
+fi
+if [ "$cVer" -gt 7 ]; then
+    echo "--- Executing: $0 $@"
+
+    echo 'Updating OS. This will take a around 10 minutes'
+
+    dnf -y --quiet update
+fi

linux-common/main.tf

@@ -13,26 +13,40 @@ data "azurerm_key_vault_secret" "cert_password" {
   key_vault_id = var.key_vault_id
 }
 
+#data "azurerm_key_vault_certificate_data" "cert-data" {
+#  name         = "star-qmi-qlikpoc-com-cert"
+#  key_vault_id = var.key_vault_id
+#}
+
+
 locals {
   falcon_id     = nonsensitive(data.azurerm_key_vault_secret.cid.value)
   tenable_key   = nonsensitive(data.azurerm_key_vault_secret.tenable-key.value)
   cert_password = nonsensitive(data.azurerm_key_vault_secret.cert_password.value)
-}
+  #cert_pem =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.pem)
+  #cert_key =  nonsensitive(data.azurerm_key_vault_certificate_data.cert-data.key)
+} 
 
-resource "null_resource" "post-linux-vm" {
-    
-    provisioner "file" {
+resource "null_resource" "files" {
+  provisioner "file" {
         connection {
             type     = "ssh"
             host     = var.private_ip_address
             user     = var.admin_username
             password = var.admin_password
-            timeout  = "60s"
+            timeout  = "10m"
             #private_key = "${file("~/.ssh/id_rsa")}"   
         }
         source      = "${path.module}/${var.os_type}/common"
         destination = "/home/${var.admin_username}"
     }
+}
+
+resource "null_resource" "post-linux-vm" {
+    
+    depends_on = [
+      null_resource.files
+    ]
 
     provisioner "remote-exec" {
         connection {
@@ -40,16 +54,69 @@ resource "null_resource" "post-linux-vm" {
             host     = var.private_ip_address
             user     = var.admin_username
             password = var.admin_password
-            timeout  = "60s"
+            timeout  = "3m"
             #private_key = "${file("~/.ssh/id_rsa")}"
         }
 
         inline = [
             "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
             "sudo /home/${var.admin_username}/common/falcon.sh '${local.falcon_id}'",
-            "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
             "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
+            #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
         ]
     }
-}
+}
+
+resource "null_resource" "update" {
+
+  count = var.update? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.post-linux-vm
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "3m"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/update.sh",
+        ]
+  }
+}
+
+resource "null_resource" "resize" {
+
+  count = var.resize? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.update
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "3m"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/resizedisk.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
 
 BASEDIR=$(dirname "$0")
 
+mkdir -p $BASEDIR/qmicerts
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
+
 PASSWORD=$1
 # Extract the private key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nodes -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.key -passin pass:$PASSWORD

linux-common/ubuntu/common/falcon.sh

@@ -9,7 +9,7 @@ binary="falcon-sensor_5.17.0-8103_amd64.deb"
 echo "--- Installing CrowdStrike falcon agent --> $binary"
 echo "CID=$1"
 
-sudo apt -qq -y update
+#sudo apt -qq -y update
 sudo apt install libnl-3-200 libnl-genl-3-200 -qq -y
 
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary

linux-common/ubuntu/common/tenable.sh

@@ -7,7 +7,7 @@ BASEDIR=$(dirname "$0")
 KEY=$1
 echo "KEY=$KEY"
 
-FILE="NessusAgent-10.2.1-ubuntu1404_amd64.deb"
+FILE="NessusAgent-10.8.2-ubuntu1604_amd64.deb"
 
 echo "--- Installing Tenable Nessus Agent --> $FILE"
 

linux-common/ubuntu/common/update.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "--- Executing: $0 $@"
+
+echo 'Updating OS. This will take a around 10 minutes'
+apt -qq -y update
+apt --fix-broken -qq -y upgrade

linux-common/variables.tf

@@ -16,3 +16,13 @@ variable "key_vault_id" {
   default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-Machines/providers/Microsoft.KeyVault/vaults/qmisecrets"
 }
 
+variable "update" {
+  type = bool
+  default = true
+}
+
+variable "resize" {
+  type = bool
+  default = true
+}
+

qmi-nic/mainf.tf

@@ -71,7 +71,7 @@
   address_prefix       = "10.0.2.0/24"
 }*/
 
-resource "azurerm_public_ip" "publicip" {
+/*resource "azurerm_public_ip" "publicip" {
 
   count = var.isExternal? 1 : 0
   
@@ -80,7 +80,7 @@ resource "azurerm_public_ip" "publicip" {
   location            = var.location
   allocation_method   = "Dynamic"
   domain_name_label   = lower(var.prefix)
-}
+}*/
 
 resource "azurerm_network_interface" "nic" {
   name                = "${var.prefix}-nic"
@@ -92,13 +92,13 @@ resource "azurerm_network_interface" "nic" {
     name                          = "${var.prefix}-ip-cfg"
     subnet_id                     = var.subnet_id
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = var.isExternal? azurerm_public_ip.publicip[0].id : null
   }
 
   tags = {
     Deployment = "QMI PoC"
     "Cost Center" = "3100"
     QMI_user = var.user_id
+    Owner    = var.user_id
   }
 }
 

qmi-nic/outputs.tf

@@ -8,12 +8,4 @@ output "private_ip_address" {
 
 output "ip_configuration_name" {
   value = "${var.prefix}-ip-cfg"
-}
-
-output "public_ip_address" {
-  value = var.isExternal? azurerm_public_ip.publicip[0].ip_address : null
-}
-
-output "fqdn" {
-  value = var.isExternal? azurerm_public_ip.publicip[0].fqdn : null
 }

qmi-storage-account/main.tf

@@ -11,12 +11,16 @@ resource "azurerm_storage_account" "qmi-storage-account" {
     name                = "diag${random_id.randomId.hex}"
     resource_group_name = var.resource_group_name
     location            = var.location
-    account_replication_type = "LRS"
-    account_tier = "Standard"
+    
+    account_kind              = "StorageV2"
+    account_replication_type  = "LRS"
+    account_tier              = "Standard"
+    access_tier               = "Hot"
 
     tags = {
         Deployment = "QMI PoC"
         "Cost Center" = "3100"
         QMI_user = var.user_id
+        Owner    = var.user_id
     }
 }

qmicerts/wildcard_qmi_qlik-poc_com.pem

@@ -0,0 +1,89 @@
+-----BEGIN CERTIFICATE-----
+MIIG4TCCBcmgAwIBAgIQAxAE3FY3y74JI7dXp7IdNjANBgkqhkiG9w0BAQsFADBZ
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTMwMQYDVQQDEypE
+aWdpQ2VydCBHbG9iYWwgRzIgVExTIFJTQSBTSEEyNTYgMjAyMCBDQTEwHhcNMjUw
+NDA3MDAwMDAwWhcNMjYwNDA3MjM1OTU5WjBdMQswCQYDVQQGEwJTRTENMAsGA1UE
+BxMETHVuZDEiMCAGA1UEChMZUWxpa1RlY2ggSW50ZXJuYXRpb25hbCBBQjEbMBkG
+A1UEAwwSKi5xbWkucWxpay1wb2MuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAoZ8Fts4HoStfVxezUqPBPtZz9CMLHRFDd1OILtM7Fn23L13Ks+a9
+uL6wnvp1anV4z4Hx5pAEo2u5/nquLcX+raU5I7AnoOdKcyeI8pJYtTjKPqQbMX1t
+Ngbzk8Y0440sAG2FtXkqH1XLL79e/YYZ5s2QK4ueJn7Xwp1Avo1pgy9/SU+1fTHa
+Udlj7mxCyVymoOucoikqRHsV5VkTzgwnL3MAvOO6G7XMWVOpRZI++yJvEjkq/B9b
+SCVH2lp4mbYdKAtovusM217sVT8Gm7ap/AWuYvJcdqIKCsNvSqu3ZyJr1xUpf3O/
+toGzH9hHiOGsdpzSInbxVNW7Z+i4FW6rKQIDAQABo4IDnzCCA5swHwYDVR0jBBgw
+FoAUdIWAwGbH3zfez70pN6oDHb7tzRcwHQYDVR0OBBYEFCy+ODAFw4YErvNnnnW6
++X1Jo0CGMC8GA1UdEQQoMCaCEioucW1pLnFsaWstcG9jLmNvbYIQcW1pLnFsaWst
+cG9jLmNvbTA+BgNVHSAENzA1MDMGBmeBDAECAjApMCcGCCsGAQUFBwIBFhtodHRw
+Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW
+MBQGCCsGAQUFBwMBBggrBgEFBQcDAjCBnwYDVR0fBIGXMIGUMEigRqBEhkJodHRw
+Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxHMlRMU1JTQVNIQTI1
+NjIwMjBDQTEtMS5jcmwwSKBGoESGQmh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9E
+aWdpQ2VydEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNybDCBhwYIKwYB
+BQUHAQEEezB5MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20w
+UQYIKwYBBQUHMAKGRWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2Vy
+dEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNydDAMBgNVHRMBAf8EAjAA
+MIIBfQYKKwYBBAHWeQIEAgSCAW0EggFpAWcAdQCWl2S/VViXrfdDh2g3CEJ36fA6
+1fak8zZuRqQ/D8qpxgAAAZYQZtz0AAAEAwBGMEQCIA9bkdnKRWyHdJKovDloGgVC
+PEayqXugtDH9ElCPjxz9AiAsK5eEF6LhU7NgxJ+kl/7VCObUOSx2BLX2KfkxBdId
+qgB2AGQRxGykEuyniRyiAi4AvKtPKAfUHjUnq+r+1QPJfc3wAAABlhBm3P0AAAQD
+AEcwRQIhAPpECY9PYjhxBITyYjFNl6gUd/94CBRd70bSqajIDEh8AiBcuBXsgM4n
+eW4OknJ/6Vp5cOstxg5+WTdbjZt3o5W4/AB2AEmcm2neHXzs/DbezYdkprhbrwqH
+gBnRVVL76esp3fjDAAABlhBm3QgAAAQDAEcwRQIgMxJ/w1C7eItarCkVKjYSs7jE
+6TVUfBohNMYsYceha5ICIQCNZGVKxi/sQuciyqyhG6AuSd8JbmSX80o+feNhSaRu
+GzANBgkqhkiG9w0BAQsFAAOCAQEAd4Os9L/h/mDXrElwbPBxY+ILysk2T4hzRrye
+yYvdGlmqB+I84bac5+eDA6HvmQX1rjNEzdybQuzdjLoUIvZN1dYeoUf66+tW+xhR
+mCZ6DWU1QOKabJlwwyxIA8wmfE9YpTxMt1SCHfRtFBazyir2nL7wAWGWiMPLc8ZX
+H5N2uJrt+jH4cLN+MnLWgvY5CsC2g3IimKfqRNLrs3L1ug2vIgzQVbfBm+5OkoCx
+JVZJ8WqoF9oPd8Kl0xWQwocjwzhhYtJ6MffmrkyFs7098OXgpYZMFnoveA/85F+b
+bQ74NzbCsXXuevhszg7tgtPyGEE4VpOA4ileC4sr0BVNgJcA9g==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEyDCCA7CgAwIBAgIQDPW9BitWAvR6uFAsI8zwZjANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0yMTAzMzAwMDAwMDBaFw0zMTAzMjkyMzU5NTlaMFkxCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxMzAxBgNVBAMTKkRpZ2lDZXJ0IEdsb2Jh
+bCBHMiBUTFMgUlNBIFNIQTI1NiAyMDIwIENBMTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMz3EGJPprtjb+2QUlbFbSd7ehJWivH0+dbn4Y+9lavyYEEV
+cNsSAPonCrVXOFt9slGTcZUOakGUWzUb+nv6u8W+JDD+Vu/E832X4xT1FE3LpxDy
+FuqrIvAxIhFhaZAmunjZlx/jfWardUSVc8is/+9dCopZQ+GssjoP80j812s3wWPc
+3kbW20X+fSP9kOhRBx5Ro1/tSUZUfyyIxfQTnJcVPAPooTncaQwywa8WV0yUR0J8
+osicfebUTVSvQpmowQTCd5zWSOTOEeAqgJnwQ3DPP3Zr0UxJqyRewg2C/Uaoq2yT
+zGJSQnWS+Jr6Xl6ysGHlHx+5fwmY6D36g39HaaECAwEAAaOCAYIwggF+MBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHSFgMBmx9833s+9KTeqAx2+7c0XMB8G
+A1UdIwQYMBaAFE4iVCAYlebjbuYP+vq5Eu0GF485MA4GA1UdDwEB/wQEAwIBhjAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdgYIKwYBBQUHAQEEajBoMCQG
+CCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQAYIKwYBBQUHMAKG
+NGh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RH
+Mi5jcnQwQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29t
+L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNybDA9BgNVHSAENjA0MAsGCWCGSAGG/WwC
+ATAHBgVngQwBATAIBgZngQwBAgEwCAYGZ4EMAQICMAgGBmeBDAECAzANBgkqhkiG
+9w0BAQsFAAOCAQEAkPFwyyiXaZd8dP3A+iZ7U6utzWX9upwGnIrXWkOH7U1MVl+t
+wcW1BSAuWdH/SvWgKtiwla3JLko716f2b4gp/DA/JIS7w7d7kwcsr4drdjPtAFVS
+slme5LnQ89/nD/7d+MS5EHKBCQRfz5eeLjJ1js+aWNJXMX43AYGyZm0pGrFmCW3R
+bpD0ufovARTFXFZkAdl9h6g4U5+LXUZtXMYnhIHUfoyMo5tS58aI7Dd8KvvwVVo4
+chDYABPPTHPbqjc1qCmBaZx2vN4Ye5DUys/vZwP9BFohFrH/6j/f3IL16/RZkiMN
+JCqVJUzKoZHm1Lesh3Sz8W2jmdv51b2EQJ8HmA==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
+2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
+1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
+q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
+tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
+vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
+5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
+1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
+NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
+Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
+8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
+pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
+MrY=
+-----END CERTIFICATE-----

replicate-install/scripts/Replicate_install.iss

@@ -1,29 +1,29 @@
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
-Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
-Count=7
-Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0
-Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
-Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
-Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
-Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
-Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
-szDir=C:\Program Files\Attunity\Replicate\
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
-szDir=C:\Program Files\Attunity\Replicate\data
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
-Result=1
-Sel-0=1
-Sel-1=0
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
-Result=1
-[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
-Result=1
-bOpt1=0
-bOpt2=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
+Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
+Count=7
+Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0
+Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
+Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
+Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
+Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
+Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicenseAgreement-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
+szDir=C:\Program Files\Attunity\Replicate\
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
+szDir=C:\Program Files\Attunity\Replicate\data
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
+Result=1
+Sel-0=1
+Sel-1=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

replicate-install/scripts/Replicate_install_2022.11.0.208.iss

@@ -0,0 +1,29 @@
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-DlgOrder]
+Dlg0={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0
+Count=7
+Dlg1={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0
+Dlg2={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0
+Dlg3={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1
+Dlg4={9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0
+Dlg5={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0
+Dlg6={9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdWelcome-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdLicense2Rtf-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-0]
+szDir=C:\Program Files\Attunity\Replicate\
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdAskDestPath-1]
+szDir=C:\Program Files\Attunity\Replicate\data
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-AskOptions-0]
+Result=1
+Sel-0=1
+Sel-1=0
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdStartCopy-0]
+Result=1
+[{9C614355-28A0-4C2A-98DF-DB9FD674826F}-SdFinish-0]
+Result=1
+bOpt1=0
+bOpt2=0

replicate-install/scripts/di-replicate-installQMICertificate.ps1

@@ -7,6 +7,11 @@ Param(
 try {
 
     Write-Host "Install qmi_qlik-poc_com certificate on Windows"
+    
+    $ProgressPreference = 'SilentlyContinue'
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    (New-Object System.Net.WebClient).DownloadFile("https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx", "$PSScriptRoot\wildcard_qmi_qlik-poc_com.pfx") 
+
     $secpasswd = ConvertTo-SecureString $CertPwd -AsPlainText -Force
     $sslCert = Import-PfxCertificate -FilePath $PSScriptRoot/wildcard_qmi_qlik-poc_com.pfx -CertStoreLocation Cert:\LocalMachine\My -Password $secpasswd
     $thumb=$($sslCert.Thumbprint)
@@ -16,7 +21,7 @@ try {
     Write-Host "Set SSL qmi_qlik-poc_com for Replicate"
 
     Stop-Service AttunityReplicateConsole
-    if ( $url -Match "2022.11" ) {  
+    if ( $url -Match "2022.11" -or $url -Match "2023.5" ) {  
         Stop-Service QlikReplicateServer
     } else {
         Stop-Service AttunityReplicateServer
@@ -27,7 +32,7 @@ try {
     netsh http add sslcert ipport=0.0.0.0:443 certhash=$thumb appid='{4dc3e181-e14b-4a21-b022-59fc669b0914}'
 
     Start-Service AttunityReplicateConsole 
-    if ( $url -Match "2022.11" ) {  
+    if ( $url -Match "2022.11" -or $url -Match "2023.5" ) {  
         Start-Service QlikReplicateServer
     } else {
         Start-Service AttunityReplicateServer