Merge branch 'master' into dev

.gitignore

@@ -1 +0,0 @@
-.DS_Store

adls/outputs.tf

@@ -31,5 +31,5 @@ output "Azure_Application_Registration_Client_ID" {
 }
 
 output "Azure_Application_Registration_Secret" {
-  value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
 }

aws-olh-prereqs/kinesis.tf

@@ -0,0 +1,9 @@
+resource "aws_kinesis_stream" "qlik_stream" {
+  name             = "qlik-data-stream-${var.provision_id}"  # Change the name as needed
+  shard_count      = 1
+  stream_mode_details {
+    stream_mode = "PROVISIONED"
+  }
+
+  retention_period = 24  # Optional: Retain records for 24 hours (default)
+}

aws-olh-prereqs/kms.tf

@@ -0,0 +1,14 @@
+resource "aws_kms_key" "qlik_key" {
+  description             = "QMIOpenLakehouse - Customer managed key for Qlik resources"
+  deletion_window_in_days = 10
+  enable_key_rotation     = true
+
+  tags = {
+    Alias = "kms-for-qmi-olh-${var.provision_id}"
+  }
+}
+
+resource "aws_kms_alias" "qlik_key_alias" {
+  name          = "alias/kms-for-qmi-olh-${var.provision_id}"
+  target_key_id = aws_kms_key.qlik_key.key_id
+}

aws-olh-prereqs/main.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 6.0.0"
+    }
+  }
+}

aws-olh-prereqs/output.tf

@@ -0,0 +1,71 @@
+output "AWS_ACCOUNT_ID" {
+  value = var.aws_account_id
+}
+
+output "VPC_ID" {
+  value = aws_vpc.main.id
+  
+}
+
+output "VPC_CIDR" {
+  value = var.vpc_cidr
+}
+
+output "PUBLIC_SUBNET_CIDRS" {
+  value = var.public_subnet_cidrs
+}
+
+output "SUBNETS_IDS" {
+  value = aws_subnet.public[*].id
+  
+}
+
+output "PUBLIC_SUBNET_AZS" {
+  value = [
+    for subnet in aws_subnet.public : subnet.availability_zone
+  ]
+}
+
+output "KMS_KEY_ARN" {
+  value = aws_kms_key.qlik_key.arn
+}
+
+output "KINESIS_STREAM_NAME" {
+  value = aws_kinesis_stream.qlik_stream.name
+}
+
+output "SECURITY_GROUP_ID" {
+  value = aws_security_group.allow_tls.id
+}
+
+output "MANAGEMENT_ROLE_ARN" {
+  value = aws_iam_role.olh_mgmt_role.arn
+}
+
+output "EC2_INTANCE_ROLE_ARN" {
+  value = aws_iam_role.olh_ec2_role.arn
+}
+
+output "EC2_INTANCE_PROFILE_ARN" {
+  value = aws_iam_instance_profile.ec2_instance_profile.arn
+}
+
+output "S3_BUCKET_NAME" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_id    
+}
+
+output "S3_BUCKET_REGION" {
+  value = module.qmi-s3-bucket.bucket.s3_bucket_region 
+}
+
+output "S3_BUCKET_ACCESS_KEY" {
+  value = module.qmi-s3-bucket.iam_access_key   
+}
+
+output "S3_BUCKET_ACCESS_SECRET" {
+  value = module.qmi-s3-bucket.iam_access_secret
+}
+
+output "TAGS" {
+  value = "Environment = QMI-${var.provision_id}"
+}

aws-olh-prereqs/role-ec2.tf

@@ -0,0 +1,156 @@
+resource "aws_iam_role" "olh_ec2_role" {  
+    name = "qmi_olh_ec2_${var.provision_id}"
+
+    assume_role_policy = jsonencode({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": "sts:AssumeRole",
+                "Principal": {
+                    "Service" : "ec2.amazonaws.com"
+                }
+            }
+        ]
+    })
+
+}
+
+
+resource "aws_iam_role_policy" "server_policy" {
+
+  name = "qmi-olh-${var.provision_id}_s3_policy"
+  role = aws_iam_role.olh_ec2_role.id
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:AttachVolume",
+                "ec2:DetachVolume",
+                "autoscaling:SetInstanceHealth"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:PutMetricData"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:kinesis:*:*:stream/${aws_kinesis_stream.qlik_stream.name}"
+            ],
+            "Action": [
+                "kinesis:PutRecord",
+                "kinesis:DescribeStreamSummary",
+                "kinesis:ListShards",
+                "kinesis:PutRecords",
+                "kinesis:GetShardIterator",
+                "kinesis:GetRecords",
+                "kinesis:DescribeStream",
+                "kinesis:ListStreamConsumers"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}"
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}",
+                "arn:aws:s3:::${module.qmi-s3-bucket.bucket.s3_bucket_id}/*"
+            ],
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:AbortMultipartUpload",
+                "s3:DeleteObjectVersion",
+                "s3:ListBucket",
+                "s3:DeleteObject",
+                "s3:GetObjectVersion"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ssm:*:*:parameter/qlik/*"
+            ],
+            "Action": [
+                "ssm:GetParameter",
+                "ssm:PutParameter"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "${aws_kms_key.qlik_key.arn}"
+            ],
+            "Action": [
+                "kms:*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "sts:AssumeRole"
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "sts:ExternalId": "6b69f9c3c8f502f2ddfc8434d443f172ebe4c032f4ef1a3c51215d27a58ca799"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "glue:CreateTable",
+                "glue:UpdateTable",
+                "glue:DeleteTable",
+                "glue:BatchDeleteTable",
+                "glue:GetTable",
+                "glue:GetTables",
+                "glue:CreateDatabase",
+                "glue:UpdateDatabase",
+                "glue:GetDatabase",
+                "glue:GetDatabases",
+                "glue:GetUserDefinedFunction"
+            ],
+            "Resource": [
+                "arn:aws:glue:us-east-1:*:catalog",
+                "arn:aws:glue:us-east-1:*:database/*",
+                "arn:aws:glue:us-east-1:*:table/*/*"
+            ]
+        }
+    ]
+}
+  )
+}
+
+
+resource "aws_iam_instance_profile" "ec2_instance_profile" {
+  name = "qmi-olh-instance-profile-${var.provision_id}"
+  role = aws_iam_role.olh_ec2_role.name
+}

aws-olh-prereqs/role-mgmt.tf

@@ -0,0 +1,233 @@
+resource "aws_iam_role" "olh_mgmt_role" {  
+    name = "qmi_olh_mgmt_${var.provision_id}"
+
+    assume_role_policy = jsonencode({
+        "Version": "2012-10-17",
+        "Statement": [
+              {
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": "arn:aws:iam::072398622971:role/upsolver-proxy-assume-role"
+                },
+                "Action": [
+                    "sts:AssumeRole"
+                ],
+                "Condition": {
+                    "StringEquals": {
+                        "sts:ExternalId": "6b69f9c3c8f502f2ddfc8434d443f172ebe4c032f4ef1a3c51215d27a58ca799"
+                    }
+                }
+            },
+            {
+                "Effect": "Allow",
+                "Action": "sts:AssumeRole",
+                "Principal": {
+                    "Service" : "ec2.amazonaws.com"
+                }
+            }
+        ]
+    })
+
+}
+
+resource "aws_iam_role_policy" "inline_policy" {
+
+  name = "qmi-olh-role-policy-${var.provision_id}"
+  role = aws_iam_role.olh_mgmt_role.id
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:CancelSpotInstanceRequests",
+                "ec2:CreateLaunchTemplate",
+                "ec2:CreateTags",
+                "ec2:CreateVolume",
+                "ec2:DescribeAddresses",
+                "ec2:DescribeImageAttribute",
+                "ec2:DescribeImages",
+                "ec2:DescribeInstanceStatus",
+                "ec2:DescribeInstanceTypeOfferings",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
+                "ec2:DescribeLaunchTemplates",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSpotInstanceRequests",
+                "ec2:DescribeSpotPriceHistory",
+                "ec2:DescribeTags",
+                "ec2:DescribeRegions",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups",
+                "ec2:RequestSpotInstances",
+                "ec2:DescribeVolumes"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ec2:*::image/*",
+                "arn:aws:ec2:*:*:instance/*",
+                "arn:aws:ec2:*:*:key-pair/*",
+                "arn:aws:ec2:*:*:launch-template/*",
+                "arn:aws:ec2:*:*:network-interface/*",
+                "arn:aws:ec2:*:*:security-group/*",
+                "arn:aws:ec2:*:*:subnet/*",
+                "arn:aws:ec2:*:*:volume/*"
+            ],
+            "Action": [
+                "ec2:RunInstances"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:AttachVolume",
+                "ec2:DeleteVolume",
+                "ec2:DetachVolume",
+                "ec2:DeleteLaunchTemplate",
+                "ec2:TerminateInstances",
+                "ec2:StartInstances",
+                "ec2:ModifyLaunchTemplate",
+                "ec2:DeleteLaunchTemplateVersions",
+                "ec2:CreateLaunchTemplateVersion"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "autoscaling:DeleteAutoScalingGroup",
+                "autoscaling:DeletePolicy",
+                "autoscaling:PutScalingPolicy",
+                "autoscaling:StartInstanceRefresh",
+                "autoscaling:TerminateInstanceInAutoScalingGroup",
+                "autoscaling:UpdateAutoScalingGroup"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "autoscaling:CreateAutoScalingGroup",
+                "autoscaling:CreateOrUpdateTags",
+                "autoscaling:DescribeAutoScalingGroups",
+                "autoscaling:DescribeInstanceRefreshes",
+                "autoscaling:DescribePolicies",
+                "autoscaling:DescribeScalingActivities"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "ec2:DescribeRegions",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:DescribeAlarmHistory",
+                "cloudwatch:DescribeAlarms",
+                "cloudwatch:DescribeAlarmsForMetric",
+                "cloudwatch:GetMetricStatistics",
+                "cloudwatch:ListMetrics",
+                "cloudwatch:PutMetricData"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "cloudwatch:PutMetricAlarm",
+                "cloudwatch:DeleteAlarms"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/qlik_cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "iam:AddRoleToInstanceProfile",
+                "iam:CreateServiceLinkedRole",
+                "iam:GetPolicy",
+                "iam:GetPolicyVersion",
+                "iam:ListAccountAliases",
+                "iam:ListAttachedRolePolicies",
+                "iam:ListInstanceProfiles",
+                "iam:ListInstanceProfilesForRole",
+                "iam:ListPolicies",
+                "iam:ListRoles",
+                "iam:PassRole"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+                "sts:DecodeAuthorizationMessage"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:ssm:*:${var.aws_account_id}:parameter/qlik/*"
+            ],
+            "Action": [
+                "ssm:PutParameter"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "${aws_kms_key.qlik_key.arn}"
+            ],
+            "Action": [
+                "kms:GenerateDataKeyPairWithoutPlaintext",
+                "kms:Encrypt"
+            ]
+        }
+    ]
+  })
+}

aws-olh-prereqs/s3.tf

@@ -0,0 +1,10 @@
+module "qmi-s3-bucket" {          
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
+  
+  bucket_name                   = "qmi-olh-bucket-${var.provision_id}"
+  provision_id                  = var.provision_id
+  user_id                       = var.user_id
+  forced_destroy                = var.forced_destroy
+  
+}

aws-olh-prereqs/variables.tf

@@ -0,0 +1,30 @@
+
+variable "aws_account_id" {
+    description = "AWS account ID"
+}
+
+variable "region" {
+    description = "AWS region"
+    default     = "us-east-1"
+}
+
+variable "provision_id" {
+}
+
+variable "user_id" {
+}
+
+variable "vpc_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "public_subnet_cidrs" {
+  default = ["10.0.1.0/24", "10.0.2.0/24"]
+}
+
+variable "forced_destroy" {
+  default = null
+}
+
+
+

aws-olh-prereqs/vpc.tf

@@ -0,0 +1,74 @@
+resource "aws_vpc" "main" {
+  cidr_block           = var.vpc_cidr
+  enable_dns_support   = true
+  enable_dns_hostnames = true
+
+  tags = {
+    Name = "qmi-olh-vpc-${var.provision_id}"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "qmi-olh-igw-${var.provision_id}"
+  }
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+resource "aws_subnet" "public" {
+  count                   = length(var.public_subnet_cidrs)
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = var.public_subnet_cidrs[count.index]
+  availability_zone       = data.aws_availability_zones.available.names[count.index]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "qmi-olh-subnet-${var.provision_id}-${count.index + 1}"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+}
+
+resource "aws_route_table_association" "public_assoc" {
+  count          = length(var.public_subnet_cidrs)
+  subnet_id      = aws_subnet.public[count.index].id
+  route_table_id = aws_route_table.public.id
+}
+
+
+resource "aws_security_group" "allow_tls" {
+  name        = "olh-${var.provision_id}-SG"
+  description = "olh-${var.provision_id}-SG"
+  vpc_id      = aws_vpc.main.id
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {  
+    security_group_id = aws_security_group.allow_tls.id
+
+    cidr_ipv4         = var.vpc_cidr
+    ip_protocol           = "tcp"
+    from_port          = 0
+    to_port            = 65535
+
+    description = "olh-${var.provision_id}-ingress-rule"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  security_group_id = aws_security_group.allow_tls.id
+  cidr_ipv4          = "0.0.0.0/0"
+  ip_protocol       = "-1"
+  description = "olh-${var.provision_id}-egress-rule"
+}

compose-install/scripts/compose-license.txt

@@ -1,14 +1,14 @@
 {
   "$type": "ComposeLicense",
   "product": "QlikCompose",
-  "issued_to": "Qlik Internal Use only",
+  "issued_to": "Qlik Internal",
   "issued_by": "Attunity US",
   "license_type": "EVALUATION",
-  "serial_no": "85010344",
-  "expiration_date": "2024-12-31",
+  "serial_no": "85011700",
+  "expiration_date": "2025-12-31",
   "hosts": "",
-  "product_version": "2023.11",
-  "notes": "Qlik Internal Use only",
+  "product_version": "2024.12",
+  "notes": "",
   "host_role": "",
   "source_db_types": "",
   "dwh_type": "*",
@@ -16,6 +16,6 @@
   "number_of_dms": "0",
   "number_of_developers": "0",
   "managed_dwh_size": "0",
-  "issue_time": "12/5/2023 9:36:23 PM",
-  "signature": "LnAeWQPSDsQw0quG6h5HxvAdnzerIbW/KwOCYvFeq6T8Tp8i6hxAgnKpeVeqTKNiA3t5ovkvIjBvH5n0JGjG75r4OohWiNLFaWYrSvgf6d/fPzFK/4tgyhdImmbS8dZm3tInY3Y8ZNM+kEGi8Hze2hPoHZGwh5nZMM0eCo2E4Sk="
+  "issue_time": "12/18/2024 8:31:25 PM",
+  "signature": "bMVg+X1Saxvcux5I96eU+oHndPWHv4E1QzxYwgo/cGpfVltneqGgBrVPmjotMGxp705aj29jGQLHNSx/0bBlwJd9DcfLAKaS/mH9mfph50KWRV1+ZLgPVQmmDHv/vFYLoIQLoyV2RqhMGN2rbVEXrQLzXRa+FWQEbulVa05Uvog="
 }

databases/aws-emr/main.tf

@@ -136,6 +136,8 @@ module "emr" {
   #iam_instance_profile_name = "AmazonEMR-InstanceProfile-20230622T122640"
 
   tags = {
+    Environment = "QMI-${var.provision_id}"
+    Deployment = "QMI-${var.provision_id}"
     Terraform   = "true"
     Environment = "dev"
     QMI_user = var.user_id

databases/aws-kinesis/main.tf

@@ -23,7 +23,8 @@ locals {
   scnamelower = "${local.container_n1}${local.container_n2}${local.container_n3}"
 
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     Owner    = var.user_id

databases/aws-rds/main.tf

@@ -36,7 +36,8 @@ locals {
   family               = var.family[var.engine]
 
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
@@ -49,96 +50,6 @@ locals {
 }
 
 
-module "fw-ips" {
-    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-}
-
-module "security_group_2" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG2"
-  description = "${var.provision_id}-SG2"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
-  
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
-module "security_group" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG"
-  description = "${var.provision_id}-SG"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-  
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "RDS"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
 module "common_rds_instance" {
   source  = "terraform-aws-modules/rds/aws"
   version = "= 6.1.1"
@@ -166,8 +77,8 @@ module "common_rds_instance" {
   multi_az               = false
   subnet_ids             = local.subnet_ids
   vpc_security_group_ids = [
-    module.security_group.security_group_id,
-    module.security_group_2.security_group_id
+    aws_security_group.allow_tls.id,
+    aws_security_group.allow_tls_2.id
   ]
   publicly_accessible    = true
 
@@ -265,8 +176,6 @@ module "aurora_rds_instance" {
   engine_version = local.engine_version
   instance_class  = var.instance_size
 
-  apply_immediately = true
-
   database_name = local.name
 
   instances       = { 1 = {} }
@@ -279,8 +188,8 @@ module "aurora_rds_instance" {
   subnets                = local.subnet_ids
   create_security_group  = false
   vpc_security_group_ids = [
-      module.security_group.security_group_id,
-      module.security_group_2.security_group_id
+      aws_security_group.allow_tls.id,
+      aws_security_group.allow_tls_2.id
   ]
   port                   = local.port
   publicly_accessible    = true

databases/aws-rds/output.tf

@@ -1,4 +1,4 @@
- output "db_username" {
+output "db_username" {
   description = "The master username for the database"
   value       = (local.aurora)? nonsensitive(module.aurora_rds_instance[0].cluster_master_username) : nonsensitive(module.common_rds_instance[0].db_instance_username)
 } 

databases/aws-rds/sec_groups.tf

@@ -0,0 +1,69 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "aws_security_group" "allow_tls" {
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {
+  for_each = toset(module.fw-ips.cidr_blocks)
+  
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  for_each = toset(module.fw-ips.cidr_blocks)
+
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "dbport"
+}
+
+resource "aws_security_group" "allow_tls_2" {
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4_2" {
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+  
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "Others - dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv_2" {
+
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "Others - dbport"
+}

databases/aws-rds/variables.tf

@@ -43,20 +43,24 @@ variable "instance_size" {
 }
 
 variable "storage" {
-  default = "20"
+  default = "100"
 }
 
 variable "engine_version" {
   type = map
 
   default = {
+<<<<<<< HEAD
     "mysql"  = "8.0.32"
-    "postgres"  = "14.9" 
+=======
+    "mysql"  = "8.0.36"
+>>>>>>> master
+    "postgres"  = "14.12" 
     "oracle-se2"  = "19.0.0.0.ru-2023-04.rur-2023-04.r1"
     "sqlserver-ex" = "15.00.4236.7.v1"
     "mariadb" = "10.5"
-    "aurora-mysql" = "8.0.mysql_aurora.3.05.2" #"5.7.mysql_aurora.2.11.2"
-    "aurora-postgresql"  = "14.9"
+    "aurora-mysql" = "8.0.mysql_aurora.3.02.3" #"5.7.mysql_aurora.2.11.2"
+    "aurora-postgresql"  = "14.5"
   }
   
 }

databases/aws-redshift/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.49.0"
+      version = ">= 6.0.0"
     }
   }
 }
@@ -30,7 +30,8 @@ locals {
   subnet_ids           = (var.region == "eu-west-1") ? var.subnet_ids_eu : (var.region == "us-east-1") ? var.subnet_ids_us : var.subnet_ids_ap
   port                 = "5439"
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     ProvID   = var.provision_id
@@ -39,139 +40,30 @@ locals {
   }
 }
 
-module "fw-ips" {
-    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-}
-
-module "security_group" {
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG"
-  description = "${var.provision_id}-SG"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks
-
-  
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
-module "security_group_2" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG2"
-  description = "${var.provision_id}-SG2"
-  vpc_id      = local.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
-  
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Redshift"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
 module "qmi-s3-bucket" {
 
   source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//s3-bucket"
   
   provision_id                  = var.provision_id
-  region                        = var.region
   user_id                       = var.user_id
 }
 
-module "redshift" {
 
-  source  = "terraform-aws-modules/redshift/aws"
-  version = "~> 5.0.0"
+resource "aws_redshift_cluster" "qmi" {
+  cluster_identifier = "qmi-${var.provision_id}"
+  database_name      = var.cluster_database_name
+  master_username    = var.cluster_master_username
+  master_password    = random_password.password.result
+  node_type          = "ra3.large"
+  cluster_type       = "single-node"
 
-  cluster_identifier      = "qmi-${var.provision_id}"
-  node_type       = "dc2.large" #"dc1.large"
-  number_of_nodes = 1
+  skip_final_snapshot = true
 
-  database_name   = var.cluster_database_name
-  master_username = var.cluster_master_username
-  create_random_password = false
-  master_password = random_password.password.result
+  vpc_security_group_ids = [aws_security_group.allow_tls.id, aws_security_group.allow_tls_2.id]
 
-  # Group parameters
-  #wlm_json_configuration = "[{\"query_concurrency\": 5}]"
-
-  # DB Subnet Group Inputs
-  subnet_ids = local.subnet_ids
-  vpc_security_group_ids = [
-      module.security_group.security_group_id,
-      module.security_group_2.security_group_id
-  ]
   publicly_accessible = true
 
-  /*logging = {
-    enable        = true
-    bucket_name   = module.qmi-s3-bucket.bucket.s3_bucket_id
-    s3_key_prefix = local.s3_prefix
-  }*/
-
-  # IAM Roles
-  #cluster_iam_roles = ["arn:aws:iam::225367859851:role/developer"]
-
   tags = local.tags
-
 }
 
+

databases/aws-redshift/outputs.tf

@@ -1,25 +1,20 @@
 output "redshift_cluster_id" {
   description = "The availability zone of the RDS instance"
-  value       = module.redshift.cluster_id
+  value       = aws_redshift_cluster.qmi.id
 }
 
 output "redshift_cluster_endpoint" {
   description = "Redshift endpoint"
-  value       = module.redshift.cluster_endpoint
-}
-
-output "redshift_cluster_hostname" {
-  description = "Redshift hostname"
-  value       = module.redshift.cluster_hostname
+  value       = aws_redshift_cluster.qmi.endpoint
 }
 
 output "redshift_cluster_port" {
   description = "Redshift port"
-  value       = module.redshift.cluster_port
+  value       = aws_redshift_cluster.qmi.port
 }
 
 output "redshift_cluster_database_name" {
-    value = module.redshift.cluster_database_name
+    value = aws_redshift_cluster.qmi.database_name
 }
 
 output "redshift_cluster_master_username" {

databases/aws-redshift/sec_groups.tf

@@ -0,0 +1,69 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "aws_security_group" "allow_tls" {
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {
+  for_each = toset(module.fw-ips.cidr_blocks)
+  
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  for_each = toset(module.fw-ips.cidr_blocks)
+
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "dbport"
+}
+
+resource "aws_security_group" "allow_tls_2" {
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = local.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4_2" {
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+  
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "Others - dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv_2" {
+
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "Others - dbport"
+}

databases/azure-eventhub/main.tf

@@ -22,7 +22,8 @@ resource "azurerm_eventhub_namespace" "ehbnamespace" {
   maximum_throughput_units = 2
 
   tags = {
-    Deployment = "QMI PoC"
+    Deployment = "QMI-${var.provision_id}"
+    Environment = "QMI-${var.provision_id}"
     "Cost Center" = "3100"
     QMI_user = var.user_id
     Owner    = var.user_id

databases/azure-hdinsight-hadoop/outputs.tf

@@ -30,7 +30,11 @@ output "Azure_Application_Registration_Client_ID" {
 }
 
 output "Azure_Application_Registration_Secret" {
+<<<<<<< HEAD
   value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+=======
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
+>>>>>>> master
 }
 
 
@@ -55,5 +59,9 @@ output "adls_Azure_Application_Registration_Client_ID" {
 }
 
 output "adls_Azure_Application_Registration_Secret" {
+<<<<<<< HEAD
   value = "~qp8Q~utl~YJ3skNM9kAuq25VY~rKxxOWpaVYcnQ"
+=======
+  value = "i3F8Q~FxhoyOP1-4r9sstaohnjxXaf~ulhVJFav_"
+>>>>>>> master
 }

databases/azure-rds-flexmysql/variables.tf

@@ -25,12 +25,16 @@ variable "admin_login" {
 variable "mysql-version" {
   type = string
   description = "MySQL Server version to deploy"
-  default = "5.7"
+  default = "8.0.21"
 }
 variable "mysql-sku-name" {
   type = string
   description = "MySQL SKU Name"
-  default = "GP_Standard_D2ds_v4"
+<<<<<<< HEAD
+  default = "GP_Standard_D4ds_v4"
+=======
+  default = "B_Standard_B2s"
+>>>>>>> master
 }
 variable "mysql-storage" {
   type = string

databases/azure-rds-flexpostgres/firewall.tf

@@ -1,28 +1,13 @@
-/*resource "azurerm_postgresql_flexible_server_firewall_rule" "all-azure-services" {
-  name                = "AllAzureServices"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "0.0.0.0"
-  end_ip_address      = "0.0.0.0"
-}*/
-
-
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule1" {
-  name                = "a1"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "52.249.189.38"
-  end_ip_address      = "52.249.189.38"
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
 }
 
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule2" {
-  name                = "a2"
-  server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "13.67.39.86"
-  end_ip_address      = "13.67.39.86"
-}
+resource "azurerm_postgresql_flexible_server_firewall_rule" "fw_rule" {
+  for_each = module.fw-ips.ips_az_qcs
 
-resource "azurerm_postgresql_flexible_server_firewall_rule" "fw-a-rule3" {
-  name                = "a3"
+  name                = each.key
   server_id = azurerm_postgresql_flexible_server.postgresql-server.id
-  start_ip_address    = "20.67.110.207"
-  end_ip_address      = "20.67.110.207"
+  start_ip_address    = each.value.0
+  end_ip_address      = each.value.1
+
 }

databases/azure-rds-flexpostgres/variables.tf

@@ -23,12 +23,12 @@ variable "admin_login" {
 variable "postgresql-version" {
   type        = string
   description = "PostgreSQL Server version to deploy"
-  default     = "12"
+  default     = "14"
 }
 variable "postgresql-sku-name" {
   type        = string
   description = "PostgreSQL SKU Name"
-  default     = "GP_Standard_D2s_v3"
+  default     = "B_Standard_B2s"
 }
 variable "postgresql-storage" {
   type        = string

databases/dbricks/main.tf

@@ -5,10 +5,8 @@ terraform {
   required_providers {
     databricks = {
       source = "databricks/databricks"
-      version = ">= 1.47.0"
-      configuration_aliases = [ databricks.main, databricks.accounts ]
+      version = ">= 1.13.0"
     }
-    
   }
 }
 
@@ -17,82 +15,13 @@ locals {
   cluster_nameUpper = upper(var.cluster_name)
 }
 
-locals {
-  resource_regex            = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)"
-  subscription_id           = regex(local.resource_regex, var.databricks_resource_id)[0]
-  resource_group            = regex(local.resource_regex, var.databricks_resource_id)[1]
-  databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2]
-  tenant_id                 = data.azurerm_client_config.current.tenant_id
-  databricks_workspace_host = data.azurerm_databricks_workspace.this.workspace_url
-  databricks_workspace_id   = data.azurerm_databricks_workspace.this.workspace_id
-  prefix                    = replace(replace(lower(data.azurerm_resource_group.this.name), "rg", ""), "-", "")
-}
-
-data "azurerm_resource_group" "this" {
-  name = local.resource_group
-}
-
-data "azurerm_client_config" "current" {
-}
-
-data "azurerm_databricks_workspace" "this" {
-  name                = local.databricks_workspace_name
-  resource_group_name = local.resource_group
-}
-
-resource "databricks_metastore" "this" {
-  provider      = databricks.accounts
-  name          = "primary"
-  force_destroy = true
-  region        = data.azurerm_resource_group.this.location
-}
-
-resource "databricks_metastore_assignment" "this" {
-  provider             = databricks.accounts
-  workspace_id         = local.databricks_workspace_id
-  metastore_id         = databricks_metastore.this.id
-  default_catalog_name = "hive_metastore"
-}
-
 /*resource "databricks_workspace_conf" "this" {
   
   custom_config = {
     "enableIpAccessLists" : true
   }
 }
-
-resource "databricks_ip_access_list" "allowed-list" {
-
-  label     = "allow_in"
-  list_type = "ALLOW"
-  ip_addresses = [
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
-    "14.98.59.168/29", 
-    "182.74.33.8/29", 
-    "188.65.156.32/28", 
-    "212.73.252.96/29", 
-    "194.90.96.176/29", 
-    "213.57.84.160/29", 
-    "4.4.97.104/29", 
-    "206.196.17.32/27", 
-    #QCS
-    "18.205.71.36/32", 
-    "18.232.32.199/32", 
-    "34.237.68.254/32", 
-    "34.247.21.179/32", 
-    "52.31.212.214/32", 
-    "54.154.95.18/32", 
-    "13.210.43.241/32", 
-    "13.236.104.42/32", 
-    "13.236.206.172/32",
-    "18.138.163.172/32",
-    "18.142.157.182/32",
-    "54.179.13.251/32"
-  ]
-
-}*/
+*/
 
 
 data "databricks_group" "admins" {
@@ -101,7 +30,6 @@ data "databricks_group" "admins" {
 
 
 resource "databricks_user" "aor" {
-  provider      = databricks.main
   count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
   
   allow_cluster_create = true
@@ -112,7 +40,7 @@ resource "databricks_user" "aor" {
 }
 
 resource "databricks_user" "me" {
-  provider      = databricks.main
+
   allow_cluster_create = true
   databricks_sql_access = true
 
@@ -122,27 +50,25 @@ resource "databricks_user" "me" {
 
 
 resource "databricks_group_member" "i-am-admin" {
-  provider      = databricks.main
   group_id  = data.databricks_group.admins.id
   member_id = databricks_user.me.id
 }
 
 
 resource "databricks_group_member" "i-am-admin-aor" {
-  provider      = databricks.main
   count = lower(var.user_email) != "aor@qlik.com"? 1 : 0
   group_id  = data.databricks_group.admins.id
   member_id = databricks_user.aor[0].id
 }
 
 resource "databricks_secret_scope" "scopeapp" {
-    provider      = databricks.main
+
     count = (var.sku == "premium")? 1 : 0
 
     name = "${var.cluster_name}scope"
 }
 resource "databricks_secret" "publishing_api" {
-    provider      = databricks.main
+
     count = (var.sku == "premium")? 1 : 0
 
     key = "accesskey"
@@ -151,7 +77,7 @@ resource "databricks_secret" "publishing_api" {
 }
 
 resource "databricks_sql_endpoint" "sqlep" {
-  provider      = databricks.main
+
   count = (var.sku == "premium")? 1 : 0
 
   name             = "${local.cluster_nameUpper} SQL warehouse"
@@ -161,7 +87,7 @@ resource "databricks_sql_endpoint" "sqlep" {
 }
 
 resource "databricks_sql_global_config" "this" {
-  provider      = databricks.main
+
   count = (var.sku == "premium")? 1 : 0
 
   security_policy      = "DATA_ACCESS_CONTROL"
@@ -180,7 +106,6 @@ resource "databricks_sql_global_config" "this" {
 
 
 resource "databricks_token" "pat" {  
-  provider      = databricks.main
   depends_on = [
     databricks_user.me,
     databricks_user.aor
@@ -190,7 +115,6 @@ resource "databricks_token" "pat" {
 }
 
 resource "databricks_cluster" "dbrickscluster" {
-  provider      = databricks.main
   count = (var.cluster_name != null)? 1 : 0
 
 

databases/dbricks/varaibles.tf

@@ -36,7 +36,4 @@ variable "app_reg_secret" {
 
 variable "cluster_size" {
   default = "Small"
-}
-
-variable "databricks_resource_id" {
 }

databases/firewall_ips/output.tf

@@ -1,14 +1,18 @@
 locals {
 
   az = {
-      az1          = ["52.249.189.38","52.249.189.38"]
-      az2          = ["13.67.39.86", "13.67.39.86"]
-      az3          = ["20.67.110.207", "20.67.110.207"]
+      #az1          = ["52.249.189.38","52.249.189.38"] # OLD USA
+      #az2          = ["13.67.39.86", "13.67.39.86"]    # OLD APAC
+      #az3          = ["20.67.110.207", "20.67.110.207"] # OLD EUROPE
+      azeurope          = ["20.160.170.99", "20.160.170.99"] # NEW EUROPE
+      azusa          = ["20.169.241.157", "20.169.241.157"] # NEW USA
+      azapac          = ["52.163.112.12", "52.163.112.12"] # NEW APAC
   }
 
   q_routes = {
     ITG         = ["188.65.156.32", "188.65.156.47"]
     ITG1        = ["212.73.252.96", "212.73.252.103"]
+    USDC4       = ["155.204.23.130","155.204.23.130"]
     kfar_saba   = ["194.90.96.176", "194.90.96.183"]
     kfar_saba1  = ["213.57.84.160", "213.57.84.167"]
     peak_10     = ["4.4.97.104", "4.4.97.111"]
@@ -27,10 +31,6 @@ locals {
     T_SV2_3     = ["149.97.160.217","149.97.160.218"]
     T_SV2_4     = ["149.97.160.221","149.97.160.222"]
     T_SV2_5     = ["149.97.185.1","149.97.185.30"]
-    T_Blr_1     = ["14.97.166.193","14.97.166.198"]
-    T_Blr_2     = ["14.97.167.21","14.97.167.22"]
-    T_Blr_3     = ["122.185.123.61","122.185.123.62"]
-    T_Blr_4     = ["182.79.10.25","182.79.10.26"]
   }
 
   fullvpn = {
@@ -58,6 +58,13 @@ locals {
       QCS_uk1          = ["13.42.141.246", "13.42.141.246"]
       QCS_uk2          = ["18.135.245.97", "18.135.245.97"]
       QCS_uk3          = ["35.179.0.171", "35.179.0.171"]
+      QCS_jp1          = ["54.238.168.131", "54.238.168.131"]
+      QCS_jp2          = ["3.113.68.20", "3.113.68.20"]
+      QCS_jp3          = ["35.73.207.58", "35.73.207.58"]
+      QCS_in1          = ["52.66.18.8", "52.66.18.8"]
+      QCS_in2          = ["13.201.250.59", "13.201.250.59"]
+      QCS_in3          = ["15.207.252.34", "15.207.252.34"]
+
   }
 
   qcs_qaa = {
@@ -79,6 +86,13 @@ locals {
       QCS_QAA_uk1     = ["18.168.120.199", "18.168.120.199"]
       QCS_QAA_uk2     = ["18.169.91.38", "18.169.91.38"]
       QCS_QAA_uk3     = ["13.42.135.168", "13.42.135.168"]
+      QCS_QAA_jp1     = ["35.77.238.13", "35.77.238.13"]
+      QCS_QAA_jp2     = ["13.115.58.233", "13.115.58.233"]
+      QCS_QAA_jp3     = ["35.74.220.230", "35.74.220.230"]
+      QCS_QAA_in1     = ["3.109.34.226", "3.109.34.226"]
+      QCS_QAA_in2     = ["15.206.64.196", "15.206.64.196"]
+      QCS_QAA_in3     = ["3.6.11.209", "3.6.11.209"]
+    
   }
 
   stitch = {
@@ -114,10 +128,14 @@ output "cidr_blocks" {
     "193.15.228.246/32",
     "50.239.179.6/32",
     # Azure QMI machines
-    "52.249.189.38/32", 
-    "13.67.39.86/32", 
-    "20.67.110.207/32", 
+    #"52.249.189.38/32", # OLD USA
+    #"13.67.39.86/32",   # OLD APAC
+    #"20.67.110.207/32", # OLD EUROPE
+    "20.160.170.99/32", # NEW EUROPE
+    "20.169.241.157/32", # NEW USA
+    "52.163.112.12/32", # NEW APAC
     # Qlik network routers
+    "155.204.23.130/32",
     "14.98.59.168/29", 
     "182.74.33.8/29", 
     "188.65.156.32/28", 
@@ -145,11 +163,6 @@ output "cidr_blocks" {
     "149.97.160.216/30",
     "149.97.160.220/30",
     "149.97.185.0/27",
-    # T Blr Office
-    "14.97.166.192/29",
-    "14.97.167.20/30",
-    "122.185.123.60/30",
-    "182.79.10.24/29",
     # QCS Amercias (us)
     "18.205.71.36/32", 
     "18.232.32.199/32", 
@@ -170,6 +183,18 @@ output "cidr_blocks" {
     "3.76.162.169/32",
     "3.77.162.68/32",
     "3.122.137.91/32",
+    # QCS Sweden (se)
+    "13.51.129.105/32",
+    "16.170.33.251/32",
+    "16.170.27.83/32",
+    # QCS Japan (jp)
+    "54.238.168.131/32",
+    "3.113.68.20/32",
+    "35.73.207.58/32",
+    # QCS India (in)
+    "52.66.18.8/32", 
+    "13.201.250.59/32",
+    "15.207.252.34/32",
     # QCS London (uk)
     "13.42.141.246/32",
     "18.135.245.97/32",
@@ -198,6 +223,18 @@ output "cidr_blocks" {
     "18.168.120.199/32",
     "18.169.91.38/32",
     "13.42.135.168/32",
+    # QCS-QAA Japan (jp)
+    "35.77.238.13/32",
+    "13.115.58.233/32",
+    "35.74.220.230/32",
+    # QCS-QAA India (in)
+    "3.109.34.226/32",
+    "15.206.64.196/32",
+    "3.6.11.209/32",
+    # QCS-QAA Sweden (se)
+    "13.53.211.145/32",
+    "13.50.63.235/32",
+    "13.60.101.12/32",
     # QCS STAGING
     "18.233.22.130/32",
     "18.205.135.40/32",
@@ -206,6 +243,8 @@ output "cidr_blocks" {
     "18.155.181.25/32",
     "18.155.181.128/32",
     "18.155.181.76/32",
+    "52.16.133.167/32",
+    "34.248.105.199/32", # europe
   ]
 }
 
@@ -221,4 +260,4 @@ output "cidr_blocks_others" {
     "18.158.251.55/32",
     "52.57.235.168/32",
   ]
-}
+}

databases/synapse-ws/firewall.tf

@@ -6,16 +6,18 @@ resource "azurerm_synapse_firewall_rule" "azureservices" {
   end_ip_address       = "0.0.0.0"
 }
 
-
+/*
+# OLD USA
 resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
 
-  name                = "az1"
+  name                = "az1" 
   synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
   start_ip_address    = "52.249.189.38"
   end_ip_address      = "52.249.189.38"
 
 }
 
+# OLD APAC
 resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
 
   name                = "az2"
@@ -25,6 +27,7 @@ resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
 
 }
 
+# OLD EUROPE
 resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
 
   name                = "az3"
@@ -32,6 +35,35 @@ resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
   start_ip_address    = "20.67.110.207"
   end_ip_address      = "20.67.110.207"
 
+}
+*/
+
+# NEW EUROPE
+resource "azurerm_synapse_firewall_rule" "fw-a-rule1" {
+
+  name                = "azeurope"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "20.160.170.99"
+  end_ip_address      = "20.160.170.99"
+
+}
+
+# NEW USA
+resource "azurerm_synapse_firewall_rule" "fw-a-rule2" {
+  name                = "azusa"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "20.169.241.157"
+  end_ip_address      = "20.169.241.157"
+
+}
+
+# NEW USA
+resource "azurerm_synapse_firewall_rule" "fw-a-rule3" {
+  name                = "azapac"
+  synapse_workspace_id = azurerm_synapse_workspace.synapsews.id
+  start_ip_address    = "52.163.112.12"
+  end_ip_address      = "52.163.112.12"
+
 }
 
 module "fw-ips" {

linux-common/centos/common/extract-certs.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
 
 BASEDIR=$(dirname "$0")
 
 mkdir -p $BASEDIR/qmicerts
 wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
-wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
-
 
 PASSWORD=$1
 # Extract the private key
@@ -14,7 +13,7 @@ openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nod
 # Extract the public key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
 # Extract the CA cert chain
-#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
 
 cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
 

linux-common/centos/common/falcon.sh

@@ -16,7 +16,7 @@ echo "CID=$1"
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$binary -O $BASEDIR/$binary
 
 if ! rpm -qa | grep -qw falcon-sensor; then
-    sudo yum -y --quiet install $BASEDIR/$binary
+    sudo dnf -y --quiet install $BASEDIR/$binary
 fi
 sudo /opt/CrowdStrike/falconctl -s -f --cid=$1
 sudo systemctl start falcon-sensor 

linux-common/centos/common/resizedisk.sh

@@ -2,6 +2,7 @@
 
 echo "--- Executing: $0 $@"
 
+<<<<<<< HEAD
 echo "Resizing main partition to max available disk"
 (
 echo u # Change the units to sectors
@@ -18,6 +19,34 @@ echo w # Write changes
 
 sudo partprobe
 sudo xfs_growfs -d /
+=======
+if  grep -Pq '/dev/(mapper/|disk/by-id/dm)' /etc/fstab  ||  mount | grep -q /dev/mapper/
+then
+    echo "LVM is in use (Oracle Linux)"
+    #work out what disk we need.  About 1 in 5 times we get sdb instead on sda
+    rootdisk=$(df --type=xfs|grep \/dev\/sd | sed -e's/[0-9].*//')
+    sudo gdisk -l $rootdisk
+    sudo growpart $rootdisk 2
+    sudo pvresize $rootdisk"2"
+    sudo lvextend -l +49%FREE /dev/rootvg/rootlv
+    sudo lvextend -l +100%FREE /dev/mapper/rootvg-crashlv
+    sudo xfs_growfs /dev/rootvg/rootlv
+    sudo xfs_growfs /dev/mapper/rootvg-crashlv
+else
+    echo "LVM not in use.  Resizing main partition to max available disk"
+    (
+    echo u # Change the units to sectors
+    echo p # List the partitions details
+    echo d # Delete  partition
+    echo 2 # Partition number
+    echo n # Add a new partition
+    echo p # Primary partition
+    echo 2 # Partition number
+    echo   # First sector (Accept default)
+    echo   # Last sector (Accept default: varies)
+    echo w # Write changes
+    ) | sudo fdisk /dev/sda
+>>>>>>> master
 
 echo "Done resize!"
 echo ""

linux-common/centos/common/save-cert.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
+
 BASEDIR=$(dirname "$0")
 
 echo $1 > $BASEDIR/qmicerts/myserver.crt

linux-common/centos/common/tenable.sh

@@ -9,13 +9,18 @@ echo "KEY=$KEY"
 
 cVer=`rpm -E %{rhel}`
 
-FILE="NessusAgent-10.2.1-es$cVer.x86_64.rpm"
+FILE="NessusAgent-10.8.2-el$cVer.x86_64.rpm"
 
 echo "--- Installing Tenable Nessus Agent --> $FILE"
 
 wget --quiet https://d7ipctdjxxii4.cloudfront.net/others/$FILE -O $BASEDIR/$FILE
 
-sudo /bin/yum -y --quiet install $BASEDIR/$FILE -y
+
+sudo yum-config-manager --disable mysql80-community > /dev/null 2>&1
+sudo yum-config-manager --disable mysql-connectors-community > /dev/null 2>&1
+sudo yum-config-manager --disable mysql-tools-community > /dev/null 2>&1
+
+sudo dnf -y --quiet install $BASEDIR/$FILE -y
 
 echo "--- Linking Tenable Nessus Agent..."
 sudo /bin/systemctl start nessusagent.service

linux-common/centos/common/update.sh

@@ -1,2 +1,15 @@
-echo 'Updating OS. This will take a around 10 minutes'
-yum -y --quiet update
+#!/bin/bash
+
+
+cVer=`rpm -E %{rhel}`
+if [ -z "$cVer" ]; then
+    echo "Error: Unable to determine CentOS version."
+    exit 0
+fi
+if [ "$cVer" -gt 7 ]; then
+    echo "--- Executing: $0 $@"
+
+    echo 'Updating OS. This will take a around 10 minutes'
+
+    dnf -y --quiet update
+fi

linux-common/main.tf

@@ -34,7 +34,7 @@ resource "null_resource" "files" {
             host     = var.private_ip_address
             user     = var.admin_username
             password = var.admin_password
-            timeout  = "60s"
+            timeout  = "10m"
             #private_key = "${file("~/.ssh/id_rsa")}"   
         }
         source      = "${path.module}/${var.os_type}/common"
@@ -54,14 +54,13 @@ resource "null_resource" "post-linux-vm" {
             host     = var.private_ip_address
             user     = var.admin_username
             password = var.admin_password
-            timeout  = "60s"
+            timeout  = "3m"
             #private_key = "${file("~/.ssh/id_rsa")}"
         }
 
         inline = [
             "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
             "sudo /home/${var.admin_username}/common/falcon.sh '${local.falcon_id}'",
-            "sudo /home/${var.admin_username}/common/resizedisk.sh",
             "sudo /home/${var.admin_username}/common/tenable.sh '${local.tenable_key}'",
             "sudo /home/${var.admin_username}/common/extract-certs.sh '${local.cert_password}'",
             #"sudo /home/${var.admin_username}/common/save-cert.sh '${local.cert_pem}' '${local.cert_key}'",
@@ -84,7 +83,7 @@ resource "null_resource" "update" {
             host     = var.private_ip_address
             user     = var.admin_username
             password = var.admin_password
-            timeout  = "60s"
+            timeout  = "3m"
             #private_key = "${file("~/.ssh/id_rsa")}"
         }
 
@@ -95,3 +94,29 @@ resource "null_resource" "update" {
   }
 }
 
+resource "null_resource" "resize" {
+
+  count = var.resize? 1 : 0
+
+  depends_on = [
+      null_resource.files,
+      null_resource.update
+  ]
+
+  provisioner "remote-exec" {
+        connection {
+            type     = "ssh"
+            host     = var.private_ip_address
+            user     = var.admin_username
+            password = var.admin_password
+            timeout  = "3m"
+            #private_key = "${file("~/.ssh/id_rsa")}"
+        }
+
+        inline = [
+            "echo ${var.admin_password} | sudo -S chmod u+x /home/${var.admin_username}/common/*.sh",
+            "sudo /home/${var.admin_username}/common/resizedisk.sh",
+        ]
+  }
+}
+

linux-common/ubuntu/common/extract-certs.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 
+echo "--- Executing: $0 $@"
 
 BASEDIR=$(dirname "$0")
 
 mkdir -p $BASEDIR/qmicerts
-wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-full.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
-wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt
-wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/star_qmi_qlik-poc_com.pem -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pem
+wget --quiet https://gitlab.com/qmi/qmi-cloud-tf-modules/-/raw/master/qmicerts/wildcard_qmi_qlik-poc_com.pfx -O $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx
 
 PASSWORD=$1
 # Extract the private key
@@ -14,7 +13,7 @@ openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -nocerts -nod
 # Extract the public key
 openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -clcerts -nokeys -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt -passin pass:$PASSWORD
 # Extract the CA cert chain
-#openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
+openssl pkcs12 -in $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.pfx -cacerts -nokeys -chain -out $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt -passin pass:$PASSWORD
 
 cat $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com.crt $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-ca.crt >> $BASEDIR/qmicerts/wildcard_qmi_qlik-poc_com-fullchain.crt
 

linux-common/ubuntu/common/tenable.sh

@@ -7,7 +7,7 @@ BASEDIR=$(dirname "$0")
 KEY=$1
 echo "KEY=$KEY"
 
-FILE="NessusAgent-10.2.1-ubuntu1404_amd64.deb"
+FILE="NessusAgent-10.8.2-ubuntu1604_amd64.deb"
 
 echo "--- Installing Tenable Nessus Agent --> $FILE"
 

linux-common/ubuntu/common/update.sh

@@ -1,3 +1,7 @@
+#!/bin/bash
+
+echo "--- Executing: $0 $@"
+
 echo 'Updating OS. This will take a around 10 minutes'
 apt -qq -y update
 apt --fix-broken -qq -y upgrade

linux-common/variables.tf

@@ -21,3 +21,8 @@ variable "update" {
   default = true
 }
 
+variable "resize" {
+  type = bool
+  default = true
+}
+

qmicerts/wildcard_qmi_qlik-poc_com-ca.crt

@@ -1,57 +0,0 @@
-Bag Attributes
-    friendlyName: DigiCert Global Root G2
-subject=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
-issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
------BEGIN CERTIFICATE-----
-MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
-MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
-d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
-MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
-MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
-b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
-9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
-2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
-1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
-q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
-tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
-vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
-BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
-5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
-1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
-NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
-Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
-8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
-pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
-MrY=
------END CERTIFICATE-----
-Bag Attributes: <Empty Attributes>
-subject=/C=US/O=DigiCert Inc/CN=DigiCert Global G2 TLS RSA SHA256 2020 CA1
-issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert Global Root G2
------BEGIN CERTIFICATE-----
-MIIEyDCCA7CgAwIBAgIQDPW9BitWAvR6uFAsI8zwZjANBgkqhkiG9w0BAQsFADBh
-MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
-d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
-MjAeFw0yMTAzMzAwMDAwMDBaFw0zMTAzMjkyMzU5NTlaMFkxCzAJBgNVBAYTAlVT
-MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxMzAxBgNVBAMTKkRpZ2lDZXJ0IEdsb2Jh
-bCBHMiBUTFMgUlNBIFNIQTI1NiAyMDIwIENBMTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAMz3EGJPprtjb+2QUlbFbSd7ehJWivH0+dbn4Y+9lavyYEEV
-cNsSAPonCrVXOFt9slGTcZUOakGUWzUb+nv6u8W+JDD+Vu/E832X4xT1FE3LpxDy
-FuqrIvAxIhFhaZAmunjZlx/jfWardUSVc8is/+9dCopZQ+GssjoP80j812s3wWPc
-3kbW20X+fSP9kOhRBx5Ro1/tSUZUfyyIxfQTnJcVPAPooTncaQwywa8WV0yUR0J8
-osicfebUTVSvQpmowQTCd5zWSOTOEeAqgJnwQ3DPP3Zr0UxJqyRewg2C/Uaoq2yT
-zGJSQnWS+Jr6Xl6ysGHlHx+5fwmY6D36g39HaaECAwEAAaOCAYIwggF+MBIGA1Ud
-EwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHSFgMBmx9833s+9KTeqAx2+7c0XMB8G
-A1UdIwQYMBaAFE4iVCAYlebjbuYP+vq5Eu0GF485MA4GA1UdDwEB/wQEAwIBhjAd
-BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdgYIKwYBBQUHAQEEajBoMCQG
-CCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQAYIKwYBBQUHMAKG
-NGh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RH
-Mi5jcnQwQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29t
-L0RpZ2lDZXJ0R2xvYmFsUm9vdEcyLmNybDA9BgNVHSAENjA0MAsGCWCGSAGG/WwC
-ATAHBgVngQwBATAIBgZngQwBAgEwCAYGZ4EMAQICMAgGBmeBDAECAzANBgkqhkiG
-9w0BAQsFAAOCAQEAkPFwyyiXaZd8dP3A+iZ7U6utzWX9upwGnIrXWkOH7U1MVl+t
-wcW1BSAuWdH/SvWgKtiwla3JLko716f2b4gp/DA/JIS7w7d7kwcsr4drdjPtAFVS
-slme5LnQ89/nD/7d+MS5EHKBCQRfz5eeLjJ1js+aWNJXMX43AYGyZm0pGrFmCW3R
-bpD0ufovARTFXFZkAdl9h6g4U5+LXUZtXMYnhIHUfoyMo5tS58aI7Dd8KvvwVVo4
-chDYABPPTHPbqjc1qCmBaZx2vN4Ye5DUys/vZwP9BFohFrH/6j/f3IL16/RZkiMN
-JCqVJUzKoZHm1Lesh3Sz8W2jmdv51b2EQJ8HmA==
------END CERTIFICATE-----

qmicerts/wildcard_qmi_qlik-poc_com.pem

@@ -1,18 +1,18 @@
 -----BEGIN CERTIFICATE-----
-MIIG4DCCBcigAwIBAgIQAxl66Ob3+2PBk9UFiseUtDANBgkqhkiG9w0BAQsFADBZ
+MIIG4TCCBcmgAwIBAgIQAxAE3FY3y74JI7dXp7IdNjANBgkqhkiG9w0BAQsFADBZ
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTMwMQYDVQQDEypE
-aWdpQ2VydCBHbG9iYWwgRzIgVExTIFJTQSBTSEEyNTYgMjAyMCBDQTEwHhcNMjQw
-MzI5MDAwMDAwWhcNMjUwNDAzMjM1OTU5WjBdMQswCQYDVQQGEwJTRTENMAsGA1UE
+aWdpQ2VydCBHbG9iYWwgRzIgVExTIFJTQSBTSEEyNTYgMjAyMCBDQTEwHhcNMjUw
+NDA3MDAwMDAwWhcNMjYwNDA3MjM1OTU5WjBdMQswCQYDVQQGEwJTRTENMAsGA1UE
 BxMETHVuZDEiMCAGA1UEChMZUWxpa1RlY2ggSW50ZXJuYXRpb25hbCBBQjEbMBkG
 A1UEAwwSKi5xbWkucWxpay1wb2MuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
-MIIBCgKCAQEAy6c9dY5juLMMjCMrKxITmTRFDwAS8igaRo0DZQl3ga8sy2BT0+pv
-CfisX95utappgaOH5EbvjN7yok6YvXQKL9rPz3jFrtv8eAkeUr8zr8DZjIl2OvAw
-5n/gCEhVfJUyKwWjowVa0K2m0Dc07NSt7ZuSmP2ubAC+qP8sirg6JjacEB1PNjRt
-xUSns9EIO8wRfPgRYPeZzjpqGTFMhdM2xQfOrGlB8PTnyGG5UfiLVrD6xN+rxF5u
-jCOIHlsbn3lXLMGOqNwmwO58leXJm6CVnG1O/94KqNnc8YxELYs0d2HcJS1V03rV
-tYWyuepT3xd2pLy6H7GqDgQsomEyGarxbQIDAQABo4IDnjCCA5owHwYDVR0jBBgw
-FoAUdIWAwGbH3zfez70pN6oDHb7tzRcwHQYDVR0OBBYEFA9gOoFIR0J2G2yxCySa
-bP0fiWPAMC8GA1UdEQQoMCaCEioucW1pLnFsaWstcG9jLmNvbYIQcW1pLnFsaWst
+MIIBCgKCAQEAoZ8Fts4HoStfVxezUqPBPtZz9CMLHRFDd1OILtM7Fn23L13Ks+a9
+uL6wnvp1anV4z4Hx5pAEo2u5/nquLcX+raU5I7AnoOdKcyeI8pJYtTjKPqQbMX1t
+Ngbzk8Y0440sAG2FtXkqH1XLL79e/YYZ5s2QK4ueJn7Xwp1Avo1pgy9/SU+1fTHa
+Udlj7mxCyVymoOucoikqRHsV5VkTzgwnL3MAvOO6G7XMWVOpRZI++yJvEjkq/B9b
+SCVH2lp4mbYdKAtovusM217sVT8Gm7ap/AWuYvJcdqIKCsNvSqu3ZyJr1xUpf3O/
+toGzH9hHiOGsdpzSInbxVNW7Z+i4FW6rKQIDAQABo4IDnzCCA5swHwYDVR0jBBgw
+FoAUdIWAwGbH3zfez70pN6oDHb7tzRcwHQYDVR0OBBYEFCy+ODAFw4YErvNnnnW6
++X1Jo0CGMC8GA1UdEQQoMCaCEioucW1pLnFsaWstcG9jLmNvbYIQcW1pLnFsaWst
 cG9jLmNvbTA+BgNVHSAENzA1MDMGBmeBDAECAjApMCcGCCsGAQUFBwIBFhtodHRw
 Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW
 MBQGCCsGAQUFBwMBBggrBgEFBQcDAjCBnwYDVR0fBIGXMIGUMEigRqBEhkJodHRw
@@ -22,20 +22,20 @@ aWdpQ2VydEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNybDCBhwYIKwYB
 BQUHAQEEezB5MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20w
 UQYIKwYBBQUHMAKGRWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2Vy
 dEdsb2JhbEcyVExTUlNBU0hBMjU2MjAyMENBMS0xLmNydDAMBgNVHRMBAf8EAjAA
-MIIBfAYKKwYBBAHWeQIEAgSCAWwEggFoAWYAdgDPEVbu1S58r/OHW9lpLpvpGnFn
-SrAX7KwB0lt3zsw7CAAAAY6LXgYuAAAEAwBHMEUCIQD4iDMCfzcat85RUyRrU535
-2bhUWcjhj8N7k74Z0gMNzwIgCjWhfTgY5d8sOcGb/TharJfYzmQrlS8kMp7SEw1o
-aO0AdQB9WR4S4XgqexxhZ3xe/fjQh1wUoE6VnrkDL9kOjC55uAAAAY6LXgX2AAAE
-AwBGMEQCIDIzZ2lvtqDASkCmNWGNuucy7cxZyPHutwR+W3ONu2M7AiBpk4k2E8Xh
-eaBDsp8SDPxgRIqvfVxg7Gc861A/HKJk6AB1AObSMWNAd4zBEEEG13G5zsHSQPaW
-hIb7uocyHf0eN45QAAABjoteBh8AAAQDAEYwRAIgRIsJ9bny4B86Wi2bgfsFDSQ0
-XnZl0dowBMUQN+06C/8CIGFrEWk0x6GDbjPkFjhy2C0OuaxqxhJ+UFhnMaanXU6b
-MA0GCSqGSIb3DQEBCwUAA4IBAQAydZmIYplxaTFpJl4yXXRd5+SaW5PtM6Y8l+dF
-x8HuENuvLNB4e2Ebl3vE8HqcXF10v2vQ8cwldW85Qov5SSNgXLM4NN4VcLCDub0e
-ArQXzs7OK/uVx+Qd0tIBMD1Ruv/Y8qTOWRlrvfoIduPgS+zjkc1Q0ACex9mCdm3l
-yC0lqB+wtOeCwkZU9yRP4sMZsRiGAtVm2G+fTlQ5/GDxaBI/fOgUHHQZSDICRWvI
-FPqTAgyk+xqPRliTlM9XM22bIxTCtBJceKD3PNwMC2KAhvLV1gP3hrPdMWH0TooD
-uW6bllK/309bMIB5YNlycgBDebddeA4aXWDLP6ldkv72kBo7
+MIIBfQYKKwYBBAHWeQIEAgSCAW0EggFpAWcAdQCWl2S/VViXrfdDh2g3CEJ36fA6
+1fak8zZuRqQ/D8qpxgAAAZYQZtz0AAAEAwBGMEQCIA9bkdnKRWyHdJKovDloGgVC
+PEayqXugtDH9ElCPjxz9AiAsK5eEF6LhU7NgxJ+kl/7VCObUOSx2BLX2KfkxBdId
+qgB2AGQRxGykEuyniRyiAi4AvKtPKAfUHjUnq+r+1QPJfc3wAAABlhBm3P0AAAQD
+AEcwRQIhAPpECY9PYjhxBITyYjFNl6gUd/94CBRd70bSqajIDEh8AiBcuBXsgM4n
+eW4OknJ/6Vp5cOstxg5+WTdbjZt3o5W4/AB2AEmcm2neHXzs/DbezYdkprhbrwqH
+gBnRVVL76esp3fjDAAABlhBm3QgAAAQDAEcwRQIgMxJ/w1C7eItarCkVKjYSs7jE
+6TVUfBohNMYsYceha5ICIQCNZGVKxi/sQuciyqyhG6AuSd8JbmSX80o+feNhSaRu
+GzANBgkqhkiG9w0BAQsFAAOCAQEAd4Os9L/h/mDXrElwbPBxY+ILysk2T4hzRrye
+yYvdGlmqB+I84bac5+eDA6HvmQX1rjNEzdybQuzdjLoUIvZN1dYeoUf66+tW+xhR
+mCZ6DWU1QOKabJlwwyxIA8wmfE9YpTxMt1SCHfRtFBazyir2nL7wAWGWiMPLc8ZX
+H5N2uJrt+jH4cLN+MnLWgvY5CsC2g3IimKfqRNLrs3L1ug2vIgzQVbfBm+5OkoCx
+JVZJ8WqoF9oPd8Kl0xWQwocjwzhhYtJ6MffmrkyFs7098OXgpYZMFnoveA/85F+b
+bQ74NzbCsXXuevhszg7tgtPyGEE4VpOA4ileC4sr0BVNgJcA9g==
 -----END CERTIFICATE-----
 -----BEGIN CERTIFICATE-----
 MIIEyDCCA7CgAwIBAgIQDPW9BitWAvR6uFAsI8zwZjANBgkqhkiG9w0BAQsFADBh

replicate-install/scripts/replicate_license.txt

@@ -1,17 +1,17 @@
 #
 # Qlik  License
-# Generated on 05-Dec-2023 21:36:16.5775+02:00 
-# License Comment: Qlik Internal Use only
+# Generated on 18-Dec-2024 20:31:17.8585+02:00 
+# License Comment: 
 #
 license_type=EVALUATION_LICENSE
-licensed_to=Qlik Internal Use only
+licensed_to=Qlik Internal
 licensed_by=Attunity US
-serial_no=60036369
-expiration_date=2024-12-31
+serial_no=60040755
+expiration_date=2025-12-31
 hosts=
 source_types=
 target_types=
 features=
-version=2023.11
-issue_date=2023-12-05
-checksum=3QHR6-5F3JE-6JCF4-9R63R
+version=2024.11
+issue_date=2024-12-18
+checksum=839RC-2FK3R-3CC7C-CK26H

s3-bucket-sftp-public/main.tf

@@ -6,7 +6,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "= 3.74.1"
+      version = ">= 6.0.0"
     }
   }
 }
@@ -63,28 +63,26 @@ resource "aws_secretsmanager_secret_version" "private_key" {
 }
 
 
-module "s3_bucket" {
-  source = "terraform-aws-modules/s3-bucket/aws"
-
-  version = "~> 2.1.0"
-
+resource "aws_s3_bucket" "s3_bucket" {
   bucket = "sftp-${var.provision_id}"
-  acl    = "private"
-
-  versioning = {
-    enabled = false
-  }
-
-  force_destroy = true
 
   tags = local.tags
 
+  force_destroy = true
 }
 
+resource "aws_s3_bucket_versioning" "versioning" {
+  bucket = aws_s3_bucket.s3_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+
 ###
 
 resource "aws_s3_bucket_public_access_block" "sftp-block" {
-  bucket = module.s3_bucket.s3_bucket_id
+  bucket = aws_s3_bucket.s3_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -188,7 +186,7 @@ resource "aws_iam_role_policy" "user" {
       ],
       "Effect": "Allow",
       "Resource": [
-       "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id])}"
+       "${join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id])}"
       ]
     },
     {
@@ -214,7 +212,7 @@ resource "aws_iam_role_policy" "user" {
         "s3:DeleteObject",
         "s3:GetObjectVersion"
       ],
-      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/*"])}"
+      "Resource": "${join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"])}"
     }
   ]
 }
@@ -225,7 +223,7 @@ resource "aws_transfer_user" "this" {
   for_each       = var.sftp_users
   server_id      = aws_transfer_server.public.id
   user_name      = each.key
-  home_directory = "/${module.s3_bucket.s3_bucket_id}/${each.value}"
+  home_directory = "/${aws_s3_bucket.s3_bucket.id}/${each.value}"
   role           = aws_iam_role.user[each.key].arn
 
   tags = local.tags

s3-bucket-sftp-public/outputs.tf

@@ -26,7 +26,7 @@ output "host-key-fingerprint" {
 }
 
 output "s3-bucket-name" {
-  value = "sftp-${var.provision_id}"
+  value = aws_s3_bucket.s3_bucket.id
 }
 
 

s3-bucket-sftp/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "= 3.74.1"
+      version = ">= 6.0.0"
     }
   }
 }
@@ -42,10 +42,6 @@ resource "aws_secretsmanager_secret_version" "private_key" {
   secret_string = nonsensitive(tls_private_key.sftp-key.private_key_pem)
 }
 
-module "fw-ips" {
-    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
-}
-
 locals {
 
   port                 = "22"
@@ -59,28 +55,25 @@ locals {
 }
 
 
-module "s3_bucket" {
-  source = "terraform-aws-modules/s3-bucket/aws"
-
-  version = "~> 2.1.0"
-
-  bucket = "sftp-${var.provision_id}"
-  acl    = "private"
-
-  versioning = {
-    enabled = false
-  }
-
-  force_destroy = true
+resource "aws_s3_bucket" "s3_bucket" {
+  bucket = "qmi-bucket-${var.provision_id}"
 
   tags = local.tags
 
+  force_destroy = true
+}
+
+resource "aws_s3_bucket_versioning" "versioning" {
+  bucket = aws_s3_bucket.s3_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
 }
 
 ###
 
 resource "aws_s3_bucket_public_access_block" "sftp-block" {
-  bucket = module.s3_bucket.s3_bucket_id
+  bucket = aws_s3_bucket.s3_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -141,8 +134,8 @@ resource "aws_transfer_server" "sftp" {
     vpc_id             = var.vpc_id
     subnet_ids         = var.subnet_ids
     security_group_ids = [
-      module.security_group.security_group_id,
-      module.security_group_2.security_group_id
+      aws_security_group.allow_tls.id,
+      aws_security_group.allow_tls_2.id
     ]
   }
   tags = {
@@ -155,99 +148,6 @@ resource "aws_transfer_server" "sftp" {
   }
 }
 
-module "security_group" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG"
-  description = "${var.provision_id}-SG-SFTP"
-  vpc_id      = var.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks =  module.fw-ips.cidr_blocks
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Allow SFTP Inbound"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks =  module.fw-ips.cidr_blocks
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Allow SFTP outbound"
-
-    },
-  ]
-
-  tags = {
-    Deployment = "QMI"
-    "Cost Center" = "3100"
-    #QMI_user = var.user_id
-    ProvID   = var.provision_id
-    Name     = "sftp-${var.provision_id}"
-  }
-  
-}
-
-module "security_group_2" {
-  
-  # SGs created here as Ports differ per Engine. Only Azure Firewall IPs added for now.
-  source  = "terraform-aws-modules/security-group/aws"
-  version = "~> 4.3"
-
-  name        = "${var.provision_id}-SG2"
-  description = "${var.provision_id}-SG2-SFTP"
-  vpc_id      = var.vpc_id
-
-
-  # ingress
-
-  ingress_cidr_blocks = module.fw-ips.cidr_blocks_others
-  
-
-  ingress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Allow SFTP Inbound"
-
-    },
-  ]
-
-  # egress
-
-  egress_cidr_blocks = module.fw-ips.cidr_blocks_others
-
-  egress_with_cidr_blocks = [
-    {
-      from_port   = local.port
-      to_port     = local.port
-      protocol    = "tcp"
-      description = "Allow SFTP Inbound"
-
-    },
-  ]
-
-  tags = local.tags
-}
-
-
 resource "aws_iam_role" "user" {
   for_each           = var.sftp_users
   name               = "${var.provision_id}-sftp-user-${each.key}"
@@ -284,7 +184,7 @@ resource "aws_iam_role_policy" "user" {
       ],
       "Effect": "Allow",
       "Resource": [
-       "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id])}"
+       "${join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id])}"
       ]
     },
     {
@@ -297,7 +197,7 @@ resource "aws_iam_role_policy" "user" {
         "s3:DeleteObject",
         "s3:GetObjectVersion"
       ],
-      "Resource": "${join("", ["arn:aws:s3:::", module.s3_bucket.s3_bucket_id, "/", each.value, "/*"])}"
+      "Resource": "${join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/", each.value, "/*"])}"
     }
   ]
 }
@@ -308,7 +208,7 @@ resource "aws_transfer_user" "this" {
   for_each       = var.sftp_users
   server_id      = aws_transfer_server.sftp.id
   user_name      = each.key
-  home_directory = "/${module.s3_bucket.s3_bucket_id}/${each.value}"
+  home_directory = "/${aws_s3_bucket.s3_bucket.id}/${each.value}"
   role           = aws_iam_role.user[each.key].arn
 }
 

s3-bucket-sftp/sec_groups.tf

@@ -0,0 +1,69 @@
+module "fw-ips" {
+    source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//databases/firewall_ips"
+}
+
+resource "aws_security_group" "allow_tls" {
+  name        = "${var.provision_id}-SG"
+  description = "${var.provision_id}-SG"
+  vpc_id      = var.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4" {
+  for_each = toset(module.fw-ips.cidr_blocks)
+  
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv4" {
+
+  for_each = toset(module.fw-ips.cidr_blocks)
+
+  security_group_id = aws_security_group.allow_tls.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "dbport"
+}
+
+resource "aws_security_group" "allow_tls_2" {
+  name        = "${var.provision_id}-SG2"
+  description = "${var.provision_id}-SG2"
+  vpc_id      = var.vpc_id
+
+  tags = local.tags
+}
+
+resource "aws_vpc_security_group_ingress_rule" "allow_tls_ipv4_2" {
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+  
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4         = each.key
+  from_port         = local.port
+  ip_protocol       = "tcp"
+  to_port           = local.port
+  description = "Others - dbport"
+}
+
+resource "aws_vpc_security_group_egress_rule" "allow_tls_ipv_2" {
+
+  for_each = toset(module.fw-ips.cidr_blocks_others)
+
+  security_group_id = aws_security_group.allow_tls_2.id
+
+  cidr_ipv4   = each.key
+  from_port   = local.port
+  ip_protocol = "tcp"
+  to_port     = local.port
+  description = "Others - dbport"
+}

s3-bucket/main.tf

@@ -5,23 +5,12 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
+      version = ">= 6.0.0"
     }
   }
 }
 
-module "iam_user" {
-  source  = "terraform-aws-modules/iam/aws//modules/iam-user"
-
-  version = "~> 3.0"
-
-  name          = "qmi-user-${var.provision_id}"
-  force_destroy = true
-
-  create_iam_user_login_profile = false
-  #pgp_key = "keybase:test"
-
-  password_reset_required = false
-
+locals {
   tags = {
     Deployment = "QMI"
     "Cost Center" = "3100"
@@ -29,42 +18,36 @@ module "iam_user" {
     Owner    = var.user_id
     ProvID   = var.provision_id
     Name     = "qmi-${var.provision_id}"
+    forced_destroy = var.forced_destroy
   }
-
 }
 
-module "s3_bucket" {
-  source = "terraform-aws-modules/s3-bucket/aws"
-
-  bucket = "qmi-bucket-${var.provision_id}"
-
-  versioning = {
-    enabled = false
-  }
-
+resource "aws_iam_user" "lb" {
+  name = "qmi-user-${var.provision_id}"
   force_destroy = true
 
-  block_public_acls       = true
-  block_public_policy     = true
-  ignore_public_acls      = true
-  restrict_public_buckets = true
-
-  tags = {
-    Deployment = "QMI"
-    "Cost Center" = "3100"
-    QMI_user = var.user_id
-    Owner    = var.user_id
-    ProvID   = var.provision_id
-    Name     = "qmi-${var.provision_id}"
-  }
-
+  tags = local.tags
 }
 
+resource "aws_iam_access_key" "lb" {
+  user = aws_iam_user.lb.name
+}
+
+
+resource "aws_s3_bucket" "s3_bucket" {
+  bucket = var.bucket_name!=null? var.bucket_name : "qmi-bucket-${var.provision_id}"
+
+  tags = local.tags
+
+  force_destroy = true
+}
+
+
 
 resource "aws_iam_user_policy" "lb_ro" {
     
-  name = "s3only_policy_${module.iam_user.this_iam_user_name}"
-  user = module.iam_user.this_iam_user_name
+  name = "s3only_policy_${aws_iam_user.lb.name}"
+  user = aws_iam_user.lb.name
 
   # Terraform's "jsonencode" function converts a
   # Terraform expression result to valid JSON syntax.
@@ -83,10 +66,129 @@ resource "aws_iam_user_policy" "lb_ro" {
               "Effect": "Allow",
               "Action": "s3:*",
               "Resource": [
-                  module.s3_bucket.s3_bucket_arn,
-                  "${module.s3_bucket.s3_bucket_arn}/*"
+                  aws_s3_bucket.s3_bucket.arn,
+                  "${aws_s3_bucket.s3_bucket.arn}/*"
               ]
           }
       ]
   })
-}
+}
+
+resource "aws_iam_role" "qmi_snowflake" {
+    
+  name = "qmi_snowflake_${var.provision_id}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = "arn:aws:iam::494544507972:user/n2y3-s-ssca0544"
+        }
+      },
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = ["338144066592", "494544507972"]
+        }
+        Condition = {
+           StringEquals = {
+                  "sts:ExternalId" = "iceberg_table_external_id"
+            }
+        }
+      }
+    ]
+  })
+
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy" "qmi_snowflake_policy" {
+
+  name = "qmi-bucket-${var.provision_id}_policy"
+  role = aws_iam_role.qmi_snowflake.id
+
+  policy = jsonencode({
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Effect": "Allow",
+              "Action": [
+                          "s3:GetBucketLocation",
+                          "s3:ListAllMyBuckets"
+                        ],
+              "Resource": "arn:aws:s3:::*"
+          },
+          {
+              "Effect": "Allow",
+              "Action": "s3:*",
+              "Resource": [
+                  aws_s3_bucket.s3_bucket.arn,
+                  "${aws_s3_bucket.s3_bucket.arn}/*"
+              ]
+          }
+      ]
+  })
+}
+
+resource "aws_iam_role" "qlik_s3" {
+  
+  count = var.tenant_id != null? 1 : 0
+  
+  name = "qlik_s3_${var.tenant_id}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = "338144066592"
+        }
+        Condition = {
+           StringEquals = {
+                  "sts:ExternalId" = "qlik_connection_${var.tenant_id}"
+            }
+        }
+      }
+    ]
+  })
+
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy" "aws_s3_bucket_policy" {
+
+  count = var.tenant_id != null? 1 : 0
+
+  name = "qmi-bucket-${var.provision_id}_policy"
+  role = aws_iam_role.qlik_s3[0].id
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = ""
+        Effect   = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:ListBucket"
+        ]
+        Resource = [
+          "arn:aws:s3:::${aws_s3_bucket.s3_bucket.id}",
+          "arn:aws:s3:::${aws_s3_bucket.s3_bucket.id}/*"
+        ]
+      },
+    ]
+  })
+}

s3-bucket/main.tf_old

@@ -0,0 +1,209 @@
+terraform {
+
+  required_version = ">= 0.13"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+    }
+  }
+}
+
+locals {
+  tags = {
+    Deployment = "QMI"
+    "Cost Center" = "3100"
+    QMI_user = var.user_id
+    Owner    = var.user_id
+    ProvID   = var.provision_id
+    Name     = "qmi-${var.provision_id}"
+    forced_destroy = var.forced_destroy
+  }
+}
+
+module "iam_user" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-user"
+
+  version = "~> 3.0"
+
+  name          = "qmi-user-${var.provision_id}"
+  force_destroy = true
+
+  create_iam_user_login_profile = false
+  #pgp_key = "keybase:test"
+
+  password_reset_required = false
+
+  tags = local.tags
+
+}
+
+module "s3_bucket" {
+  source = "terraform-aws-modules/s3-bucket/aws"
+
+  bucket = "qmi-bucket-${var.provision_id}"
+
+  versioning = {
+    enabled = false
+  }
+
+  force_destroy = true
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+
+  tags = local.tags
+
+}
+
+
+resource "aws_iam_user_policy" "lb_ro" {
+    
+  name = "s3only_policy_${module.iam_user.this_iam_user_name}"
+  user = module.iam_user.this_iam_user_name
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Effect": "Allow",
+              "Action": [
+                          "s3:GetBucketLocation",
+                          "s3:ListAllMyBuckets"
+                        ],
+              "Resource": "arn:aws:s3:::*"
+          },
+          {
+              "Effect": "Allow",
+              "Action": "s3:*",
+              "Resource": [
+                  module.s3_bucket.s3_bucket_arn,
+                  "${module.s3_bucket.s3_bucket_arn}/*"
+              ]
+          }
+      ]
+  })
+}
+
+resource "aws_iam_role" "qmi_snowflake" {
+    
+  name = "qmi_snowflake_${var.provision_id}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = "arn:aws:iam::494544507972:user/n2y3-s-ssca0544"
+        }
+      },
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = ["338144066592", "494544507972"]
+        }
+        Condition = {
+           StringEquals = {
+                  "sts:ExternalId" = "iceberg_table_external_id"
+            }
+        }
+      }
+    ]
+  })
+
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy" "qmi_snowflake_policy" {
+
+  name = "qmi-bucket-${var.provision_id}_policy"
+  role = aws_iam_role.qmi_snowflake.id
+
+  policy = jsonencode({
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Effect": "Allow",
+              "Action": [
+                          "s3:GetBucketLocation",
+                          "s3:ListAllMyBuckets"
+                        ],
+              "Resource": "arn:aws:s3:::*"
+          },
+          {
+              "Effect": "Allow",
+              "Action": "s3:*",
+              "Resource": [
+                  module.s3_bucket.s3_bucket_arn,
+                  "${module.s3_bucket.s3_bucket_arn}/*"
+              ]
+          }
+      ]
+  })
+}
+
+resource "aws_iam_role" "qlik_s3" {
+  
+  count = var.tenant_id != null? 1 : 0
+  
+  name = "qlik_s3_${var.tenant_id}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = "sts:AssumeRole"
+        Principal = {
+          AWS = "338144066592"
+        }
+        Condition = {
+           StringEquals = {
+                  "sts:ExternalId" = "qlik_connection_${var.tenant_id}"
+            }
+        }
+      }
+    ]
+  })
+
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy" "aws_s3_bucket_policy" {
+
+  count = var.tenant_id != null? 1 : 0
+
+  name = "qmi-bucket-${var.provision_id}_policy"
+  role = aws_iam_role.qlik_s3[0].id
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = ""
+        Effect   = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:ListBucket"
+        ]
+        Resource = [
+          "arn:aws:s3:::${module.s3_bucket.s3_bucket_id}",
+          "arn:aws:s3:::${module.s3_bucket.s3_bucket_id}/*"
+        ]
+      },
+    ]
+  })
+}

s3-bucket/outputs.tf

@@ -1,15 +1,35 @@
 output "bucket" {
-  value = module.s3_bucket
+  value = {
+    s3_bucket_id = aws_s3_bucket.s3_bucket.id
+    s3_bucket_region = aws_s3_bucket.s3_bucket.bucket_region
+  }
 }
 
+
 output "iam_name" {
-  value = module.iam_user.this_iam_user_name
+  value = aws_iam_user.lb.name
 }
 
 output "iam_access_key" {
-  value = module.iam_user.this_iam_access_key_id
+  value = aws_iam_access_key.lb.id
 }
 
 output "iam_access_secret" {
-  value = nonsensitive(module.iam_user.this_iam_access_key_secret)
+  value = nonsensitive(aws_iam_access_key.lb.secret)
+}
+
+output "iam_role_arn" {
+  value = var.tenant_id!=null?  aws_iam_role.qlik_s3[0].arn : null
+}
+
+output "aws_account_id" {
+  value = "192018133564"
+}
+
+output "iam_role_snowflake_arn" {
+  value = aws_iam_role.qmi_snowflake.arn
+}
+
+output "iam_role_snowflake_arn_ExternalId" {
+  value = "iceberg_table_external_id"
 }

s3-bucket/variables.tf

@@ -1,7 +1,3 @@
-variable "region" {
-    default = "us-east-1"
-}
-
 variable "provision_id" { 
 }
 
@@ -9,3 +5,14 @@ variable "user_id" {
     default = null
 }
 
+variable "forced_destroy" {
+  default = null
+}
+
+variable "tenant_id" {
+  default = null
+}
+
+variable "bucket_name" {
+  default = null
+}

vm-fromsnapshot-linux/main.tf

@@ -10,7 +10,7 @@ resource "random_id" "randomMachineId" {
 resource "random_password" "password" {
   length = 16
   special = true
-  override_special = "_!"
+  override_special = "_!@"
   upper = true
   lower = true
   min_lower = 2
@@ -115,12 +115,30 @@ resource "azurerm_virtual_machine" "vm" {
 
 }
 
+module "linux-common" {
+
+  count = var.initial_password != null? 1 : 0
+  
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common"
+
+  depends_on = [
+   azurerm_virtual_machine.vm
+  ]
+  
+  os_type             = "centos"
+  private_ip_address  = module.qmi-nic.private_ip_address
+  admin_username      = local.admin_username
+  admin_password      = var.initial_password
+  resize              = false
+  update              = false
+}
+
 resource "null_resource" "post-vm-fromsnapshot-linux" {
   
   count = var.initial_password != null? 1 : 0
 
   depends_on = [
-    azurerm_virtual_machine.vm
+    module.linux-common
   ]
 
   provisioner "file" {
@@ -161,3 +179,5 @@ resource "null_resource" "post-vm-fromsnapshot-linux" {
 }
 
 
+
+

vm-oraclelinux89/main.tf

@@ -0,0 +1,97 @@
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 3
+}
+
+resource "random_password" "password" {
+  length = 16
+  special = true
+  override_special = "_!"
+  upper = true
+  lower = true
+  min_lower = 2
+  min_upper = 2
+  min_special = 2
+  min_numeric = 1
+}
+
+locals {
+  virtual_machine_name = "${var.prefix}-${random_id.randomMachineId.hex}"
+  admin_username = var.admin_username
+  admin_password = nonsensitive(random_password.password.result)
+}
+
+module "qmi-nic" {
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//qmi-nic"
+  
+  prefix = local.virtual_machine_name
+  location = var.location
+  subnet_id = var.subnet_id
+  
+  resource_group_name = var.resource_group_name
+  user_id = var.user_id
+  
+}
+
+resource "azurerm_linux_virtual_machine" "vm" {
+  name                = local.virtual_machine_name
+  resource_group_name = var.resource_group_name
+  location            = var.location
+  size                = var.vm_type
+  admin_username      = local.admin_username
+  admin_password      = local.admin_password
+  disable_password_authentication = false
+  
+  network_interface_ids = [
+    module.qmi-nic.id,
+  ]
+
+  os_disk {
+    name                 = "${local.virtual_machine_name}-osdisk"
+    caching              = "ReadWrite"
+    storage_account_type = var.managed_disk_type
+    disk_size_gb         = var.disk_size_gb
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  source_image_reference {
+    publisher = "Oracle"
+    offer     = "Oracle-Linux"
+    sku       = "ol89-lvm-gen2"
+    version   = "latest"
+  }
+
+  tags = {
+    Deployment = "QMI PoC"
+    "Cost Center" = "3100"
+    "ProvId"  = var.provId != null? var.provId : null
+    "QMI_user" = var.user_id != null? var.user_id : null
+    "Owner" = var.user_id != null? var.user_id : null
+    "24x7" = var.is_24x7 == true? "" : null
+    "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
+    "StartupTime": var.is_24x7 == false? var.startupTime : null
+  }
+}
+
+module "linux-common" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common"
+
+  depends_on = [
+    azurerm_linux_virtual_machine.vm
+  ]
+  
+  os_type             = "centos"
+  private_ip_address  = module.qmi-nic.private_ip_address
+  admin_username      = local.admin_username
+  admin_password      = local.admin_password  
+  
+}
+

vm-oraclelinux89/outputs.tf

@@ -0,0 +1,31 @@
+output "virtual_machine_id" {
+  value = azurerm_linux_virtual_machine.vm.id
+}
+
+output "virtual_machine_name" {
+    value = "${var.prefix}-${random_id.randomMachineId.hex}"
+}
+
+output "admin_username" {
+  value = var.admin_username
+}
+
+output "admin_password" {
+  value = nonsensitive(random_password.password.result)
+}
+
+output "nic_id" {
+  value = module.qmi-nic.id
+}
+
+output "nic_private_ip_address" {
+  value = module.qmi-nic.private_ip_address
+}
+
+output "nic_ip_configuration_name" {
+  value = module.qmi-nic.ip_configuration_name
+}
+
+output "principal_id" {
+  value = azurerm_linux_virtual_machine.vm.identity.0.principal_id
+}

vm-oraclelinux89/variables.tf

@@ -0,0 +1,56 @@
+variable "subnet_id" {
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-VM-Deployments"
+}
+
+variable "prefix" {
+  description = "The Prefix used for all resources in this example"
+  default = "QMI-ORACLELINUX"
+}
+
+variable "location" {
+  default = "East US"
+}
+
+variable "resource_group_name" {
+}
+
+variable "vm_type" {
+  default = "Standard_DS3_v2"
+}
+
+variable "managed_disk_type" {
+  default = "Premium_LRS"
+}
+
+variable "disk_size_gb" {
+  default = "128"
+}
+
+variable "admin_username" {
+  default = "qmi"
+}
+
+variable "user_id" {
+  default = null
+}
+
+variable "provId" {
+  default = null
+}
+
+variable "is_24x7"{
+  type = bool
+  default = null
+}
+
+variable "shutdownTime"{
+  default = null
+}
+
+variable "startupTime"{
+  default = null
+}
+
+# variable "subnet_id" {
+#   default = "/subscriptions/1f3d4c1d-6509-4c52-8dee-c15fb83f2920/resourceGroups/lkn-rg/providers/Microsoft.Network/virtualNetworks/lkn-vn/subnets/default"
+# }

vm-qs/scripts/bootstrap-qs.ps1

@@ -105,6 +105,16 @@ $shortcut.Save()  ## Save
 Rename-Item -Path $destination -NewName "Qlik Sense Hub.lnk"
 ####
 
+$Folder = 'C:\Program Files\Qlik\Sense\Tools\QlikSenseCLI'
+Write-Log "Test to see if QlikSenseCLI  exists"
+if (Test-Path -Path $Folder) {
+    Write-Log "Installing QlikSenseCLI module"
+	Install-Module QLikSenseCLI -Scope AllUsers -Force
+    #copy-item  -Path$Folder -Destination  'C:\Program Files\WindowsPowerShell\Modules' -Recurse
+} else {
+    Write-Log  "QlikSenseCLI not supported in this version."
+} 
+
 
 Write-Log "--- New Certs: CurrentUser\My"
 Get-ChildItem cert:"CurrentUser\My"

vm-rockylinux/main.tf

@@ -0,0 +1,106 @@
+resource "random_id" "randomMachineId" {
+    keepers = {
+        # Generate a new ID only when a new resource group is defined
+        resource_group = var.resource_group_name
+    }
+    
+    byte_length = 3
+}
+
+resource "random_password" "password" {
+  length = 16
+  special = false
+  override_special = "_"
+  upper = true
+  lower = true
+  min_lower = 2
+  min_upper = 2
+  min_special = 0
+  min_numeric = 2
+}
+
+locals {
+  virtual_machine_name = "${var.prefix}-${random_id.randomMachineId.hex}"
+  admin_username = var.admin_username
+  admin_password = nonsensitive(random_password.password.result)
+}
+
+module "qmi-nic" {
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//qmi-nic"
+  
+  prefix = local.virtual_machine_name
+  location = var.location
+  subnet_id = var.subnet_id
+  
+  resource_group_name = var.resource_group_name
+  user_id = var.user_id
+  
+}
+
+resource "azurerm_virtual_machine" "vm" {
+  name                  = local.virtual_machine_name
+  location              = var.location
+  resource_group_name   = var.resource_group_name
+  network_interface_ids = [module.qmi-nic.id]
+  vm_size               = var.vm_type
+
+  delete_os_disk_on_termination = true
+  delete_data_disks_on_termination = true
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  storage_image_reference {
+    # this is the Oracle linux image I found.
+    # az vm image list --all --publisher Oracle  -o table
+    publisher = "erockyenterprisesoftwarefoundationinc1653071250513"
+    offer     = "rockylinux"
+    sku       = "free"
+    version   = "8.7.20230215"
+  }
+
+  storage_os_disk {
+    name              = "${local.virtual_machine_name}-osdisk"
+    caching           = "ReadWrite"
+    create_option     = "FromImage"
+    managed_disk_type = var.managed_disk_type
+    disk_size_gb      = var.disk_size_gb
+  }
+
+  os_profile_linux_config {
+      disable_password_authentication = false
+  }
+
+  os_profile {
+    computer_name  = local.virtual_machine_name
+    admin_username = local.admin_username
+    admin_password = local.admin_password
+  }
+
+  tags = {
+    Deployment = "QMI PoC"
+    "Cost Center" = "3100"
+    "ProvId"  = var.provId != null? var.provId : null
+    "QMI_user" = var.user_id != null? var.user_id : null
+    "24x7" = var.is_24x7 == true? "" : null
+    "ShutdownTime": var.is_24x7 == false? var.shutdownTime : null
+    "StartupTime": var.is_24x7 == false? var.startupTime : null
+  }
+}
+
+module "linux-common" {
+
+  source = "git::https://gitlab.com/qmi/qmi-cloud-tf-modules.git//linux-common"
+
+  depends_on = [
+    azurerm_virtual_machine.vm
+  ]
+  
+  os_type             = "centos"
+  private_ip_address  = module.qmi-nic.private_ip_address
+  admin_username      = local.admin_username
+  admin_password      = local.admin_password  
+  
+}
+

vm-rockylinux/outputs.tf

@@ -0,0 +1,31 @@
+output "virtual_machine_id" {
+  value = azurerm_virtual_machine.vm.id
+}
+
+output "virtual_machine_name" {
+    value = "${var.prefix}-${random_id.randomMachineId.hex}"
+}
+
+output "admin_username" {
+  value = var.admin_username
+}
+
+output "admin_password" {
+  value = nonsensitive(random_password.password.result)
+}
+
+output "nic_id" {
+  value = module.qmi-nic.id
+}
+
+output "nic_private_ip_address" {
+  value = module.qmi-nic.private_ip_address
+}
+
+output "nic_ip_configuration_name" {
+  value = module.qmi-nic.ip_configuration_name
+}
+
+output "principal_id" {
+  value = azurerm_virtual_machine.vm.identity.0.principal_id
+}

vm-rockylinux/variables.tf

@@ -0,0 +1,56 @@
+variable "subnet_id" {
+  default = "/subscriptions/62ebff8f-c40b-41be-9239-252d6c0c8ad9/resourceGroups/QMI-infra-vnet/providers/Microsoft.Network/virtualNetworks/QMI-Automation-Vnet/subnets/QMI-VM-Deployments"
+}
+
+variable "prefix" {
+  description = "The Prefix used for all resources in this example"
+  default = "QMI-ROCKYLINUX"
+}
+
+variable "location" {
+  default = "East US"
+}
+
+variable "resource_group_name" {
+}
+
+variable "vm_type" {
+  default = "Standard_DS3_v2"
+}
+
+variable "managed_disk_type" {
+  default = "Premium_LRS"
+}
+
+variable "disk_size_gb" {
+  default = "128"
+}
+
+variable "admin_username" {
+  default = "qmi"
+}
+
+variable "user_id" {
+  default = null
+}
+
+variable "provId" {
+  default = null
+}
+
+variable "is_24x7"{
+  type = bool
+  default = null
+}
+
+variable "shutdownTime"{
+  default = null
+}
+
+variable "startupTime"{
+  default = null
+}
+
+# variable "subnet_id" {
+#   default = "/subscriptions/1f3d4c1d-6509-4c52-8dee-c15fb83f2920/resourceGroups/lkn-rg/providers/Microsoft.Network/virtualNetworks/lkn-vn/subnets/default"
+# }

win-common/scripts/tenable-install.ps1

@@ -2,11 +2,14 @@ Param(
     [string] $KEY
 )
 
-Write-Host "--- Installing Tenable Nessus Agent..."
+
 New-Item -ItemType Directory -Force -Path C:\Temp | Out-Null
 $ProgressPreference = 'SilentlyContinue'
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
-$version="NessusAgent-10.2.1-x64.msi"
+$version="NessusAgent-10.8.2-x64.msi"
+
+Write-Host "--- Installing Tenable Nessus Agent... --> $version"
+
 $cbBinaryUrl = "https://d7ipctdjxxii4.cloudfront.net/others/$version"
 Invoke-WebRequest -Uri $cbBinaryUrl -OutFile "C:\Temp\$version"