diff --git a/sense-conductor/.gitignore b/sense-conductor/.gitignore new file mode 100644 index 0000000..36de6f0 --- /dev/null +++ b/sense-conductor/.gitignore @@ -0,0 +1,8 @@ +*.terraform* +esoteric-parsec*json +*.tfstate* +*terraform-provider* +*.deb +*license.json +*history* +*functions.ps1 diff --git a/sense-conductor/README.md b/sense-conductor/README.md new file mode 100644 index 0000000..3b893b8 --- /dev/null +++ b/sense-conductor/README.md @@ -0,0 +1,7 @@ +# sense-conductor + +Repository for QSEoW Orchestration + +- Initial Goal is to collate disparate efforts. +- Componentize orchestration functions. +- Build framework for sustainable customer solution for orchestration of QSEoW Clusters diff --git a/sense-conductor/bt-autoscaler/.gitkeep b/sense-conductor/bt-autoscaler/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/cen.xml b/sense-conductor/bt-autoscaler/terraform/gcp/root/cen.xml new file mode 100644 index 0000000..ae9d167 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/cen.xml @@ -0,0 +1,24 @@ + + + qliksenserepository + Qlik1234! + 10.54.242.47 + 5432 + \\10.219.178.210\qlikshare + \\10.219.178.210\qlikshare\StaticContent + \\10.219.178.210\qlikshare\ArchivedLogs + \\10.219.178.210\qlikshare\Apps + true + false + false + * + 0.0.0.0/0,::/0 + 100 + + true + false + Qlik1234! + Qlik1234! + 10.54.242.47 + 5432 + \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/central/main.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/main.tf new file mode 100644 index 0000000..440c5de --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/main.tf @@ -0,0 +1,84 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + +resource "google_compute_disk" "dataCen" { + provider = google-beta + name = "central-datadisk-${random_id.vm_suffix.hex}" + type = var.disk_type + zone = var.zone + # labels = { + # environment = "dev" + # } + size = 50 +} + +resource "google_compute_instance" "central" { + provider = google-beta + name = "central-${random_id.vm_suffix.hex}" + machine_type = var.vm_type + # min_cpu_platform commented out during testing - e2 instances do not support + # min_cpu_platform = var.min_cpu + zone = var.zone + + # hostname = "central-${random_id.vm_suffix.hex}" + + # tags = ["foo", "bar"] + # timeouts { + # create = "60m" + # delete = "2h" + # } + + boot_disk { + initialize_params { + image = var.image + } + } + + attached_disk { + source = google_compute_disk.dataCen.name + } + + # lifecycle { + # ignore_changes = [attached_disk] + # } + + network_interface { + network = "tactical-qlik-vpc" + subnetwork = "compute" + + # access_config { + # network_tier = "STANDARD" + # } + } + + metadata = { + windows-startup-script-url = "gs://qliksense/scripts/bootstrap.ps1" + } + + service_account { + email = "terraform@esoteric-parsec-243510.iam.gserviceaccount.com" + scopes = ["cloud-platform"] + } +} + +# psql -h -Upostgres + +# gcloud sql instances patch qseow-psql- --authorized-networks 172.16.12.0/28,172.16.1.0/24,172.16.10.0/24,172.16.2.0/24,172.16.11.0/24,192.88.99.0/24,11.0.0.0/24 + +# gcloud compute networks peerings update cloudsql-postgres-googleapis-com --network=NETWORK --export-subnet-routes-with-public-ip --project=PROJECT + + + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/central/outputs.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/outputs.tf new file mode 100644 index 0000000..d7ed197 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/outputs.tf @@ -0,0 +1,21 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "cen_name" { + value = google_compute_instance.central.name +} + +# output "cen_ext_ip" { +# value = google_compute_instance.central.network_interface[0].access_config[0].nat_ip +# } + +output "cen_int_ip" { + value = google_compute_instance.central.network_interface[0].network_ip +} + +# output "serial_out" { +# value = data.google_compute_instance_serial_port.serial.contents +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/central/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/terraform.tfvars.json new file mode 100644 index 0000000..7883695 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/terraform.tfvars.json @@ -0,0 +1,17 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "db_user": "postgres", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/central/variables.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/variables.tf new file mode 100644 index 0000000..1dc6fe2 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/central/variables.tf @@ -0,0 +1,134 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "db_user" { + description = "The name of the DB user" + type = string + default = "postgres" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/file/main.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/main.tf new file mode 100644 index 0000000..acb6094 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/main.tf @@ -0,0 +1,32 @@ +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +resource "google_filestore_instance" "bt-filestore" { + provider = google-beta + name = "qseow-files" + zone = var.zone + tier = var.file_tier + project = var.project_id + + file_shares { + capacity_gb = 1024 + name = "qlikshare" + + # nfs_export_options { + # ip_ranges = ["0.0.0.0/0"] + # access_mode = "READ_WRITE" + # squash_mode = "NO_ROOT_SQUASH" + # anon_uid = 0 + # anon_gid = 0 + # } + } + + networks { + network = "default" + modes = ["MODE_IPV4"] + } +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/file/outputs.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/outputs.tf new file mode 100644 index 0000000..de185b0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/outputs.tf @@ -0,0 +1,17 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "fs_instance_ip" { + value = google_filestore_instance.bt-filestore.networks[0].ip_addresses +} + +output "fs_instance_name" { + value = google_filestore_instance.bt-filestore.name +} + +# output "fs_instance_path" { +# value = google_filestore_instance.bt-filestore.fileShares.name +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/file/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/terraform.tfvars.json new file mode 100644 index 0000000..7883695 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/terraform.tfvars.json @@ -0,0 +1,17 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "db_user": "postgres", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/file/variables.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/variables.tf new file mode 100644 index 0000000..e4043f0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/file/variables.tf @@ -0,0 +1,128 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "db_user" { + description = "The name of the DB user" + type = string + default = "postgres" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/main.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/main.tf new file mode 100644 index 0000000..de70525 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/main.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google" { + version = "3.10.0" + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project + region = var.region + zone = var.zone +} + +data "google_compute_instance_group" "geo" { + name = "GeoAnalyticsServer" + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/outputs.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/outputs.tf new file mode 100644 index 0000000..a31dd16 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/outputs.tf @@ -0,0 +1,9 @@ +# Configure return values from google_sql_database_instance + +output "psql_ip_address" { + value = google_sql_database_instance.qseow-psql.private_ip_address +} + +output "psql_instance_name" { + value = google_sql_database_instance.qseow-psql.connection_name +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/variables.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/variables.tf new file mode 100644 index 0000000..523d4e0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/geo/variables.tf @@ -0,0 +1,92 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/destroy.sh b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/destroy.sh new file mode 100644 index 0000000..87b46b4 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/destroy.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +terraform state rm google_sql_user.users +terraform destroy $@ -auto-approve diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/main.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/main.tf new file mode 100644 index 0000000..6d77759 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/main.tf @@ -0,0 +1,62 @@ +### Main TF - split pSQL to a module +## + +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for DB suffix +resource "random_id" "db_suffix" { + byte_length = 2 +} + +# Create res for Cloud SQL DB create +resource "google_sql_database_instance" "qseow-psql" { + provider = google-beta + name = "qseow-psql-${random_id.db_suffix.hex}" + project = var.project_id + region = var.region + database_version = var.database_version + + settings { + tier = var.db_tier + availability_type = var.availability_type + backup_configuration { + enabled = "true" + } + + ip_configuration { + ipv4_enabled = false + private_network = var.private_network + } + } +} + +# Configure postgres admin user +## Note that the SQL instance cannot be rm'd without removing the TFState for +## the SQL user (TF/PostgreSQL race condition) +# +## Use "./destroy.sh" instead of "terraform destroy" + +resource "google_sql_user" "users" { + provider = google-beta + instance = google_sql_database_instance.qseow-psql.name + name = var.db_user + password = var.user_password + project = var.project_id + + depends_on = [google_sql_database_instance.qseow-psql] + +} + +# Execute stored SQL file, prep pSQL for Qlik Sense + +resource "null_resource" "db_setup" { + provisioner "local-exec" { + command = "PGPASSWORD=${google_sql_user.users.password} /usr/bin/psql -h ${google_sql_database_instance.qseow-psql.private_ip_address} -Upostgres --dbname=postgres < qseow_db_setup.sql" + } + depends_on = [google_sql_database_instance.qseow-psql, google_sql_user.users] +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/outputs.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/outputs.tf new file mode 100644 index 0000000..a31dd16 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/outputs.tf @@ -0,0 +1,9 @@ +# Configure return values from google_sql_database_instance + +output "psql_ip_address" { + value = google_sql_database_instance.qseow-psql.private_ip_address +} + +output "psql_instance_name" { + value = google_sql_database_instance.qseow-psql.connection_name +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/qseow_db_setup.sql b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/qseow_db_setup.sql new file mode 100644 index 0000000..eac5644 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/qseow_db_setup.sql @@ -0,0 +1,39 @@ +CREATE DATABASE "QSR" ENCODING = 'UTF8'; +CREATE DATABASE "QSMQ" ENCODING = 'UTF8'; +CREATE DATABASE "Licenses" ENCODING = 'UTF8'; +CREATE DATABASE "SenseServices" ENCODING = 'UTF8'; +CREATE DATABASE "QLogs" ENCODING = 'UTF8'; + +CREATE ROLE "qliksenserepository" WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +ALTER ROLE "qliksenserepository" WITH ENCRYPTED PASSWORD 'Qlik1234!'; +GRANT "qliksenserepository" TO postgres; + +ALTER DATABASE "QSR" OWNER TO "qliksenserepository"; +ALTER DATABASE "SenseServices" OWNER TO "qliksenserepository"; +ALTER DATABASE "QSMQ" OWNER TO "qliksenserepository"; +ALTER DATABASE "Licenses" OWNER TO qliksenserepository; + +GRANT TEMPORARY, CONNECT ON DATABASE "QSMQ" TO PUBLIC; +GRANT ALL ON DATABASE "QSMQ" TO postgres; +GRANT CREATE ON DATABASE "QSMQ" TO "qliksenserepository"; +GRANT TEMPORARY, CONNECT ON DATABASE "SenseServices" TO PUBLIC; +GRANT ALL ON DATABASE "SenseServices" TO postgres; +GRANT CREATE ON DATABASE "SenseServices" TO "qliksenserepository"; + +GRANT TEMPORARY, CONNECT ON DATABASE "Licenses" TO PUBLIC; +GRANT ALL ON DATABASE "Licenses" TO postgres; +GRANT CREATE ON DATABASE "Licenses" TO qliksenserepository; + +CREATE ROLE qlogs_users WITH NOLOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +CREATE ROLE qlogs_reader WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +CREATE ROLE qlogs_writer WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; + +ALTER ROLE qlogs_reader WITH ENCRYPTED PASSWORD 'Qlik1234!'; +ALTER ROLE qlogs_writer WITH ENCRYPTED PASSWORD 'Qlik1234!'; + +GRANT qlogs_users TO qlogs_reader; +GRANT qlogs_users TO qlogs_writer; + +ALTER DATABASE "QLogs" OWNER TO qlogs_writer; --sets qlogs_writer as an owner of QLogs database + +SELECT * FROM pg_settings WHERE name = 'max_connections'; diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/terraform.tfvars.json new file mode 100644 index 0000000..7883695 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/terraform.tfvars.json @@ -0,0 +1,17 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "db_user": "postgres", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/variables.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/variables.tf new file mode 100644 index 0000000..2f8a427 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/psql/variables.tf @@ -0,0 +1,128 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "db_user" { + description = "The name of the DB user" + type = string + default = "postgres" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/rim.xml b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim.xml new file mode 100644 index 0000000..6e8c0e9 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim.xml @@ -0,0 +1,24 @@ + + + qliksenserepository + Qlik1234! + 10.54.242.47 + 5432 + \\10.219.178.210\qlikshare + \\10.219.178.210\qlikshare\StaticContent + \\10.219.178.210\qlikshare\ArchivedLogs + \\10.219.178.210\qlikshare\Apps + + false + false + * + 0.0.0.0/0,::/0 + 100 + true + true + false + Qlik1234! + Qlik1234! + 10.54.242.47 + 5432 + \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/main.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/main.tf new file mode 100644 index 0000000..8930d6d --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/main.tf @@ -0,0 +1,74 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + +resource "google_compute_disk" "dataRim" { + provider = google-beta + name = "rim-datadisk-${random_id.vm_suffix.hex}" + type = var.disk_type + zone = var.zone + # labels = { + # environment = "dev" + # } + size = 50 +} + +resource "google_compute_instance" "rim" { + provider = google-beta + name = "rim-${random_id.vm_suffix.hex}" + machine_type = var.vm_type + # min_cpu_platform commented out during testing - e2 instances do not support + # min_cpu_platform = var.min_cpu + zone = var.zone + + # hostname = "central-${random_id.vm_suffix.hex}" + + # tags = ["foo", "bar"] + # timeouts { + # create = "60m" + # delete = "2h" + # } + + boot_disk { + initialize_params { + image = var.image + } + } + + attached_disk { + source = google_compute_disk.dataRim.name + } + + # lifecycle { + # ignore_changes = [attached_disk] + # } + + network_interface { + network = "default" + + access_config { + network_tier = "STANDARD" + } + } + + metadata = { + windows-startup-script-url = "gs://qliksense/scripts/bootstrap.ps1" + } + + service_account { + email = "terraform@esoteric-parsec-243510.iam.gserviceaccount.com" + scopes = ["cloud-platform"] + } +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/outputs.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/outputs.tf new file mode 100644 index 0000000..8369613 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/outputs.tf @@ -0,0 +1,21 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "rim_name" { + value = google_compute_instance.rim.name +} + +output "rim_ext_ip" { + value = google_compute_instance.rim.network_interface[0].access_config[0].nat_ip +} + +output "rim_int_ip" { + value = google_compute_instance.rim.network_interface[0].network_ip +} + +# output "serial_out" { +# value = data.google_compute_instance_serial_port.serial.contents +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/terraform.tfvars.json new file mode 100644 index 0000000..7883695 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/terraform.tfvars.json @@ -0,0 +1,17 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "db_user": "postgres", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/variables.tf b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/variables.tf new file mode 100644 index 0000000..1dc6fe2 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/root/rim/variables.tf @@ -0,0 +1,134 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "db_user" { + description = "The name of the DB user" + type = string + default = "postgres" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/addRim.ps1 b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/addRim.ps1 new file mode 100644 index 0000000..b3775fb --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/addRim.ps1 @@ -0,0 +1,57 @@ +#!/usr/bin/env pwsh +# + +if ($env:computername -like "central-*") { + exit 0 +} + +Write-Host "------------------------------------------------------------" +Write-Host " Adding Rim to Cluster " +Write-Host "------------------------------------------------------------" + +$config = ( Read-GcsObject -Bucket "qliksense" -ObjectName "scripts/config.json" | ConvertFrom-Json ) + +# $centralNode = "central-84fe" +$centralNode = $config.cenName +$newNode = ($env:COMPUTERNAME).ToLower() +# $newNode = "rim-a8a3" +$qsAdminUser = $config.qsAdminUser +$qlikID = "$centralNode\$qsAdminUser" +$svcPwd = "Qlik1234!" +$secPwd = ConvertTo-SecureString -AsPlainText $svcPwd -Force +$secCred = New-Object System.Management.Automation.PSCredential -ArgumentList $qlikID, $secPwd + +$qsConn = New-PSSession -ComputerName $centralNode +Invoke-Command -Session $qsConn -ScriptBlock { Get-ChildItem -Path cert:\CurrentUser\My ` + | Where-Object { $_.Issuer -like "*$using:centralNode*" } ` + | Connect-Qlik -computername https://"$using:centralNode":4242 -Username $using:qlikID | Out-Null +} + +# Read-GcsObject -Bucket "qliksense" -ObjectName "certs/rim/client.pfx" -OutFile $deploy_path\certs\rim\client.pfx -Force +# Import-PfxCertificate -FilePath $deploy_path\certs\rim\client.pfx -CertStoreLocation Cert:\CurrentUser\My -Exportable + +Invoke-Command -Session $qsConn -ScriptBlock { ` + $proxyId = (Get-QlikVirtualProxy -full | Where-Object description -like "Central*").id} + +#TODO: Read existing whitelist as array, add new node to list and then re-write new list. +Invoke-Command -Session $qsConn -ScriptBlock { ` + Update-QlikVirtualProxy -id $proxyId -websocketCrossOriginWhiteList "$using:newNode", "$using:centralNode" | Out-Null +} + +Invoke-Command -Session $qsConn -ScriptBlock { ` + $password = New-QlikNode -hostname $using:newNode -name $using:newNode -nodePurpose Both -engineEnabled -proxyEnabled; + $foo = @{__pwd = "$password" } +} + +$foo = Invoke-Command -Session $qsConn -ScriptBlock { $foo } +Invoke-WebRequest -Uri "http://localhost:4570/certificateSetup" -Method Post -Body $foo -credential $secCred + +# Config Engine for better NFS Support, restart Engine to read new value +Set-Content -Path C:\ProgramData\Qlik\Sense\Engine\Settings.ini -Value "[Settings 7]`r`nMapNetworkDrives=1" +Restart-Service QlikSenseEngineService -Force + +# Tag the GCE metadata that the Node has been in initialized. +Set-GceInstance -Name ($env:computername).ToLower() -Zone $config.zone -AddTag "joined-cluster" + +# Close the remote PS session +Remove-PSSession $qsConn \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/bootstrap.ps1 b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/bootstrap.ps1 new file mode 100644 index 0000000..10562db --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/bootstrap.ps1 @@ -0,0 +1,129 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## or as a startup script via Metadata key windows-startup-script-url +## + +# Format and mount data disk +Write-Host "------------------------------------------------------------" +Write-Host " Create Data drive " +Write-Host "------------------------------------------------------------" + +Get-Disk | +Where-Object partitionstyle -eq 'raw' | +Initialize-Disk -PartitionStyle GPT -PassThru | +New-Partition -DriveLetter E -UseMaximumSize | +Format-Volume -FileSystem NTFS -NewFileSystemLabel 'Data' -Confirm:$false + +if (! (Test-Path E:\)) { + Write-Error "Drive not found" + exit 1 +} + +Write-Host "------------------------------------------------------------" +Write-Host " Create Local Accounts and add to Administrators Group " +Write-Host "------------------------------------------------------------" + +if (!(Get-LocalUser -Name qservice -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qservice' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +if (!(Get-LocalUser -Name qlikadmin -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qlikadmin' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +Add-LocalGroupMember -Group "Administrators" -Member "qservice", "qlikadmin" + +Write-Host "------------------------------------------------------------" +Write-Host " Copy tooling from Cloud Storage Bucket " +Write-Host "------------------------------------------------------------" + +# $config = ( Read-GcsObject -Bucket "qliksense" -ObjectName "scripts/config.json" | ConvertFrom-Json ) +$deploy_path = "E:\deploy" + +if (! (Test-Path $deploy_path)) { + New-Item -ItemType Directory -Path $deploy_path + New-Item -ItemType Directory -Path $deploy_path\binaries +} + +gsutil -m cp -r gs://qliksense/scripts $deploy_path\ +gsutil -m cp gs://qliksense/binaries/Qlik_Sense* $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/firefox_latest.exe $deploy_path\binaries\ + +Unblock-File -Path $deploy_path\binaries\* +Unblock-File -Path $deploy_path\scripts\* +REG ADD HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System /v ConsentPromptBehaviorAdmin /t REG_DWORD /d 0 /f + +# WinRM Connects +New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force +New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String + +Set-Item WSMan:localhost\client\trustedhosts -value * -Force + +Write-Host "------------------------------------------------------------" +Write-Host " Import PS Modules " +Write-Host "------------------------------------------------------------" + +# Installing Qlik-CLI +# Write-Host "Downloading Qlik-Cli from Github and importing the Module" +# Invoke-WebRequest "https://raw.githubusercontent.com/ahaydon/Qlik-Cli/master/Qlik-Cli.psm1" -OutFile $temp\Qlik-Cli.psm1 +# New-Item -ItemType directory -Path C:\Windows\System32\WindowsPowerShell\v1.0\Modules\Qlik-Cli -force +# Move-Item $temp\Qlik-Cli.psm1 C:\Windows\System32\WindowsPowerShell\v1.0\Modules\Qlik-Cli\ -force +# Import-Module Qlik-Cli.psm1 +# Export-QlikCertificate -machineNames rim -includeSecretsKey -exportFormat Windows + +Get-PackageProvider -Name NuGet -ForceBootstrap +Install-Module Qlik-CLI -Force + +Write-Host "------------------------------------------------------------" +Write-Host " Firefox " +Write-Host "------------------------------------------------------------" + +# Expand-Archive -Path $deploy_path\binaries\ps7.zip -DestinationPath $deploy_path\binaries\ps7 -Force +# Start-Sleep -Seconds 120 + +Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\firefox_latest.exe" -verb runAs -ArgumentList "/s" -Wait -PassThru } | Out-Null + +Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\vscode_stable.exe" -verb runAs -ArgumentList "/VERYSILENT /NORESTART /MERGETASKS=!runcode" -Wait -PassThru } | Out-Null + +Write-Host "------------------------------------------------------------" +Write-Host " Create QSEoW FW Rule " +Write-Host "------------------------------------------------------------" +New-NetFirewallRule -DisplayName "Qlik Sense" -Direction Inbound -LocalPort 80, 443, 3090, 4000, 4432, 4242, 4244, 4444, 4248, 4993, 4994, 5353, 5355, 5555, 5556 -Protocol TCP -Action Allow -ea Stop | Out-Null +# Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False + +#### Enable NFS Client, set Anon user to UID:GID 0 == root. Restart service. +Write-Host "------------------------------------------------------------" +Write-Host " Installing NFS Client " +Write-Host "------------------------------------------------------------" +Install-WindowsFeature -Name NFS-Client +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousUid" -Value "0" -PropertyType DWORD +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousGid" -Value "0" -PropertyType DWORD +nfsadmin client stop +nfsadmin client start + +# REG ADD HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System /v ConsentPromptBehaviorAdmin /t REG_DWORD /d 5 /f +# HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce + +# Create "RunOnce" registry key to install QSEoW +# New-ItemProperty -Path 'HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce' -Name 'Run' -Value 'C:\WINDOWS\system32\WindowsPowerShell\v1.0\powershell.exe -noprofile -sta -WindowStyle Hidden -executionpolicy unrestricted -file E:\deploy\scripts\qsInstall.ps1' + +# Tag the GCE metadata that the Node has been bootstrapped, remove startup-script URL. +Set-GceInstance -Name ($env:computername).ToLower() -Zone $config.zone -AddTag "bootstrapped" +Set-GceInstance -Name ($env:computername).ToLower() -Zone $config.zone -RemoveMetadata "windows-startup-script-url" + +Restart-Computer + +Exit 0 diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/config.json b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/config.json new file mode 100644 index 0000000..fb4b74d --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/config.json @@ -0,0 +1,15 @@ +[ + { + "dbUserName": "qliksenserepository", + "dbUserPassword": "Qlik1234!", + "dbInstanceIp": "10.54.242.56", + "dbPort": "5432", + "fsInstanceIp": "10.247.74.186", + "fsShareName": "qlikshare", + "deployPath": "E:\\deploy", + "cenName": "central-aa6d", + "qsAdminUser": "qlikadmin", + "qsSvcUser": "qservice", + "zone": "europe-west1-d" + } +] diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/qsInstall.ps1 b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/qsInstall.ps1 new file mode 100644 index 0000000..0afa7ee --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/qsInstall.ps1 @@ -0,0 +1,147 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## or as a startup script via Metadata key windows-startup-script-url +## + +$config = ( Read-GcsObject -Bucket "qliksense" -ObjectName "scripts/config.json" | ConvertFrom-Json ) +$qsAdminUser = $config.qsAdminUser +$centralNode = ($env:COMPUTERNAME).ToLower() +$qlikID = "$centralNode\$qsAdminUser" +$deploy_path = $config.deployPath + +#### Functions + +Function waitProxy { + $statusCode = 0 + Write-Host "Waiting for Proxy..." + while ($StatusCode -ne 200) { + try { $statusCode = (Invoke-WebRequest https://$($env:COMPUTERNAME)/qps/user -usebasicParsing).statusCode } + Catch { + Write-Host "Proxy unavailable - retrying in 30s" + Start-Sleep -s 30 + } + } + Write-Host "Proxy responding on $env:COMPUTERNAME, status code: $statusCode" +} + +# TODO: Archive XMLs to GCS Bucket? +Function writeXmlCen { + $xmlsettings = New-Object System.Xml.XmlWriterSettings + $xmlsettings.Indent = $true + $xmlsettings.IndentChars = " " + $filePath = "$deploy_path\scripts\cen.xml" # Set the File Name + $xml = [System.XML.XmlWriter]::Create($filePath, $Null) # Create The Document + $xml.WriteStartDocument() # Write the XML Decleration + $xml.WriteStartElement("SharedPersistenceConfiguration") # Write Root Element + $xml.WriteAttributeString("xmlns", "xsi", "", "http://www.w3.org/2001/XMLSchema-instance") + $xml.WriteAttributeString("xmlns", "xsd", "", "http://www.w3.org/2001/XMLSchema") + $xml.WriteElementString("DbUserName", "$($config.dbUserName)") # <-- Begin writing the XML file + $xml.WriteElementString("DbUserPassword", "$($config.dbUserPassword)") + $xml.WriteElementString("DbHost", "$($config.dbInstanceIp)") + $xml.WriteElementString("DbPort", "$($config.dbPort)") + $xml.WriteElementString("RootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)") + $xml.WriteElementString("StaticContentRootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\StaticContent") + $xml.WriteElementString("ArchivedLogsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\ArchivedLogs") + $xml.WriteElementString("AppsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\Apps") + $xml.WriteElementString("CreateCluster", "true") + $xml.WriteElementString("InstallLocalDb", "false") + $xml.WriteElementString("ConfigureDbListener", "false") + $xml.WriteElementString("ListenAddresses", "*") + $xml.WriteElementString("IpRange", "0.0.0.0/0") + $xml.WriteEndElement | Out-Null # <-- Closing RootElement + $xml.WriteEndDocument() | Out-Null # End the XML Document + $xml.Finalize | Out-Null # Finish The Document + $xml.Flush | Out-Null + $xml.Close() | Out-Null +} + +# xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" +Function writeXmlRim { + $xmlsettings = New-Object System.Xml.XmlWriterSettings + $xmlsettings.Indent = $true + $xmlsettings.IndentChars = " " + $filePath = "$deploy_path\scripts\rim.xml" # Set the File Name + $xml = [System.XML.XmlWriter]::Create($filePath, $Null) # Create The Document + $xml.WriteStartDocument() # Write the XML Decleration + $xml.WriteStartElement("SharedPersistenceConfiguration") # Write Root Element + $xml.WriteAttributeString("xmlns", "xsi", "", "http://www.w3.org/2001/XMLSchema-instance") + $xml.WriteAttributeString("xmlns", "xsd", "", "http://www.w3.org/2001/XMLSchema") + $xml.WriteElementString("DbUserName", "$($config.dbUserName)") # <-- Begin writing the XML file + $xml.WriteElementString("DbUserPassword", "$($config.dbUserPassword)") + $xml.WriteElementString("DbHost", "$($config.dbInstanceIp)") + $xml.WriteElementString("DbPort", "$($config.dbPort)") + $xml.WriteElementString("RootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)") + $xml.WriteElementString("StaticContentRootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\StaticContent") + $xml.WriteElementString("ArchivedLogsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\ArchivedLogs") + $xml.WriteElementString("AppsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\Apps") + $xml.WriteElementString("JoinCluster", "true") + $xml.WriteElementString("InstallLocalDb", "false") + $xml.WriteElementString("ConfigureDbListener", "false") + $xml.WriteElementString("ListenAddresses", "*") + $xml.WriteElementString("IpRange", "0.0.0.0/0") + $xml.WriteEndElement | Out-Null # <-- Closing RootElement + $xml.WriteEndDocument() | Out-Null # End the XML Document + $xml.Finalize | Out-Null # Finish The Document + $xml.Flush | Out-Null + $xml.Close() | Out-Null +} + +#### Launch silent install +Write-Host "------------------------------------------------------------" +Write-Host " Installing QSEoW " +Write-Host "------------------------------------------------------------" + +if ($env:computername -like "central-*") { + + writeXmlCen + + Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log $deploy_path\deploy.log accepteula=1 skipvalidation=1 installdir=$deploy_path\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=$deploy_path\scripts\cen.xml" -Wait -PassThru } | Out-Null + + Set-Item WSMan:localhost\client\trustedhosts -value * -Force + + # Connect to Qlik Sense + waitProxy + Get-ChildItem -Path cert:\CurrentUser\My | Where-Object { $_.Issuer -like "*$centralNode*" } | Connect-Qlik -computername https://"$centralNode":4242 -Username $qlikID | Out-Null + + # Set the SLK + $license = (Get-Content $deploy_path\scripts\license.json -raw) | ConvertFrom-Json + Set-QlikLicense -key "$($license.signed.key)" -name "$($license.signed.name)" -organization "$($license.signed.organization)" | Out-Null + + If (Test-Path "$deploy_path\binaries\Qlik_Sense_update.exe") { + Write-Host "------------------------------------------------------------" + Write-Host " Installing QSEoW Patch " + Write-Host "------------------------------------------------------------" + Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\Qlik_Sense_Update.exe" -ArgumentList "install" -Wait -Passthru } | Out-Null + } + + # Config Engine for better NFS Support, restart Engine to read new value + Set-Content -Path C:\ProgramData\Qlik\Sense\Engine\Settings.ini -Value "[Settings 7]`r`nMapNetworkDrives=1" + Restart-Service QlikSenseEngineService -Force + + # Tag the GCE metadata that the Node has been in initialized. + Set-GceInstance -Name ($env:computername).ToLower() -Zone $config.zone -AddTag "qseInstalled" +} + +else { + + writeXmlRim + + Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log $deploy_path\deploy.log accepteula=1 skipvalidation=1 installdir=$deploy_path\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=$deploy_path\scripts\rim.xml" -Wait -PassThru } | Out-Null + + Set-Item WSMan:localhost\client\trustedhosts -value * -Force + + If (Test-Path "$deploy_path\binaries\Qlik_Sense_update.exe") { + Write-Host "------------------------------------------------------------" + Write-Host " Installing QSEoW Patch " + Write-Host "------------------------------------------------------------" + Invoke-Command -ScriptBlock { Start-Process -FilePath "$deploy_path\binaries\Qlik_Sense_Update.exe" ` + -ArgumentList "install" -Wait -Passthru } | Out-Null + } + + Invoke-Command -ScriptBlock { Start-Process C:\WINDOWS\system32\WindowsPowerShell\v1.0\powershell.exe "-noprofile -sta -WindowStyle Hidden -executionpolicy unrestricted -file $deploy_path\scripts\addRim.ps1" -Wait -PassThru } | Out-Null + + # Tag the GCE metadata that the Node has been initialized into the cluster + Set-GceInstance -Name ($env:computername).ToLower() -Zone $config.zone -AddTag "qse-installed" + +} diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/scratch b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/scratch new file mode 100644 index 0000000..6052e8e --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/scratch @@ -0,0 +1,48 @@ +REG ADD HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System /v ConsentPromptBehaviorAdmin /t REG_DWORD /d 0 /f + +Invoke-Command -ScriptBlock {Start-Process -FilePath "E:\deploy\binaries\Qlik_Sense_setup.exe" -verb runAs -ArgumentList "-s -log E:\deploy\deploy.log accepteula=1 installdir=E:\deploy\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=E:\deploy\scripts\cen.xml" -Wait -PassThru} | Out-Null + +$license = (Get-Content E:\scripts\license.json -raw) | ConvertFrom-Json +Set-QlikLicense -key "$($license.signed.key)" -name "$($license.signed.name)" -organization "$($license.signed.organization)"| Out-Null +# qlikSenseProfessionalAccess + + +New-GcsObject -Bucket "qliksense" -Folder "C:\ProgramData\Qlik\Sense\Repository\Exported Certificates" -Force +Write-GcsObject -Bucket "qliksense" -File "C:\ProgramData\Qlik\Sense\Repository\Exported Certificates\rim\client.pfx" -ObjectName "certs/rim/client.pfx" + +REG ADD HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System /v ConsentPromptBehaviorAdmin /t REG_DWORD /d 5 /f +######################################## + +# 6 $license = (Get-Content $deploy_path\scripts\license.json -raw) | ConvertFrom-Json +# 7 $centralNode = ($env:COMPUTERNAME).ToLower() +# 9 $qlikID = ".\qlikadmin" +# 10 $deploy_path = "E:\deploy" + +# 14 Set-QlikLicense -key "$($license.signed.key)" -name "$($license.signed.name)" -organization "$($license.signed.... + +# 18 Export-QlikCertificate -machineNames rim -includeSecretsKey -exportFormat Windows + +# 1 $deploy_path = "E:\deploy" +# 2 $centralNode = ($env:COMPUTERNAME).ToLower() +# 3 $qlikID = ".\qlikadmin" +# Write-GcsObject -Bucket "qliksense" -File "C:\ProgramData\Qlik\Sense\Repository\Exported Certificates\rim\client.pfx" -ObjectName "certs/rim/client.pfx" + + +# Installing Qlik-CLI +# Write-Host "Downloading Qlik-Cli from Github and importing the Module" +# Invoke-WebRequest "https://raw.githubusercontent.com/ahaydon/Qlik-Cli/master/Qlik-Cli.psm1" -OutFile $temp\Qlik-Cli.psm1 +# New-Item -ItemType directory -Path C:\Windows\System32\WindowsPowerShell\v1.0\Modules\Qlik-Cli -force +# Move-Item $temp\Qlik-Cli.psm1 C:\Windows\System32\WindowsPowerShell\v1.0\Modules\Qlik-Cli\ -force +# Import-Module Qlik-Cli.psm1 +# Export-QlikCertificate -machineNames rim -includeSecretsKey -exportFormat Windows + +Read-GcsObject -Bucket "qliksense" -ObjectName "certs/rim/client.pfx" -OutFile $deploy_path\certs\rim\client.pfx -Force + +Import-PfxCertificate -FilePath $deploy_path\certs\rim\client.pfx -CertStoreLocation Cert:\CurrentUser\My -Exportable + +$nodeid = Get-QlikNode -filter "(name eq $newNode)" +Invoke-QlikGet -path /qrs/servernoderegistration/start/$($nodeid.id) + +# Invoke-WebRequest -Uri https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.zip -OutFile $deploy_path\binaries\ps7.zip +# Invoke-WebRequest -Uri https://aka.ms/win32-x64-user-stable -Outfile $deploy_path\binaries\vscode_stable.exe +# Invoke-WebRequest -Uri "https://download.mozilla.org/?product=firefox-latest-ssl&os=win64&lang=en-US" -Outfile $deploy_path\binaries\firefox_latest.exe \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/gcp/scripts/xml-test.ps1 b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/xml-test.ps1 new file mode 100644 index 0000000..4d69f54 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/gcp/scripts/xml-test.ps1 @@ -0,0 +1,39 @@ +$config = ( Read-GcsObject -Bucket "qliksense" -ObjectName "scripts/config.json" | ConvertFrom-Json ) +$qsAdminUser = $config.qsAdminUser +$centralNode = ($env:COMPUTERNAME).ToLower() +$qlikID = "$centralNode\$qsAdminUser" +$deploy_path = $config.deployPath + +#### Functions + +Function writeXmlCen { + $xmlsettings = New-Object System.Xml.XmlWriterSettings + $xmlsettings.Indent = $true + $xmlsettings.IndentChars = " " + $filePath = "$deploy_path\scripts\cen.xml" # Set the File Name + $xml = [System.XML.XmlWriter]::Create($filePath, $Null) # Create The Document + $xml.WriteStartDocument() # Write the XML Decleration + $xml.WriteStartElement("SharedPersistenceConfiguration") # Write Root Element + $xml.WriteAttributeString("xmlns", "xsi", "", "http://www.w3.org/2001/XMLSchema-instance") + $xml.WriteAttributeString("xmlns", "xsd", "", "http://www.w3.org/2001/XMLSchema") + $xml.WriteElementString("DbUserName", "$($config.dbUserName)") # <-- Begin writing the XML file + $xml.WriteElementString("DbUserPassword", "$($config.dbUserPassword)") + $xml.WriteElementString("DbHost", "$($config.dbInstanceIp)") + $xml.WriteElementString("DbPort", "$($config.dbPort)") + $xml.WriteElementString("RootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)") + $xml.WriteElementString("StaticContentRootDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\StaticContent") + $xml.WriteElementString("ArchivedLogsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\ArchivedLogs") + $xml.WriteElementString("AppsDir", "$("\\" + $config.fsInstanceIp + "\" + $config.fsShareName)" + "\Apps") + $xml.WriteElementString("CreateCluster", "true") + $xml.WriteElementString("InstallLocalDb", "false") + $xml.WriteElementString("ConfigureDbListener", "false") + $xml.WriteElementString("ListenAddresses", "*") + $xml.WriteElementString("IpRange", "0.0.0.0/0") + $xml.WriteEndElement | Out-Null # <-- Closing RootElement + $xml.WriteEndDocument() | Out-Null # End the XML Document + $xml.Finalize | Out-Null # Finish The Document + $xml.Flush | Out-Null + $xml.Close() | Out-Null +} + +writeXmlCen \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/graph.svg b/sense-conductor/bt-autoscaler/terraform/roots/central/graph.svg new file mode 100644 index 0000000..89d6b68 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/graph.svg @@ -0,0 +1,367 @@ + + + + + + +%3 + + + +[root] google_compute_disk.data (expand) + +google_compute_disk.data + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"] + +provider["registry.terraform.io/hashicorp/google-beta"] + + + +[root] google_compute_disk.data (expand)->[root] provider["registry.terraform.io/hashicorp/google-beta"] + + + + + +[root] random_id.vm_suffix (expand) + +random_id.vm_suffix + + + +[root] google_compute_disk.data (expand)->[root] random_id.vm_suffix (expand) + + + + + +[root] var.disk_type + + + +var.disk_type + + + +[root] google_compute_disk.data (expand)->[root] var.disk_type + + + + + +[root] google_compute_instance.default (expand) + +google_compute_instance.default + + + +[root] google_compute_instance.default (expand)->[root] google_compute_disk.data (expand) + + + + + +[root] var.image + + + +var.image + + + +[root] google_compute_instance.default (expand)->[root] var.image + + + + + +[root] var.vm_type + + + +var.vm_type + + + +[root] google_compute_instance.default (expand)->[root] var.vm_type + + + + + +[root] var.project_id + + + +var.project_id + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"]->[root] var.project_id + + + + + +[root] var.region + + + +var.region + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"]->[root] var.region + + + + + +[root] var.zone + + + +var.zone + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"]->[root] var.zone + + + + + +[root] provider["registry.terraform.io/hashicorp/random"] + +provider["registry.terraform.io/hashicorp/random"] + + + +[root] random_id.vm_suffix (expand)->[root] provider["registry.terraform.io/hashicorp/random"] + + + + + +[root] var.additional_users + + + +var.additional_users + + + +[root] var.availability_type + + + +var.availability_type + + + +[root] var.backup_configuration + + + +var.backup_configuration + + + +[root] var.database_version + + + +var.database_version + + + +[root] var.min_cpu + + + +var.min_cpu + + + +[root] var.private_network + + + +var.private_network + + + +[root] var.tier + + + +var.tier + + + +[root] var.user_name + + + +var.user_name + + + +[root] var.user_password + + + +var.user_password + + + +[root] meta.count-boundary (EachMode fixup) + +[root] meta.count-boundary (EachMode fixup) + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.additional_users + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.availability_type + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.backup_configuration + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.database_version + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.min_cpu + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.private_network + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.tier + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.user_name + + + + + +[root] meta.count-boundary (EachMode fixup)->[root] var.user_password + + + + + +[root] output.gce_external_ip (expand) + +[root] output.gce_external_ip (expand) + + + +[root] meta.count-boundary (EachMode fixup)->[root] output.gce_external_ip (expand) + + + + + +[root] output.gce_internal_ip (expand) + +[root] output.gce_internal_ip (expand) + + + +[root] meta.count-boundary (EachMode fixup)->[root] output.gce_internal_ip (expand) + + + + + +[root] output.gce_external_ip (expand)->[root] google_compute_instance.default (expand) + + + + + +[root] output.gce_internal_ip (expand)->[root] google_compute_instance.default (expand) + + + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"] (close) + +[root] provider["registry.terraform.io/hashicorp/google-beta"] (close) + + + +[root] provider["registry.terraform.io/hashicorp/google-beta"] (close)->[root] google_compute_instance.default (expand) + + + + + +[root] provider["registry.terraform.io/hashicorp/random"] (close) + +[root] provider["registry.terraform.io/hashicorp/random"] (close) + + + +[root] provider["registry.terraform.io/hashicorp/random"] (close)->[root] random_id.vm_suffix (expand) + + + + + +[root] root + +[root] root + + + +[root] root->[root] meta.count-boundary (EachMode fixup) + + + + + +[root] root->[root] provider["registry.terraform.io/hashicorp/google-beta"] (close) + + + + + +[root] root->[root] provider["registry.terraform.io/hashicorp/random"] (close) + + + + + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/iam.json b/sense-conductor/bt-autoscaler/terraform/roots/central/iam.json new file mode 100644 index 0000000..77c4aec --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/iam.json @@ -0,0 +1,19 @@ +{ + "bindings": [ + { + "members": [ + "user:jprdonnelly@gmail.com" + ], + "role": "roles/iam.serviceAccountAdmin" + }, + { + "members": [ + "serviceAccount:593942668809-compute@developer.gserviceaccount.com", + "user:jprdonnelly@gmail.com" + ], + "role": "roles/iam.serviceAccountUser" + } + ], + "etag": "BwWvglMDG2Y=", + "version": 1 +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/iam.yaml b/sense-conductor/bt-autoscaler/terraform/roots/central/iam.yaml new file mode 100644 index 0000000..c16a6f8 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/iam.yaml @@ -0,0 +1,14 @@ +bindings: +- members: + - user:jprdonnelly@gmail.com + role: roles/iam.serviceAccountAdmin +- members: + - serviceAccount:593942668809-compute@developer.gserviceaccount.com + - user:jprdonnelly@gmail.com + role: roles/iam.serviceAccountUser +- members: + - serviceAccount:593942668809-compute@developer.gserviceaccount.com + - user:jprdonnelly@gmail.com + role: roles/compute.instanceAdmin.v1 +etag: BwWvglMDG2Y= +version: 1 diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/main.tf b/sense-conductor/bt-autoscaler/terraform/roots/central/main.tf new file mode 100644 index 0000000..da8eeaa --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/main.tf @@ -0,0 +1,107 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + +resource "google_compute_disk" "dataCen" { + provider = google-beta + name = "central-datadisk-${random_id.vm_suffix.hex}" + type = var.disk_type + zone = var.zone + # labels = { + # environment = "dev" + # } + size = 50 +} + +resource "google_compute_instance" "central" { + provider = google-beta + name = "central-${random_id.vm_suffix.hex}" + machine_type = var.vm_type + # min_cpu_platform commented out during testing - e2 instances do not support + # min_cpu_platform = var.min_cpu + zone = var.zone + + # hostname = "central-${random_id.vm_suffix.hex}" + + # tags = ["foo", "bar"] + # timeouts { + # create = "60m" + # delete = "2h" + # } + + boot_disk { + initialize_params { + image = var.image + } + } + + attached_disk { + source = google_compute_disk.dataCen.name + } + + # lifecycle { + # ignore_changes = [attached_disk] + # } + + network_interface { + network = "default" + + access_config { + network_tier = "STANDARD" + } + } + + metadata = { + windows-startup-script-url = "gs://qliksense/scripts/bootstrap-cen.ps1" + } + + service_account { + email = "terraform@esoteric-parsec-243510.iam.gserviceaccount.com" + scopes = ["cloud-platform"] + } +} + +# data "google_compute_instance_serial_port" "serial" { +# provider = google-beta +# instance = google_compute_instance.central.name +# zone = var.zone +# port = 4 +# } + +# resource "local_file" "log" { +# filename = "${path.module}/tf-deploy.log" +# content = join("\n", "${data.google_compute_instance_serial_port.serial.contents}") +# } + +# resource "null_resource" "bootstrap" { + +# connection { +# type = "winrm" +# https = true +# insecure = true +# use_ntlm = true +# user = var.user_name +# password = var.user_password +# host = google_compute_instance.central.name +# timeout = "20m" +# } + +# provisioner "remote-exec" { +# inline = [ +# "powershell.exe -ExecutionPolicy Bypass -File E:\\deploy\\scripts\\win-nfs_client-install.ps1" +# ] +# } + +# } diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/outputs.tf b/sense-conductor/bt-autoscaler/terraform/roots/central/outputs.tf new file mode 100644 index 0000000..4683a15 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/outputs.tf @@ -0,0 +1,21 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "gce_name" { + value = google_compute_instance.central.name +} + +output "gce_external_ip" { + value = google_compute_instance.central.network_interface[0].access_config[0].nat_ip +} + +output "gce_internal_ip" { + value = google_compute_instance.central.network_interface[0].network_ip +} + +# output "serial_out" { +# value = data.google_compute_instance_serial_port.serial.contents +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/roots/central/terraform.tfvars.json new file mode 100644 index 0000000..3da0f41 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/terraform.tfvars.json @@ -0,0 +1,14 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/central/variables.tf b/sense-conductor/bt-autoscaler/terraform/roots/central/variables.tf new file mode 100644 index 0000000..ac51dd5 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/central/variables.tf @@ -0,0 +1,116 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/file/main.tf b/sense-conductor/bt-autoscaler/terraform/roots/file/main.tf new file mode 100644 index 0000000..acb6094 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/file/main.tf @@ -0,0 +1,32 @@ +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +resource "google_filestore_instance" "bt-filestore" { + provider = google-beta + name = "qseow-files" + zone = var.zone + tier = var.file_tier + project = var.project_id + + file_shares { + capacity_gb = 1024 + name = "qlikshare" + + # nfs_export_options { + # ip_ranges = ["0.0.0.0/0"] + # access_mode = "READ_WRITE" + # squash_mode = "NO_ROOT_SQUASH" + # anon_uid = 0 + # anon_gid = 0 + # } + } + + networks { + network = "default" + modes = ["MODE_IPV4"] + } +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/file/outputs.tf b/sense-conductor/bt-autoscaler/terraform/roots/file/outputs.tf new file mode 100644 index 0000000..de185b0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/file/outputs.tf @@ -0,0 +1,17 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "fs_instance_ip" { + value = google_filestore_instance.bt-filestore.networks[0].ip_addresses +} + +output "fs_instance_name" { + value = google_filestore_instance.bt-filestore.name +} + +# output "fs_instance_path" { +# value = google_filestore_instance.bt-filestore.fileShares.name +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/file/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/roots/file/terraform.tfvars.json new file mode 100644 index 0000000..0064502 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/file/terraform.tfvars.json @@ -0,0 +1,12 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "availability_type": "REGIONAL", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/file/variables.tf b/sense-conductor/bt-autoscaler/terraform/roots/file/variables.tf new file mode 100644 index 0000000..debd2db --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/file/variables.tf @@ -0,0 +1,98 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/geo/main.tf b/sense-conductor/bt-autoscaler/terraform/roots/geo/main.tf new file mode 100644 index 0000000..de70525 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/geo/main.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google" { + version = "3.10.0" + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project + region = var.region + zone = var.zone +} + +data "google_compute_instance_group" "geo" { + name = "GeoAnalyticsServer" + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/geo/outputs.tf b/sense-conductor/bt-autoscaler/terraform/roots/geo/outputs.tf new file mode 100644 index 0000000..a31dd16 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/geo/outputs.tf @@ -0,0 +1,9 @@ +# Configure return values from google_sql_database_instance + +output "psql_ip_address" { + value = google_sql_database_instance.qseow-psql.private_ip_address +} + +output "psql_instance_name" { + value = google_sql_database_instance.qseow-psql.connection_name +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/geo/variables.tf b/sense-conductor/bt-autoscaler/terraform/roots/geo/variables.tf new file mode 100644 index 0000000..523d4e0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/geo/variables.tf @@ -0,0 +1,92 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/destroy.sh b/sense-conductor/bt-autoscaler/terraform/roots/psql/destroy.sh new file mode 100644 index 0000000..87b46b4 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/destroy.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +terraform state rm google_sql_user.users +terraform destroy $@ -auto-approve diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/main.tf b/sense-conductor/bt-autoscaler/terraform/roots/psql/main.tf new file mode 100644 index 0000000..76fb589 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/main.tf @@ -0,0 +1,60 @@ +### Main TF - split pSQL to a module +## + +provider "google" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for DB suffix +resource "random_id" "db_suffix" { + byte_length = 2 +} + +# Create res for Cloud SQL DB create +resource "google_sql_database_instance" "qseow-psql" { + name = "qseow-psql-${random_id.db_suffix.hex}" + project = var.project_id + region = var.region + database_version = var.database_version + + settings { + tier = var.db_tier + availability_type = var.availability_type + backup_configuration { + enabled = "true" + } + + ip_configuration { + ipv4_enabled = false + private_network = var.private_network + } + } +} + +# Configure postgres admin user +## Note that the SQL instance cannot be rm'd without removing the TFState for +## the SQL user (TF/PostgreSQL race condition) +# +## Use "./destroy.sh" instead of "terraform destroy" + +resource "google_sql_user" "users" { + instance = google_sql_database_instance.qseow-psql.name + name = var.db_user + password = var.user_password + project = var.project_id + + depends_on = [google_sql_database_instance.qseow-psql] + +} + +# Execute stored SQL file, prep pSQL for Qlik Sense + +resource "null_resource" "db_setup" { + provisioner "local-exec" { + command = "PGPASSWORD=${google_sql_user.users.password} /usr/bin/psql -h ${google_sql_database_instance.qseow-psql.private_ip_address} -Upostgres --dbname=postgres < qseow_db_setup.sql" + } + depends_on = [google_sql_database_instance.qseow-psql, google_sql_user.users] +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/outputs.tf b/sense-conductor/bt-autoscaler/terraform/roots/psql/outputs.tf new file mode 100644 index 0000000..a31dd16 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/outputs.tf @@ -0,0 +1,9 @@ +# Configure return values from google_sql_database_instance + +output "psql_ip_address" { + value = google_sql_database_instance.qseow-psql.private_ip_address +} + +output "psql_instance_name" { + value = google_sql_database_instance.qseow-psql.connection_name +} \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/qseow_db_setup.sql b/sense-conductor/bt-autoscaler/terraform/roots/psql/qseow_db_setup.sql new file mode 100644 index 0000000..61b296e --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/qseow_db_setup.sql @@ -0,0 +1,37 @@ +CREATE DATABASE "QSR" ENCODING = 'UTF8'; +CREATE DATABASE "QSMQ" ENCODING = 'UTF8'; +CREATE DATABASE "Licenses" ENCODING = 'UTF8'; +CREATE DATABASE "SenseServices" ENCODING = 'UTF8'; +CREATE DATABASE "QLogs" ENCODING = 'UTF8'; + +CREATE ROLE "qliksenserepository" WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +ALTER ROLE "qliksenserepository" WITH ENCRYPTED PASSWORD 'Qlik1234!'; +GRANT "qliksenserepository" TO postgres; + +ALTER DATABASE "QSR" OWNER TO "qliksenserepository"; +ALTER DATABASE "SenseServices" OWNER TO "qliksenserepository"; +ALTER DATABASE "QSMQ" OWNER TO "qliksenserepository"; +ALTER DATABASE "Licenses" OWNER TO qliksenserepository; + +GRANT TEMPORARY, CONNECT ON DATABASE "QSMQ" TO PUBLIC; +GRANT ALL ON DATABASE "QSMQ" TO postgres; +GRANT CREATE ON DATABASE "QSMQ" TO "qliksenserepository"; +GRANT TEMPORARY, CONNECT ON DATABASE "SenseServices" TO PUBLIC; +GRANT ALL ON DATABASE "SenseServices" TO postgres; +GRANT CREATE ON DATABASE "SenseServices" TO "qliksenserepository"; + +GRANT TEMPORARY, CONNECT ON DATABASE "Licenses" TO PUBLIC; +GRANT ALL ON DATABASE "Licenses" TO postgres; +GRANT CREATE ON DATABASE "Licenses" TO qliksenserepository; + +CREATE ROLE qlogs_users WITH NOLOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +CREATE ROLE qlogs_reader WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; +CREATE ROLE qlogs_writer WITH LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION VALID UNTIL 'infinity'; + +ALTER ROLE qlogs_reader WITH ENCRYPTED PASSWORD 'Qlik1234!'; +ALTER ROLE qlogs_writer WITH ENCRYPTED PASSWORD 'Qlik1234!'; + +GRANT qlogs_users TO qlogs_reader; +GRANT qlogs_users TO qlogs_writer; + +ALTER DATABASE "QLogs" OWNER TO qlogs_writer; --sets qlogs_writer as an owner of QLogs database \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars b/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars new file mode 100644 index 0000000..795bd24 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars @@ -0,0 +1,5 @@ + project = "esoteric-parsec-243510" + region = "us-central1" + zone = "us-central1-a" + aclCIDR = "71.164.77.198/32" + postgresPwd = "Qlik1234!" \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars.json new file mode 100644 index 0000000..3816731 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/terraform.tfvars.json @@ -0,0 +1,13 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "db_tier": "db-g1-small", + "file_tier": "BASIC_HDD", + "availability_type": "REGIONAL", + "db_user": "postgres", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/variables.tf b/sense-conductor/bt-autoscaler/terraform/roots/psql/variables.tf new file mode 100644 index 0000000..9beee6a --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/variables.tf @@ -0,0 +1,104 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "db_tier" { + description = "The tier for the primary SQL instance." + type = string + default = "db-f1-micro" +} + +variable "file_tier" { + description = "The tier for the Filestore instance." + type = string + default = "BASIC_HDD" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "db_user" { + description = "The name of the DB user" + type = string + default = "postgres" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/psql/versions.tf b/sense-conductor/bt-autoscaler/terraform/roots/psql/versions.tf new file mode 100644 index 0000000..ffdead5 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/psql/versions.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + postgresql = { + source = "terraform-providers/postgresql" + } + random = { + source = "hashicorp/random" + } + template = { + source = "hashicorp/template" + } + null = { + source = "hashicorp/null" + } + } + required_version = ">= 0.13" +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/rim/main.tf b/sense-conductor/bt-autoscaler/terraform/roots/rim/main.tf new file mode 100644 index 0000000..21ff516 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/rim/main.tf @@ -0,0 +1,106 @@ +terraform { + required_version = ">= 0.12" +} + +provider "google-beta" { + credentials = file("esoteric-parsec-243510-a8f93bb5a906.json") + project = var.project_id + region = var.region + zone = var.zone +} + +# Create random ID for VM suffix +resource "random_id" "vm_suffix" { + byte_length = 2 +} + +resource "google_compute_disk" "dataRim" { + provider = google-beta + name = "rim-datadisk-${random_id.vm_suffix.hex}" + type = var.disk_type + zone = var.zone + # labels = { + # environment = "dev" + # } + size = 50 +} + +resource "google_compute_instance" "rim" { + provider = google-beta + name = "rim-${random_id.vm_suffix.hex}" + machine_type = var.vm_type + # min_cpu_platform commented out during testing - e2 instances do not support + # min_cpu_platform = var.min_cpu + zone = var.zone + + # hostname = "central-${random_id.vm_suffix.hex}" + + # tags = ["foo", "bar"] + # timeouts { + # create = "60m" + # delete = "2h" + # } + + boot_disk { + initialize_params { + image = var.image + } + } + + attached_disk { + source = google_compute_disk.dataRim.name + } + + # lifecycle { + # ignore_changes = [attached_disk] + # } + + network_interface { + network = "default" + + access_config { + network_tier = "STANDARD" + } + } + + metadata = { + windows-startup-script-url = "gs://qliksense/scripts/bootstrap-rim.ps1" + } + + service_account { + email = "terraform@esoteric-parsec-243510.iam.gserviceaccount.com" + scopes = ["userinfo-email", "compute-ro", "storage-rw"] + } +} +# data "google_compute_instance_serial_port" "serial" { +# provider = google-beta +# instance = google_compute_instance.central.name +# zone = var.zone +# port = 4 +# } + +# resource "local_file" "log" { +# filename = "${path.module}/tf-deploy.log" +# content = join("\n", "${data.google_compute_instance_serial_port.serial.contents}") +# } + +# resource "null_resource" "bootstrap" { + +# connection { +# type = "winrm" +# https = true +# insecure = true +# use_ntlm = true +# user = var.user_name +# password = var.user_password +# host = google_compute_instance.central.name +# timeout = "20m" +# } + +# provisioner "remote-exec" { +# inline = [ +# "powershell.exe -ExecutionPolicy Bypass -File E:\\deploy\\scripts\\win-nfs_client-install.ps1" +# ] +# } + +# } diff --git a/sense-conductor/bt-autoscaler/terraform/roots/rim/outputs.tf b/sense-conductor/bt-autoscaler/terraform/roots/rim/outputs.tf new file mode 100644 index 0000000..e8cb5e0 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/rim/outputs.tf @@ -0,0 +1,21 @@ +# Configure return values from google_sql_database_instance + +# output "psql_ip_address" { +# value = google_sql_database_instance.qseow-psql.private_ip_address +# } + +output "gce_name" { + value = google_compute_instance.rim.name +} + +output "gce_external_ip" { + value = google_compute_instance.rim.network_interface[0].access_config[0].nat_ip +} + +output "gce_internal_ip" { + value = google_compute_instance.rim.network_interface[0].network_ip +} + +# output "serial_out" { +# value = data.google_compute_instance_serial_port.serial.contents +# } \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/rim/terraform.tfvars.json b/sense-conductor/bt-autoscaler/terraform/roots/rim/terraform.tfvars.json new file mode 100644 index 0000000..3da0f41 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/rim/terraform.tfvars.json @@ -0,0 +1,14 @@ +{ + "project_id": "esoteric-parsec-243510", + "region": "europe-west1", + "zone": "europe-west1-d", + "private_network": "projects/esoteric-parsec-243510/global/networks/default", + "database_version": "POSTGRES_9_6", + "vm_type": "e2-highmem-4", + "min_cpu": "Intel Skylake", + "image": "gce-uefi-images/windows-2019", + "disk_type": "pd-ssd", + "availability_type": "REGIONAL", + "user_name": "qlikadmin", + "user_password": "Qlik1234!" +} diff --git a/sense-conductor/bt-autoscaler/terraform/roots/rim/variables.tf b/sense-conductor/bt-autoscaler/terraform/roots/rim/variables.tf new file mode 100644 index 0000000..ac51dd5 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/rim/variables.tf @@ -0,0 +1,116 @@ +variable "project_id" { + type = string + description = "The project ID to manage the Cloud SQL resources" +} + +variable "database_version" { + description = "The database version to use" + type = string +} + +variable "region" { + type = string + description = "GCP Region" + default = "us-central1" +} + +variable "tier" { + description = "The tier for the master instance." + type = string + default = "db-f1-micro" +} + +variable "zone" { + type = string + description = "Zone target" +} + +variable "disk_type" { + description = "GCE Boot/Attached Disk Type" + type = string + default = "pd-ssd" +} + +variable "min_cpu" { + description = "GCE Minimum CPU Family" + type = string + default = "AMD Rome " +} + +variable "image" { + description = "Path to GCE Image Type" + type = string + default = "windows-2019/windows-server-2019-dc-v20200908" +} + +variable "vm_type" { + description = "The GCE machine type" + type = string + default = "n2d-highmem-8" +} + +variable "availability_type" { + description = "The availability type for the master instance.This is only used to set up high availability for the PostgreSQL instance. Can be either `ZONAL` or `REGIONAL`." + type = string + default = "REGIONAL" +} + +variable "backup_configuration" { + description = "The backup_configuration settings subblock for the database setings" + type = object({ + enabled = bool + start_time = string + location = string + }) + default = { + enabled = false + start_time = null + location = null + } +} + +# variable "authorized_networks_a" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "71.164.77.198/32" +# } + +# variable "authorized_networks_b" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +# variable "authorized_networks_c" { +# description = "CIDR Block to add to network ACL" +# type = string +# default = "127.0.0.1/32" +# } + +variable "private_network" { + description = "Full path to private network ID" + type = string + default = "projects/esoteric-parsec-243510/global/networks/private-network" +} + +variable "user_name" { + description = "The name of the default user" + type = string + default = "default" +} + +variable "user_password" { + description = "The password for the default user. If not set, a random one will be generated and available in the generated_user_password output variable." + type = string + default = "" +} + +variable "additional_users" { + description = "A list of users to be created in your cluster" + type = list(object({ + name = string + password = string + })) + default = [] +} + diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-cen.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-cen.ps1 new file mode 100644 index 0000000..6d57d15 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-cen.ps1 @@ -0,0 +1,112 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## or as a startup script via Metadata key windows-startup-script-url +## + +if ($env:computername -notlike "central-*") { + exit 0 +} + +# Format and mount data disk +Write-Host "## ========================>> Create Data drive & dirs" + +Get-Disk | + Where-Object partitionstyle -eq 'raw' | + Initialize-Disk -PartitionStyle GPT -PassThru | + New-Partition -DriveLetter E -UseMaximumSize | + Format-Volume -FileSystem NTFS -NewFileSystemLabel 'Data' -Confirm:$false + +if (! (Test-Path E:\)) { + Write-Error "Drive not found" + exit 1 +} + +$deploy_path = "E:\deploy" + +if (! (Test-Path $deploy_path)) { + New-Item -ItemType Directory -Path $deploy_path + New-Item -ItemType Directory -Path $deploy_path\binaries + New-Item -ItemType Directory -Path $deploy_path\modules + New-Item -ItemType Directory -Path $deploy_path\modules\tf + New-Item -ItemType Directory -Path $deploy_path\modules\ps +} + +Write-Host "## ========================>> Create Local Accounts and add to Administrators Group" + +if(!(Get-LocalUser -Name qservice -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qservice' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +if(!(Get-LocalUser -Name qlikadmin -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qlikadmin' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +Add-LocalGroupMember -Group "Administrators" -Member "qservice", "qlikadmin" + +Write-Host "## ========================>> Copy scripts and binaries from Cloud Storage Bucket" +gsutil -m cp -r gs://qliksense/scripts $deploy_path\ +gsutil -m cp gs://qliksense/binaries/Qlik_Sense* $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/vscode_stable.exe $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/firefox_latest.exe $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/ps7.zip $deploy_path\binaries\ +# gsutil -m cp -r gs://qliksense/modules 'C:\Program Files\WindowsPowerShell\Modules' + +# Invoke-WebRequest -Uri https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.zip -OutFile $deploy_path\binaries\ps7.zip +# Invoke-WebRequest -Uri https://aka.ms/win32-x64-user-stable -Outfile $deploy_path\binaries\vscode_stable.exe +# Invoke-WebRequest -Uri "https://download.mozilla.org/?product=firefox-latest-ssl&os=win64&lang=en-US" -Outfile $deploy_path\binaries\firefox_latest.exe + +Unblock-File -Path $deploy_path\binaries\* +Unblock-File -Path $deploy_path\scripts\* + +# WinRM Connects +New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force +New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String + +Write-Host "## ========================>> Import PS Modules" +Get-PackageProvider -Name NuGet -ForceBootstrap + +Install-Module PSDscResources -Force +Install-Module QlikResources -Force + +Write-Host "## ========================>> Deploy PS7 | Firefox | VS Code" +Expand-Archive -Path $deploy_path\binaries\ps7.zip -DestinationPath $deploy_path\binaries\ps7 +Invoke-Command -ScriptBlock {Start-Process -FilePath "$deploy_path\binaries\vscode_stable.exe" -ArgumentList "/VERYSILENT /MERGETASKS=!runcode" -Wait -PassThru} | Out-Null +Invoke-Command -ScriptBlock {Start-Process -FilePath "$deploy_path\binaries\firefox_latest.exe" -ArgumentList "/s" -Wait -PassThru} | Out-Null + +# QSEoW FW Rule +Write-Host "## ========================>> Create QSEoW FW Rule" +New-NetFirewallRule -DisplayName "Qlik Sense" -Direction Inbound -LocalPort 443, 4244,4242, 4432, 4444, 5355, 5353, 80, 4248, 3090, 4000, 5555, 5556, 4993, 4994 -Protocol TCP -Action Allow -ea Stop | Out-Null +# Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False + +#### Enable NFS Client, set Anon user to UID:GID 0 == root. Restart service. +Write-Host "## ========================>> Installing NFS Client" +Install-WindowsFeature -Name NFS-Client +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousUid" -Value "0" -PropertyType DWORD +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousGid" -Value "0" -PropertyType DWORD +nfsadmin client stop +nfsadmin client start + +#### Launch silent install +Write-Host "========================>> Installing QSEoW" +Invoke-Command -ScriptBlock {Start-Process -FilePath "E:\deploy\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log E:\deploy\deploy.log accepteula=1 installdir=E:\deploy\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=E:\deploy\scripts\cen.xml" -Wait -PassThru} #| Out-Null + +# HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce +# New-ItemProperty -Path 'HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce' -Name 'Run' -Value '' + +Set-GceInstance -Name ($env:computername).ToLower() -Zone europe-west1-d -AddTag "bootstrapped" +Set-GceInstance -Name ($env:computername).ToLower() -Zone europe-west1-d -RemoveMetadata "windows-startup-script-url" + +Exit 0 diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-rim.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-rim.ps1 new file mode 100644 index 0000000..64c973f --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap-rim.ps1 @@ -0,0 +1,112 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## or as a startup script via Metadata key windows-startup-script-url +## + +if ($env:computername -notlike "rim-*") { + exit 0 +} + +# Format and mount data disk +Write-Host "## ========================>> Create Data drive & dirs" + +Get-Disk | + Where-Object partitionstyle -eq 'raw' | + Initialize-Disk -PartitionStyle GPT -PassThru | + New-Partition -DriveLetter E -UseMaximumSize | + Format-Volume -FileSystem NTFS -NewFileSystemLabel 'Data' -Confirm:$false + +if (! (Test-Path E:\)) { + Write-Error "Drive not found" + exit 1 +} + +$deploy_path = "E:\deploy" + +if (! (Test-Path $deploy_path)) { + New-Item -ItemType Directory -Path $deploy_path + New-Item -ItemType Directory -Path $deploy_path\binaries + New-Item -ItemType Directory -Path $deploy_path\modules + New-Item -ItemType Directory -Path $deploy_path\modules\tf + New-Item -ItemType Directory -Path $deploy_path\modules\ps +} + +Write-Host "## ========================>> Create Local Accounts and add to Administrators Group" + +if(!(Get-LocalUser -Name qservice -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qservice' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +if(!(Get-LocalUser -Name qlikadmin -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qlikadmin' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +Add-LocalGroupMember -Group "Administrators" -Member "qservice", "qlikadmin" + +Write-Host "## ========================>> Copy scripts and binaries from Cloud Storage Bucket" +gsutil -m cp -r gs://qliksense/scripts $deploy_path\ +gsutil -m cp gs://qliksense/binaries/Qlik_Sense* $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/vscode_stable.exe $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/firefox_latest.exe $deploy_path\binaries\ +gsutil -m cp gs://qliksense/binaries/ps7.zip $deploy_path\binaries\ +# gsutil -m cp -r gs://qliksense/modules 'C:\Program Files\WindowsPowerShell\Modules' + +# Invoke-WebRequest -Uri https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.zip -OutFile $deploy_path\binaries\ps7.zip +# Invoke-WebRequest -Uri https://aka.ms/win32-x64-user-stable -Outfile $deploy_path\binaries\vscode_stable.exe +# Invoke-WebRequest -Uri "https://download.mozilla.org/?product=firefox-latest-ssl&os=win64&lang=en-US" -Outfile $deploy_path\binaries\firefox_latest.exe + +Unblock-File -Path $deploy_path\binaries\* +Unblock-File -Path $deploy_path\scripts\* + +# WinRM Connects +New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force +New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String + +Write-Host "## ========================>> Import PS Modules" +Get-PackageProvider -Name NuGet -ForceBootstrap + +Install-Module PSDscResources -Force +Install-Module QlikResources -Force + +Write-Host "## ========================>> Deploy PS7 | Firefox | VS Code" +Expand-Archive -Path $deploy_path\binaries\ps7.zip -DestinationPath $deploy_path\binaries\ps7 +Invoke-Command -ScriptBlock {Start-Process -FilePath "$deploy_path\binaries\vscode_stable.exe" -ArgumentList "/VERYSILENT /MERGETASKS=!runcode" -Wait -PassThru} | Out-Null +Invoke-Command -ScriptBlock {Start-Process -FilePath "$deploy_path\binaries\firefox_latest.exe" -ArgumentList "/s" -Wait -PassThru} | Out-Null + +# QSEoW FW Rule +Write-Host "## ========================>> Create QSEoW FW Rule" +New-NetFirewallRule -DisplayName "Qlik Sense" -Direction Inbound -LocalPort 443, 4244,4242, 4432, 4444, 5355, 5353, 80, 4248, 3090, 4000, 5555, 5556, 4993, 4994 -Protocol TCP -Action Allow -ea Stop | Out-Null +# Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False + +#### Enable NFS Client, set Anon user to UID:GID 0 == root. Restart service. +Write-Host "## ========================>> Installing NFS Client" +Install-WindowsFeature -Name NFS-Client +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousUid" -Value "0" -PropertyType DWORD +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousGid" -Value "0" -PropertyType DWORD +nfsadmin client stop +nfsadmin client start + +#### Launch silent install +Write-Host "========================>> Installing QSEoW" +Invoke-Command -ScriptBlock {Start-Process -FilePath "E:\deploy\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log E:\deploy\deploy.log accepteula=1 installdir=E:\deploy\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=E:\deploy\scripts\rim.xml" -Wait -PassThru} #| Out-Null + +# HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce +# New-ItemProperty -Path 'HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce' -Name 'Run' -Value '' + +Set-GceInstance -Name $shortname -Zone europe-west1-d -AddTag "bootstrapped" +Set-GceInstance -Name $shortname -Zone europe-west1-d -RemoveMetadata "windows-startup-script-url" + +Exit 0 diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap.ps1 new file mode 100644 index 0000000..ee478e2 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/bootstrap.ps1 @@ -0,0 +1,79 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## +Enable-PSRemoting +winrm set 'winrm/config/client' '@{TrustedHosts="*"}' + +if(!(Get-LocalUser -Name qservice -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qservice' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False + +New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force +New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String + +# if ($env:computername -ne 'central') { +# exit 0 +# } + +# Get-WmiObject -Class Win32_Volume -Filter "DriveType=5" | Set-WmiInstance -Arguments @{DriveLetter="Z:"} + +Get-Disk | + Where-Object partitionstyle -eq 'raw' | + Initialize-Disk -PartitionStyle GPT -PassThru | + New-Partition -DriveLetter E -UseMaximumSize | + Format-Volume -FileSystem NTFS -NewFileSystemLabel 'Data' -Confirm:$false + +if (! (Test-Path E:\)) { + Write-Error "Drive not found" + exit 1 +} + +$deploy_path = "E:\deploy" + +if (-Not (Test-Path $deploy_path)) { + New-Item -ItemType Directory -Path $deploy_path + New-Item -ItemType Directory -Path $deploy_path\binaries + New-Item -ItemType Directory -Path $deploy_path\modules + New-Item -ItemType Directory -Path $deploy_path\modules\tf + New-Item -ItemType Directory -Path $deploy_path\modules\ps +} + +# Update these gcloud commands to be vars, allows portability +# +# CLOUDCLI_UPDATE +# OBJCLI_COPY + +# gcloud components update -Force +gsutil -m cp gs://qliksense/Qlik_Sense* $deploy_path\binaries\ +gsutil -m cp -r gs://qliksense/scripts $deploy_path\ +gsutil -m cp -r gs://qliksense/modules 'C:\Program Files\WindowsPowerShell\Modules' + +# Push-Location $deploy_path\modules\ps +Get-PackageProvider -Name NuGet -ForceBootstrap +# Below will also download and save the QlikResources requirements modules +# Save-Module -Path $deploy_path/modules/ps/ -Name QlikResources -RequiredVersion 1.9.2 +# Save-Module -Path $deploy_path/modules/ps/ -Name GoogleCloud +# HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce +# New-ItemProperty -Path 'HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce' -Name 'Run' -Value '' + + +Set-ExecutionPolicy Bypass -Scope Process -Force; E:\deploy\scripts\win-nfs_client-install.ps1 +gcloud compute instances add-tags central-12d5 --tags=bootstrapped --zone europe-west1-d + + +# Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +# choco install git --no-progress +# Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +# Update-SessionEnvironment +# git clone https://gitlab+deploy-token-52544:YH-1SVsTrNmzj7AXz7oh@gitlab.com/ahaydon/cityclarity-psm.git ./CitiClarity +# Pop-Location + +Exit 0 diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/cen.xml b/sense-conductor/bt-autoscaler/terraform/roots/scripts/cen.xml new file mode 100644 index 0000000..efda2bc --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/cen.xml @@ -0,0 +1,24 @@ + + + qliksenserepository + Qlik1234! + 10.54.242.14 + 5432 + \\10.159.210.26\qlikshare + \\10.159.210.26\qlikshare\StaticContent + \\10.159.210.26\qlikshare\ArchivedLogs + \\10.159.210.26\qlikshare\Apps + true + false + false + * + 0.0.0.0/0,::/0 + 100 + + true + false + Qlik1234! + Qlik1234! + 10.54.242.14 + 5432 + \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/centralNode.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/centralNode.ps1 new file mode 100644 index 0000000..11b9684 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/centralNode.ps1 @@ -0,0 +1,96 @@ +$configData = @{ + AllNodes = @(@{ + NodeName = 'localhost' + PSDscAllowPlainTextPassword = $true + }) +} + +$password = ConvertTo-SecureString -String 'password' -AsPlainText -Force +$SenseService = New-Object System.Management.Automation.PSCredential("$env:computername\qservice", $password) +$QlikAdmin = New-Object System.Management.Automation.PSCredential("$env:computername\qlikadmin", $password) + +Configuration QlikConfig +{ + Import-DscResource -ModuleName QlikResources, PSDscResources + + Node localhost + { + # User QlikAdmin + # { + # UserName = $QlikAdmin.GetNetworkCredential().UserName + # Password = $QlikAdmin + # FullName = 'Qlik User' + # PasswordChangeRequired = $false + # PasswordNeverExpires = $true + # Ensure = 'Present' + # DependsOn = "[Windows]local" + # } + + # User SenseService + # { + # UserName = $SenseService.GetNetworkCredential().UserName + # Password = $SenseService + # FullName = 'Qlik Sense Service Account' + # PasswordChangeNotAllowed = $true + # PasswordChangeRequired = $false + # PasswordNeverExpires = $true + # Ensure = 'Present' + # DependsOn = "[Windows]local" + # } + + # Group Administrators + # { + # GroupName = 'Administrators' + # MembersToInclude = $QlikAdmin.GetNetworkCredential().UserName, $SenseService.GetNetworkCredential().UserName + # DependsOn = "[User]QlikAdmin", "[User]SenseService" + # } + + QlikCentral CentralNode + { + SenseService = $SenseService + QlikAdmin = $QlikAdmin + ProductName = 'Qlik Sense September 2020 Patch 2' + SetupPath = 'E:\deploy\binaries\Qlik_Sense_setup.exe' + PatchPath = 'E:\deploy\binaries\Qlik_Sense_update.exe' + ClusterShareHost = "" + License = @{ + Serial = '1234567890' + Control = '12345' + Name = 'User' + Organization = 'Organization' + } + PSDscRunasCredential = $QlikAdmin + DependsOn = "[MSFT_GroupResource]Administrators" + } + + # QlikVirtualProxy SAML + # { + # Prefix = "saml" + # Description = "SAML" + # SessionCookieHeaderName = "X-Qlik-Session-SAML" + # LoadBalancingServerNodes = "name eq 'Central'" + # AuthenticationMethod = "saml" + # SamlMetadataIdp = (Get-Content -raw E:\deploy\idp-metadata.xml) + # SamlHostUri = "https://$($env:computername)" + # SamlEntityId = "https://$($env:computername)/saml" + # SamlAttributeUserId = "uid" + # SamlAttributeUserDirectory = "[SAML]" + # SamlAttributeMapMandatory = @{ + # mail = 'email' + # } + # samlSlo = $true + # SamlMetadataExportPath = "E:\deploy\saml_metadata_sp.xml" + # Proxy = $env:computername + # PSDscRunasCredential = $QlikAdmin + # Ensure = "Present" + # } + } +} + +QlikConfig -ConfigurationData $configData + +if (! (Test-Path E:\deploy\scripts\QlikConfig)) { + New-Item -ItemType Directory -Path E:\deploy\scripts\QlikConfig +} + +Start-DscConfiguration -Path E:\deploy\scripts\QlikConfig -Wait -Verbose -Force diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/pave.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/pave.ps1 new file mode 100644 index 0000000..997ecaa --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/pave.ps1 @@ -0,0 +1,109 @@ +#!/usr/bin/env pwsh +# +## Intended to be executed in a GitOps pipeline on the new GCE resource by remote-exec in TF +## or as a startup script via Metadata key windows-startup-script-url +## + +if ($env:computername -notlike "central-*") { + exit 0 +} + +#### Format and mount data disk +Write-Host "<---- Create Data drive & dirs" + +Get-Disk | + Where-Object partitionstyle -eq 'raw' | + Initialize-Disk -PartitionStyle GPT -PassThru | + New-Partition -DriveLetter E -UseMaximumSize | + Format-Volume -FileSystem NTFS -NewFileSystemLabel 'Data' -Confirm:$false + +if (! (Test-Path E:\)) { + Write-Error "Drive not found" + exit 1 +} + +$deploy_path = "E:\deploy" + +if (! (Test-Path $deploy_path)) { + New-Item -ItemType Directory -Path $deploy_path + New-Item -ItemType Directory -Path $deploy_path\binaries + New-Item -ItemType Directory -Path $deploy_path\modules + New-Item -ItemType Directory -Path $deploy_path\modules\tf + New-Item -ItemType Directory -Path $deploy_path\modules\ps +} + +Write-Host "<---- Copy scripts and binaries from Cloud Storage Bucket" +gcloud components update -Force +gsutil -m cp -r gs://qliksense/scripts $deploy_path\ +gsutil -m cp gs://qliksense/binaries/Qlik_Sense* $deploy_path\binaries\ +# gsutil -m cp -r gs://qliksense/modules 'C:\Program Files\WindowsPowerShell\Modules' + +Invoke-WebRequest -Uri https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.zip -OutFile $deploy_path\binaries\ps7.zip +Invoke-WebRequest -Uri https://aka.ms/win32-x64-user-stable -Outfile $deploy_path\binaries\vscode_stable.exe +Invoke-WebRequest -Uri "https://download.mozilla.org/?product=firefox-latest-ssl&os=win64&lang=en-US" -Outfile $deploy_path\binaries\firefox_latest.exe + +Unblock-File -Path $deploy_path\binaries\* +Unblock-File -Path $deploy_path\scripts\* + +Write-Host "<---- Create Local Accounts and add to Administrators Group" + +if(!(Get-LocalUser -Name qservice -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qservice' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +if(!(Get-LocalUser -Name qlikadmin -ErrorAction Ignore)) { + $password = ConvertTo-SecureString -String 'Qlik1234!' -AsPlainText -Force + New-LocalUser ` + -Name 'qlikadmin' ` + -Password $password ` + -PasswordNeverExpires ` + -UserMayNotChangePassword +} + +Add-LocalGroupMember -Group "Administrators" -Member "qservice", "qlikadmin" + +#### WinRM Connects +New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force +New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String + +Write-Host "<---- Import PS Modules and Install Firefox / VS Code" +Get-PackageProvider -Name NuGet -ForceBootstrap + +Install-Module PSDesiredStateConfiguration -Force +Install-Module PSDscResources -Force +Install-Module QlikResources -Force + +Expand-Archive -Path $deploy_path\binaries\ps7.zip -DestinationPath $deploy_path\binaries\ps7 +& $deploy_path\binaries\vscode_stable.exe /VERYSILENT /MERGETASKS=!runcode +& $deploy_path\binaries\firefox_latest.exe /s +# HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce +# New-ItemProperty -Path 'HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce' -Name 'Run' -Value '' + +#### QSEoW FW Rule +Write-Host "<---- Create QSEoW FW Rule" +New-NetFirewallRule -DisplayName "Qlik Sense" -Direction Inbound -LocalPort 443, 4244,4242, 4432, 4444, 5355, 5353, 80, 4248, 3090, 4000, 5555, 5556, 4993, 4994 -Protocol TCP -Action Allow -ea Stop | Out-Null +# Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False + +#### Enable NFS Client, set Anon user to UID:GID 0 == root. Restart service. +Write-Host -Message "<---- Installing NFS Client" +Install-WindowsFeature -Name NFS-Client +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousUid" -Value "0" -PropertyType DWORD +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousGid" -Value "0" -PropertyType DWORD +nfsadmin client stop +nfsadmin client start + +#### Launch silent install +# Write-Host "<---- Installing QSEoW" +# Invoke-Command -ScriptBlock {Start-Process -FilePath "E:\deploy\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log E:\deploy\deploy.log accepteula=1 installdir=E:\deploy\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=E:\deploy\scripts\spc.xml" -Wait -PassThru} | Out-Null + +Set-GceInstance -Name $shortname -Zone europe-west1-d -AddTag "bootstrapped" +Set-GceInstance -Name $shortname -Zone europe-west1-d -RemoveMetadata "windows-startup-script-url" + +Exit 0 diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/rim.xml b/sense-conductor/bt-autoscaler/terraform/roots/scripts/rim.xml new file mode 100644 index 0000000..ee84587 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/rim.xml @@ -0,0 +1,24 @@ + + + qliksenserepository + Qlik1234! + 10.54.242.14 + 5432 + \\10.159.210.26\qlikshare + \\10.159.210.26\qlikshare\StaticContent + \\10.159.210.26\qlikshare\ArchivedLogs + \\10.159.210.26\qlikshare\Apps + false + false + false + * + 0.0.0.0/0,::/0 + 100 + + true + false + Qlik1234! + Qlik1234! + 10.54.242.14 + 5432 + \ No newline at end of file diff --git a/sense-conductor/bt-autoscaler/terraform/roots/scripts/win-nfs_client-install.ps1 b/sense-conductor/bt-autoscaler/terraform/roots/scripts/win-nfs_client-install.ps1 new file mode 100644 index 0000000..1607054 --- /dev/null +++ b/sense-conductor/bt-autoscaler/terraform/roots/scripts/win-nfs_client-install.ps1 @@ -0,0 +1,22 @@ +#!/usr/bin/env pwsh +# Install NFS Client on Windows via PS +# + +# Wait for bootstrap.ps1 to finish before proceeding. +# $bootScript = "bootstrap.ps1" +# Wait-Process -Name $bootScript -ErrorAction SilentlyContinue -Timeout 1200 + +# Enable NFS Client, set Anon user to UID:GID 0 == root. Restart service. +Write-Host -Message "<---- Installing NFS Client" +Install-WindowsFeature -Name NFS-Client +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousUid" -Value "0" -PropertyType DWORD +New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\ClientForNFS\CurrentVersion\Default" ` + -Name "AnonymousGid" -Value "0" -PropertyType DWORD +nfsadmin client stop +nfsadmin client start + +# & E:\deploy\binaries\Qlik_Sense_setup.exe -s -l E:\deploy\deploy.log spc="E:\deploy\scripts\spc.xml" installdir="E:\deploy\Qlik" userwithdomain="central-b445\qservice" userpassword="Qlik1234!" dbpassword="Qlik1234!" accepteula=1 skipvalidation=1 bundleinstall=1 skipdbconfig=1 + +Write-Host "<---- Installing QSEoW" +Invoke-Command -ScriptBlock {Start-Process -FilePath "E:\deploy\binaries\Qlik_Sense_setup.exe" -ArgumentList "-s -log E:\deploy\deploy.log accepteula=1 installdir=E:\deploy\Qlik dbpassword=Qlik1234! hostname=$($env:computername) userwithdomain=$($env:computername)\qservice password=Qlik1234! bundleinstall=dashboard,visualization spc=E:\deploy\scripts\spc.xml" -Wait -PassThru} | Out-Null