From e81d57b9dadcc60a2c87d9d4f770ba9700cf19e6 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 22:03:51 +0200 Subject: [PATCH 01/11] All in with PG --- examples/cloud-sql-postgres/.gitkeep | 0 examples/postgres-private-ip/README.md | 18 ++ examples/postgres-private-ip/main.tf | 99 +++++++++ examples/postgres-private-ip/outputs.tf | 34 +++ examples/postgres-private-ip/variables.tf | 49 +++++ examples/postgres-public-ip/README.md | 17 ++ examples/postgres-public-ip/main.tf | 88 ++++++++ examples/postgres-public-ip/outputs.tf | 39 ++++ examples/postgres-public-ip/variables.tf | 56 +++++ examples/postgres-replicas/README.md | 18 ++ examples/postgres-replicas/main.tf | 86 ++++++++ examples/postgres-replicas/outputs.tf | 74 +++++++ examples/postgres-replicas/variables.tf | 69 ++++++ modules/mysql/compute_outputs.tf | 12 +- modules/mysql/main.tf | 12 +- modules/mysql/outputs.tf | 4 +- modules/mysql/variables.tf | 11 +- test/Gopkg.lock | 14 ++ test/example_cloud_sql_postgres_test.go | 13 -- test/example_mysql_private_ip_test.go | 2 +- test/example_mysql_public_ip_test.go | 6 +- test/example_mysql_replicas_test.go | 10 +- test/example_postgres_private_ip_test.go | 73 +++++++ test/example_postgres_public_ip_test.go | 249 ++++++++++++++++++++++ test/example_postgres_replicas_test.go | 177 +++++++++++++++ test/test_util.go | 26 ++- 26 files changed, 1209 insertions(+), 47 deletions(-) delete mode 100644 examples/cloud-sql-postgres/.gitkeep create mode 100644 examples/postgres-private-ip/README.md create mode 100644 examples/postgres-private-ip/main.tf create mode 100644 examples/postgres-private-ip/outputs.tf create mode 100644 examples/postgres-private-ip/variables.tf create mode 100644 examples/postgres-public-ip/README.md create mode 100644 examples/postgres-public-ip/main.tf create mode 100644 examples/postgres-public-ip/outputs.tf create mode 100644 examples/postgres-public-ip/variables.tf create mode 100644 examples/postgres-replicas/README.md create mode 100644 examples/postgres-replicas/main.tf create mode 100644 examples/postgres-replicas/outputs.tf create mode 100644 examples/postgres-replicas/variables.tf delete mode 100644 test/example_cloud_sql_postgres_test.go create mode 100644 test/example_postgres_private_ip_test.go create mode 100644 test/example_postgres_public_ip_test.go create mode 100644 test/example_postgres_replicas_test.go diff --git a/examples/cloud-sql-postgres/.gitkeep b/examples/cloud-sql-postgres/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/examples/postgres-private-ip/README.md b/examples/postgres-private-ip/README.md new file mode 100644 index 0000000..7d9b983 --- /dev/null +++ b/examples/postgres-private-ip/README.md @@ -0,0 +1,18 @@ +# PostgreSQL Cloud SQL Private IP Example + +This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a private IP. + +## How do you run this example? + +To run this example, you need to: + +1. Install [Terraform](https://www.terraform.io/). +1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in + the file that don't have defaults. +1. `terraform init`. +1. `terraform plan`. +1. If the plan looks good, run `terraform apply`. + +When the templates are applied, Terraform will output the IP address of the instance +and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). diff --git a/examples/postgres-private-ip/main.tf b/examples/postgres-private-ip/main.tf new file mode 100644 index 0000000..26d100d --- /dev/null +++ b/examples/postgres-private-ip/main.tf @@ -0,0 +1,99 @@ +# ------------------------------------------------------------------------------ +# LAUNCH A MYSQL CLOUD SQL PRIVATE IP INSTANCE +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# CONFIGURE OUR GCP CONNECTION +# ------------------------------------------------------------------------------ + +provider "google-beta" { + region = "${var.region}" + project = "${var.project}" +} + +# Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via +# https://github.com/terraform-providers/terraform-provider-google +terraform { + required_version = ">= 0.10.3" +} + +# ------------------------------------------------------------------------------ +# CREATE A RANDOM SUFFIX AND PREPARE RESOURCE NAMES +# ------------------------------------------------------------------------------ + +resource "random_id" "name" { + byte_length = 2 +} + +locals { + # If name_override is specified, use that - otherwise use the name_prefix with a random string + instance_name = "${length(var.name_override) == 0 ? format("%s-%s", var.name_prefix, random_id.name.hex) : var.name_override}" + private_network_name = "private-network-${random_id.name.hex}" + private_ip_name = "private-ip-${random_id.name.hex}" +} + +# ------------------------------------------------------------------------------ +# CREATE COMPUTE NETWORKS +# ------------------------------------------------------------------------------ + +# Simple network, auto-creates subnetworks +resource "google_compute_network" "private_network" { + provider = "google-beta" + name = "${local.private_network_name}" +} + +# Reserve global internal address range for the peering +resource "google_compute_global_address" "private_ip_address" { + provider = "google-beta" + name = "${local.private_ip_name}" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = "${google_compute_network.private_network.self_link}" +} + +# Establish VPC network peering connection using the reserved address range +resource "google_service_networking_connection" "private_vpc_connection" { + provider = "google-beta" + network = "${google_compute_network.private_network.self_link}" + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = ["${google_compute_global_address.private_ip_address.name}"] +} + +# ------------------------------------------------------------------------------ +# CREATE DATABASE INSTANCE WITH PRIVATE IP +# ------------------------------------------------------------------------------ + +module "mysql" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" + source = "../../modules/mysql" + + project = "${var.project}" + region = "${var.region}" + name = "${local.instance_name}" + db_name = "${var.db_name}" + + engine = "${var.postgres_version}" + machine_type = "${var.machine_type}" + + # These together will construct the master_user privileges, i.e. + # 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. + # These should typically be set as the environment variable TF_VAR_master_user_password, etc. + # so you don't check these into source control." + master_user_password = "${var.master_user_password}" + + master_user_name = "${var.master_user_name}" + master_user_host = "%" + + # Pass the private network link to the module + private_network = "${google_compute_network.private_network.self_link}" + + # Wait for the vpc connection to complete + wait_for = "${google_service_networking_connection.private_vpc_connection.network}" + + custom_labels = { + test-id = "postgres-private-ip-example" + } +} diff --git a/examples/postgres-private-ip/outputs.tf b/examples/postgres-private-ip/outputs.tf new file mode 100644 index 0000000..3268020 --- /dev/null +++ b/examples/postgres-private-ip/outputs.tf @@ -0,0 +1,34 @@ +output "master_instance_name" { + description = "The name of the database instance" + value = "${module.mysql.master_instance_name}" +} + +output "master_ip_addresses" { + description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" + value = "${module.mysql.master_ip_addresses}" +} + +output "master_private_ip" { + description = "The first IPv4 address of the addresses assigned to the instance. As this instance has only private IP, it is the private IP address." + value = "${module.mysql.master_first_ip_address}" +} + +output "master_instance" { + description = "Self link to the master instance" + value = "${module.mysql.master_instance}" +} + +output "master_proxy_connection" { + description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" + value = "${module.mysql.master_proxy_connection}" +} + +output "db_name" { + description = "Name of the default database" + value = "${module.mysql.db_name}" +} + +output "db" { + description = "Self link to the default database" + value = "${module.mysql.db}" +} diff --git a/examples/postgres-private-ip/variables.tf b/examples/postgres-private-ip/variables.tf new file mode 100644 index 0000000..1d86733 --- /dev/null +++ b/examples/postgres-private-ip/variables.tf @@ -0,0 +1,49 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These variables are expected to be passed in by the operator +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The project ID to host the database in." +} + +variable "region" { + description = "The region to host the database in." +} + +# Note, after a name db instance is used, it cannot be reused for up to one week. +variable "name_prefix" { + description = "The name prefix for the database instance. Will be appended with a random string. Use lowercase letters, numbers, and hyphens. Start with a letter." +} + +variable "master_user_name" { + description = "The username part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_name so you don't check it into source control." +} + +variable "master_user_password" { + description = "The password part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_password so you don't check it into source control." +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# Generally, these values won't need to be changed. +# --------------------------------------------------------------------------------------------------------------------- +variable "postgres_version" { + description = "The engine version of the database, e.g. `POSTGRES_9_6`. See https://cloud.google.com/sql/docs/features for supported versions." + default = "POSTGRES_9_6" +} + +variable "machine_type" { + description = "The machine type to use, see https://cloud.google.com/sql/pricing for more details" + default = "db-f1-micro" +} + +variable "db_name" { + description = "Name for the db" + default = "default" +} + +variable "name_override" { + description = "You may optionally override the name_prefix + random string by specifying an override" + default = "" +} diff --git a/examples/postgres-public-ip/README.md b/examples/postgres-public-ip/README.md new file mode 100644 index 0000000..d60c6f1 --- /dev/null +++ b/examples/postgres-public-ip/README.md @@ -0,0 +1,17 @@ +# PostgreSQL Cloud SQL Public IP Example + +This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a public IP. + +## How do you run this example? + +To run this example, you need to: + +1. Install [Terraform](https://www.terraform.io/). +1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in + the file that don't have defaults. +1. `terraform init`. +1. `terraform plan`. +1. If the plan looks good, run `terraform apply`. + +When the templates are applied, Terraform will output the IP address of the instance and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). \ No newline at end of file diff --git a/examples/postgres-public-ip/main.tf b/examples/postgres-public-ip/main.tf new file mode 100644 index 0000000..cd1b2e5 --- /dev/null +++ b/examples/postgres-public-ip/main.tf @@ -0,0 +1,88 @@ +# ------------------------------------------------------------------------------ +# LAUNCH A POSTGRESQL CLOUD SQL PUBLIC IP INSTANCE +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# CONFIGURE OUR GCP CONNECTION +# ------------------------------------------------------------------------------ + +provider "google-beta" { + region = "${var.region}" + project = "${var.project}" +} + +# Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via +# https://github.com/terraform-providers/terraform-provider-google +terraform { + required_version = ">= 0.10.3" +} + +# ------------------------------------------------------------------------------ +# CREATE A RANDOM SUFFIX AND PREPARE RESOURCE NAMES +# ------------------------------------------------------------------------------ + +resource "random_id" "name" { + byte_length = 2 +} + +locals { + # If name_override is specified, use that - otherwise use the name_prefix with a random string + instance_name = "${length(var.name_override) == 0 ? format("%s-%s", var.name_prefix, random_id.name.hex) : var.name_override}" +} + +# ------------------------------------------------------------------------------ +# CREATE DATABASE INSTANCE WITH PUBLIC IP +# ------------------------------------------------------------------------------ + +module "mysql" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" + source = "../../modules/mysql" + + project = "${var.project}" + region = "${var.region}" + name = "${local.instance_name}" + db_name = "${var.db_name}" + + engine = "${var.postgres_version}" + machine_type = "${var.machine_type}" + + # These together will construct the master_user privileges, i.e. + # 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. + # These should typically be set as the environment variable TF_VAR_master_user_password, etc. + # so you don't check these into source control." + master_user_password = "${var.master_user_password}" + + master_user_name = "${var.master_user_name}" + master_user_host = "%" + + # To make it easier to test this example, we are giving the servers public IP addresses and allowing inbound + # connections from anywhere. In real-world usage, your servers should live in private subnets, only have private IP + # addresses, and only allow access from specific trusted networks, servers or applications in your VPC. + enable_public_internet_access = true + + # Default setting for this is 'false' in 'variables.tf' + # In the test cases, we're setting this to true, to test forced SSL. + require_ssl = "${var.require_ssl}" + + authorized_networks = [ + { + name = "allow-all-inbound" + value = "0.0.0.0/0" + }, + ] + + # Set auto-increment flags to test the + # feature during automated testing + database_flags = [ + { + name = "autovacuum_naptime" + value = "2" + }, + ] + + custom_labels = { + test-id = "postgres-public-ip-example" + } +} diff --git a/examples/postgres-public-ip/outputs.tf b/examples/postgres-public-ip/outputs.tf new file mode 100644 index 0000000..95fc3ce --- /dev/null +++ b/examples/postgres-public-ip/outputs.tf @@ -0,0 +1,39 @@ +output "master_instance_name" { + description = "The name of the database instance" + value = "${module.mysql.master_instance_name}" +} + +output "master_ip_addresses" { + description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" + value = "${module.mysql.master_ip_addresses}" +} + +output "master_public_ip" { + description = "The first IPv4 address of the addresses assigned to the instance. As this instance has only public IP, it is the public IP address." + value = "${module.mysql.master_first_ip_address}" +} + +output "master_ca_cert" { + value = "${module.mysql.master_ca_cert}" + description = "The CA Certificate used to connect to the SQL Instance via SSL" +} + +output "master_instance" { + description = "Self link to the master instance" + value = "${module.mysql.master_instance}" +} + +output "master_proxy_connection" { + description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" + value = "${module.mysql.master_proxy_connection}" +} + +output "db_name" { + description = "Name of the default database" + value = "${module.mysql.db_name}" +} + +output "db" { + description = "Self link to the default database" + value = "${module.mysql.db}" +} diff --git a/examples/postgres-public-ip/variables.tf b/examples/postgres-public-ip/variables.tf new file mode 100644 index 0000000..eafe075 --- /dev/null +++ b/examples/postgres-public-ip/variables.tf @@ -0,0 +1,56 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These variables are expected to be passed in by the operator +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The project ID to host the database in." +} + +variable "region" { + description = "The region to host the database in." +} + +# Note, after a name db instance is used, it cannot be reused for up to one week. +variable "name_prefix" { + description = "The name prefix for the database instance. Will be appended with a random string. Use lowercase letters, numbers, and hyphens. Start with a letter." +} + +variable "master_user_name" { + description = "The username part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_name so you don't check it into source control." +} + +variable "master_user_password" { + description = "The password part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_password so you don't check it into source control." +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# Generally, these values won't need to be changed. +# --------------------------------------------------------------------------------------------------------------------- +variable "postgres_version" { + description = "The engine version of the database, e.g. `POSTGRES_9_6`. See https://cloud.google.com/sql/docs/features for supported versions." + default = "POSTGRES_9_6" +} + +variable "machine_type" { + description = "The machine type to use, see https://cloud.google.com/sql/pricing for more details" + default = "db-f1-micro" +} + +variable "db_name" { + description = "Name for the db" + default = "default" +} + +variable "name_override" { + description = "You may optionally override the name_prefix + random string by specifying an override" + default = "" +} + +# When configuring a public IP instance, you should only allow secure connections +# For testing purposes, we're initially allowing unsecured connections. +variable "require_ssl" { + description = "True if the instance should require SSL/TLS for users connecting over IP. Note: SSL/TLS is needed to provide security when you connect to Cloud SQL using IP addresses. If you are connecting to your instance only by using the Cloud SQL Proxy or the Java Socket Library, you do not need to configure your instance to use SSL/TLS." + default = false +} diff --git a/examples/postgres-replicas/README.md b/examples/postgres-replicas/README.md new file mode 100644 index 0000000..18440cc --- /dev/null +++ b/examples/postgres-replicas/README.md @@ -0,0 +1,18 @@ +# MySQL Cloud SQL Replica Example + +This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[MySQL](https://cloud.google.com/sql/docs/mysql/) database cluster with a public IP and failover and read replicas. + +## How do you run this example? + +To run this example, you need to: + +1. Install [Terraform](https://www.terraform.io/). +1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in + the file that don't have defaults. +1. `terraform init`. +1. `terraform plan`. +1. If the plan looks good, run `terraform apply`. + +When the templates are applied, Terraform will output the IP address of the instance +and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). diff --git a/examples/postgres-replicas/main.tf b/examples/postgres-replicas/main.tf new file mode 100644 index 0000000..db06315 --- /dev/null +++ b/examples/postgres-replicas/main.tf @@ -0,0 +1,86 @@ +# ------------------------------------------------------------------------------ +# LAUNCH A POSTGRES CLUSTER WITH HA AND READ REPLICAS +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# CONFIGURE OUR GCP CONNECTION +# ------------------------------------------------------------------------------ + +provider "google-beta" { + region = "${var.region}" + project = "${var.project}" +} + +# Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via +# https://github.com/terraform-providers/terraform-provider-google +terraform { + required_version = ">= 0.10.3" +} + +# ------------------------------------------------------------------------------ +# CREATE A RANDOM SUFFIX AND PREPARE RESOURCE NAMES +# ------------------------------------------------------------------------------ + +resource "random_id" "name" { + byte_length = 2 +} + +locals { + # If name_override is specified, use that - otherwise use the name_prefix with a random string + instance_name = "${length(var.name_override) == 0 ? format("%s-%s", var.name_prefix, random_id.name.hex) : var.name_override}" + private_network_name = "private-network-${random_id.name.hex}" + private_ip_name = "private-ip-${random_id.name.hex}" +} + +# ------------------------------------------------------------------------------ +# CREATE DATABASE CLUSTER WITH PUBLIC IP +# ------------------------------------------------------------------------------ + +module "mysql" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" + source = "../../modules/mysql" + + project = "${var.project}" + region = "${var.region}" + name = "${local.instance_name}" + db_name = "${var.db_name}" + + engine = "${var.postgres_version}" + machine_type = "${var.machine_type}" + + master_zone = "${var.master_zone}" + + # To make it easier to test this example, we are giving the servers public IP addresses and allowing inbound + # connections from anywhere. In real-world usage, your servers should live in private subnets, only have private IP + # addresses, and only allow access from specific trusted networks, servers or applications in your VPC. + enable_public_internet_access = true + + authorized_networks = [ + { + name = "allow-all-inbound" + value = "0.0.0.0/0" + }, + ] + + # Indicate that we want to create a failover replica + enable_failover_replica = true + + # Indicate we want read replicas to be created + num_read_replicas = "${var.num_read_replicas}" + read_replica_zones = ["${var.read_replica_zones}"] + + # These together will construct the master_user privileges, i.e. + # 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. + # These should typically be set as the environment variable TF_VAR_master_user_password, etc. + # so you don't check these into source control." + master_user_password = "${var.master_user_password}" + + master_user_name = "${var.master_user_name}" + master_user_host = "%" + + custom_labels = { + test-id = "postgres-replicas-example" + } +} diff --git a/examples/postgres-replicas/outputs.tf b/examples/postgres-replicas/outputs.tf new file mode 100644 index 0000000..52a54d1 --- /dev/null +++ b/examples/postgres-replicas/outputs.tf @@ -0,0 +1,74 @@ +# ------------------------------------------------------------------------------ +# MASTER OUTPUTS +# ------------------------------------------------------------------------------ + +output "master_instance_name" { + description = "The name of the database instance" + value = "${module.mysql.master_instance_name}" +} + +output "master_ip_addresses" { + description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" + value = "${module.mysql.master_ip_addresses}" +} + +output "master_public_ip" { + description = "The first IPv4 address of the addresses assigned to the master instance. As this instance has only public IP, it is the public IP address." + value = "${module.mysql.master_first_ip_address}" +} + +output "master_instance" { + description = "Self link to the master instance" + value = "${module.mysql.master_instance}" +} + +output "master_proxy_connection" { + description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" + value = "${module.mysql.master_proxy_connection}" +} + +# ------------------------------------------------------------------------------ +# DB OUTPUTS +# ------------------------------------------------------------------------------ + +output "db_name" { + description = "Name of the default database" + value = "${module.mysql.db_name}" +} + +output "db" { + description = "Self link to the default database" + value = "${module.mysql.db}" +} + +# ------------------------------------------------------------------------------ +# READ REPLICA OUTPUTS +# ------------------------------------------------------------------------------ + +output "read_replica_instance_names" { + description = "List of names for the read replica instances" + value = ["${module.mysql.read_replica_instance_names}"] +} + +output "read_replica_public_ips" { + description = "List of first IPv4 addresses of the addresses assigned to the read replica instances. As the instances have only public IP in the example, the are the public IP addresses." + value = ["${module.mysql.read_replica_first_ip_addresses}"] +} + +output "read_replica_instances" { + description = "List of self links to the read replica instances" + value = ["${module.mysql.read_replica_instances}"] +} + +output "read_replica_proxy_connections" { + description = "List of read replica instance paths for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" + value = ["${module.mysql.read_replica_proxy_connections}"] +} + +# Although we don't use the values, this output highlights the JSON encoded output we use in certain +# cases where the resource output cannot properly be computed. +# See https://github.com/hashicorp/terraform/issues/17048 +output "read_replica_server_ca_certs" { + description = "JSON encoded list of CA Certificates used to connect to the read replica instances via SSL" + value = "${module.mysql.read_replica_server_ca_certs}" +} diff --git a/examples/postgres-replicas/variables.tf b/examples/postgres-replicas/variables.tf new file mode 100644 index 0000000..8aefcb7 --- /dev/null +++ b/examples/postgres-replicas/variables.tf @@ -0,0 +1,69 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These variables are expected to be passed in by the operator +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The project ID to host the database in." +} + +variable "region" { + description = "The region to host the database in (e.g. 'us-central1')." +} + +variable "master_zone" { + description = "The preferred zone for the master instance (e.g. 'us-central1-a'). Must be different than 'failover_replica_zone'." +} + +variable "failover_replica_zone" { + description = "The preferred zone for the failover instance (e.g. 'us-central1-b'). Must be different than 'master_zone'." +} + +variable "num_read_replicas" { + description = "The number of read replicas to create. Cloud SQL will replicate all data from the master to these replicas, which you can use to horizontally scale read traffic." +} + +variable "read_replica_zones" { + description = "A list of compute zones where read replicas should be created. List size should match 'num_read_replicas'" + type = "list" + + # Example: + # default = ["us-central1-b", "us-central1-c"] +} + +# Note, after a name db instance is used, it cannot be reused for up to one week. +variable "name_prefix" { + description = "The name prefix for the database instance. Will be appended with a random string. Use lowercase letters, numbers, and hyphens. Start with a letter." +} + +variable "master_user_name" { + description = "The username part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_name so you don't check it into source control." +} + +variable "master_user_password" { + description = "The password part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_password so you don't check it into source control." +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# Generally, these values won't need to be changed. +# --------------------------------------------------------------------------------------------------------------------- +variable "postgres_version" { + description = "The engine version of the database, e.g. `POSTGRES_9_6`. See https://cloud.google.com/sql/docs/features for supported versions." + default = "POSTGRES_9_6" +} + +variable "machine_type" { + description = "The machine type to use, see https://cloud.google.com/sql/pricing for more details" + default = "db-f1-micro" +} + +variable "db_name" { + description = "Name for the db" + default = "default" +} + +variable "name_override" { + description = "You may optionally override the name_prefix + random string by specifying an override" + default = "" +} diff --git a/modules/mysql/compute_outputs.tf b/modules/mysql/compute_outputs.tf index ebba7bc..a6ddea9 100644 --- a/modules/mysql/compute_outputs.tf +++ b/modules/mysql/compute_outputs.tf @@ -23,7 +23,7 @@ locals { # ------------------------------------------------------------------------------ data "template_file" "failover_proxy_connection" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${var.project}:${var.region}:${google_sql_database_instance.failover_replica.0.name}" } @@ -36,27 +36,27 @@ data "template_file" "failover_proxy_connection" { # ------------------------------------------------------------------------------ data "template_file" "failover_certificate" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${google_sql_database_instance.failover_replica.0.server_ca_cert.0.cert}" } data "template_file" "failover_certificate_common_name" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${google_sql_database_instance.failover_replica.0.server_ca_cert.0.common_name}" } data "template_file" "failover_certificate_create_time" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${google_sql_database_instance.failover_replica.0.server_ca_cert.0.create_time}" } data "template_file" "failover_certificate_expiration_time" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${google_sql_database_instance.failover_replica.0.server_ca_cert.0.expiration_time}" } data "template_file" "failover_certificate_sha1_fingerprint" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" template = "${google_sql_database_instance.failover_replica.0.server_ca_cert.0.sha1_fingerprint}" } diff --git a/modules/mysql/main.tf b/modules/mysql/main.tf index cccf5a7..5b23bcc 100644 --- a/modules/mysql/main.tf +++ b/modules/mysql/main.tf @@ -12,9 +12,15 @@ # ------------------------------------------------------------------------------ locals { + # Determine the engine type is_postgres = "${replace(var.engine, "POSTGRES", "") != var.engine}" is_mysql = "${replace(var.engine, "MYSQL", "") != var.engine}" + # Calculate actuals, so we get expected behavior for each engine + actual_binary_log = "${local.is_postgres ? false : var.mysql_binary_log_enabled}" + actual_availability_type = "${local.is_postgres && var.enable_failover_replica ? "REGIONAL" : "ZONAL"}" + actual_failover_replica_count = "${local.is_postgres ? 0 : var.enable_failover_replica ? 1 : 0}" + # Terraform does not allow using lists of maps with coditionals, so we have to # trick terraform by creating a string conditional first. # See https://github.com/hashicorp/terraform/issues/12453 @@ -71,7 +77,7 @@ resource "google_sql_database_instance" "master" { } backup_configuration { - binary_log_enabled = "${var.binary_log_enabled}" + binary_log_enabled = "${local.actual_binary_log}" enabled = "${var.backup_enabled}" start_time = "${var.backup_start_time}" } @@ -85,7 +91,7 @@ resource "google_sql_database_instance" "master" { disk_size = "${var.disk_size}" disk_type = "${var.disk_type}" database_flags = ["${var.database_flags}"] - availability_type = "${var.availability_type}" + availability_type = "${local.actual_availability_type}" user_labels = "${var.custom_labels}" } @@ -138,7 +144,7 @@ resource "null_resource" "wait_for" { # ------------------------------------------------------------------------------ resource "google_sql_database_instance" "failover_replica" { - count = "${var.enable_failover_replica}" + count = "${local.actual_failover_replica_count}" depends_on = [ "google_sql_database_instance.master", diff --git a/modules/mysql/outputs.tf b/modules/mysql/outputs.tf index 28deb1c..b31dd8d 100644 --- a/modules/mysql/outputs.tf +++ b/modules/mysql/outputs.tf @@ -71,7 +71,7 @@ output "db_name" { } # ------------------------------------------------------------------------------ -# FAILOVER REPLICA OUTPUTS +# FAILOVER REPLICA OUTPUTS - ONLY APPLICABLE TO MYSQL # ------------------------------------------------------------------------------ output "failover_instance_name" { @@ -101,7 +101,7 @@ output "failover_proxy_connection" { } # ------------------------------------------------------------------------------ -# FAILOVER CERT OUTPUTS +# FAILOVER CERT OUTPUTS - ONLY APPLICABLE TO MYSQL # ------------------------------------------------------------------------------ output "failover_replica_ca_cert" { diff --git a/modules/mysql/variables.tf b/modules/mysql/variables.tf index 7082c50..cec6a62 100644 --- a/modules/mysql/variables.tf +++ b/modules/mysql/variables.tf @@ -66,11 +66,6 @@ variable "authorized_gae_applications" { default = [] } -variable "availability_type" { - description = "This specifies whether a PostgreSQL instance should be set up for high availability (REGIONAL) or single zone (ZONAL)." - default = "ZONAL" -} - variable "backup_enabled" { description = "Set to false if you want to disable backup." default = true @@ -81,8 +76,8 @@ variable "backup_start_time" { default = "04:00" } -variable "binary_log_enabled" { - description = "Set to false if you want to disable binary logs. Note, when using failover or read replicas, master and existing backups need to have binary_log_enabled=true set." +variable "mysql_binary_log_enabled" { + description = "Set to false if you want to disable binary logs - only applicable to MySQL. Note, when using failover or read replicas, master and existing backups need to have binary_log_enabled=true set." default = true } @@ -172,7 +167,7 @@ variable "enable_failover_replica" { } variable "failover_replica_zone" { - description = "The preferred zone for the failover instance (e.g. 'us-central1-b'). Must be different than 'master_zone'." + description = "The preferred zone for the failover instance (e.g. 'us-central1-b'). Must be different than 'master_zone'. Only applicable to MySQL, Postgres will determine this automatically." default = "" } diff --git a/test/Gopkg.lock b/test/Gopkg.lock index 84416d6..95f8b84 100644 --- a/test/Gopkg.lock +++ b/test/Gopkg.lock @@ -24,6 +24,7 @@ "logging", "proxy/certs", "proxy/dialers/mysql", + "proxy/dialers/postgres", "proxy/proxy", "proxy/util", ] @@ -182,6 +183,17 @@ pruneopts = "" revision = "c2b33e84" +[[projects]] + digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10" + name = "github.com/lib/pq" + packages = [ + ".", + "oid", + ] + pruneopts = "" + revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79" + version = "v1.0.0" + [[projects]] digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" name = "github.com/pmezard/go-difflib" @@ -413,11 +425,13 @@ analyzer-version = 1 input-imports = [ "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/mysql", + "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/postgres", "github.com/go-sql-driver/mysql", "github.com/gruntwork-io/terratest/modules/gcp", "github.com/gruntwork-io/terratest/modules/logger", "github.com/gruntwork-io/terratest/modules/terraform", "github.com/gruntwork-io/terratest/modules/test-structure", + "github.com/lib/pq", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/require", ] diff --git a/test/example_cloud_sql_postgres_test.go b/test/example_cloud_sql_postgres_test.go deleted file mode 100644 index 603cb9b..0000000 --- a/test/example_cloud_sql_postgres_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCloudSQLPostgres(t *testing.T) { - t.Parallel() - - assert.Equal(t, "5432", "5432") -} diff --git a/test/example_mysql_private_ip_test.go b/test/example_mysql_private_ip_test.go index 09a6ea9..4f3d00a 100644 --- a/test/example_mysql_private_ip_test.go +++ b/test/example_mysql_private_ip_test.go @@ -42,7 +42,7 @@ func TestMySqlPrivateIP(t *testing.T) { test_structure.RunTestStage(t, "deploy", func() { region := test_structure.LoadString(t, exampleDir, KEY_REGION) projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) - terraformOptions := createTerratestOptionsForMySql(projectId, region, exampleDir, NAME_PREFIX_PRIVATE, "", "", 0, "") + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_PRIVATE, "", "", 0, "") test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) terraform.InitAndApply(t, terraformOptions) diff --git a/test/example_mysql_public_ip_test.go b/test/example_mysql_public_ip_test.go index 582c538..942812c 100644 --- a/test/example_mysql_public_ip_test.go +++ b/test/example_mysql_public_ip_test.go @@ -65,7 +65,7 @@ func TestMySqlPublicIP(t *testing.T) { test_structure.RunTestStage(t, "deploy", func() { region := test_structure.LoadString(t, exampleDir, KEY_REGION) projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) - terraformOptions := createTerratestOptionsForMySql(projectId, region, exampleDir, NAME_PREFIX_PUBLIC, "", "", 0, "") + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_PUBLIC, "", "", 0, "") test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) terraform.InitAndApply(t, terraformOptions) @@ -119,8 +119,8 @@ func TestMySqlPublicIP(t *testing.T) { } // Clean up - logger.Logf(t, "Empty table: %s", MYSQL_EMPTY_TEST_TABLE_STATEMENT) - if _, err = db.Exec(MYSQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { + logger.Logf(t, "Empty table: %s", SQL_EMPTY_TEST_TABLE_STATEMENT) + if _, err = db.Exec(SQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { t.Fatalf("Failed to clean up table: %v", err) } diff --git a/test/example_mysql_replicas_test.go b/test/example_mysql_replicas_test.go index 6e1a4e3..89bd79a 100644 --- a/test/example_mysql_replicas_test.go +++ b/test/example_mysql_replicas_test.go @@ -58,7 +58,7 @@ func TestMySqlReplicas(t *testing.T) { masterZone := test_structure.LoadString(t, exampleDir, KEY_MASTER_ZONE) failoverReplicaZone := test_structure.LoadString(t, exampleDir, KEY_FAILOVER_REPLICA_ZONE) readReplicaZone := test_structure.LoadString(t, exampleDir, KEY_READ_REPLICA_ZONE) - terraformOptions := createTerratestOptionsForMySql(projectId, region, exampleDir, NAME_PREFIX_REPLICAS, masterZone, failoverReplicaZone, 1, readReplicaZone) + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_REPLICAS, masterZone, failoverReplicaZone, 1, readReplicaZone) test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) terraform.InitAndApply(t, terraformOptions) @@ -132,8 +132,8 @@ func TestMySqlReplicas(t *testing.T) { } // Clean up - logger.Logf(t, "Empty table: %s", MYSQL_EMPTY_TEST_TABLE_STATEMENT) - if _, err = db.Exec(MYSQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { + logger.Logf(t, "Empty table: %s", SQL_EMPTY_TEST_TABLE_STATEMENT) + if _, err = db.Exec(SQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { t.Fatalf("Failed to clean up table: %v", err) } @@ -190,11 +190,11 @@ func TestMySqlReplicas(t *testing.T) { logger.Logf(t, "Failed to insert data to read replica as expected: %v", err) // Prepare statement for reading data - stmtOut, err := db.Prepare(MYSQL_QUERY_ROW_COUNT) + stmtOut, err := db.Prepare(SQL_QUERY_ROW_COUNT) require.NoError(t, err, "Failed to prepare readonly count statement") // Query data, results don't matter... - logger.Logf(t, "Query r/o data: %s", MYSQL_QUERY_ROW_COUNT) + logger.Logf(t, "Query r/o data: %s", SQL_QUERY_ROW_COUNT) var numResults int diff --git a/test/example_postgres_private_ip_test.go b/test/example_postgres_private_ip_test.go new file mode 100644 index 0000000..96a65b9 --- /dev/null +++ b/test/example_postgres_private_ip_test.go @@ -0,0 +1,73 @@ +package test + +import ( + "fmt" + "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/assert" + "path/filepath" + "strings" + "testing" +) + +const NAME_PREFIX_POSTGRES_PRIVATE = "postgres-private" +const EXAMPLE_NAME_POSTGRES_PRIVATE = "postgres-private-ip" + +func TestPostgresPrivateIP(t *testing.T) { + t.Parallel() + + //os.Setenv("SKIP_bootstrap", "true") + //os.Setenv("SKIP_deploy", "true") + //os.Setenv("SKIP_validate_outputs", "true") + //os.Setenv("SKIP_teardown", "true") + + _examplesDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples") + exampleDir := filepath.Join(_examplesDir, EXAMPLE_NAME_POSTGRES_PRIVATE) + + test_structure.RunTestStage(t, "bootstrap", func() { + projectId := gcp.GetGoogleProjectIDFromEnvVar(t) + region := getRandomRegion(t, projectId) + + test_structure.SaveString(t, exampleDir, KEY_REGION, region) + test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId) + }) + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer test_structure.RunTestStage(t, "teardown", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + terraform.Destroy(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "deploy", func() { + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_POSTGRES_PRIVATE, "", "", 0, "") + test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) + + terraform.InitAndApply(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "validate_outputs", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + + instanceNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_INSTANCE_NAME) + ipAddressesFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_IP_ADDRESSES) + privateIPFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PRIVATE_IP) + + assert.Contains(t, ipAddressesFromOutput, "PRIVATE", "IP Addresses output has to contain 'PRIVATE'") + assert.Contains(t, ipAddressesFromOutput, privateIPFromOutput, "IP Addresses output has to contain 'private_ip' from output") + + dbNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_DB_NAME) + proxyConnectionFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PROXY_CONNECTION) + + expectedDBConn := fmt.Sprintf("%s:%s:%s", projectId, region, instanceNameFromOutput) + + assert.True(t, strings.HasPrefix(instanceNameFromOutput, NAME_PREFIX_POSTGRES_PRIVATE)) + assert.Equal(t, DB_NAME, dbNameFromOutput) + assert.Equal(t, expectedDBConn, proxyConnectionFromOutput) + }) +} diff --git a/test/example_postgres_public_ip_test.go b/test/example_postgres_public_ip_test.go new file mode 100644 index 0000000..f561189 --- /dev/null +++ b/test/example_postgres_public_ip_test.go @@ -0,0 +1,249 @@ +package test + +import ( + "database/sql" + "fmt" + _ "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/postgres" + "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/test-structure" + _ "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "os" + "path/filepath" + "strings" + "testing" +) + +const NAME_PREFIX_POSTGRES_PUBLIC = "postgres-public" +const EXAMPLE_NAME_POSTGRES_PUBLIC = "postgres-public-ip" + +func TestPostgresPublicIP(t *testing.T) { + t.Parallel() + + //os.Setenv("SKIP_bootstrap", "true") + //os.Setenv("SKIP_deploy", "true") + //os.Setenv("SKIP_validate_outputs", "true") + //os.Setenv("SKIP_sql_tests", "true") + //os.Setenv("SKIP_proxy_tests", "true") + //os.Setenv("SKIP_deploy_cert", "true") + //os.Setenv("SKIP_redeploy", "true") + //os.Setenv("SKIP_ssl_sql_tests", "true") + //os.Setenv("SKIP_teardown_cert", "true") + //os.Setenv("SKIP_teardown", "true") + + _examplesDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples") + exampleDir := filepath.Join(_examplesDir, EXAMPLE_NAME_POSTGRES_PUBLIC) + certExampleDir := filepath.Join(_examplesDir, EXAMPLE_NAME_CERT) + + // BOOTSTRAP VARIABLES FOR THE TESTS + test_structure.RunTestStage(t, "bootstrap", func() { + projectId := gcp.GetGoogleProjectIDFromEnvVar(t) + region := getRandomRegion(t, projectId) + + test_structure.SaveString(t, exampleDir, KEY_REGION, region) + test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId) + }) + + // AT THE END OF THE TESTS, RUN `terraform destroy` + // TO CLEAN UP ANY RESOURCES THAT WERE CREATED + defer test_structure.RunTestStage(t, "teardown", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + terraform.Destroy(t, terraformOptions) + }) + + defer test_structure.RunTestStage(t, "teardown_cert", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, certExampleDir) + terraform.Destroy(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "deploy", func() { + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_POSTGRES_PUBLIC, "", "", 0, "") + test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) + + terraform.InitAndApply(t, terraformOptions) + }) + + // VALIDATE MODULE OUTPUTS + test_structure.RunTestStage(t, "validate_outputs", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + + instanceNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_INSTANCE_NAME) + dbNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_DB_NAME) + proxyConnectionFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PROXY_CONNECTION) + + expectedDBConn := fmt.Sprintf("%s:%s:%s", projectId, region, instanceNameFromOutput) + + assert.True(t, strings.HasPrefix(instanceNameFromOutput, NAME_PREFIX_POSTGRES_PUBLIC)) + assert.Equal(t, DB_NAME, dbNameFromOutput) + assert.Equal(t, expectedDBConn, proxyConnectionFromOutput) + }) + + // TEST REGULAR SQL CLIENT + test_structure.RunTestStage(t, "sql_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + publicIp := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PUBLIC_IP) + + connectionString := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", DB_USER, DB_PASS, publicIp, DB_NAME) + + // Does not actually open up the connection - just returns a DB ref + logger.Logf(t, "Connecting to: %s", publicIp) + db, err := sql.Open("postgres", connectionString) + require.NoError(t, err, "Failed to open DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB: %v", err) + } + + // Create table if not exists + logger.Logf(t, "Create table: %s", POSTGRES_CREATE_TEST_TABLE_WITH_SERIAL) + if _, err = db.Exec(POSTGRES_CREATE_TEST_TABLE_WITH_SERIAL); err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + // Clean up + logger.Logf(t, "Empty table: %s", SQL_EMPTY_TEST_TABLE_STATEMENT) + if _, err = db.Exec(SQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { + t.Fatalf("Failed to clean up table: %v", err) + } + + logger.Logf(t, "Insert data: %s", POSTGRES_INSERT_TEST_ROW) + var testid int + err = db.QueryRow(POSTGRES_INSERT_TEST_ROW).Scan(&testid) + require.NoError(t, err, "Failed to insert data") + + assert.True(t, testid > 0, "Data was inserted") + }) + + // TEST CLOUD SQL PROXY + test_structure.RunTestStage(t, "proxy_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + proxyConn := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PROXY_CONNECTION) + + logger.Logf(t, "Connecting to: %s via Cloud SQL Proxy", proxyConn) + + // Use the Cloud SQL Proxy for queries + // See https://cloud.google.com/sql/docs/mysql/sql-proxy + + // Note that sslmode=disable is required it does not mean that the connection + // is unencrypted. All connections via the proxy are completely encrypted. + datasourceName := fmt.Sprintf("host=%s user=%s dbname=%s password=%s sslmode=disable", proxyConn, DB_USER, DB_NAME, DB_PASS) + db, err := sql.Open("cloudsqlpostgres", datasourceName) + + require.NoError(t, err, "Failed to open Proxy DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB via Proxy") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB via Proxy: %v", err) + } + + logger.Logf(t, "Insert data via Proxy: %s", POSTGRES_INSERT_TEST_ROW) + var testid int + err = db.QueryRow(POSTGRES_INSERT_TEST_ROW).Scan(&testid) + require.NoError(t, err, "Failed to insert data via Proxy") + + assert.True(t, testid > 0, "Assert data was inserted") + }) + + // CREATE CLIENT CERT + test_structure.RunTestStage(t, "deploy_cert", func() { + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + instanceNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_INSTANCE_NAME) + commonName := fmt.Sprintf("%s-client", instanceNameFromOutput) + + terraformOptionsForCert := createTerratestOptionsForClientCert(projectId, region, certExampleDir, commonName, instanceNameFromOutput) + test_structure.SaveTerraformOptions(t, certExampleDir, terraformOptionsForCert) + + terraform.InitAndApply(t, terraformOptionsForCert) + }) + + // REDEPLOY WITH FORCED SSL SETTINGS + test_structure.RunTestStage(t, "redeploy", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + // Force secure connections + terraformOptions.Vars["require_ssl"] = true + terraform.InitAndApply(t, terraformOptions) + }) + + // RUN TESTS WITH SECURED CONNECTION + test_structure.RunTestStage(t, "ssl_sql_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + terraformOptionsForCert := test_structure.LoadTerraformOptions(t, certExampleDir) + + //******************************************************** + // First test that we're not allowed to connect over insecure connection + //******************************************************** + + publicIp := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PUBLIC_IP) + + connectionString := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", DB_USER, DB_PASS, publicIp, DB_NAME) + + // Does not actually open up the connection - just returns a DB ref + logger.Logf(t, "Connecting to: %s", publicIp) + db, err := sql.Open("postgres", + connectionString) + require.NoError(t, err, "Failed to open DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB with forced SSL") + if err = db.Ping(); err != nil { + logger.Logf(t, "Not allowed to ping %s as expected.", publicIp) + } else { + t.Fatalf("Ping %v succeeded against the odds.", publicIp) + } + + //******************************************************** + // Test connection over secure connection + //******************************************************** + + // Prepare certificates + serverCertB := []byte(terraform.Output(t, terraformOptions, OUTPUT_MASTER_CA_CERT)) + clientCertB := []byte(terraform.Output(t, terraformOptionsForCert, OUTPUT_CLIENT_CA_CERT)) + clientPKB := []byte(terraform.Output(t, terraformOptionsForCert, OUTPUT_CLIENT_PRIVATE_KEY)) + + serverCertFile := createTempFile(t, serverCertB) + defer os.Remove(serverCertFile.Name()) + + clientCertFile := createTempFile(t, clientCertB) + defer os.Remove(clientCertFile.Name()) + + clientPKFile := createTempFile(t, clientPKB) + defer os.Remove(clientPKFile.Name()) + + // Prepare the secure connection string and ping the DB + sslConnectionString := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=require&sslrootcert=%s&sslcert=%s&sslkey=%s", DB_USER, DB_PASS, publicIp, DB_NAME, serverCertFile.Name(), clientCertFile.Name(), clientPKFile.Name()) + + db, err = sql.Open("postgres", sslConnectionString) + + // Run ping to actually test the connection with the SSL config + logger.Log(t, "Ping the DB with forced SSL") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB with forced SSL: %v", err) + } + }) +} diff --git a/test/example_postgres_replicas_test.go b/test/example_postgres_replicas_test.go new file mode 100644 index 0000000..f1991d4 --- /dev/null +++ b/test/example_postgres_replicas_test.go @@ -0,0 +1,177 @@ +package test + +import ( + "database/sql" + "fmt" + "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/test-structure" + _ "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "path/filepath" + "strings" + "testing" +) + +const NAME_PREFIX_POSTGRES_REPLICAS = "postgres-replicas" +const EXAMPLE_NAME_POSTGRES_REPLICAS = "postgres-replicas" + +func TestPostgresReplicas(t *testing.T) { + t.Parallel() + + //os.Setenv("SKIP_bootstrap", "true") + //os.Setenv("SKIP_deploy", "true") + //os.Setenv("SKIP_validate_outputs", "true") + //os.Setenv("SKIP_sql_tests", "true") + //os.Setenv("SKIP_read_replica_tests", "true") + //os.Setenv("SKIP_teardown", "true") + + _examplesDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples") + exampleDir := filepath.Join(_examplesDir, EXAMPLE_NAME_POSTGRES_REPLICAS) + + // BOOTSTRAP VARIABLES FOR THE TESTS + test_structure.RunTestStage(t, "bootstrap", func() { + projectId := gcp.GetGoogleProjectIDFromEnvVar(t) + region := getRandomRegion(t, projectId) + + masterZone, readReplicaZone := getTwoDistinctRandomZonesForRegion(t, projectId, region) + + test_structure.SaveString(t, exampleDir, KEY_REGION, region) + test_structure.SaveString(t, exampleDir, KEY_MASTER_ZONE, masterZone) + test_structure.SaveString(t, exampleDir, KEY_READ_REPLICA_ZONE, readReplicaZone) + test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId) + }) + + // AT THE END OF THE TESTS, RUN `terraform destroy` + // TO CLEAN UP ANY RESOURCES THAT WERE CREATED + defer test_structure.RunTestStage(t, "teardown", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + terraform.Destroy(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "deploy", func() { + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + masterZone := test_structure.LoadString(t, exampleDir, KEY_MASTER_ZONE) + readReplicaZone := test_structure.LoadString(t, exampleDir, KEY_READ_REPLICA_ZONE) + terraformOptions := createTerratestOptionsForCloudSql(projectId, region, exampleDir, NAME_PREFIX_POSTGRES_REPLICAS, masterZone, "", 1, readReplicaZone) + test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) + + terraform.InitAndApply(t, terraformOptions) + }) + + // VALIDATE MODULE OUTPUTS + test_structure.RunTestStage(t, "validate_outputs", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + + instanceNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_INSTANCE_NAME) + dbNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_DB_NAME) + proxyConnectionFromOutput := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PROXY_CONNECTION) + + expectedDBConn := fmt.Sprintf("%s:%s:%s", projectId, region, instanceNameFromOutput) + + assert.True(t, strings.HasPrefix(instanceNameFromOutput, NAME_PREFIX_POSTGRES_REPLICAS)) + assert.Equal(t, DB_NAME, dbNameFromOutput) + assert.Equal(t, expectedDBConn, proxyConnectionFromOutput) + + // Read replica outputs + readReplicaInstanceNameFromOutputList := terraform.OutputList(t, terraformOptions, OUTPUT_READ_REPLICA_INSTANCE_NAMES) + readReplicaProxyConnectionFromOutputList := terraform.OutputList(t, terraformOptions, OUTPUT_READ_REPLICA_PROXY_CONNECTIONS) + + readReplicaInstanceNameFromOutput := readReplicaInstanceNameFromOutputList[0] + readReplicaProxyConnectionFromOutput := readReplicaProxyConnectionFromOutputList[0] + + expectedReadReplicaDBConn := fmt.Sprintf("%s:%s:%s", projectId, region, readReplicaInstanceNameFromOutput) + + assert.True(t, strings.HasPrefix(readReplicaInstanceNameFromOutput, NAME_PREFIX_POSTGRES_REPLICAS)) + assert.Equal(t, expectedReadReplicaDBConn, readReplicaProxyConnectionFromOutput) + }) + + // TEST REGULAR SQL CLIENT + test_structure.RunTestStage(t, "sql_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + publicIp := terraform.Output(t, terraformOptions, OUTPUT_MASTER_PUBLIC_IP) + + connectionString := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", DB_USER, DB_PASS, publicIp, DB_NAME) + + // Does not actually open up the connection - just returns a DB ref + logger.Logf(t, "Connecting to: %s", publicIp) + db, err := sql.Open("postgres", connectionString) + require.NoError(t, err, "Failed to open DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB: %v", err) + } + + // Create table if not exists + logger.Logf(t, "Create table: %s", POSTGRES_CREATE_TEST_TABLE_WITH_SERIAL) + if _, err = db.Exec(POSTGRES_CREATE_TEST_TABLE_WITH_SERIAL); err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + // Clean up + logger.Logf(t, "Empty table: %s", SQL_EMPTY_TEST_TABLE_STATEMENT) + if _, err = db.Exec(SQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { + t.Fatalf("Failed to clean up table: %v", err) + } + + logger.Logf(t, "Insert data: %s", POSTGRES_INSERT_TEST_ROW) + var testid int + err = db.QueryRow(POSTGRES_INSERT_TEST_ROW).Scan(&testid) + require.NoError(t, err, "Failed to insert data") + + assert.True(t, testid > 0, "Data was inserted") + }) + + // TEST READ REPLICA WITH REGULAR SQL CLIENT + test_structure.RunTestStage(t, "read_replica_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + readReplicaPublicIpList := terraform.OutputList(t, terraformOptions, OUTPUT_READ_REPLICA_PUBLIC_IPS) + readReplicaPublicIp := readReplicaPublicIpList[0] + + connectionString := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", DB_USER, DB_PASS, readReplicaPublicIp, DB_NAME) + + // Does not actually open up the connection - just returns a DB ref + logger.Logf(t, "Connecting to: %s", readReplicaPublicIp) + db, err := sql.Open("postgres", connectionString) + require.NoError(t, err, "Failed to open DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB: %v", err) + } + + // Try to insert data to verify we cannot write + logger.Logf(t, "Insert data: %s", POSTGRES_INSERT_TEST_ROW) + var testid int + err = db.QueryRow(POSTGRES_INSERT_TEST_ROW).Scan(&testid) + + // This time we actually expect an error: + // 'cannot execute INSERT in a read-only transaction' + require.Error(t, err, "Should not be able to write to read replica") + logger.Logf(t, "Failed to insert data to read replica as expected: %v", err) + + // Query data, results don't matter... + logger.Logf(t, "Query r/o data: %s", SQL_QUERY_ROW_COUNT) + rows, err := db.Query(SQL_QUERY_ROW_COUNT) + require.NoError(t, err, "Failed to execute query statement on read replica") + + assert.True(t, rows.Next(), "We have a result") + }) +} diff --git a/test/test_util.go b/test/test_util.go index 2d3ff6d..8e24cd2 100644 --- a/test/test_util.go +++ b/test/test_util.go @@ -3,6 +3,9 @@ package test import ( "github.com/gruntwork-io/terratest/modules/gcp" "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" "testing" ) @@ -16,8 +19,6 @@ const KEY_MASTER_ZONE = "masterZone" const KEY_FAILOVER_REPLICA_ZONE = "failoverReplicaZone" const KEY_READ_REPLICA_ZONE = "readReplicaZone" -const MYSQL_VERSION = "MYSQL_5_7" - const OUTPUT_MASTER_IP_ADDRESSES = "master_ip_addresses" const OUTPUT_MASTER_INSTANCE_NAME = "master_instance_name" const OUTPUT_FAILOVER_INSTANCE_NAME = "failover_instance_name" @@ -35,9 +36,13 @@ const OUTPUT_CLIENT_PRIVATE_KEY = "client_private_key" const OUTPUT_DB_NAME = "db_name" const MYSQL_CREATE_TEST_TABLE_WITH_AUTO_INCREMENT_STATEMENT = "CREATE TABLE IF NOT EXISTS test (id int NOT NULL AUTO_INCREMENT, name varchar(10) NOT NULL, PRIMARY KEY (ID))" -const MYSQL_EMPTY_TEST_TABLE_STATEMENT = "DELETE FROM test" const MYSQL_INSERT_TEST_ROW = "INSERT INTO test(name) VALUES(?)" -const MYSQL_QUERY_ROW_COUNT = "SELECT count(*) FROM test" + +const SQL_EMPTY_TEST_TABLE_STATEMENT = "DELETE FROM test" +const SQL_QUERY_ROW_COUNT = "SELECT count(*) FROM test" + +const POSTGRES_CREATE_TEST_TABLE_WITH_SERIAL = "CREATE TABLE IF NOT EXISTS test (id SERIAL, name varchar(10) NOT NULL, PRIMARY KEY (ID))" +const POSTGRES_INSERT_TEST_ROW = "INSERT INTO test(name) VALUES('Grunty') RETURNING id" func getRandomRegion(t *testing.T, projectID string) string { approvedRegions := []string{"europe-north1", "europe-west1", "europe-west2", "europe-west3", "us-central1", "us-east1", "us-west1"} @@ -58,7 +63,7 @@ func getTwoDistinctRandomZonesForRegion(t *testing.T, projectID string, region s return firstZone, secondZone } -func createTerratestOptionsForMySql(projectId string, region string, exampleDir string, namePrefix string, masterZone string, failoverReplicaZone string, numReadReplicas int, readReplicaZone string) *terraform.Options { +func createTerratestOptionsForCloudSql(projectId string, region string, exampleDir string, namePrefix string, masterZone string, failoverReplicaZone string, numReadReplicas int, readReplicaZone string) *terraform.Options { terratestOptions := &terraform.Options{ // The path to where your Terraform code is located @@ -71,7 +76,6 @@ func createTerratestOptionsForMySql(projectId string, region string, exampleDir "failover_replica_zone": failoverReplicaZone, "project": projectId, "name_prefix": namePrefix, - "mysql_version": MYSQL_VERSION, "db_name": DB_NAME, "master_user_name": DB_USER, "master_user_password": DB_PASS, @@ -96,3 +100,13 @@ func createTerratestOptionsForClientCert(projectId string, region string, exampl return terratestOptions } + +func createTempFile(t *testing.T, content []byte) *os.File { + tmpFile, err := ioutil.TempFile(os.TempDir(), "temp-") + require.NoError(t, err, "Failed to create temp file") + _, err = tmpFile.Write(content) + require.NoError(t, err, "Failed to write to temp file") + err = tmpFile.Close() + require.NoError(t, err, "Failed to close temp file") + return tmpFile +} From 5d6561bada9d5a0b103aa2a1477702c53b744fd7 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 22:19:58 +0200 Subject: [PATCH 02/11] Add reference to the single module. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 99ac946..da07cb5 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ This repo contains modules for running relational databases such as MySQL and Po ## Code included in this Module -* [mysql](/modules/mysql): Deploy a Cloud SQL MySQL cluster. -* [postgresql](/modules/postgresql): Deploy a Cloud SQL PostgreSQL cluster. +* [cloud-sql](/modules/cloud-sql): Deploy a Cloud SQL MySQL or PostgreSQL cluster. ## What is Cloud SQL? From 53c2e6b204918b7da5ffbd34a568229597f524f8 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 22:29:33 +0200 Subject: [PATCH 03/11] Rename mysql to cloud-sql --- examples/mysql-private-ip/main.tf | 4 ++-- examples/mysql-public-ip/main.tf | 4 ++-- examples/mysql-replicas/main.tf | 4 ++-- examples/postgres-private-ip/main.tf | 6 ++--- examples/postgres-private-ip/outputs.tf | 14 +++++------ examples/postgres-public-ip/main.tf | 6 ++--- examples/postgres-public-ip/outputs.tf | 16 ++++++------- examples/postgres-replicas/main.tf | 6 ++--- examples/postgres-replicas/outputs.tf | 24 +++++++++---------- modules/cloud-sql/.gitkeep | 0 modules/{mysql => cloud-sql}/README.md | 0 .../{mysql => cloud-sql}/compute_outputs.tf | 0 modules/{mysql => cloud-sql}/main.tf | 0 modules/{mysql => cloud-sql}/outputs.tf | 0 modules/{mysql => cloud-sql}/variables.tf | 0 modules/postgresql/.gitkeep | 0 16 files changed, 42 insertions(+), 42 deletions(-) delete mode 100644 modules/cloud-sql/.gitkeep rename modules/{mysql => cloud-sql}/README.md (100%) rename modules/{mysql => cloud-sql}/compute_outputs.tf (100%) rename modules/{mysql => cloud-sql}/main.tf (100%) rename modules/{mysql => cloud-sql}/outputs.tf (100%) rename modules/{mysql => cloud-sql}/variables.tf (100%) delete mode 100644 modules/postgresql/.gitkeep diff --git a/examples/mysql-private-ip/main.tf b/examples/mysql-private-ip/main.tf index dda75b3..662f45e 100644 --- a/examples/mysql-private-ip/main.tf +++ b/examples/mysql-private-ip/main.tf @@ -67,8 +67,8 @@ resource "google_service_networking_connection" "private_vpc_connection" { module "mysql" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/mysql-public-ip/main.tf b/examples/mysql-public-ip/main.tf index 1cc3067..0adc979 100644 --- a/examples/mysql-public-ip/main.tf +++ b/examples/mysql-public-ip/main.tf @@ -37,8 +37,8 @@ locals { module "mysql" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/mysql-replicas/main.tf b/examples/mysql-replicas/main.tf index a7c0a2e..3426048 100644 --- a/examples/mysql-replicas/main.tf +++ b/examples/mysql-replicas/main.tf @@ -39,8 +39,8 @@ locals { module "mysql" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/postgres-private-ip/main.tf b/examples/postgres-private-ip/main.tf index 26d100d..417a085 100644 --- a/examples/postgres-private-ip/main.tf +++ b/examples/postgres-private-ip/main.tf @@ -64,11 +64,11 @@ resource "google_service_networking_connection" "private_vpc_connection" { # CREATE DATABASE INSTANCE WITH PRIVATE IP # ------------------------------------------------------------------------------ -module "mysql" { +module "postgres" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/postgres-private-ip/outputs.tf b/examples/postgres-private-ip/outputs.tf index 3268020..9dad428 100644 --- a/examples/postgres-private-ip/outputs.tf +++ b/examples/postgres-private-ip/outputs.tf @@ -1,34 +1,34 @@ output "master_instance_name" { description = "The name of the database instance" - value = "${module.mysql.master_instance_name}" + value = "${module.postgres.master_instance_name}" } output "master_ip_addresses" { description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" - value = "${module.mysql.master_ip_addresses}" + value = "${module.postgres.master_ip_addresses}" } output "master_private_ip" { description = "The first IPv4 address of the addresses assigned to the instance. As this instance has only private IP, it is the private IP address." - value = "${module.mysql.master_first_ip_address}" + value = "${module.postgres.master_first_ip_address}" } output "master_instance" { description = "Self link to the master instance" - value = "${module.mysql.master_instance}" + value = "${module.postgres.master_instance}" } output "master_proxy_connection" { description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" - value = "${module.mysql.master_proxy_connection}" + value = "${module.postgres.master_proxy_connection}" } output "db_name" { description = "Name of the default database" - value = "${module.mysql.db_name}" + value = "${module.postgres.db_name}" } output "db" { description = "Self link to the default database" - value = "${module.mysql.db}" + value = "${module.postgres.db}" } diff --git a/examples/postgres-public-ip/main.tf b/examples/postgres-public-ip/main.tf index cd1b2e5..de00732 100644 --- a/examples/postgres-public-ip/main.tf +++ b/examples/postgres-public-ip/main.tf @@ -34,11 +34,11 @@ locals { # CREATE DATABASE INSTANCE WITH PUBLIC IP # ------------------------------------------------------------------------------ -module "mysql" { +module "postgres" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/postgres-public-ip/outputs.tf b/examples/postgres-public-ip/outputs.tf index 95fc3ce..5e1ccf0 100644 --- a/examples/postgres-public-ip/outputs.tf +++ b/examples/postgres-public-ip/outputs.tf @@ -1,39 +1,39 @@ output "master_instance_name" { description = "The name of the database instance" - value = "${module.mysql.master_instance_name}" + value = "${module.postgres.master_instance_name}" } output "master_ip_addresses" { description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" - value = "${module.mysql.master_ip_addresses}" + value = "${module.postgres.master_ip_addresses}" } output "master_public_ip" { description = "The first IPv4 address of the addresses assigned to the instance. As this instance has only public IP, it is the public IP address." - value = "${module.mysql.master_first_ip_address}" + value = "${module.postgres.master_first_ip_address}" } output "master_ca_cert" { - value = "${module.mysql.master_ca_cert}" + value = "${module.postgres.master_ca_cert}" description = "The CA Certificate used to connect to the SQL Instance via SSL" } output "master_instance" { description = "Self link to the master instance" - value = "${module.mysql.master_instance}" + value = "${module.postgres.master_instance}" } output "master_proxy_connection" { description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" - value = "${module.mysql.master_proxy_connection}" + value = "${module.postgres.master_proxy_connection}" } output "db_name" { description = "Name of the default database" - value = "${module.mysql.db_name}" + value = "${module.postgres.db_name}" } output "db" { description = "Self link to the default database" - value = "${module.mysql.db}" + value = "${module.postgres.db}" } diff --git a/examples/postgres-replicas/main.tf b/examples/postgres-replicas/main.tf index db06315..dd5005d 100644 --- a/examples/postgres-replicas/main.tf +++ b/examples/postgres-replicas/main.tf @@ -36,11 +36,11 @@ locals { # CREATE DATABASE CLUSTER WITH PUBLIC IP # ------------------------------------------------------------------------------ -module "mysql" { +module "postgres" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: - # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" - source = "../../modules/mysql" + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/cloud-sql?ref=v0.1.0" + source = "../../modules/cloud-sql" project = "${var.project}" region = "${var.region}" diff --git a/examples/postgres-replicas/outputs.tf b/examples/postgres-replicas/outputs.tf index 52a54d1..e57016d 100644 --- a/examples/postgres-replicas/outputs.tf +++ b/examples/postgres-replicas/outputs.tf @@ -4,27 +4,27 @@ output "master_instance_name" { description = "The name of the database instance" - value = "${module.mysql.master_instance_name}" + value = "${module.postgres.master_instance_name}" } output "master_ip_addresses" { description = "All IP addresses of the instance as list of maps, see https://www.terraform.io/docs/providers/google/r/sql_database_instance.html#ip_address-0-ip_address" - value = "${module.mysql.master_ip_addresses}" + value = "${module.postgres.master_ip_addresses}" } output "master_public_ip" { description = "The first IPv4 address of the addresses assigned to the master instance. As this instance has only public IP, it is the public IP address." - value = "${module.mysql.master_first_ip_address}" + value = "${module.postgres.master_first_ip_address}" } output "master_instance" { description = "Self link to the master instance" - value = "${module.mysql.master_instance}" + value = "${module.postgres.master_instance}" } output "master_proxy_connection" { description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" - value = "${module.mysql.master_proxy_connection}" + value = "${module.postgres.master_proxy_connection}" } # ------------------------------------------------------------------------------ @@ -33,12 +33,12 @@ output "master_proxy_connection" { output "db_name" { description = "Name of the default database" - value = "${module.mysql.db_name}" + value = "${module.postgres.db_name}" } output "db" { description = "Self link to the default database" - value = "${module.mysql.db}" + value = "${module.postgres.db}" } # ------------------------------------------------------------------------------ @@ -47,22 +47,22 @@ output "db" { output "read_replica_instance_names" { description = "List of names for the read replica instances" - value = ["${module.mysql.read_replica_instance_names}"] + value = ["${module.postgres.read_replica_instance_names}"] } output "read_replica_public_ips" { description = "List of first IPv4 addresses of the addresses assigned to the read replica instances. As the instances have only public IP in the example, the are the public IP addresses." - value = ["${module.mysql.read_replica_first_ip_addresses}"] + value = ["${module.postgres.read_replica_first_ip_addresses}"] } output "read_replica_instances" { description = "List of self links to the read replica instances" - value = ["${module.mysql.read_replica_instances}"] + value = ["${module.postgres.read_replica_instances}"] } output "read_replica_proxy_connections" { description = "List of read replica instance paths for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" - value = ["${module.mysql.read_replica_proxy_connections}"] + value = ["${module.postgres.read_replica_proxy_connections}"] } # Although we don't use the values, this output highlights the JSON encoded output we use in certain @@ -70,5 +70,5 @@ output "read_replica_proxy_connections" { # See https://github.com/hashicorp/terraform/issues/17048 output "read_replica_server_ca_certs" { description = "JSON encoded list of CA Certificates used to connect to the read replica instances via SSL" - value = "${module.mysql.read_replica_server_ca_certs}" + value = "${module.postgres.read_replica_server_ca_certs}" } diff --git a/modules/cloud-sql/.gitkeep b/modules/cloud-sql/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/modules/mysql/README.md b/modules/cloud-sql/README.md similarity index 100% rename from modules/mysql/README.md rename to modules/cloud-sql/README.md diff --git a/modules/mysql/compute_outputs.tf b/modules/cloud-sql/compute_outputs.tf similarity index 100% rename from modules/mysql/compute_outputs.tf rename to modules/cloud-sql/compute_outputs.tf diff --git a/modules/mysql/main.tf b/modules/cloud-sql/main.tf similarity index 100% rename from modules/mysql/main.tf rename to modules/cloud-sql/main.tf diff --git a/modules/mysql/outputs.tf b/modules/cloud-sql/outputs.tf similarity index 100% rename from modules/mysql/outputs.tf rename to modules/cloud-sql/outputs.tf diff --git a/modules/mysql/variables.tf b/modules/cloud-sql/variables.tf similarity index 100% rename from modules/mysql/variables.tf rename to modules/cloud-sql/variables.tf diff --git a/modules/postgresql/.gitkeep b/modules/postgresql/.gitkeep deleted file mode 100644 index e69de29..0000000 From b6f53ab027217de1318d96d04b6b681cc9c4cd3d Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 22:32:48 +0200 Subject: [PATCH 04/11] Better documentation for outputs --- examples/client-certificate/outputs.tf | 4 ++++ examples/mysql-private-ip/outputs.tf | 8 ++++++++ examples/mysql-public-ip/outputs.tf | 8 ++++++++ examples/postgres-private-ip/outputs.tf | 8 ++++++++ examples/postgres-public-ip/outputs.tf | 8 ++++++++ 5 files changed, 36 insertions(+) diff --git a/examples/client-certificate/outputs.tf b/examples/client-certificate/outputs.tf index 21dc370..82e0c14 100644 --- a/examples/client-certificate/outputs.tf +++ b/examples/client-certificate/outputs.tf @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------ +# CLIENT CERTIFICATE OUTPUTS +# ------------------------------------------------------------------------------ + output "client_ca_cert" { description = "Certificate data for the client certificate." value = "${google_sql_ssl_cert.client_cert.cert}" diff --git a/examples/mysql-private-ip/outputs.tf b/examples/mysql-private-ip/outputs.tf index 3268020..ef89a99 100644 --- a/examples/mysql-private-ip/outputs.tf +++ b/examples/mysql-private-ip/outputs.tf @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------ +# MASTER OUTPUTS +# ------------------------------------------------------------------------------ + output "master_instance_name" { description = "The name of the database instance" value = "${module.mysql.master_instance_name}" @@ -23,6 +27,10 @@ output "master_proxy_connection" { value = "${module.mysql.master_proxy_connection}" } +# ------------------------------------------------------------------------------ +# DB OUTPUTS +# ------------------------------------------------------------------------------ + output "db_name" { description = "Name of the default database" value = "${module.mysql.db_name}" diff --git a/examples/mysql-public-ip/outputs.tf b/examples/mysql-public-ip/outputs.tf index 95fc3ce..20d103c 100644 --- a/examples/mysql-public-ip/outputs.tf +++ b/examples/mysql-public-ip/outputs.tf @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------ +# MASTER OUTPUTS +# ------------------------------------------------------------------------------ + output "master_instance_name" { description = "The name of the database instance" value = "${module.mysql.master_instance_name}" @@ -28,6 +32,10 @@ output "master_proxy_connection" { value = "${module.mysql.master_proxy_connection}" } +# ------------------------------------------------------------------------------ +# DB OUTPUTS +# ------------------------------------------------------------------------------ + output "db_name" { description = "Name of the default database" value = "${module.mysql.db_name}" diff --git a/examples/postgres-private-ip/outputs.tf b/examples/postgres-private-ip/outputs.tf index 9dad428..edc31db 100644 --- a/examples/postgres-private-ip/outputs.tf +++ b/examples/postgres-private-ip/outputs.tf @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------ +# MASTER OUTPUTS +# ------------------------------------------------------------------------------ + output "master_instance_name" { description = "The name of the database instance" value = "${module.postgres.master_instance_name}" @@ -23,6 +27,10 @@ output "master_proxy_connection" { value = "${module.postgres.master_proxy_connection}" } +# ------------------------------------------------------------------------------ +# DB OUTPUTS +# ------------------------------------------------------------------------------ + output "db_name" { description = "Name of the default database" value = "${module.postgres.db_name}" diff --git a/examples/postgres-public-ip/outputs.tf b/examples/postgres-public-ip/outputs.tf index 5e1ccf0..9f37b83 100644 --- a/examples/postgres-public-ip/outputs.tf +++ b/examples/postgres-public-ip/outputs.tf @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------ +# MASTER OUTPUTS +# ------------------------------------------------------------------------------ + output "master_instance_name" { description = "The name of the database instance" value = "${module.postgres.master_instance_name}" @@ -28,6 +32,10 @@ output "master_proxy_connection" { value = "${module.postgres.master_proxy_connection}" } +# ------------------------------------------------------------------------------ +# DB OUTPUTS +# ------------------------------------------------------------------------------ + output "db_name" { description = "Name of the default database" value = "${module.postgres.db_name}" From 58f9e9e2db9c6de7a56c175b9701ee0c5b670548 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 22:45:37 +0200 Subject: [PATCH 05/11] Documentation linting for examples. --- examples/mysql-private-ip/README.md | 4 ++-- examples/mysql-public-ip/README.md | 4 ++-- examples/mysql-replicas/README.md | 4 ++-- examples/postgres-private-ip/README.md | 4 ++-- examples/postgres-public-ip/README.md | 4 ++-- examples/postgres-replicas/README.md | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/mysql-private-ip/README.md b/examples/mysql-private-ip/README.md index 2ea5710..f1b98b3 100644 --- a/examples/mysql-private-ip/README.md +++ b/examples/mysql-private-ip/README.md @@ -1,7 +1,7 @@ # MySQL Cloud SQL Private IP Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a private IP. +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a private IP address. ## How do you run this example? diff --git a/examples/mysql-public-ip/README.md b/examples/mysql-public-ip/README.md index c58d9f9..f60ad2e 100644 --- a/examples/mysql-public-ip/README.md +++ b/examples/mysql-public-ip/README.md @@ -1,7 +1,7 @@ # MySQL Cloud SQL Public IP Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a public IP. +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a public IP address. ## How do you run this example? diff --git a/examples/mysql-replicas/README.md b/examples/mysql-replicas/README.md index 18440cc..c06e511 100644 --- a/examples/mysql-replicas/README.md +++ b/examples/mysql-replicas/README.md @@ -1,6 +1,6 @@ -# MySQL Cloud SQL Replica Example +# MySQL Cloud SQL HA Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [High Availability](https://cloud.google.com/sql/docs/mysql/configure-ha) [Google Cloud SQL](https://cloud.google.com/sql/) [MySQL](https://cloud.google.com/sql/docs/mysql/) database cluster with a public IP and failover and read replicas. ## How do you run this example? diff --git a/examples/postgres-private-ip/README.md b/examples/postgres-private-ip/README.md index 7d9b983..76b6ac2 100644 --- a/examples/postgres-private-ip/README.md +++ b/examples/postgres-private-ip/README.md @@ -1,7 +1,7 @@ # PostgreSQL Cloud SQL Private IP Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a private IP. +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a private IP address. ## How do you run this example? diff --git a/examples/postgres-public-ip/README.md b/examples/postgres-public-ip/README.md index d60c6f1..9ef6594 100644 --- a/examples/postgres-public-ip/README.md +++ b/examples/postgres-public-ip/README.md @@ -1,7 +1,7 @@ # PostgreSQL Cloud SQL Public IP Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a public IP. +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a public IP address. ## How do you run this example? diff --git a/examples/postgres-replicas/README.md b/examples/postgres-replicas/README.md index 18440cc..c6809ff 100644 --- a/examples/postgres-replicas/README.md +++ b/examples/postgres-replicas/README.md @@ -1,7 +1,7 @@ -# MySQL Cloud SQL Replica Example +# PostgreSQL Cloud SQL HA Example -This folder contains an example of how to use the [MySQL module](/modules/mysql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database cluster with a public IP and failover and read replicas. +This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [High Availability](https://cloud.google.com/sql/docs/postgres/high-availability) [Google Cloud SQL](https://cloud.google.com/sql/) +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database cluster with a public IP and a read replica. ## How do you run this example? From ee9fafb9427eed2045d86a87ffccd7a1746f44ab Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Thu, 14 Feb 2019 23:58:35 +0200 Subject: [PATCH 06/11] Improved module documentation --- modules/cloud-sql/README.md | 59 ++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/modules/cloud-sql/README.md b/modules/cloud-sql/README.md index 72c92a7..3520331 100644 --- a/modules/cloud-sql/README.md +++ b/modules/cloud-sql/README.md @@ -1,9 +1,9 @@ -# MySQL Module +# Cloud SQL Module -This module creates a [Google Cloud SQL](https://cloud.google.com/sql/) [MySQL](https://cloud.google.com/sql/docs/mysql/) cluster. +This module creates a [Google Cloud SQL](https://cloud.google.com/sql/) cluster. The cluster is managed by Google, automating backups, replication, patches, and updates. -This module helps you run [MySQL](https://cloud.google.com/sql/docs/mysql/), see [postgres](../postgresql) for running [PostgreSQL](https://cloud.google.com/sql/docs/postgres/). +This module helps you run [MySQL](https://cloud.google.com/sql/docs/mysql/) and [PostgreSQL](https://cloud.google.com/sql/docs/postgres/) databases in [Google Cloud](https://cloud.google.com/). ## How do you use this module? @@ -11,40 +11,71 @@ See the [examples](/examples) folder for an example. ## How do you configure this module? -This module allows you to configure a number of parameters, such as backup windows, maintenance window, replicas -and encryption. For a list of all available variables and their descriptions, see [variables.tf](./variables.tf). +This module allows you to configure a number of parameters, such as high availability, backup windows, maintenance window and replicas. +For a list of all available variables and their descriptions, see [variables.tf](./variables.tf). ## How do you connect to the database? **Cloud SQL instances are created in a producer network (a VPC network internal to Google). They are not created in your VPC network. See https://cloud.google.com/sql/docs/mysql/private-ip** -You can use both [public IP](https://cloud.google.com/sql/docs/mysql/connect-admin-ip) and [private IP](https://cloud.google.com/sql/docs/mysql/private-ip) to connect to a Cloud SQL instance. +You can use both public IP and private IP to connect to a Cloud SQL instance. Neither connection method affects the other; you must protect the public IP connection whether the instance is configured to use private IP or not. -You can also use the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy) to connect to an instance that is also configured to use private IP. The proxy can connect using either the private IP address or a public IP address. +You can also use the [Cloud SQL Proxy for MySQL](https://cloud.google.com/sql/docs/mysql/sql-proxy) and [Cloud SQL Proxy for PostgreSQL](https://cloud.google.com/sql/docs/postgres/sql-proxy) +to connect to an instance that is also configured to use private IP. The proxy can connect using either the private IP address or a public IP address. This module provides the connection details as [Terraform output variables](https://www.terraform.io/intro/getting-started/outputs.html): -1. **First IP Address** `first_ip_address`: The first IPv4 address of the addresses assigned to the instance. If the instance has only public IP, it is the [public IP address](https://cloud.google.com/sql/docs/mysql/connect-admin-ip). If it has only private IP, it the [private IP address](https://cloud.google.com/sql/docs/mysql/private-ip). If it has both, it is the first item in the list and full IP address details are in `instance_ip_addresses`. -1. **Proxy connection** `proxy_connection`: Instance path for connecting with Cloud SQL Proxy; see [Connecting mysql Client Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). -1. TODO: **Replica endpoints** `replica_endpoints`: A comma-separated list of all DB instance URLs in the cluster, including the primary and all - read replicas. Use these URLs for reads (see "How do you scale this DB?" below). +1. **Master First IP Address** `master_first_ip_address`: The first IPv4 address of the addresses assigned to the instance. If the instance has only public IP, it is the [public IP address](https://cloud.google.com/sql/docs/mysql/connect-admin-ip). If it has only private IP, it the [private IP address](https://cloud.google.com/sql/docs/mysql/private-ip). If it has both, it is the first item in the list and full IP address details are in `instance_ip_addresses`. +1. **Master Proxy connection** `master_proxy_connection`: Instance path for connecting with Cloud SQL Proxy; see [Connecting mysql Client Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). +1. **Read Replica First IP Addresses** `read_replica_first_ip_addresses`: A list of all read replica IP addresses in the cluster. Use these addresses for reads (see "How do you scale this database?" below). +1. **Read Replica Proxy Connections** `read_replica_proxy_connections`: A list of instance paths for connecting with Cloud SQL Proxy; see [Connecting Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). You can programmatically extract these variables in your Terraform templates and pass them to other resources. You'll also see the variables at the end of each `terraform apply` call or if you run `terraform output`. -For full connectivity options and detailed documentation, see [Connecting to Cloud SQL from External Applications](https://cloud.google.com/sql/docs/mysql/connect-external-app). +For full connectivity options and detailed documentation, see [Connecting to Cloud SQL MySQL from External Applications](https://cloud.google.com/sql/docs/mysql/connect-external-app) and [Connecting to Cloud SQL PostgreSQL from External Applications](https://cloud.google.com/sql/docs/postgres/external-connection-methods). + +## How do you configure High Availability? + +You can enable High Availability using the `enable_failover_replica` input variable. + +### High Availability for MySQL + +The configuration is made up of a primary instance (master) in the primary zone (`master_zone` input variable) and a failover replica in the secondary zone (`failover_replica_zone` input variable). +The failover replica is configured with the same database flags, users and passwords, authorized applications and networks, and databases as the primary instance. + +For full details about MySQL High Availability, see https://cloud.google.com/sql/docs/mysql/high-availability + +### High Availability for PostgreSQL + +A Cloud SQL PostgreSQL instance configured for HA is also called a _regional instance_ and is located in a primary and secondary zone within the configured region. Within a regional instance, +the configuration is made up of a primary instance (master) and a standby instance. You control the primary zone for the master instance +with input variable `master_zone` and Google will automatically place the standby instance in another zone. + +For full details about PostgreSQL High Availability, see https://cloud.google.com/sql/docs/postgres/high-availability + + +## How do you secure this database? + +Cloud SQL customer data is encrypted when stored in database tables, temporary files, and backups. +External connections can be encrypted by using SSL, or by using the Cloud SQL Proxy, which automatically encrypts traffic to and from the database. +If you do not use the proxy, you can enforce SSL for external connections using the `require_ssl` input variable. + +For further information, see https://cloud.google.com/blog/products/gcp/best-practices-for-securing-your-google-cloud-databases and +https://cloud.google.com/sql/faq#encryption ## How do you scale this database? -* **Storage**: Cloud SQL manages storage for you, automatically growing cluster volume up to 10TB. +* **Storage**: Cloud SQL manages storage for you, automatically growing cluster volume up to 10TB You can set the +initial disk size using the `disk_size` input variable. * **Vertical scaling**: To scale vertically (i.e. bigger DB instances with more CPU and RAM), use the `machine_type` input variable. For a list of Cloud SQL Machine Types, see [Cloud SQL Pricing](https://cloud.google.com/sql/pricing#2nd-gen-pricing). -* **Horizontal scaling**: To scale horizontally, you can add more replicas using the `instance_count` input variable, +* **Horizontal scaling**: To scale horizontally, you can add more replicas using the `num_read_replicas` and `read_replica_zones` input variables, and the module will automatically deploy the new instances, sync them to the master, and make them available as read replicas. From 7a276ead3373369e7a09bd5f078b389cd4565608 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Fri, 15 Feb 2019 10:41:47 +0200 Subject: [PATCH 07/11] [skip ci] add links to database products --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index da07cb5..3eba277 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This repo contains modules for running relational databases such as MySQL and Po ## Code included in this Module -* [cloud-sql](/modules/cloud-sql): Deploy a Cloud SQL MySQL or PostgreSQL cluster. +* [cloud-sql](/modules/cloud-sql): Deploy a Cloud SQL [MySQL](https://cloud.google.com/sql/docs/mysql/) or [PostgreSQL](https://cloud.google.com/sql/docs/postgres/) cluster. ## What is Cloud SQL? From e542a6940642db0aedd86d5425a9b0de4a0d3743 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Fri, 15 Feb 2019 10:59:35 +0200 Subject: [PATCH 08/11] [skip ci] Documentation fixes and improvements for examples --- examples/mysql-private-ip/README.md | 7 +++++-- examples/mysql-public-ip/README.md | 4 ++-- examples/mysql-replicas/README.md | 4 ++-- examples/postgres-private-ip/README.md | 9 ++++++--- examples/postgres-public-ip/README.md | 4 ++-- examples/postgres-replicas/README.md | 4 ++-- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/examples/mysql-private-ip/README.md b/examples/mysql-private-ip/README.md index f1b98b3..3760f80 100644 --- a/examples/mysql-private-ip/README.md +++ b/examples/mysql-private-ip/README.md @@ -1,14 +1,14 @@ # MySQL Cloud SQL Private IP Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a private IP address. +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a [private IP address](https://cloud.google.com/sql/docs/mysql/private-ip). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. @@ -16,3 +16,6 @@ To run this example, you need to: When the templates are applied, Terraform will output the IP address of the instance and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). + +Note that you cannot connect to the private IP instance from outside Google Cloud Platform. +If you want to experiment with connecting from your own workstation, see the [public IP example](../mysql-public-ip) diff --git a/examples/mysql-public-ip/README.md b/examples/mysql-public-ip/README.md index f60ad2e..b582c7d 100644 --- a/examples/mysql-public-ip/README.md +++ b/examples/mysql-public-ip/README.md @@ -1,14 +1,14 @@ # MySQL Cloud SQL Public IP Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a public IP address. +[MySQL](https://cloud.google.com/sql/docs/mysql/) database instance with a [public IP address](https://cloud.google.com/sql/docs/mysql/connect-external-app#appaccessIP). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. diff --git a/examples/mysql-replicas/README.md b/examples/mysql-replicas/README.md index c06e511..849bc66 100644 --- a/examples/mysql-replicas/README.md +++ b/examples/mysql-replicas/README.md @@ -1,14 +1,14 @@ # MySQL Cloud SQL HA Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [High Availability](https://cloud.google.com/sql/docs/mysql/configure-ha) [Google Cloud SQL](https://cloud.google.com/sql/) -[MySQL](https://cloud.google.com/sql/docs/mysql/) database cluster with a public IP and failover and read replicas. +[MySQL](https://cloud.google.com/sql/docs/mysql/) database cluster with a [public IP](https://cloud.google.com/sql/docs/mysql/connect-external-app#appaccessIP) and failover and [read replicas](https://cloud.google.com/sql/docs/mysql/replication/). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. diff --git a/examples/postgres-private-ip/README.md b/examples/postgres-private-ip/README.md index 76b6ac2..7fa4c01 100644 --- a/examples/postgres-private-ip/README.md +++ b/examples/postgres-private-ip/README.md @@ -1,18 +1,21 @@ # PostgreSQL Cloud SQL Private IP Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a private IP address. +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a [private IP address](https://cloud.google.com/sql/docs/postgres/private-ip). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. 1. If the plan looks good, run `terraform apply`. When the templates are applied, Terraform will output the IP address of the instance -and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). +and the instance path for [connecting using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/postgres/sql-proxy). + +Note that you cannot connect to the private IP instance from outside Google Cloud Platform. +If you want to experiment with connecting from your own workstation, see the [public IP example](../postgres-public-ip) diff --git a/examples/postgres-public-ip/README.md b/examples/postgres-public-ip/README.md index 9ef6594..572e17d 100644 --- a/examples/postgres-public-ip/README.md +++ b/examples/postgres-public-ip/README.md @@ -1,14 +1,14 @@ # PostgreSQL Cloud SQL Public IP Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [Google Cloud SQL](https://cloud.google.com/sql/) -[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a public IP address. +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database instance with a [public IP address](https://cloud.google.com/sql/docs/postgres/connect-external-app#appaccessIP). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. diff --git a/examples/postgres-replicas/README.md b/examples/postgres-replicas/README.md index c6809ff..6dd83d0 100644 --- a/examples/postgres-replicas/README.md +++ b/examples/postgres-replicas/README.md @@ -1,14 +1,14 @@ # PostgreSQL Cloud SQL HA Example This folder contains an example of how to use the [Cloud SQL module](/modules/cloud-sql) to create a [High Availability](https://cloud.google.com/sql/docs/postgres/high-availability) [Google Cloud SQL](https://cloud.google.com/sql/) -[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database cluster with a public IP and a read replica. +[PostgreSQL](https://cloud.google.com/sql/docs/postgres/) database cluster with a [public IP address](https://cloud.google.com/sql/docs/postgres/connect-external-app#appaccessIP) and a [read replica](https://cloud.google.com/sql/docs/postgres/replication/). ## How do you run this example? To run this example, you need to: 1. Install [Terraform](https://www.terraform.io/). -1. Open up `vars.tf` and set secrets at the top of the file as environment variables and fill in any other variables in +1. Open up `variables.tf` and set secrets at the top of the file as environment variables and fill in any other variables in the file that don't have defaults. 1. `terraform init`. 1. `terraform plan`. From 008cd36cf660d4b4269c39533cb4a29ad87eadb6 Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Fri, 15 Feb 2019 11:05:22 +0200 Subject: [PATCH 09/11] [skip ci] Documentation fixes --- modules/cloud-sql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloud-sql/README.md b/modules/cloud-sql/README.md index 3520331..c3059bb 100644 --- a/modules/cloud-sql/README.md +++ b/modules/cloud-sql/README.md @@ -28,7 +28,7 @@ This module provides the connection details as [Terraform output variables](https://www.terraform.io/intro/getting-started/outputs.html): -1. **Master First IP Address** `master_first_ip_address`: The first IPv4 address of the addresses assigned to the instance. If the instance has only public IP, it is the [public IP address](https://cloud.google.com/sql/docs/mysql/connect-admin-ip). If it has only private IP, it the [private IP address](https://cloud.google.com/sql/docs/mysql/private-ip). If it has both, it is the first item in the list and full IP address details are in `instance_ip_addresses`. +1. **Master First IP Address** `master_first_ip_address`: The first IPv4 address of the addresses assigned to the instance. If the instance has only public IP, it is the [public IP address](https://cloud.google.com/sql/docs/mysql/connect-admin-ip). If it has only private IP, it the [private IP address](https://cloud.google.com/sql/docs/mysql/private-ip). If it has both, it is the first item in the list and full IP address details are in `master_ip_addresses`. 1. **Master Proxy connection** `master_proxy_connection`: Instance path for connecting with Cloud SQL Proxy; see [Connecting mysql Client Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). 1. **Read Replica First IP Addresses** `read_replica_first_ip_addresses`: A list of all read replica IP addresses in the cluster. Use these addresses for reads (see "How do you scale this database?" below). 1. **Read Replica Proxy Connections** `read_replica_proxy_connections`: A list of instance paths for connecting with Cloud SQL Proxy; see [Connecting Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). From 691757195518e390e62eb474087d597f3d79352c Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Fri, 15 Feb 2019 17:42:40 +0200 Subject: [PATCH 10/11] [skip ci] Configurable resource timeout --- modules/cloud-sql/main.tf | 18 +++++++++--------- modules/cloud-sql/variables.tf | 7 +++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/cloud-sql/main.tf b/modules/cloud-sql/main.tf index 5b23bcc..87635a4 100644 --- a/modules/cloud-sql/main.tf +++ b/modules/cloud-sql/main.tf @@ -100,9 +100,9 @@ resource "google_sql_database_instance" "master" { # Sometimes the database creation can, however, take longer, so we # increase the timeouts slightly. timeouts { - create = "30m" - delete = "30m" - update = "30m" + create = "${var.resource_timeout}" + delete = "${var.resource_timeout}" + update = "${var.resource_timeout}" } } @@ -191,9 +191,9 @@ resource "google_sql_database_instance" "failover_replica" { # Sometimes the database creation can, however, take longer, so we # increase the timeouts slightly. timeouts { - create = "30m" - delete = "30m" - update = "30m" + create = "${var.resource_timeout}" + delete = "${var.resource_timeout}" + update = "${var.resource_timeout}" } } @@ -249,9 +249,9 @@ resource "google_sql_database_instance" "read_replica" { # to allow successful creation of multiple read replicas without having to # fear the operation timing out. timeouts { - create = "60m" - delete = "60m" - update = "60m" + create = "${var.resource_timeout}" + delete = "${var.resource_timeout}" + update = "${var.resource_timeout}" } } diff --git a/modules/cloud-sql/variables.tf b/modules/cloud-sql/variables.tf index cec6a62..0ba19b1 100644 --- a/modules/cloud-sql/variables.tf +++ b/modules/cloud-sql/variables.tf @@ -201,6 +201,13 @@ variable "custom_labels" { default = {} } +# Resources are created sequentially. Therefore we increase the default timeouts considerably +# to not have the operations time out. +variable "resource_timeout" { + description = "Timeout for creating, updating and deleting database instances. Valid units of time are s, m, h." + default = "60m" +} + variable "wait_for" { description = "By passing a value to this variable, you can effectively tell this module to wait to deploy until the given variable's value is resolved, which is a way to require that this module depend on some other module. Note that the actual value of this variable doesn't matter." default = "" From 6629371e1cd7a16761a3347b441faa79dad209ce Mon Sep 17 00:00:00 2001 From: Petri Autero Date: Sat, 16 Feb 2019 09:12:07 +0200 Subject: [PATCH 11/11] Fixes based on PR review --- examples/mysql-replicas/main.tf | 4 ++-- examples/postgres-private-ip/main.tf | 2 +- examples/postgres-private-ip/variables.tf | 2 +- examples/postgres-public-ip/main.tf | 4 ++-- examples/postgres-public-ip/outputs.tf | 2 +- modules/cloud-sql/main.tf | 6 +++--- modules/cloud-sql/variables.tf | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/mysql-replicas/main.tf b/examples/mysql-replicas/main.tf index 3426048..7df6faf 100644 --- a/examples/mysql-replicas/main.tf +++ b/examples/mysql-replicas/main.tf @@ -65,8 +65,8 @@ module "mysql" { ] # Indicate that we want to create a failover replica - enable_failover_replica = true - failover_replica_zone = "${var.failover_replica_zone}" + enable_failover_replica = true + mysql_failover_replica_zone = "${var.failover_replica_zone}" # Indicate we want read replicas to be created num_read_replicas = "${var.num_read_replicas}" diff --git a/examples/postgres-private-ip/main.tf b/examples/postgres-private-ip/main.tf index 417a085..c467e1f 100644 --- a/examples/postgres-private-ip/main.tf +++ b/examples/postgres-private-ip/main.tf @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------ -# LAUNCH A MYSQL CLOUD SQL PRIVATE IP INSTANCE +# LAUNCH A POSTGRES CLOUD SQL PRIVATE IP INSTANCE # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ diff --git a/examples/postgres-private-ip/variables.tf b/examples/postgres-private-ip/variables.tf index 1d86733..68250ff 100644 --- a/examples/postgres-private-ip/variables.tf +++ b/examples/postgres-private-ip/variables.tf @@ -29,7 +29,7 @@ variable "master_user_password" { # Generally, these values won't need to be changed. # --------------------------------------------------------------------------------------------------------------------- variable "postgres_version" { - description = "The engine version of the database, e.g. `POSTGRES_9_6`. See https://cloud.google.com/sql/docs/features for supported versions." + description = "The engine version of the database, e.g. `POSTGRES_9_6`. See https://cloud.google.com/sql/docs/db-versions for supported versions." default = "POSTGRES_9_6" } diff --git a/examples/postgres-public-ip/main.tf b/examples/postgres-public-ip/main.tf index de00732..7ea0a08 100644 --- a/examples/postgres-public-ip/main.tf +++ b/examples/postgres-public-ip/main.tf @@ -73,8 +73,8 @@ module "postgres" { }, ] - # Set auto-increment flags to test the - # feature during automated testing + # Set test flags + # Cloud SQL will complain if they're not applicable to the engine database_flags = [ { name = "autovacuum_naptime" diff --git a/examples/postgres-public-ip/outputs.tf b/examples/postgres-public-ip/outputs.tf index 9f37b83..d3c41aa 100644 --- a/examples/postgres-public-ip/outputs.tf +++ b/examples/postgres-public-ip/outputs.tf @@ -18,8 +18,8 @@ output "master_public_ip" { } output "master_ca_cert" { - value = "${module.postgres.master_ca_cert}" description = "The CA Certificate used to connect to the SQL Instance via SSL" + value = "${module.postgres.master_ca_cert}" } output "master_instance" { diff --git a/modules/cloud-sql/main.tf b/modules/cloud-sql/main.tf index 87635a4..8e5e7d7 100644 --- a/modules/cloud-sql/main.tf +++ b/modules/cloud-sql/main.tf @@ -17,7 +17,7 @@ locals { is_mysql = "${replace(var.engine, "MYSQL", "") != var.engine}" # Calculate actuals, so we get expected behavior for each engine - actual_binary_log = "${local.is_postgres ? false : var.mysql_binary_log_enabled}" + actual_binary_log_enabled = "${local.is_postgres ? false : var.mysql_binary_log_enabled}" actual_availability_type = "${local.is_postgres && var.enable_failover_replica ? "REGIONAL" : "ZONAL"}" actual_failover_replica_count = "${local.is_postgres ? 0 : var.enable_failover_replica ? 1 : 0}" @@ -77,7 +77,7 @@ resource "google_sql_database_instance" "master" { } backup_configuration { - binary_log_enabled = "${local.actual_binary_log}" + binary_log_enabled = "${local.actual_binary_log_enabled}" enabled = "${var.backup_enabled}" start_time = "${var.backup_start_time}" } @@ -177,7 +177,7 @@ resource "google_sql_database_instance" "failover_replica" { location_preference { follow_gae_application = "${var.follow_gae_application}" - zone = "${var.failover_replica_zone}" + zone = "${var.mysql_failover_replica_zone}" } disk_size = "${var.disk_size}" diff --git a/modules/cloud-sql/variables.tf b/modules/cloud-sql/variables.tf index 0ba19b1..a99489a 100644 --- a/modules/cloud-sql/variables.tf +++ b/modules/cloud-sql/variables.tf @@ -166,7 +166,7 @@ variable "enable_failover_replica" { default = false } -variable "failover_replica_zone" { +variable "mysql_failover_replica_zone" { description = "The preferred zone for the failover instance (e.g. 'us-central1-b'). Must be different than 'master_zone'. Only applicable to MySQL, Postgres will determine this automatically." default = "" }