diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..cfa1109 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,102 @@ +defaults: &defaults + machine: true + environment: + GRUNTWORK_INSTALLER_VERSION: v0.0.21 + TERRATEST_LOG_PARSER_VERSION: v0.13.24 + MODULE_CI_VERSION: v0.13.3 + TERRAFORM_VERSION: 0.11.8 + TERRAGRUNT_VERSION: NONE + PACKER_VERSION: NONE + GOLANG_VERSION: 1.11.2 + +install_gruntwork_utils: &install_gruntwork_utils + name: install gruntwork utils + command: | + curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}" + gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}" + gruntwork-install --binary-name "terratest_log_parser" --repo "https://github.com/gruntwork-io/terratest" --tag "${TERRATEST_LOG_PARSER_VERSION}" + configure-environment-for-gruntwork-module \ + --circle-ci-2-machine-executor \ + --terraform-version ${TERRAFORM_VERSION} \ + --terragrunt-version ${TERRAGRUNT_VERSION} \ + --packer-version ${PACKER_VERSION} \ + --use-go-dep \ + --go-version ${GOLANG_VERSION} \ + --go-src-path test +version: 2 +jobs: + build: + <<: *defaults + steps: + - checkout + - restore_cache: + keys: + - dep-v1-{{ checksum "test/Gopkg.lock" }} + + # Install gruntwork utilities + - run: + <<: *install_gruntwork_utils + + - save_cache: + key: dep-v1-{{ checksum "test/Gopkg.lock" }} + paths: + - ./test/vendor + + # Fail the build if the pre-commit hooks don't pass. Note: if you run pre-commit install locally, these hooks will + # execute automatically every time before you commit, ensuring the build never fails at this step! + - run: pip install pre-commit==1.11.2 + - run: pre-commit install + - run: pre-commit run --all-files + + - persist_to_workspace: + root: /home/circleci + paths: + - project + - terraform + - packer + + test: + <<: *defaults + steps: + - attach_workspace: + at: /home/circleci + - checkout + - run: echo 'export PATH=$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV + - run: + <<: *install_gruntwork_utils + - run: + name: update gcloud + command: | + sudo apt-get remove -y google-cloud-sdk + sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update + sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update beta + - run: + name: run tests + command: | + mkdir -p /tmp/logs + # required for gcloud to authenticate correctly + echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=- + gcloud --quiet config set project ${GOOGLE_PROJECT_ID} + gcloud --quiet config set compute/zone ${GOOGLE_COMPUTE_ZONE} + # required for terraform and terratest to authenticate correctly + echo $GCLOUD_SERVICE_KEY > /tmp/gcloud.json + export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcloud.json" + # run the tests + run-go-tests --path test --timeout 60m | tee /tmp/logs/all.log + no_output_timeout: 3600s + - run: + command: terratest_log_parser --testlog /tmp/logs/all.log --outputdir /tmp/logs + when: always + - store_artifacts: + path: /tmp/logs + - store_test_results: + path: /tmp/logs + +workflows: + version: 2 + build-and-test: + jobs: + - build + - test: + requires: + - build \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d7a06c --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Terraform files +.terraform +terraform.tfstate +terraform.tfvars +*.tfstate* +*.zip + +# OS X files +.history +.DS_Store + +# IntelliJ files +.idea_modules +*.iml +*.iws +*.ipr +.idea/ +build/ +*/build/ +out/ + +# Go best practices dictate that libraries should not include the vendor directory +vendor + +#VIM swap files +*.swp + +.test-data \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1752b28 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/gruntwork-io/pre-commit + rev: v0.0.4 + hooks: + - id: terraform-fmt +# \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..20b9986 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,99 @@ +# Contribution Guidelines + +Contributions to this Module are very welcome! We follow a fairly standard [pull request process]( +https://help.github.com/articles/about-pull-requests/) for contributions, subject to the following guidelines: + +1. [File a GitHub issue](#file-a-github-issue) +1. [Update the documentation](#update-the-documentation) +1. [Update the tests](#update-the-tests) +1. [Update the code](#update-the-code) +1. [Create a pull request](#create-a-pull-request) +1. [Merge and release](#merge-and-release) + +## File a GitHub issue + +Before starting any work, we recommend filing a GitHub issue in this repo. This is your chance to ask questions and +get feedback from the maintainers and the community before you sink a lot of time into writing (possibly the wrong) +code. If there is anything you're unsure about, just ask! + +## Update the documentation + +We recommend updating the documentation *before* updating any code (see [Readme Driven +Development](http://tom.preston-werner.com/2010/08/23/readme-driven-development.html)). This ensures the documentation +stays up to date and allows you to think through the problem at a high level before you get lost in the weeds of +coding. + +## Update the tests + +We also recommend updating the automated tests *before* updating any code (see [Test Driven +Development](https://en.wikipedia.org/wiki/Test-driven_development)). That means you add or update a test case, +verify that it's failing with a clear error message, and *then* make the code changes to get that test to pass. This +ensures the tests stay up to date and verify all the functionality in this Module, including whatever new +functionality you're adding in your contribution. Check out the [tests](https://github.com/gruntwork-io/terraform-google-sql/tree/master/test) folder for instructions on running the +automated tests. + +## Update the code + +At this point, make your code changes and use your new test case to verify that everything is working. As you work, +keep in mind two things: + +1. Backwards compatibility +1. Downtime + +### Backwards compatibility + +Please make every effort to avoid unnecessary backwards incompatible changes. With Terraform code, this means: + +1. Do not delete, rename, or change the type of input variables. +1. If you add an input variable, it should have a `default`. +1. Do not delete, rename, or change the type of output variables. +1. Do not delete or rename a module in the `modules` folder. + +If a backwards incompatible change cannot be avoided, please make sure to call that out when you submit a pull request, +explaining why the change is absolutely necessary. + +### Downtime + +Bear in mind that the Terraform code in this Module is used by real companies to run real infrastructure in +production, and certain types of changes could cause downtime. For example, consider the following: + +1. If you rename a resource (e.g. `google_sql_database_instance "foo"` -> `google_sql_database_instance "bar"`), Terraform will see that as deleting + the old resource and creating a new one. +1. If you change certain attributes of a resource (e.g. the `name` of an `google_compute_instance`), the cloud provider (e.g. Google) may + treat that as an instruction to delete the old resource and a create a new one. + +Deleting certain types of resources (e.g. virtual servers, load balancers) can cause downtime, so when making code +changes, think carefully about how to avoid that. For example, can you avoid downtime by using +[create_before_destroy](https://www.terraform.io/docs/configuration/resources.html#create_before_destroy)? Or via +the `terraform state` command? If so, make sure to note this in our pull request. If downtime cannot be avoided, +please make sure to call that out when you submit a pull request. + + +### Formatting and pre-commit hooks + +You must run `terraform fmt` on the code before committing. You can configure your computer to do this automatically +using pre-commit hooks managed using [pre-commit](http://pre-commit.com/): + +1. [Install pre-commit](http://pre-commit.com/#install). E.g.: `brew install pre-commit`. +1. Install the hooks: `pre-commit install`. + +That's it! Now just write your code, and every time you commit, `terraform fmt` will be run on the files you're +committing. + + +## Create a pull request + +[Create a pull request](https://help.github.com/articles/creating-a-pull-request/) with your changes. Please make sure +to include the following: + +1. A description of the change, including a link to your GitHub issue. +1. The output of your automated test run, preferably in a [GitHub Gist](https://gist.github.com/). We cannot run + automated tests for pull requests automatically due to [security + concerns](https://circleci.com/docs/fork-pr-builds/#security-implications), so we need you to manually provide this + test output so we can verify that everything is working. +1. Any notes on backwards incompatibility or downtime. + +## Merge and release + +The maintainers for this repo will review your code and provide feedback. If everything looks good, they will merge the +code and release a new version, which you'll be able to find in the [releases page](../../releases). \ No newline at end of file diff --git a/GRUNTWORK_PHILOSOPHY.md b/GRUNTWORK_PHILOSOPHY.md new file mode 100644 index 0000000..22a28a8 --- /dev/null +++ b/GRUNTWORK_PHILOSOPHY.md @@ -0,0 +1,64 @@ +# Gruntwork Philosophy + +At Gruntwork, we strive to accelerate the deployment of production grade infrastructure by prodiving a library of +stable, reusable, and battle tested infrastructure as code organized into a series of [modules](#what-is-a-module) with +[submodules](#what-is-a-submodule). Each module represents a particular set of infrastructure that is componentized into +smaller pieces represented by the submodules within the module. By doing so, we have built a composable library that can +be combined into building out everything from simple single service deployments to complicated microservice setups so +that your infrastructure can grow with your business needs. Every module we provide is built with the [production grade +infrastruture checklist](#production-grade-infrastructure-checklist) in mind, ensuring that the services you deploy are +resilient, fault tolerant, and scalable. + + +## What is a Module? + +A Module is a reusable, tested, documented, configurable, best-practices definition of a single piece of Infrastructure +(e.g., Docker cluster, VPC, Jenkins, Consul), written using a combination of [Terraform](https://www.terraform.io/), Go, +and Bash. A module contains a set of automated tests, documentation, and examples that have been proven in production, +providing the underlying infrastructure for [Gruntwork's customers](https://www.gruntwork.io/customers). + +Instead of figuring out the details of how to run a piece of infrastructure from scratch, you can reuse existing code +that has been proven in production. And instead of maintaining all that infrastructure code yourself, you can leverage +the work of the community to pick up infrastructure improvements through a version number bump. + + +## What is a Submodule? + +Each Infrastructure Module consists of one or more orthogonal Submodules that handle some specific aspect of that +Infrastructure Module's functionality. Breaking the code up into multiple submodules makes it easier to reuse and +compose to handle many different use cases. Although Modules are designed to provide an end to end solution to manage +the relevant infrastructure by combining the Submodules defined in the Module, Submodules can be used independently for +specific functionality that you need in your infrastructure code. + + +## Production Grade Infrastructure Checklist + +At Gruntwork, we have learned over the years that it is not enough to just get the services up and running in a publicly +accessible space to call your application "production-ready." There are many more things to consider, and oftentimes +many of these considerations are missing in the deployment plan of applications. These topics come up as afterthoughts, +and are learned the hard way after the fact. That is why we codified all of them into a checklist that can be used as a +reference to help ensure that they are considered before your application goes to production, and conscious decisions +are made to neglect particular components if needed, as opposed to accidentally omitting them from consideration. + + + +| Task | Description | Example tools | +|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------| +| Install | Install the software binaries and all dependencies. | Bash, Chef, Ansible, Puppet | +| Configure | Configure the software at runtime. Includes port settings, TLS certs, service discovery, leaders, followers, replication, etc. | Bash, Chef, Ansible, Puppet | +| Provision | Provision the infrastructure. Includes EC2 instances, load balancers, network topology, security gr oups, IAM permissions, etc. | Terraform, CloudFormation | +| Deploy | Deploy the service on top of the infrastructure. Roll out updates with no downtime. Includes blue-green, rolling, and canary deployments. | Scripts, Orchestration tools (ECS, k8s, Nomad) | +| High availability | Withstand outages of individual processes, EC2 instances, services, Availability Zones, and regions. | Multi AZ, multi-region, replication, ASGs, ELBs | +| Scalability | Scale up and down in response to load. Scale horizontally (more servers) and/or vertically (bigger servers). | ASGs, replication, sharding, caching, divide and conquer | +| Performance | Optimize CPU, memory, disk, network, GPU, and usage. Includes query tuning, benchmarking, load testing, and profiling. | Dynatrace, valgrind, VisualVM, ab, Jmeter | +| Networking | Configure static and dynamic IPs, ports, service discovery, firewalls, DNS, SSH access, and VPN access. | EIPs, ENIs, VPCs, NACLs, SGs, Route 53, OpenVPN | +| Security | Encryption in transit (TLS) and on disk, authentication, authorization, secrets management, server hardening. | ACM, EBS Volumes, Cognito, Vault, CIS | +| Metrics | Availability metrics, business metrics, app metrics, server metrics, events, observability, tracing, and alerting. | CloudWatch, DataDog, New Relic, Honeycomb | +| Logs | Rotate logs on disk. Aggregate log data to a central location. | CloudWatch logs, ELK, Sumo Logic, Papertrail | +| Backup and Restore | Make backups of DBs, caches, and other data on a scheduled basis. Replicate to separate region/account. | RDS, ElastiCache, ec2-snapper, Lambda | +| Cost optimization | Pick proper instance types, use spot and reserved instances, use auto scaling, and nuke unused resources. | ASGs, spot instances, reserved instances | +| Documentation | Document your code, architecture, and practices. Create playbooks to respond to incidents. | READMEs, wikis, Slack | +| Tests | Write automated tests for your infrastructure code. Run tests after every commit and nightly. | Terratest | \ No newline at end of file diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..24a7708 --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +terraform-google-sql +Copyright 2019 Gruntwork, Inc. + +This product includes software developed at Gruntwork (https://www.gruntwork.io/). diff --git a/README.md b/README.md new file mode 100644 index 0000000..99ac946 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +[![Maintained by Gruntwork.io](https://img.shields.io/badge/maintained%20by-gruntwork.io-%235849a6.svg)](https://gruntwork.io/?ref=repo_google_cloudsql) + +# Cloud SQL Modules + +This repo contains modules for running relational databases such as MySQL and PostgreSQL on Google's +[Cloud SQL](https://cloud.google.com/sql/) on [GCP](https://cloud.google.com/). + +## Code included in this Module + +* [mysql](/modules/mysql): Deploy a Cloud SQL MySQL cluster. +* [postgresql](/modules/postgresql): Deploy a Cloud SQL PostgreSQL cluster. + + +## What is Cloud SQL? + +Cloud SQL is Google's fully-managed database service that makes it easy to set up, maintain, manage, and administer +your relational databases on Google Cloud Platform. Cloud SQL automatically includes: + +* Data replication between multiple zones with automatic failover. +* Automated and on-demand backups, and point-in-time recovery. +* Data encryption on networks, database tables, temporary files, and backups. +* Secure external connections with the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/sql-proxy) or with the SSL/TLS protocol. + +You can learn more about Cloud SQL from [the official documentation](https://cloud.google.com/sql/docs/). + +## Who maintains this Module? + +This Module and its Submodules are maintained by [Gruntwork](http://www.gruntwork.io/). Read the [Gruntwork Philosophy](/GRUNTWORK_PHILOSOPHY.md) document to learn more about how Gruntwork builds production grade infrastructure code. If you are looking for help or +commercial support, send an email to +[support@gruntwork.io](mailto:support@gruntwork.io?Subject=Google%20SQL%20Module). + +Gruntwork can help with: + +* Setup, customization, and support for this Module. +* Modules and submodules for other types of infrastructure, such as VPCs, Docker clusters, databases, and continuous + integration. +* Modules and Submodules that meet compliance requirements, such as HIPAA. +* Consulting & Training on GCP, AWS, Terraform, and DevOps. + + +## How do I contribute to this Module? + +Contributions are very welcome! Check out the [Contribution Guidelines](/CONTRIBUTING.md) for instructions. + + +## How is this Module versioned? + +This Module follows the principles of [Semantic Versioning](http://semver.org/). You can find each new release, along +with the changelog, in the [Releases Page](../../releases). + +During initial development, the major version will be 0 (e.g., `0.x.y`), which indicates the code does not yet have a +stable API. Once we hit `1.0.0`, we will make every effort to maintain a backwards compatible API and use the MAJOR, +MINOR, and PATCH versions on each release to indicate any incompatibilities. + + +## License + +Please see [LICENSE.txt](/LICENSE.txt) for details on how the code in this repo is licensed. diff --git a/examples/cloud-sql-mysql/main.tf b/examples/cloud-sql-mysql/main.tf new file mode 100644 index 0000000..60b844a --- /dev/null +++ b/examples/cloud-sql-mysql/main.tf @@ -0,0 +1,72 @@ +provider "google-beta" { + region = "${var.region}" + project = "${var.project}" +} + +# Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via +# https://github.com/terraform-providers/terraform-provider-google +terraform { + required_version = ">= 0.10.3" +} + +resource "random_id" "name" { + byte_length = 2 +} + +locals { + # If name_override is specified, use that - otherwise use the name_prefix with a random string + instance_name = "${length(var.name_override) == 0 ? format("%s-%s", var.name_prefix, random_id.name.hex) : var.name_override}" +} + +module "mysql" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/terraform-google-sql.git//modules/mysql?ref=v0.1.0" + source = "../../modules/mysql" + + project = "${var.project}" + region = "${var.region}" + name = "${local.instance_name}" + db_name = "${var.db_name}" + + engine = "${var.mysql_version}" + machine_type = "${var.machine_type}" + + # These together will construct the master_user privileges, i.e. + # 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. + # These should typically be set as the environment variable TF_VAR_master_user_password, etc. + # so you don't check these into source control." + master_user_password = "${var.master_user_password}" + + master_user_name = "${var.master_user_name}" + master_user_host = "%" + + # To make it easier to test this example, we are giving the servers public IP addresses and allowing inbound + # connections from anywhere. In real-world usage, your servers should live in private subnets, only have private IP + # addresses, and only allow access from specific trusted networks, servers or applications in your VPC. + enable_public_internet_access = true + + authorized_networks = [ + { + name = "allow-all-inbound" + value = "0.0.0.0/0" + }, + ] + + # Set auto-increment flags to test the + # feature during automated testing + database_flags = [ + { + name = "auto_increment_increment" + value = "5" + }, + { + name = "auto_increment_offset" + value = "5" + }, + ] + + custom_labels = { + project = "mysql-example" + } +} diff --git a/examples/cloud-sql-mysql/outputs.tf b/examples/cloud-sql-mysql/outputs.tf new file mode 100644 index 0000000..c7e71d4 --- /dev/null +++ b/examples/cloud-sql-mysql/outputs.tf @@ -0,0 +1,28 @@ +output "instance_name" { + description = "The name of the database instance" + value = "${module.mysql.instance_name}" +} + +output "public_ip" { + description = "The IPv4 address of the master database instance" + value = "${module.mysql.public_ip}" +} + +output "instance" { + description = "Self link to the master instance" + value = "${module.mysql.instance}" +} + +output "db_name" { + description = "Name of the default database" + value = "${module.mysql.db_name}" +} + +output "proxy_connection" { + value = "${module.mysql.proxy_connection}" +} + +output "db" { + description = "Self link to the default database" + value = "${module.mysql.db}" +} diff --git a/examples/cloud-sql-mysql/variables.tf b/examples/cloud-sql-mysql/variables.tf new file mode 100644 index 0000000..0eecc43 --- /dev/null +++ b/examples/cloud-sql-mysql/variables.tf @@ -0,0 +1,49 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These variables are expected to be passed in by the operator +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The project ID to host the database in." +} + +variable "region" { + description = "The region to host the database in." +} + +# Note, after a name db instance is used, it cannot be reused for up to one week. +variable "name_prefix" { + description = "The name prefix for the database instance. Will be appended with a random string. Use lowercase letters, numbers, and hyphens. Start with a letter." +} + +variable "master_user_name" { + description = "The username part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_name so you don't check it into source control." +} + +variable "master_user_password" { + description = "The password part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_password so you don't check it into source control." +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# Generally, these values won't need to be changed. +# --------------------------------------------------------------------------------------------------------------------- +variable "mysql_version" { + description = "The engine version of the database, e.g. `MYSQL_5_6` or `MYSQL_5_7`. See https://cloud.google.com/sql/docs/features for supported versions." + default = "MYSQL_5_7" +} + +variable "machine_type" { + description = "The machine type to use, see https://cloud.google.com/sql/pricing for more details" + default = "db-f1-micro" +} + +variable "db_name" { + description = "Name for the db" + default = "default" +} + +variable "name_override" { + description = "You may optionally override the name_prefix + random string by specifying an override" + default = "" +} diff --git a/examples/cloud-sql-postgres/.gitkeep b/examples/cloud-sql-postgres/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/modules/cloud-sql/.gitkeep b/modules/cloud-sql/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/modules/mysql/.gitkeep b/modules/mysql/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/modules/mysql/README.md b/modules/mysql/README.md new file mode 100644 index 0000000..6b3d2cd --- /dev/null +++ b/modules/mysql/README.md @@ -0,0 +1,59 @@ +# MySQL Module + +This module creates a [Google Cloud SQL](https://cloud.google.com/sql/) [MySQL](https://cloud.google.com/sql/docs/mysql/) cluster. +The cluster is managed by Google, automating backups, replication, patches, and updates. + +This module helps you run [MySQL](https://cloud.google.com/sql/docs/mysql/), see [postgres](../postgresql) for running [PostgreSQL](https://cloud.google.com/sql/docs/postgres/). + +## How do you use this module? + +See the [examples](/examples) folder for an example. + +## How do you configure this module? + +This module allows you to configure a number of parameters, such as backup windows, maintenance window, replicas +and encryption. For a list of all available variables and their descriptions, see [variables.tf](./variables.tf). + +## How do you connect to the database? + +**Cloud SQL instances are created in a producer network (a VPC network internal to Google). They are not created in your VPC network. See https://cloud.google.com/sql/docs/mysql/private-ip** + +You can use both [public IP](https://cloud.google.com/sql/docs/mysql/connect-admin-ip) and [private IP](https://cloud.google.com/sql/docs/mysql/private-ip) to connect to a Cloud SQL instance. +Neither connection method affects the other; you must protect the public IP connection whether the instance is configured to use private IP or not. + +You can also use the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy) to connect to an instance that is also configured to use private IP. The proxy can connect using either the private IP address or a public IP address. + +This module provides the connection details as [Terraform output +variables](https://www.terraform.io/intro/getting-started/outputs.html): + + +1. TODO: **Private IP** `private_ip`: The public endpoint for the cluster. +1. **Public IP** `public_ip`: The public endpoint for the cluster; see [Connecting mysql Client Using Public IP](https://cloud.google.com/sql/docs/mysql/connect-admin-ip). +1. **Proxy connection** `proxy_connection`: Instance path for connecting with Cloud SQL Proxy; see [Connecting mysql Client Using the Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy). +1. TODO: **Replica endpoints** `replica_endpoints`: A comma-separated list of all DB instance URLs in the cluster, including the primary and all + read replicas. Use these URLs for reads (see "How do you scale this DB?" below). + + + +You can programmatically extract these variables in your Terraform templates and pass them to other resources. +You'll also see the variables at the end of each `terraform apply` call or if you run `terraform output`. + +For full connectivity options and detailed documentation, see [Connecting to Cloud SQL from External Applications](https://cloud.google.com/sql/docs/mysql/connect-external-app). + +## How do you scale this database? + +* **Storage**: Cloud SQL manages storage for you, automatically growing cluster volume up to 10TB. +* **Vertical scaling**: To scale vertically (i.e. bigger DB instances with more CPU and RAM), use the `machine_type` + input variable. For a list of Cloud SQL Machine Types, see [Cloud SQL Pricing](https://cloud.google.com/sql/pricing#2nd-gen-pricing). +* **Horizontal scaling**: To scale horizontally, you can add more replicas using the `instance_count` input variable, + and the module will automatically deploy the new instances, sync them to the master, and make them available as read + replicas. + +## Known Issues + +### Instance Recovery + +Due to limitations on the current `terraform` provider for Google, it is not possible to restore backups with `terraform`. +See https://github.com/terraform-providers/terraform-provider-google/issues/2446 + + diff --git a/modules/mysql/main.tf b/modules/mysql/main.tf new file mode 100644 index 0000000..828901e --- /dev/null +++ b/modules/mysql/main.tf @@ -0,0 +1,64 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# DEPLOY A CLOUD SQL CLUSTER +# This module deploys a Cloud SQL MySQL cluster. The cluster is managed by Google and automatically handles leader +# election, replication, failover, backups, patching, and encryption. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# ------------------------------------------------------------------------------ +# CREATE THE CLOUD SQL MYSQL CLUSTER +# +# NOTE: We have multiple google_sql_database_instance resources, based on +# HA, encryption and replication configuration options. +# ------------------------------------------------------------------------------ + +resource "google_sql_database_instance" "master" { + provider = "google-beta" + name = "${var.name}" + project = "${var.project}" + region = "${var.region}" + database_version = "${var.engine}" + + settings { + tier = "${var.machine_type}" + activation_policy = "${var.activation_policy}" + authorized_gae_applications = ["${var.authorized_gae_applications}"] + disk_autoresize = "${var.disk_autoresize}" + + ip_configuration { + authorized_networks = ["${var.authorized_networks}"] + ipv4_enabled = "${var.enable_public_internet_access}" + } + + location_preference { + follow_gae_application = "${var.follow_gae_application}" + zone = "${var.zone}" + } + + disk_size = "${var.disk_size}" + disk_type = "${var.disk_type}" + database_flags = ["${var.database_flags}"] + availability_type = "${var.availability_type}" + + user_labels = "${var.custom_labels}" + } +} + +# ------------------------------------------------------------------------------ +# CREATE A DATABASE +# ------------------------------------------------------------------------------ + +resource "google_sql_database" "default" { + name = "${var.db_name}" + project = "${var.project}" + instance = "${google_sql_database_instance.master.name}" + charset = "${var.db_charset}" + collation = "${var.db_collation}" +} + +resource "google_sql_user" "default" { + name = "${var.master_user_name}" + project = "${var.project}" + instance = "${google_sql_database_instance.master.name}" + host = "${var.master_user_host}" + password = "${var.master_user_password}" +} diff --git a/modules/mysql/outputs.tf b/modules/mysql/outputs.tf new file mode 100644 index 0000000..ffd22c1 --- /dev/null +++ b/modules/mysql/outputs.tf @@ -0,0 +1,29 @@ +output "instance_name" { + description = "The name of the database instance" + value = "${google_sql_database_instance.master.name}" +} + +output "public_ip" { + description = "The IPv4 address of the master database instance" + value = "${var.enable_public_internet_access ? google_sql_database_instance.master.ip_address.0.ip_address : ""}" +} + +output "instance" { + description = "Self link to the master instance" + value = "${google_sql_database_instance.master.self_link}" +} + +output "db_name" { + description = "Name of the default database" + value = "${google_sql_database.default.name}" +} + +output "proxy_connection" { + description = "Instance path for connecting with Cloud SQL Proxy. Read more at https://cloud.google.com/sql/docs/mysql/sql-proxy" + value = "${var.project}:${var.region}:${google_sql_database_instance.master.name}" +} + +output "db" { + description = "Self link to the default database" + value = "${google_sql_database.default.self_link}" +} diff --git a/modules/mysql/variables.tf b/modules/mysql/variables.tf new file mode 100644 index 0000000..eab8a9a --- /dev/null +++ b/modules/mysql/variables.tf @@ -0,0 +1,149 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These variables are expected to be passed in by the operator +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The project ID to host the database in." +} + +variable "region" { + description = "The region to host the database in." +} + +variable "name" { + description = "The name of the database instance. Note, after a name is used, it cannot be reused for up to one week. Use lowercase letters, numbers, and hyphens. Start with a letter." +} + +variable "engine" { + description = "The engine version of the database, e.g. `MYSQL_5_6` or `MYSQL_5_7`. See https://cloud.google.com/sql/docs/features for supported versions." +} + +# TODO: Depending on how the replicas are set up, tweak this. +#variable "master_instance_name" { +# description = "The name of the instance that will act as the master in the replication setup. Note, this requires the master to have binary_log_enabled set, as well as existing backups." +# default = "" +#} + +variable "machine_type" { + description = "The machine type for the instance. See this page for supported tiers and pricing: https://cloud.google.com/sql/pricing" +} + +variable "db_name" { + description = "Name of your database. Needs to follow MySQL identifier rules: https://dev.mysql.com/doc/refman/5.7/en/identifiers.html" +} + +variable "master_user_name" { + description = "The username part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_name so you don't check it into source control." +} + +variable "master_user_password" { + description = "The password part for the default user credentials, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password'. This should typically be set as the environment variable TF_VAR_master_user_password so you don't check it into source control." +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# Generally, these values won't need to be changed. +# --------------------------------------------------------------------------------------------------------------------- + +variable "activation_policy" { + description = "This specifies when the instance should be active. Can be either `ALWAYS`, `NEVER` or `ON_DEMAND`." + default = "ALWAYS" +} + +variable "authorized_networks" { + description = "A list of authorized CIDR-formatted IP address ranges that can connect to this DB." + type = "list" + default = [] + + # Example: + # + # authorized_networks = [ + # { + # name = "all-inbound" # optional + # value = "0.0.0.0/0" + # } + # ] +} + +variable "authorized_gae_applications" { + description = "A list of Google App Engine (GAE) project names that are allowed to access this instance." + type = "list" + default = [] +} + +variable "availability_type" { + description = "This specifies whether a PostgreSQL instance should be set up for high availability (REGIONAL) or single zone (ZONAL)." + default = "ZONAL" +} + +variable "db_charset" { + description = "The charset for the default database." + default = "" +} + +variable "db_collation" { + description = "The collation for the default database. Example for MySQL databases: 'utf8_general_ci'." + default = "" +} + +variable "database_flags" { + description = "List of Cloud SQL flags that are applied to the database server" + type = "list" + default = [] + + # Example: + # + # database_flags = [ + # { + # name = "auto_increment_increment" + # value = "10" + # }, + # { + # name = "auto_increment_offset" + # value = "5" + # }, + #] +} + +variable "disk_autoresize" { + description = "Second Generation only. Configuration to increase storage size automatically." + default = true +} + +variable "disk_size" { + description = "Second generation only. The size of data disk, in GB. Size of a running instance cannot be reduced but can be increased." + default = 10 +} + +variable "disk_type" { + description = "The type of storage to use. Must be one of `PD_SSD` or `PD_HDD`." + default = "PD_SSD" +} + +variable "follow_gae_application" { + description = "A GAE application whose zone to remain in. Must be in the same region as this instance." + default = "" +} + +variable "zone" { + description = "Preferred zone for the instance." + default = "" +} + +variable "master_user_host" { + description = "The host part for the default user, i.e. 'master_user_name'@'master_user_host' IDENTIFIED BY 'master_user_password' " + default = "%" +} + +# In nearly all cases, databases should NOT be publicly accessible, however if you're migrating from a PAAS provider like Heroku to GCP, this needs to remain open to the internet. +variable "enable_public_internet_access" { + description = "WARNING: - In nearly all cases a database should NOT be publicly accessible. Only set this to true if you want the database open to the internet." + default = false +} + +variable "custom_labels" { + description = "A map of custom labels to apply to the instance. The key is the label name and the value is the label value." + type = "map" + default = {} +} diff --git a/modules/postgresql/.gitkeep b/modules/postgresql/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/Gopkg.lock b/test/Gopkg.lock new file mode 100644 index 0000000..f1061a0 --- /dev/null +++ b/test/Gopkg.lock @@ -0,0 +1,408 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:8b95956b70e181b19025c7ba3578fdfd8efbec4ce916490700488afb9218972c" + name = "cloud.google.com/go" + packages = [ + "compute/metadata", + "iam", + "internal", + "internal/optional", + "internal/trace", + "internal/version", + "storage", + ] + pruneopts = "" + revision = "64a2037ec6be8a4b0c1d1f706ed35b428b989239" + version = "v0.26.0" + +[[projects]] + digest = "1:4b6ba994aa18b6c6a871cd700a66525a182f2c0cd78022e25e01508fc8559122" + name = "github.com/aws/aws-sdk-go" + packages = [ + "aws", + "aws/awserr", + "aws/awsutil", + "aws/client", + "aws/client/metadata", + "aws/corehandlers", + "aws/credentials", + "aws/credentials/ec2rolecreds", + "aws/credentials/endpointcreds", + "aws/credentials/processcreds", + "aws/credentials/stscreds", + "aws/csm", + "aws/defaults", + "aws/ec2metadata", + "aws/endpoints", + "aws/request", + "aws/session", + "aws/signer/v4", + "internal/ini", + "internal/s3err", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", + "internal/shareddefaults", + "private/protocol", + "private/protocol/ec2query", + "private/protocol/eventstream", + "private/protocol/eventstream/eventstreamapi", + "private/protocol/json/jsonutil", + "private/protocol/jsonrpc", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/restxml", + "private/protocol/xml/xmlutil", + "service/acm", + "service/autoscaling", + "service/cloudwatchlogs", + "service/ec2", + "service/iam", + "service/kms", + "service/rds", + "service/s3", + "service/s3/s3iface", + "service/s3/s3manager", + "service/sns", + "service/sqs", + "service/sts", + ] + pruneopts = "" + revision = "949cbce4e4443b72f6da12adbeb5d416dd506fbe" + version = "v1.16.27" + +[[projects]] + digest = "1:b529f4bf748979caa18b599d40d13e8b6e591a74b340f315ce4f95e119c288c2" + name = "github.com/boombuler/barcode" + packages = [ + ".", + "qr", + "utils", + ] + pruneopts = "" + revision = "3cfea5ab600ae37946be2b763b8ec2c1cf2d272d" + version = "v1.0.0" + +[[projects]] + digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + digest = "1:e692d16fdfbddb94e9e4886aaf6c08bdbae5cb4ac80651445de9181b371c6e46" + name = "github.com/go-sql-driver/mysql" + packages = ["."] + pruneopts = "" + revision = "72cd26f257d44c1114970e19afddcd812016007e" + source = "git@github.com:go-sql-driver/mysql" + version = "v1.4.1" + +[[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + name = "github.com/golang/protobuf" + packages = [ + "proto", + "protoc-gen-go/descriptor", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp", + ] + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986" + name = "github.com/google/uuid" + packages = ["."] + pruneopts = "" + revision = "064e2069ce9c359c118179501254f67d7d37ba24" + version = "0.2" + +[[projects]] + digest = "1:55c1b46a80db2baf4d762c1d0b5cb4946e46125baa02b8959310abab15b54aee" + name = "github.com/googleapis/gax-go" + packages = [ + ".", + "v2", + ] + pruneopts = "" + revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" + version = "v2.0.3" + +[[projects]] + digest = "1:4b6c36dd91add683751cb4670cc26d63c7e7db259695ffd723670c666797c62c" + name = "github.com/gruntwork-io/terratest" + packages = [ + "modules/aws", + "modules/collections", + "modules/customerrors", + "modules/environment", + "modules/files", + "modules/gcp", + "modules/logger", + "modules/packer", + "modules/random", + "modules/retry", + "modules/shell", + "modules/ssh", + "modules/terraform", + "modules/test-structure", + ] + pruneopts = "" + revision = "425f7f1c2c72c607142c67aee02d1c14f874cf0a" + source = "git@github.com:gruntwork-io/terratest" + version = "v0.13.24" + +[[projects]] + digest = "1:13fe471d0ed891e8544eddfeeb0471fd3c9f2015609a1c000aefdedf52a19d40" + name = "github.com/jmespath/go-jmespath" + packages = ["."] + pruneopts = "" + revision = "c2b33e84" + +[[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:d8c398c75a666d415196ba289402c3f52b595d8cede451a9e57278354e327a93" + name = "github.com/pquerna/otp" + packages = [ + ".", + "hotp", + "totp", + ] + pruneopts = "" + revision = "be78767b3e392ce45ea73444451022a6fc32ad0d" + version = "v1.1.0" + +[[projects]] + digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c" + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require", + ] + pruneopts = "" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" + +[[projects]] + digest = "1:b1bb9332f6cb7821a730e1681819b2813340eba5e873f05381815a7c6807d172" + name = "go.opencensus.io" + packages = [ + ".", + "exemplar", + "internal", + "internal/tagencoding", + "plugin/ochttp", + "plugin/ochttp/propagation/b3", + "stats", + "stats/internal", + "stats/view", + "tag", + "trace", + "trace/internal", + "trace/propagation", + "trace/tracestate", + ] + pruneopts = "" + revision = "2b5032d79456124f42db6b7eb19ac6c155449dc2" + version = "v0.19.0" + +[[projects]] + branch = "master" + digest = "1:cbc1ebc01ec2ceb2c3cc7b9e33357dbc3eff173335f02d9f01603f14102602a7" + name = "golang.org/x/crypto" + packages = [ + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "internal/chacha20", + "internal/subtle", + "poly1305", + "ssh", + "ssh/agent", + ] + pruneopts = "" + revision = "b8fe1690c61389d7d2a8074a507d1d40c5d30448" + +[[projects]] + branch = "master" + digest = "1:1e9704e5379e68ac473c28aeb3f7e7cd4036ae8a246bf0285b5ebdbb8e0cfacf" + name = "golang.org/x/net" + packages = [ + "context", + "context/ctxhttp", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace", + ] + pruneopts = "" + revision = "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf" + +[[projects]] + digest = "1:b697592485cb412be4188c08ca0beed9aab87f36b86418e21acc4a3998f63734" + name = "golang.org/x/oauth2" + packages = [ + ".", + "google", + "internal", + "jws", + "jwt", + ] + pruneopts = "" + revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" + +[[projects]] + branch = "master" + digest = "1:489610147902fe0c7229218c749bb25a8a9ecce0d726ae4f8662517319f32554" + name = "golang.org/x/sys" + packages = ["unix"] + pruneopts = "" + revision = "41f3e6584952bb034a481797859f6ab34b6803bd" + +[[projects]] + digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable", + ] + pruneopts = "" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + digest = "1:e53b3650fadc76377e90fa596a939233cff79f8606cae71e0b6c73e5e6829ecd" + name = "google.golang.org/api" + packages = [ + "compute/v1", + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "googleapi/transport", + "internal", + "iterator", + "option", + "storage/v1", + "transport/http", + "transport/http/internal/propagation", + ] + pruneopts = "" + revision = "697424a9b4245c525a11d14a2460c76f35f8db55" + +[[projects]] + digest = "1:bc09e719c4e2a15d17163f5272d9a3131c45d77542b7fdc53ff518815bc19ab3" + name = "google.golang.org/appengine" + packages = [ + ".", + "cloudsql", + "internal", + "internal/app_identity", + "internal/base", + "internal/datastore", + "internal/log", + "internal/modules", + "internal/remote_api", + "internal/urlfetch", + "urlfetch", + ] + pruneopts = "" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" + +[[projects]] + branch = "master" + digest = "1:91f75dc679abcf04b29b064a5b9c40a0b0561c86c322010fbf6fc08040cc48bc" + name = "google.golang.org/genproto" + packages = [ + "googleapis/api/annotations", + "googleapis/iam/v1", + "googleapis/rpc/code", + "googleapis/rpc/status", + ] + pruneopts = "" + revision = "4b09977fb92221987e99d190c8f88f2c92727a29" + +[[projects]] + digest = "1:39d4d828b87d58d114fdc211f0638f32dcae84019fe17d6b48f9b697f4b60213" + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", + "codes", + "connectivity", + "credentials", + "credentials/internal", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/binarylog", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + ] + pruneopts = "" + revision = "a02b0774206b209466313a0b525d2c738fe407eb" + version = "v1.18.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/go-sql-driver/mysql", + "github.com/gruntwork-io/terratest/modules/gcp", + "github.com/gruntwork-io/terratest/modules/random", + "github.com/gruntwork-io/terratest/modules/terraform", + "github.com/gruntwork-io/terratest/modules/test-structure", + "github.com/stretchr/testify/assert", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/test/Gopkg.toml b/test/Gopkg.toml new file mode 100644 index 0000000..75c9934 --- /dev/null +++ b/test/Gopkg.toml @@ -0,0 +1,30 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + +[[constraint]] + name = "github.com/gruntwork-io/terratest" + source = "git@github.com:gruntwork-io/terratest" + version = "0.13.24" + +[[constraint]] + name = "github.com/go-sql-driver/mysql" + source = "git@github.com:go-sql-driver/mysql" + version = "1.4.1" diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..b428e57 --- /dev/null +++ b/test/README.md @@ -0,0 +1,59 @@ +# Tests + +This folder contains automated tests for this Module. All of the tests are written in [Go](https://golang.org/). +Most of these are "integration tests" that deploy real infrastructure using Terraform and verify that infrastructure +works as expected using a helper library called [Terratest](https://github.com/gruntwork-io/terratest). + + + +## WARNING WARNING WARNING + +**Note #1**: Many of these tests create real resources in a GCP project and then try to clean those resources up at +the end of a test run. That means these tests may cost you money to run! When adding tests, please be considerate of +the resources you create and take extra care to clean everything up when you're done! + +**Note #2**: Never forcefully shut the tests down (e.g. by hitting `CTRL + C`) or the cleanup tasks won't run! + +**Note #3**: We set `-timeout 60m` on all tests not because they necessarily take that long, but because Go has a +default test timeout of 10 minutes, after which it forcefully kills the tests with a `SIGQUIT`, preventing the cleanup +tasks from running. Therefore, we set an overlying long timeout to make sure all tests have enough time to finish and +clean up. + + + +## Running the tests + +### Prerequisites + +- Install the latest version of [Go](https://golang.org/). +- Install [dep](https://github.com/golang/dep) for Go dependency management. +- Install [Terraform](https://www.terraform.io/downloads.html). +- Configure your Google credentials using one of the [options supported by GCP](https://cloud.google.com/docs/authentication/getting-started). + + +### One-time setup + +Download Go dependencies using dep: + +``` +cd test +dep ensure +``` + + +### Run all the tests + +```bash +cd test +go test -v -timeout 60m +``` + + +### Run a specific test + +To run a specific test called `TestFoo`: + +```bash +cd test +go test -v -timeout 60m -run TestFoo +``` \ No newline at end of file diff --git a/test/example_cloud_sql_mysql_test.go b/test/example_cloud_sql_mysql_test.go new file mode 100644 index 0000000..6fa6cbe --- /dev/null +++ b/test/example_cloud_sql_mysql_test.go @@ -0,0 +1,154 @@ +package test + +import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" + "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "path/filepath" + "strings" + "testing" +) + +const DB_NAME = "testdb" +const DB_USER = "testuser" +const DB_PASS = "testpassword" +const NAME_PREFIX = "mysql-test" +const MYSQL_VERSION = "MYSQL_5_7" +const EXAMPLE_NAME = "cloud-sql-mysql" + +const KEY_REGION = "region" +const KEY_PROJECT = "project" + +const OUTPUT_INSTANCE_NAME = "instance_name" +const OUTPUT_PROXY_CONNECTION = "proxy_connection" +const OUTPUT_DB_NAME = "db_name" +const OUTPUT_PUBLIC_IP = "public_ip" + +func TestCloudSQLMySql(t *testing.T) { + t.Parallel() + + //os.Setenv("SKIP_bootstrap", "true") + //os.Setenv("SKIP_deploy", "true") + //os.Setenv("SKIP_validate_outputs", "true") + //os.Setenv("SKIP_sql_tests", "true") + //os.Setenv("SKIP_teardown", "true") + + _examplesDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples") + exampleDir := filepath.Join(_examplesDir, EXAMPLE_NAME) + + test_structure.RunTestStage(t, "bootstrap", func() { + projectId := gcp.GetGoogleProjectIDFromEnvVar(t) + region := getRandomRegion(t, projectId) + + test_structure.SaveString(t, exampleDir, KEY_REGION, region) + test_structure.SaveString(t, exampleDir, KEY_PROJECT, projectId) + }) + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer test_structure.RunTestStage(t, "teardown", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + terraform.Destroy(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "deploy", func() { + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + terraformOptions := createTerratestOptionsForMySql(projectId, region, exampleDir) + test_structure.SaveTerraformOptions(t, exampleDir, terraformOptions) + + terraform.InitAndApply(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "validate_outputs", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + region := test_structure.LoadString(t, exampleDir, KEY_REGION) + projectId := test_structure.LoadString(t, exampleDir, KEY_PROJECT) + + instanceNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_INSTANCE_NAME) + dbNameFromOutput := terraform.Output(t, terraformOptions, OUTPUT_DB_NAME) + proxyConnectionFromOutput := terraform.Output(t, terraformOptions, OUTPUT_PROXY_CONNECTION) + + expectedDBConn := fmt.Sprintf("%s:%s:%s", projectId, region, instanceNameFromOutput) + + assert.True(t, strings.HasPrefix(instanceNameFromOutput, NAME_PREFIX)) + assert.Equal(t, DB_NAME, dbNameFromOutput) + assert.Equal(t, expectedDBConn, proxyConnectionFromOutput) + }) + + test_structure.RunTestStage(t, "sql_tests", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, exampleDir) + + publicIp := terraform.Output(t, terraformOptions, OUTPUT_PUBLIC_IP) + + connectionString := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", DB_USER, DB_PASS, publicIp, DB_NAME) + + // Does not actually open up the connection - just returns a DB ref + logger.Logf(t, "Connecting to: %s", publicIp) + db, err := sql.Open("mysql", + connectionString) + require.NoError(t, err, "Failed to open DB connection") + + // Make sure we clean up properly + defer db.Close() + + // Run ping to actually test the connection + logger.Log(t, "Ping the DB") + if err = db.Ping(); err != nil { + t.Fatalf("Failed to ping DB: %v", err) + } + + // Create table if not exists + logger.Logf(t, "Create table: %s", MYSQL_CREATE_TEST_TABLE_WITH_AUTO_INCREMENT_STATEMENT) + if _, err = db.Exec(MYSQL_CREATE_TEST_TABLE_WITH_AUTO_INCREMENT_STATEMENT); err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + // Clean up + logger.Logf(t, "Empty table: %s", MYSQL_EMPTY_TEST_TABLE_STATEMENT) + if _, err = db.Exec(MYSQL_EMPTY_TEST_TABLE_STATEMENT); err != nil { + t.Fatalf("Failed to clean up table: %v", err) + } + + // Insert data to check that our auto-increment flags worked + logger.Logf(t, "Insert data: %s", MYSQL_INSERT_TEST_ROW) + stmt, err := db.Prepare(MYSQL_INSERT_TEST_ROW) + require.NoError(t, err, "Failed to prepare statement") + + // Execute the statement + res, err := stmt.Exec("Grunt") + require.NoError(t, err, "Failed to execute statement") + + // Get the last insert id + lastId, err := res.LastInsertId() + require.NoError(t, err, "Failed to get last insert id") + + // Since we set the auto increment to 5, modulus should always be 0 + assert.Equal(t, int64(0), int64(lastId%5)) + }) +} + +func createTerratestOptionsForMySql(projectId string, region string, exampleDir string) *terraform.Options { + + terratestOptions := &terraform.Options{ + // The path to where your Terraform code is located + TerraformDir: exampleDir, + Vars: map[string]interface{}{ + "region": region, + "project": projectId, + "name_prefix": NAME_PREFIX, + "mysql_version": MYSQL_VERSION, + "db_name": DB_NAME, + "master_user_name": DB_USER, + "master_user_password": DB_PASS, + }, + } + + return terratestOptions +} diff --git a/test/example_cloud_sql_postgres_test.go b/test/example_cloud_sql_postgres_test.go new file mode 100644 index 0000000..603cb9b --- /dev/null +++ b/test/example_cloud_sql_postgres_test.go @@ -0,0 +1,13 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCloudSQLPostgres(t *testing.T) { + t.Parallel() + + assert.Equal(t, "5432", "5432") +} diff --git a/test/test_util.go b/test/test_util.go new file mode 100644 index 0000000..c6b482c --- /dev/null +++ b/test/test_util.go @@ -0,0 +1,16 @@ +package test + +import ( + "github.com/gruntwork-io/terratest/modules/gcp" + "testing" +) + +const MYSQL_CREATE_TEST_TABLE_WITH_AUTO_INCREMENT_STATEMENT = "CREATE TABLE IF NOT EXISTS test (id int NOT NULL AUTO_INCREMENT, name varchar(10) NOT NULL, PRIMARY KEY (ID))" +const MYSQL_EMPTY_TEST_TABLE_STATEMENT = "DELETE FROM test" +const MYSQL_INSERT_TEST_ROW = "INSERT INTO test(name) VALUES(?)" + +func getRandomRegion(t *testing.T, projectID string) string { + //approvedRegions := []string{"europe-north1", "europe-west1", "europe-west2", "europe-west3", "us-central1", "us-east1", "us-west1"} + approvedRegions := []string{"europe-north1"} + return gcp.GetRandomRegion(t, projectID, approvedRegions, []string{}) +}