148 lines
3.5 KiB
HCL
148 lines
3.5 KiB
HCL
terraform {
|
|
|
|
required_version = ">= 1.1"
|
|
|
|
required_providers {
|
|
aws = {
|
|
source = "hashicorp/aws"
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
module "s3_bucket" {
|
|
source = "terraform-aws-modules/s3-bucket/aws"
|
|
version = "~> 3.0"
|
|
|
|
bucket_prefix = "qmi-bucket-${var.provision_id}"
|
|
|
|
# Allow deletion of non-empty bucket
|
|
# Example usage only - not recommended for production
|
|
force_destroy = true
|
|
|
|
attach_deny_insecure_transport_policy = true
|
|
attach_require_latest_tls_policy = true
|
|
|
|
block_public_acls = true
|
|
block_public_policy = true
|
|
ignore_public_acls = true
|
|
restrict_public_buckets = true
|
|
|
|
server_side_encryption_configuration = {
|
|
rule = {
|
|
apply_server_side_encryption_by_default = {
|
|
sse_algorithm = "AES256"
|
|
}
|
|
}
|
|
}
|
|
|
|
tags = {
|
|
QMI_user = var.user_id
|
|
ProvID = var.provision_id
|
|
Name = "qmi-emr-${var.provision_id}"
|
|
Owner = var.user_id
|
|
}
|
|
}
|
|
|
|
module "emr" {
|
|
source = "terraform-aws-modules/emr/aws"
|
|
|
|
name = var.provision_id
|
|
|
|
release_label = "emr-6.11.0"
|
|
applications = ["spark", "hadoop", "hive", "hue"]
|
|
auto_termination_policy = {
|
|
idle_timeout = 3600
|
|
}
|
|
|
|
bootstrap_action = {
|
|
example = {
|
|
name = "Just an example",
|
|
path = "file:/bin/echo",
|
|
args = ["Hello World!"]
|
|
}
|
|
}
|
|
|
|
configurations_json = jsonencode([
|
|
{
|
|
"classification" : "spark-env",
|
|
"configurations" : [
|
|
{
|
|
"classification" : "export",
|
|
"properties" : {
|
|
"JAVA_HOME" : "/usr/lib/jvm/java-1.8.0"
|
|
}
|
|
}
|
|
],
|
|
"properties" : {}
|
|
},
|
|
{
|
|
"classification": "hive-site",
|
|
"properties": {
|
|
"hive.support.concurrency": "true",
|
|
"hive.exec.dynamic.partition.mode": "nonstrict",
|
|
"hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"
|
|
}
|
|
}
|
|
])
|
|
|
|
master_instance_group = {
|
|
name = "master-group"
|
|
instance_count = 1
|
|
instance_type = "m5.xlarge"
|
|
}
|
|
|
|
core_instance_group = {
|
|
name = "core-group"
|
|
instance_count = 1
|
|
instance_type = "c4.large"
|
|
}
|
|
|
|
task_instance_group = {
|
|
name = "task-group"
|
|
instance_count = 1
|
|
instance_type = "c5.xlarge"
|
|
bid_price = "0.1"
|
|
|
|
ebs_config = {
|
|
size = 64
|
|
type = "gp3"
|
|
volumes_per_instance = 1
|
|
}
|
|
ebs_optimized = true
|
|
}
|
|
|
|
ebs_root_volume_size = 64
|
|
ec2_attributes = {
|
|
# Instance groups only support one Subnet/AZ
|
|
# Subnets should be private subnets and tagged with
|
|
# { "for-use-with-amazon-emr-managed-policies" = true }
|
|
subnet_id = var.subnet_ids_us[0]
|
|
}
|
|
vpc_id = var.vpc_id_us
|
|
|
|
list_steps_states = ["PENDING", "RUNNING", "CANCEL_PENDING", "CANCELLED", "FAILED", "INTERRUPTED", "COMPLETED"]
|
|
log_uri = "s3://${module.s3_bucket.s3_bucket_id}/"
|
|
|
|
scale_down_behavior = "TERMINATE_AT_TASK_COMPLETION"
|
|
step_concurrency_level = 3
|
|
termination_protection = false
|
|
visible_to_all_users = true
|
|
|
|
is_private_cluster = false
|
|
#create_service_iam_role = false
|
|
#service_iam_role_arn = "arn:aws:iam::192018133564:role/service-role/AmazonEMR-ServiceRole-20230622T122656"
|
|
#create_iam_instance_profile = false
|
|
#iam_instance_profile_name = "AmazonEMR-InstanceProfile-20230622T122640"
|
|
|
|
tags = {
|
|
Environment = "QMI-${var.provision_id}"
|
|
Deployment = "QMI-${var.provision_id}"
|
|
Terraform = "true"
|
|
Environment = "dev"
|
|
QMI_user = var.user_id
|
|
Owner = var.user_id
|
|
ProvID = var.provision_id
|
|
Name = "qmi-emr-${var.provision_id}"
|
|
}
|
|
} |