terraform cleanup: tags, formatting, better defaults (#1080)

oavdeev · web-flow · commit 54b0104c05ab · 2020-10-21T17:47:21.000+08:00
Signed-off-by: Oleg Avdeev &lt;oleg.v.avdeev@gmail.com&gt;
diff --git a/infra/terraform/aws/eks.tf b/infra/terraform/aws/eks.tf
@@ -54,6 +54,8 @@ resource "random_string" "suffix" {
 resource "aws_security_group" "all_worker_mgmt" {
   name_prefix = "${var.name_prefix}-worker"
   vpc_id      = module.vpc.vpc_id
+
+  tags = var.tags
 }
 
 module "vpc" {
@@ -78,40 +80,38 @@ module "vpc" {
     "kubernetes.io/cluster/${local.cluster_name}" = "shared"
     "kubernetes.io/role/internal-elb"             = "1"
   }
+
+  tags = var.tags
 }
 
 module "eks" {
-  source          = "terraform-aws-modules/eks/aws"
+  source  = "terraform-aws-modules/eks/aws"
   version = "12.2.0"
 
   cluster_name    = local.cluster_name
   cluster_version = "1.17"
   subnets         = module.vpc.private_subnets
 
-  tags = {
-    Environment = "test"
-    GithubRepo  = "terraform-aws-eks"
-    GithubOrg   = "terraform-aws-modules"
-  }
+  tags = var.tags
 
   vpc_id = module.vpc.vpc_id
 
   worker_groups = [
     {
-      name                          = "worker-group-1"
-      instance_type                 = "r3.large"
-      asg_desired_capacity          = 2
+      name                 = "worker-group-1"
+      instance_type        = "r3.large"
+      asg_desired_capacity = 2
     },
     {
-      name                          = "worker-group-2"
-      instance_type                 = "r3.large"
-      asg_desired_capacity          = 1
+      name                 = "worker-group-2"
+      instance_type        = "r3.large"
+      asg_desired_capacity = 1
     },
   ]
 
   worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id]
   map_roles                            = var.map_roles
   map_accounts                         = var.map_accounts
 
-  workers_additional_policies          = [aws_iam_policy.worker_policy.id]
+  workers_additional_policies = [aws_iam_policy.worker_policy.id]
 }
diff --git a/infra/terraform/aws/emr.tf b/infra/terraform/aws/emr.tf
@@ -67,4 +67,6 @@ resource "aws_emr_cluster" "persistent_cluster" {
   step_concurrency_level = 256
 
   log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/"
+
+  tags = var.tags
 }
diff --git a/infra/terraform/aws/helm.tf b/infra/terraform/aws/helm.tf
@@ -12,81 +12,81 @@ provider "helm" {
 # RDS password is stored in a configmap which is not awesome but that RDS instance is not routable
 # from the outside anyways so that'll do.
 locals {
-    feast_core_config = {
-        redis = {
-            enabled = false
-        }
-        postgresql = {
-            enabled = false
-        }
-        kafka = {
-            enabled = false
-        }
+  feast_core_config = {
+    redis = {
+      enabled = false
+    }
+    postgresql = {
+      enabled = false
+    }
+    kafka = {
+      enabled = false
+    }
 
-        "feast-core" = {
-            "application-generated.yaml" = {
-                enabled = false
-            }
+    "feast-core" = {
+      "application-generated.yaml" = {
+        enabled = false
+      }
 
-            "application-override.yaml" = {
-                spring = {
-                    datasource = {
-                        url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}"
-                        username = "${module.rds_cluster.master_username}"
-                        password = "${random_password.db_password.result}"
-                    }
-                }
-                feast = {
-                    stream = {
-                        type = "kafka"
-                        options = {
-                            bootstrapServers = ${aws_msk_cluster.msk.bootstrap_brokers}
-                            topic = "feast"
-                        }
-                    }
-                }
-                server = {
-                    port = "8080"
-                }
+      "application-override.yaml" = {
+        spring = {
+          datasource = {
+            url      = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}"
+            username = "${module.rds_cluster.master_username}"
+            password = "${random_password.db_password.result}"
+          }
+        }
+        feast = {
+          stream = {
+            type = "kafka"
+            options = {
+              bootstrapServers = aws_msk_cluster.msk.bootstrap_brokers
+              topic            = "feast"
             }
+          }
         }
+        server = {
+          port = "8080"
+        }
+      }
+    }
 
-        "feast-online-serving" = {
-            "application-override.yaml" = {
-                enabled = true
-                feast = {
-                    stores = [
-                        {
-                            name = "online"
-                            type = "REDIS"
-                            config = {
-                                host = module.redis.endpoint
-                                port = 6379
-                            }
-                            subscriptions = [
-                                {
-                                    name=  "*"
-                                    project= "*"
-                                    version= "*"
-                                }
-                            ]
-                        }
-                    ]
-                    job_store = {
-                        redis_host = module.redis.endpoint
-                        redis_port = 6379
-                    }
+    "feast-online-serving" = {
+      "application-override.yaml" = {
+        enabled = true
+        feast = {
+          stores = [
+            {
+              name = "online"
+              type = "REDIS"
+              config = {
+                host = module.redis.endpoint
+                port = 6379
+              }
+              subscriptions = [
+                {
+                  name    = "*"
+                  project = "*"
+                  version = "*"
                 }
+              ]
             }
+          ]
+          job_store = {
+            redis_host = module.redis.endpoint
+            redis_port = 6379
+          }
         }
+      }
     }
+  }
 }
 
 resource "helm_release" "feast" {
-  name       = "feast"
-  chart      = "../../charts/feast"
+  name  = "feast"
+  chart = "../../charts/feast"
 
-  wait       = false
+  wait = false
 
   values = [
     yamlencode(local.feast_core_config)
diff --git a/infra/terraform/aws/msk.tf b/infra/terraform/aws/msk.tf
@@ -3,11 +3,11 @@ resource "aws_security_group" "broker" {
   vpc_id      = module.vpc.vpc_id
 
   ingress {
-    description = "Allow connections from the worker group"
+    description     = "Allow connections from the worker group"
     security_groups = [aws_security_group.all_worker_mgmt.id]
-    protocol     = "tcp"
-    from_port    = 0
-    to_port      = 65535
+    protocol        = "tcp"
+    from_port       = 0
+    to_port         = 65535
   }
 
   egress {
@@ -16,6 +16,8 @@ resource "aws_security_group" "broker" {
     protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
+
+  tags = var.tags
 }
 
 resource "aws_msk_cluster" "msk" {
@@ -26,7 +28,7 @@ resource "aws_msk_cluster" "msk" {
   broker_node_group_info {
     instance_type   = "kafka.t3.small"
     ebs_volume_size = 100
-    client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]
+    client_subnets  = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]
     security_groups = [aws_security_group.broker.id]
   }
 
@@ -46,4 +48,5 @@ resource "aws_msk_cluster" "msk" {
     }
   }
 
+  tags = var.tags
 }
diff --git a/infra/terraform/aws/online_store.tf b/infra/terraform/aws/online_store.tf
@@ -1,8 +1,10 @@
-  module "redis" {
-    source                     = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0"
-    subnets                    = module.vpc.private_subnets
-    name                       = "${var.name_prefix}-online"
-    vpc_id                     = module.vpc.vpc_id
-    allowed_security_groups    = [aws_security_group.all_worker_mgmt.id]
-    availability_zones         = module.vpc.azs
-  }
+module "redis" {
+  source                  = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0"
+  subnets                 = module.vpc.private_subnets
+  name                    = "${var.name_prefix}-online"
+  vpc_id                  = module.vpc.vpc_id
+  allowed_security_groups = [aws_security_group.all_worker_mgmt.id]
+  availability_zones      = module.vpc.azs
+
+  tags = var.tags
+}
diff --git a/infra/terraform/aws/rds.tf b/infra/terraform/aws/rds.tf
@@ -1,25 +1,25 @@
 resource "random_password" "db_password" {
-  length = 16
-  special = true
+  length           = 16
+  special          = true
   override_special = "!#()-[]<>"
 }
 
 module "rds_cluster" {
-  source               = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0"
-  name                 = "${var.name_prefix}-db"
-  engine               = "aurora-postgresql"
-  engine_mode          = "serverless"
-  engine_version       = "10.7"
-  cluster_family       = "aurora-postgresql10"
-  cluster_size         = 0
-  admin_user           = var.postgres_db_user
-  admin_password       = random_password.db_password.result
-  db_name              = var.postgres_db_name
-  db_port              = 5432
-  instance_type        = "db.t2.small"
-  vpc_id               = module.vpc.vpc_id
-  security_groups      = [aws_security_group.all_worker_mgmt.id]
-  subnets              = module.vpc.private_subnets
+  source          = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0"
+  name            = "${var.name_prefix}-db"
+  engine          = "aurora-postgresql"
+  engine_mode     = "serverless"
+  engine_version  = "10.7"
+  cluster_family  = "aurora-postgresql10"
+  cluster_size    = 0
+  admin_user      = var.postgres_db_user
+  admin_password  = random_password.db_password.result
+  db_name         = var.postgres_db_name
+  db_port         = 5432
+  instance_type   = "db.t2.small"
+  vpc_id          = module.vpc.vpc_id
+  security_groups = [aws_security_group.all_worker_mgmt.id]
+  subnets         = module.vpc.private_subnets
 
   scaling_configuration = [
     {
@@ -30,4 +30,6 @@ module "rds_cluster" {
       timeout_action           = "ForceApplyCapacityChange"
     }
   ]
+
+  tags = var.tags
 }
diff --git a/infra/terraform/aws/s3.tf b/infra/terraform/aws/s3.tf
@@ -1,7 +1,7 @@
 resource "random_string" "s3_suffix" {
-  length = 8
-  lower = true
-  upper = false
+  length  = 8
+  lower   = true
+  upper   = false
   special = false
 }
 
@@ -17,4 +17,6 @@ resource "aws_s3_bucket" "feast_bucket" {
       }
     }
   }
+
+  tags = var.tags
 }
diff --git a/infra/terraform/aws/variables.tf b/infra/terraform/aws/variables.tf
@@ -5,11 +5,11 @@ variable "name_prefix" {
 }
 
 variable "postgres_db_name" {
-    default = "feast"
+  default = "feast"
 }
 
 variable "postgres_db_user" {
-    default = "feast"
+  default = "feast"
 }
 
 variable "map_accounts" {
@@ -34,5 +34,13 @@ variable "map_roles" {
 }
 
 variable "use_persistent_emr_cluster" {
-  default = false
+  description = "Create a persistent EMR cluster."
+  default     = true
 }
+
+variable "tags" {
+  description = "Tags"
+  type        = map(string)
+
+  default = {}
+}

Original file line number	Diff line number	Diff line change
`@@ -67,4 +67,6 @@ resource "aws_emr_cluster" "persistent_cluster" {`
`67`	`67`	`step_concurrency_level = 256`
`68`	`68`
`69`	`69`	`log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/"`
	`70`	`+`
	`71`	`+ tags = var.tags`
`70`	`72`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,11 +3,11 @@ resource "aws_security_group" "broker" {`
`3`	`3`	`vpc_id = module.vpc.vpc_id`
`4`	`4`
`5`	`5`	`ingress {`
`6`		`- description = "Allow connections from the worker group"`
	`6`	`+ description = "Allow connections from the worker group"`
`7`	`7`	`security_groups = [aws_security_group.all_worker_mgmt.id]`
`8`		`- protocol = "tcp"`
`9`		`- from_port = 0`
`10`		`- to_port = 65535`
	`8`	`+ protocol = "tcp"`
	`9`	`+ from_port = 0`
	`10`	`+ to_port = 65535`
`11`	`11`	`}`
`12`	`12`
`13`	`13`	`egress {`
`@@ -16,6 +16,8 @@ resource "aws_security_group" "broker" {`
`16`	`16`	`protocol = "-1"`
`17`	`17`	`cidr_blocks = ["0.0.0.0/0"]`
`18`	`18`	`}`
	`19`	`+`
	`20`	`+ tags = var.tags`
`19`	`21`	`}`
`20`	`22`
`21`	`23`	`resource "aws_msk_cluster" "msk" {`
`@@ -26,7 +28,7 @@ resource "aws_msk_cluster" "msk" {`
`26`	`28`	`broker_node_group_info {`
`27`	`29`	`instance_type = "kafka.t3.small"`
`28`	`30`	`ebs_volume_size = 100`
`29`		`- client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]`
	`31`	`+ client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]`
`30`	`32`	`security_groups = [aws_security_group.broker.id]`
`31`	`33`	`}`
`32`	`34`
`@@ -46,4 +48,5 @@ resource "aws_msk_cluster" "msk" {`
`46`	`48`	`}`
`47`	`49`	`}`
`48`	`50`
	`51`	`+ tags = var.tags`
`49`	`52`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`resource "random_string" "s3_suffix" {`
`2`		`- length = 8`
`3`		`- lower = true`
`4`		`- upper = false`
	`2`	`+ length = 8`
	`3`	`+ lower = true`
	`4`	`+ upper = false`
`5`	`5`	`special = false`
`6`	`6`	`}`
`7`	`7`
`@@ -17,4 +17,6 @@ resource "aws_s3_bucket" "feast_bucket" {`
`17`	`17`	`}`
`18`	`18`	`}`
`19`	`19`	`}`
	`20`	`+`
	`21`	`+ tags = var.tags`
`20`	`22`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,11 +5,11 @@ variable "name_prefix" {`
`5`	`5`	`}`
`6`	`6`
`7`	`7`	`variable "postgres_db_name" {`
`8`		`- default = "feast"`
	`8`	`+ default = "feast"`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`variable "postgres_db_user" {`
`12`		`- default = "feast"`
	`12`	`+ default = "feast"`
`13`	`13`	`}`
`14`	`14`
`15`	`15`	`variable "map_accounts" {`
`@@ -34,5 +34,13 @@ variable "map_roles" {`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`variable "use_persistent_emr_cluster" {`
`37`		`- default = false`
	`37`	`+ description = "Create a persistent EMR cluster."`
	`38`	`+ default = true`
`38`	`39`	`}`
	`40`	`+`
	`41`	`+variable "tags" {`
	`42`	`+ description = "Tags"`
	`43`	`+ type = map(string)`
	`44`	`+`
	`45`	`+ default = {}`
	`46`	`+}`