Skip to content

Commit 37c0077

Browse files
oavdeevoavdeev
committed
terraform config for aws
Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
1 parent e47903f commit 37c0077

10 files changed

Lines changed: 435 additions & 0 deletions

File tree

‎infra/terraform/aws/README.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Terraform config for feast on AWS
2+
3+
Uses terraform 0.12
4+
5+
1. Run `aws emr create-default-roles` once.
6+
7+
2. Create a tfvars file, e.g. `my.tfvars` and set name_prefix:
8+
9+
```
10+
name_prefix = "my-feast"
11+
region = "us-east-1"
12+
```
13+
14+
3. Configure tf state backend, e.g.:
15+
```
16+
terraform {
17+
backend "s3" {
18+
bucket = "my-terraform-state-bucket"
19+
key = "clusters/my-feast-test"
20+
region = "us-west-2"
21+
dynamodb_table = "terraform-state-lock"
22+
encrypt = true
23+
}
24+
}
25+
```
26+
27+
3. Use `terraform apply -var-file="my.tfvars"` to deploy.
28+

‎infra/terraform/aws/eks.tf‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
terraform {
2+
required_version = ">= 0.12.0"
3+
}
4+
5+
provider "aws" {
6+
version = ">= 2.28.1"
7+
region = var.region
8+
}
9+
10+
provider "random" {
11+
version = "~> 2.1"
12+
}
13+
14+
provider "local" {
15+
version = "~> 1.2"
16+
}
17+
18+
provider "null" {
19+
version = "~> 2.1"
20+
}
21+
22+
provider "template" {
23+
version = "~> 2.1"
24+
}
25+
26+
data "aws_eks_cluster" "cluster" {
27+
name = module.eks.cluster_id
28+
}
29+
30+
data "aws_eks_cluster_auth" "cluster" {
31+
name = module.eks.cluster_id
32+
}
33+
34+
provider "kubernetes" {
35+
host = data.aws_eks_cluster.cluster.endpoint
36+
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
37+
token = data.aws_eks_cluster_auth.cluster.token
38+
load_config_file = false
39+
version = "~> 1.11"
40+
}
41+
42+
data "aws_availability_zones" "available" {
43+
}
44+
45+
locals {
46+
cluster_name = "${var.name_prefix}-${random_string.suffix.result}"
47+
}
48+
49+
resource "random_string" "suffix" {
50+
length = 8
51+
special = false
52+
}
53+
54+
resource "aws_security_group" "all_worker_mgmt" {
55+
name_prefix = "${var.name_prefix}-worker"
56+
vpc_id = module.vpc.vpc_id
57+
}
58+
59+
module "vpc" {
60+
source = "terraform-aws-modules/vpc/aws"
61+
version = "2.47.0"
62+
63+
name = "${var.name_prefix}-vpc"
64+
cidr = "10.0.0.0/16"
65+
azs = data.aws_availability_zones.available.names
66+
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
67+
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
68+
enable_nat_gateway = true
69+
single_nat_gateway = true
70+
enable_dns_hostnames = true
71+
72+
public_subnet_tags = {
73+
"kubernetes.io/cluster/${local.cluster_name}" = "shared"
74+
"kubernetes.io/role/elb" = "1"
75+
}
76+
77+
private_subnet_tags = {
78+
"kubernetes.io/cluster/${local.cluster_name}" = "shared"
79+
"kubernetes.io/role/internal-elb" = "1"
80+
}
81+
}
82+
83+
module "eks" {
84+
source = "terraform-aws-modules/eks/aws"
85+
version = "12.2.0"
86+
87+
cluster_name = local.cluster_name
88+
cluster_version = "1.17"
89+
subnets = module.vpc.private_subnets
90+
91+
tags = {
92+
Environment = "test"
93+
GithubRepo = "terraform-aws-eks"
94+
GithubOrg = "terraform-aws-modules"
95+
}
96+
97+
vpc_id = module.vpc.vpc_id
98+
99+
worker_groups = [
100+
{
101+
name = "worker-group-1"
102+
instance_type = "r3.large"
103+
asg_desired_capacity = 2
104+
},
105+
{
106+
name = "worker-group-2"
107+
instance_type = "r3.large"
108+
asg_desired_capacity = 1
109+
},
110+
]
111+
112+
worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id]
113+
map_roles = var.map_roles
114+
map_accounts = var.map_accounts
115+
116+
workers_additional_policies = [aws_iam_policy.worker_policy.id]
117+
}

‎infra/terraform/aws/emr.tf‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
data "aws_iam_instance_profile" "emr_default_role" {
2+
name = "EMR_EC2_DefaultRole"
3+
}
4+
5+
resource "aws_emr_cluster" "persistent_cluster" {
6+
count = var.use_persistent_emr_cluster ? 1 : 0
7+
8+
name = "${var.name_prefix}-persistent-emr"
9+
keep_job_flow_alive_when_no_steps = true
10+
release_label = "emr-6.0.0"
11+
12+
ec2_attributes {
13+
subnet_id = module.vpc.private_subnets[0]
14+
additional_master_security_groups = aws_security_group.all_worker_mgmt.id
15+
additional_slave_security_groups = aws_security_group.all_worker_mgmt.id
16+
instance_profile = data.aws_iam_instance_profile.emr_default_role.arn
17+
}
18+
19+
applications = ["Hadoop", "Hive", "Spark", "Livy"]
20+
service_role = "EMR_DefaultRole"
21+
22+
bootstrap_action {
23+
path = "s3://aws-bigdata-blog/artifacts/resize_storage/resize_storage.sh"
24+
name = "runif"
25+
args = ["--scaling-factor", "1.5"]
26+
}
27+
28+
master_instance_fleet {
29+
instance_type_configs {
30+
instance_type = "m4.xlarge"
31+
ebs_config {
32+
size = "100"
33+
type = "gp2"
34+
volumes_per_instance = 1
35+
}
36+
}
37+
launch_specifications {
38+
spot_specification {
39+
timeout_action = "SWITCH_TO_ON_DEMAND"
40+
timeout_duration_minutes = 10
41+
allocation_strategy = "capacity-optimized"
42+
}
43+
}
44+
target_spot_capacity = 1
45+
}
46+
core_instance_fleet {
47+
instance_type_configs {
48+
bid_price_as_percentage_of_on_demand_price = 100
49+
ebs_config {
50+
size = "100"
51+
type = "gp2"
52+
volumes_per_instance = 1
53+
}
54+
instance_type = "m4.xlarge"
55+
weighted_capacity = 1
56+
}
57+
launch_specifications {
58+
spot_specification {
59+
timeout_action = "SWITCH_TO_ON_DEMAND"
60+
timeout_duration_minutes = 10
61+
allocation_strategy = "capacity-optimized"
62+
}
63+
}
64+
target_spot_capacity = 2
65+
}
66+
67+
step_concurrency_level = 256
68+
69+
log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/"
70+
}

‎infra/terraform/aws/helm.tf‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
provider "helm" {
2+
kubernetes {
3+
host = data.aws_eks_cluster.cluster.endpoint
4+
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
5+
token = data.aws_eks_cluster_auth.cluster.token
6+
load_config_file = false
7+
}
8+
}
9+
10+
# Construct feast configs that need to point to RDS and Redis.
11+
#
12+
# RDS password is stored in a configmap which is not awesome but that RDS instance is not routable
13+
# from the outside anyways so that'll do.
14+
locals {
15+
feast_core_config = {
16+
redis = {
17+
enabled = false
18+
}
19+
postgresql = {
20+
enabled = false
21+
}
22+
kafka = {
23+
enabled = false
24+
}
25+
26+
"feast-core" = {
27+
"application-generated.yaml" = {
28+
enabled = false
29+
}
30+
31+
"application-override.yaml" = {
32+
spring = {
33+
datasource = {
34+
url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}"
35+
username = "${module.rds_cluster.master_username}"
36+
password = "${random_password.db_password.result}"
37+
}
38+
}
39+
feast = {
40+
stream = {
41+
type = "kafka"
42+
options = {
43+
bootstrapServers = "myrelease-kafka:9092"
44+
topic = "feast"
45+
}
46+
}
47+
}
48+
server = {
49+
port = "8080"
50+
}
51+
}
52+
}
53+
54+
"feast-online-serving" = {
55+
"application-override.yaml" = {
56+
enabled = true
57+
feast = {
58+
stores = [
59+
{
60+
name = "online"
61+
type = "REDIS"
62+
config = {
63+
host = module.redis.endpoint
64+
port = 6379
65+
}
66+
subscriptions = [
67+
{
68+
name= "*"
69+
project= "*"
70+
version= "*"
71+
}
72+
]
73+
}
74+
]
75+
job_store = {
76+
redis_host = module.redis.endpoint
77+
redis_port = 6379
78+
}
79+
}
80+
}
81+
}
82+
}
83+
}
84+
85+
resource "helm_release" "feast" {
86+
name = "feast"
87+
chart = "../../charts/feast"
88+
89+
wait = false
90+
91+
values = [
92+
yamlencode(local.feast_core_config)
93+
]
94+
}

‎infra/terraform/aws/iam.tf‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
data "aws_iam_policy_document" "worker_policy_document" {
2+
statement {
3+
sid = "1"
4+
5+
actions = [
6+
"s3:*",
7+
"elasticmapreduce:*",
8+
"glue:*",
9+
"cloudwatch:*",
10+
"ecr:*",
11+
"iam:PassRole",
12+
]
13+
14+
resources = [
15+
"*",
16+
]
17+
}
18+
19+
}
20+
21+
resource "aws_iam_policy" "worker_policy" {
22+
name = "${var.name_prefix}_feast_worker_policy"
23+
path = "/"
24+
description = "Worker IAM policy"
25+
26+
policy = data.aws_iam_policy_document.worker_policy_document.json
27+
}

‎infra/terraform/aws/online_store.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
module "redis" {
2+
source = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0"
3+
subnets = module.vpc.private_subnets
4+
name = "${var.name_prefix}-online"
5+
vpc_id = module.vpc.vpc_id
6+
allowed_security_groups = [aws_security_group.all_worker_mgmt.id]
7+
availability_zones = module.vpc.azs
8+
}

‎infra/terraform/aws/outputs.tf‎

Whitespace-only changes.

‎infra/terraform/aws/rds.tf‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
resource "random_password" "db_password" {
2+
length = 16
3+
special = true
4+
override_special = "!#()-[]<>"
5+
}
6+
7+
module "rds_cluster" {
8+
source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0"
9+
name = "${var.name_prefix}-db"
10+
engine = "aurora-postgresql"
11+
engine_mode = "serverless"
12+
engine_version = "10.7"
13+
cluster_family = "aurora-postgresql10"
14+
cluster_size = 0
15+
admin_user = var.postgres_db_user
16+
admin_password = random_password.db_password.result
17+
db_name = var.postgres_db_name
18+
db_port = 5432
19+
instance_type = "db.t2.small"
20+
vpc_id = module.vpc.vpc_id
21+
security_groups = [aws_security_group.all_worker_mgmt.id]
22+
subnets = module.vpc.private_subnets
23+
24+
scaling_configuration = [
25+
{
26+
auto_pause = true
27+
max_capacity = 16
28+
min_capacity = 2
29+
seconds_until_auto_pause = 300
30+
timeout_action = "ForceApplyCapacityChange"
31+
}
32+
]
33+
}

0 commit comments

Comments
 (0)