-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Expand file tree
/
Copy pathemr.tf
More file actions
72 lines (63 loc) · 2.09 KB
/
Copy pathemr.tf
File metadata and controls
72 lines (63 loc) · 2.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
data "aws_iam_instance_profile" "emr_default_role" {
name = "EMR_EC2_DefaultRole"
}
resource "aws_emr_cluster" "persistent_cluster" {
count = var.use_persistent_emr_cluster ? 1 : 0
name = "${var.name_prefix}-persistent-emr"
keep_job_flow_alive_when_no_steps = true
release_label = "emr-6.0.0"
ec2_attributes {
subnet_id = module.vpc.private_subnets[0]
additional_master_security_groups = aws_security_group.all_worker_mgmt.id
additional_slave_security_groups = aws_security_group.all_worker_mgmt.id
instance_profile = data.aws_iam_instance_profile.emr_default_role.arn
}
applications = ["Hadoop", "Hive", "Spark", "Livy"]
service_role = "EMR_DefaultRole"
bootstrap_action {
path = "s3://aws-bigdata-blog/artifacts/resize_storage/resize_storage.sh"
name = "runif"
args = ["--scaling-factor", "1.5"]
}
master_instance_fleet {
instance_type_configs {
instance_type = "m4.xlarge"
ebs_config {
size = "100"
type = "gp2"
volumes_per_instance = 1
}
}
launch_specifications {
spot_specification {
timeout_action = "SWITCH_TO_ON_DEMAND"
timeout_duration_minutes = 10
allocation_strategy = "capacity-optimized"
}
}
target_spot_capacity = 1
}
core_instance_fleet {
instance_type_configs {
bid_price_as_percentage_of_on_demand_price = 100
ebs_config {
size = "100"
type = "gp2"
volumes_per_instance = 1
}
instance_type = "m4.xlarge"
weighted_capacity = 1
}
launch_specifications {
spot_specification {
timeout_action = "SWITCH_TO_ON_DEMAND"
timeout_duration_minutes = 10
allocation_strategy = "capacity-optimized"
}
}
target_spot_capacity = 2
}
step_concurrency_level = 256
log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/"
tags = var.tags
}