From 754c9ba3deb663356bded0b21e67e9ef6dcb1f29 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 11 Apr 2016 11:29:03 +0700 Subject: [PATCH 001/181] capacity = 0 when initial boot --- autoscaling.tf | 2 +- iam.tf | 2 +- scripts/terraform-enable-remote.sh | 2 +- scripts/terraform-wrapper.sh | 3 +++ security_group.tf | 1 - variables.tf | 6 +++++- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 0b467fe..513e511 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -20,7 +20,7 @@ resource "aws_autoscaling_group" "web" { min_size = 0 health_check_grace_period = 300 health_check_type = "ELB" - desired_capacity = 0 + desired_capacity = "${var.desired_capacity_web}" force_delete = true launch_configuration = "${aws_launch_configuration.web.id}" vpc_zone_identifier = [ diff --git a/iam.tf b/iam.tf index 3906316..68678d7 100644 --- a/iam.tf +++ b/iam.tf @@ -44,7 +44,7 @@ EOF } resource "aws_iam_role_policy" "letsencrypt-cache-client" { - name = "es-client" + name = "letsencrypt-cache-client" role = "${aws_iam_role.web.id}" policy = < Date: Mon, 11 Apr 2016 15:24:57 +0700 Subject: [PATCH 002/181] multi region --- autoscaling.tf | 6 +++--- elasticache.tf | 4 ++-- elb.tf | 4 ++-- rds.tf | 4 ++-- variables.tf | 22 ++++++++++++++++++++-- vpc.tf | 32 ++++++++++++++++---------------- 6 files changed, 45 insertions(+), 27 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 513e511..347e5f8 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -1,6 +1,6 @@ resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" - image_id = "${var.ami_web}" + image_id = "${lookup(var.ami_web, var.aws_region)}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", @@ -24,8 +24,8 @@ resource "aws_autoscaling_group" "web" { force_delete = true launch_configuration = "${aws_launch_configuration.web.id}" vpc_zone_identifier = [ - "${aws_subnet.public-a.id}", - "${aws_subnet.public-b.id}", + "${aws_subnet.public_primary.id}", + "${aws_subnet.public_secondary.id}", ] load_balancers = ["${aws_elb.web.name}"] } diff --git a/elasticache.tf b/elasticache.tf index b92c6a9..4b3ac60 100644 --- a/elasticache.tf +++ b/elasticache.tf @@ -16,7 +16,7 @@ resource "aws_elasticache_subnet_group" "micropost" { name = "micropost-${var.env}" description = "main subnet group" subnet_ids = [ - "${aws_subnet.private-a.id}", - "${aws_subnet.private-b.id}", + "${aws_subnet.private_primary.id}", + "${aws_subnet.private_secondary.id}", ] } \ No newline at end of file diff --git a/elb.tf b/elb.tf index c199815..f5ce953 100644 --- a/elb.tf +++ b/elb.tf @@ -1,8 +1,8 @@ resource "aws_elb" "web" { name = "web-${var.env}" subnets = [ - "${aws_subnet.public-a.id}", - "${aws_subnet.public-b.id}", + "${aws_subnet.public_primary.id}", + "${aws_subnet.public_secondary.id}", ] security_groups = [ "${aws_security_group.internal.id}", diff --git a/rds.tf b/rds.tf index a3ac802..b2faccb 100644 --- a/rds.tf +++ b/rds.tf @@ -16,7 +16,7 @@ resource "aws_db_subnet_group" "micropost" { name = "micropost-${var.env}" description = "Our main group of subnets" subnet_ids = [ - "${aws_subnet.private-a.id}", - "${aws_subnet.private-b.id}", + "${aws_subnet.private_primary.id}", + "${aws_subnet.private_secondary.id}", ] } diff --git a/variables.tf b/variables.tf index a457ff0..b504ba6 100644 --- a/variables.tf +++ b/variables.tf @@ -4,12 +4,30 @@ variable "env" { } variable "aws_account_num" { } + variable "aws_region" { - default = "ap-southeast-1" + default = "ap-northeast-1" } variable "ami_web" { - default = "ami-25c00c46" + default = { + ap-southeast-1 = "ami-25c00c46" + ap-northeast-1 = "ami-a21529cc" + } +} +variable "aws_az_primary" { + default = { + ap-southeast-1 = "ap-southeast-1a" + ap-northeast-1 = "ap-northeast-1a" + } } +variable "aws_az_secondary" { + default = { + ap-southeast-1 = "ap-southeast-1b" + ap-northeast-1 = "ap-northeast-1c" + } +} + + variable "cidr" { type = "map" default = { diff --git a/vpc.tf b/vpc.tf index ed29b9f..2820ef8 100644 --- a/vpc.tf +++ b/vpc.tf @@ -20,58 +20,58 @@ resource "aws_route_table" "public-route" { // -------- public a -resource "aws_subnet" "public-a" { +resource "aws_subnet" "public_primary" { vpc_id = "${aws_vpc.vpc.id}" cidr_block = "10.1.1.0/24" - availability_zone = "${var.aws_region}a" + availability_zone = "${lookup(var.aws_az_primary, var.aws_region)}" tags { - Name = "public-a" + Name = "public-primary" Env = "${var.env}" } } -resource "aws_route_table_association" "puclic-a" { - subnet_id = "${aws_subnet.public-a.id}" +resource "aws_route_table_association" "puclic_primary" { + subnet_id = "${aws_subnet.public_primary.id}" route_table_id = "${aws_route_table.public-route.id}" } // -------- public b -resource "aws_subnet" "public-b" { +resource "aws_subnet" "public_secondary" { vpc_id = "${aws_vpc.vpc.id}" cidr_block = "10.1.2.0/24" - availability_zone = "${var.aws_region}b" + availability_zone = "${lookup(var.aws_az_secondary, var.aws_region)}" tags { - Name = "public-b" + Name = "public-secondary" Env = "${var.env}" } } -resource "aws_route_table_association" "puclic-b" { - subnet_id = "${aws_subnet.public-b.id}" +resource "aws_route_table_association" "puclic_secondary" { + subnet_id = "${aws_subnet.public_secondary.id}" route_table_id = "${aws_route_table.public-route.id}" } // -------- private a -resource "aws_subnet" "private-a" { +resource "aws_subnet" "private_primary" { vpc_id = "${aws_vpc.vpc.id}" cidr_block = "10.1.3.0/24" - availability_zone = "${var.aws_region}a" + availability_zone = "${lookup(var.aws_az_primary, var.aws_region)}" tags { - Name = "private-a" + Name = "private-primary" Env = "${var.env}" } } // -------- private b -resource "aws_subnet" "private-b" { +resource "aws_subnet" "private_secondary" { vpc_id = "${aws_vpc.vpc.id}" cidr_block = "10.1.4.0/24" - availability_zone = "${var.aws_region}b" + availability_zone = "${lookup(var.aws_az_secondary, var.aws_region)}" tags { - Name = "private-b" + Name = "private-secondary" Env = "${var.env}" } } From cf3f1b5931738d179425216a0f7b7030ff3129a4 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 11 Apr 2016 16:57:08 +0700 Subject: [PATCH 003/181] ... --- autoscaling.tf | 4 ++-- variables.tf | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 347e5f8..3e8d59c 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -1,12 +1,12 @@ resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" - image_id = "${lookup(var.ami_web, var.aws_region)}" + image_id = "${var.ami_web}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", "${aws_security_group.ssh.id}", ] - key_name = "akirasosa" + key_name = "id_rsa" associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" lifecycle { diff --git a/variables.tf b/variables.tf index b504ba6..254b344 100644 --- a/variables.tf +++ b/variables.tf @@ -9,10 +9,7 @@ variable "aws_region" { default = "ap-northeast-1" } variable "ami_web" { - default = { - ap-southeast-1 = "ami-25c00c46" - ap-northeast-1 = "ami-a21529cc" - } + default = "ami-a21529cc" } variable "aws_az_primary" { default = { @@ -27,7 +24,6 @@ variable "aws_az_secondary" { } } - variable "cidr" { type = "map" default = { From 5845bad6210f7e3d3ed896901d6f86043d7ebb95 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 11 Apr 2016 17:10:06 +0700 Subject: [PATCH 004/181] fix min size --- autoscaling.tf | 2 +- scripts/terraform-wrapper.sh | 3 ++- variables.tf | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 3e8d59c..8f99eb3 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -17,7 +17,7 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web-${var.env}" max_size = 4 - min_size = 0 + min_size = "${var.max_size_web}" health_check_grace_period = 300 health_check_type = "ELB" desired_capacity = "${var.desired_capacity_web}" diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index 4ab45f3..e793c68 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -8,7 +8,8 @@ TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} if [ -n "${TF_VAR_ami_web}" ]; then export TF_VAR_ami_web else - # use ubuntu default ami with capacity 0. + # use ubuntu default ami with 0 instances. + export TF_VAR_max_size_web=0 export TF_VAR_desired_capacity_web=0 fi diff --git a/variables.tf b/variables.tf index 254b344..403689c 100644 --- a/variables.tf +++ b/variables.tf @@ -34,3 +34,6 @@ variable "cidr" { variable "desired_capacity_web" { default = 1 } +variable "max_size_web" { + default = 1 +} From 43289fb18b141395e2773ad458129cfc9298e893 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 08:28:59 +0700 Subject: [PATCH 005/181] create bucket for codedepoy --- s3.tf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 s3.tf diff --git a/s3.tf b/s3.tf new file mode 100644 index 0000000..988774d --- /dev/null +++ b/s3.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "web_codedeploy" { + bucket = "web-condedeploy-${var.env}.hana053.com" + region = "${var.aws_region}" +} From cea403e53bf62875c2feb758267373e04dce15c1 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 16:15:39 +0700 Subject: [PATCH 006/181] use user data --- .travis.yml | 3 --- autoscaling.tf | 9 +++++++++ templates/web_init.sh | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 templates/web_init.sh diff --git a/.travis.yml b/.travis.yml index 923b43c..8095a9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,8 +34,5 @@ script: # Save current output to compare later - terraform output | grep endpoint | tee out_after - # Compare output and take over building if endpoints were changed. - - diff out_before out_after || scripts/build-provisionings.sh - # Deploy app when initial deployment - if [ ! -s out_before ]; then scripts/build-app.sh ; fi diff --git a/autoscaling.tf b/autoscaling.tf index 8f99eb3..d8370a1 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -1,3 +1,11 @@ +resource "template_file" "web_init" { + template = "${file("templates/web_init.sh")}" + vars { + logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" + web_endpoint = "${cloudflare_record.micropost.hostname}" + } +} + resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" image_id = "${var.ami_web}" @@ -9,6 +17,7 @@ resource "aws_launch_configuration" "web" { key_name = "id_rsa" associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" + user_data = "${template_file.web_init.rendered}" lifecycle { create_before_destroy = true } diff --git a/templates/web_init.sh b/templates/web_init.sh new file mode 100644 index 0000000..dd7ea1f --- /dev/null +++ b/templates/web_init.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +cd /opt/provisioning + +cat << EOF > inventory +[web] +localhost +EOF + +ansible-playbook -i inventory --connection=local --sudo --diff \ + -e "logstash_elasticsearch_host=${logserver_endpoint}" \ + -e "letsencrypt_host=${web_endpoint}" \ + site.yml + +/usr/local/bin/update_cert.sh >/var/log/update_cert 2>&1 From 5217d136a4a5835b59deb45f693e71174ff158fe Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 18:24:11 +0700 Subject: [PATCH 007/181] test export --- templates/web_init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/web_init.sh b/templates/web_init.sh index dd7ea1f..02a2741 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -13,3 +13,5 @@ ansible-playbook -i inventory --connection=local --sudo --diff \ site.yml /usr/local/bin/update_cert.sh >/var/log/update_cert 2>&1 + +export MY_TEST=hello From 6e610b345782fc4324cea82d444264318b88f729 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 18:41:31 +0700 Subject: [PATCH 008/181] fix cyclic dependency --- autoscaling.tf | 3 +++ outputs.tf | 10 +--------- scripts/terraform-wrapper.sh | 4 +--- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index d8370a1..80dce9c 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -4,6 +4,9 @@ resource "template_file" "web_init" { logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" web_endpoint = "${cloudflare_record.micropost.hostname}" } + lifecycle { + create_before_destroy = true + } } resource "aws_launch_configuration" "web" { diff --git a/outputs.tf b/outputs.tf index 9e8bfc3..5e99a1b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,9 +1,5 @@ output "ami_web" { - value = "${aws_launch_configuration.web.image_id}" -} - -output "logserver_endpoint" { - value = "${aws_elasticsearch_domain.logserver.endpoint}" + value = "${var.ami_web}" } output "rds_endpoint" { @@ -14,7 +10,3 @@ output "rds_endpoint" { output "redis_endpoint" { value = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}:${aws_elasticache_cluster.micropost.cache_nodes.0.port}" } - -output "web_endpoint" { - value = "${cloudflare_record.micropost.hostname}" -} diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index e793c68..ea65a85 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -8,9 +8,7 @@ TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} if [ -n "${TF_VAR_ami_web}" ]; then export TF_VAR_ami_web else - # use ubuntu default ami with 0 instances. - export TF_VAR_max_size_web=0 - export TF_VAR_desired_capacity_web=0 + exit fi terraform $@ \ From 0fc16654ace9bfc329d294c1db3c82a38ca7a2f7 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 20:49:30 +0700 Subject: [PATCH 009/181] ami_web is required --- .travis.yml | 5 +---- autoscaling.tf | 4 ++-- s3.tf | 2 +- scripts/build-provisionings.sh | 19 ------------------- scripts/terraform-wrapper.sh | 8 ++++---- variables.tf | 8 -------- 6 files changed, 8 insertions(+), 38 deletions(-) delete mode 100755 scripts/build-provisionings.sh diff --git a/.travis.yml b/.travis.yml index 8095a9e..3cf32c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,11 +28,8 @@ script: - terraform output | grep endpoint | tee out_before # Execute Terraform and create tfstate on remote - - scripts/terraform-wrapper.sh plan + - scripts/terraform-wrapper.sh plan || exit - scripts/terraform-wrapper.sh apply - # Save current output to compare later - - terraform output | grep endpoint | tee out_after - # Deploy app when initial deployment - if [ ! -s out_before ]; then scripts/build-app.sh ; fi diff --git a/autoscaling.tf b/autoscaling.tf index 80dce9c..08a049f 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -29,10 +29,10 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web-${var.env}" max_size = 4 - min_size = "${var.max_size_web}" + min_size = 1 health_check_grace_period = 300 health_check_type = "ELB" - desired_capacity = "${var.desired_capacity_web}" + desired_capacity = 1 force_delete = true launch_configuration = "${aws_launch_configuration.web.id}" vpc_zone_identifier = [ diff --git a/s3.tf b/s3.tf index 988774d..77c281b 100644 --- a/s3.tf +++ b/s3.tf @@ -1,4 +1,4 @@ resource "aws_s3_bucket" "web_codedeploy" { bucket = "web-condedeploy-${var.env}.hana053.com" - region = "${var.aws_region}" + force_destroy = true } diff --git a/scripts/build-provisionings.sh b/scripts/build-provisionings.sh deleted file mode 100755 index f1c4156..0000000 --- a/scripts/build-provisionings.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -travis login --org --skip-completion-check --github-token ${GH_TOKEN} -token=$(travis token --org) -body=$(cat << EOS -{ - "request": { - "branch":"${TRAVIS_BRANCH}" - } -} -EOS -) -curl -s -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -H "Travis-API-Version: 3" \ - -H "Authorization: token ${token}" \ - -d "${body}" \ - https://api.travis-ci.org/repo/springboot-angular2-tutorial%2Fmicropost-provisionings/requests diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index ea65a85..4a6d306 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -5,11 +5,11 @@ # If TF_VAR_ami_web is defined already, use it. # If TF_VAR_ami_web is not defined, get it from tfstate. TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} -if [ -n "${TF_VAR_ami_web}" ]; then - export TF_VAR_ami_web -else - exit +if [ -z "${TF_VAR_ami_web}" ]; then + echo "ami_web is empty" + exit 1 fi +export TF_VAR_ami_web terraform $@ \ -state="${ENV}.tfstate" \ diff --git a/variables.tf b/variables.tf index 403689c..5ae695e 100644 --- a/variables.tf +++ b/variables.tf @@ -9,7 +9,6 @@ variable "aws_region" { default = "ap-northeast-1" } variable "ami_web" { - default = "ami-a21529cc" } variable "aws_az_primary" { default = { @@ -30,10 +29,3 @@ variable "cidr" { office = "42.116.5.4/32" } } - -variable "desired_capacity_web" { - default = 1 -} -variable "max_size_web" { - default = 1 -} From 789e6c6eb2cacf38616a733ae2dc560195a13644 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 22:39:30 +0700 Subject: [PATCH 010/181] fix travisyml --- .travis.yml | 4 ++-- outputs.tf | 2 +- templates/web_init.sh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3cf32c9..a1c31aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,10 +25,10 @@ script: - scripts/terraform-enable-remote.sh # Save current output to compare later - - terraform output | grep endpoint | tee out_before + - terraform output | tee out_before # Execute Terraform and create tfstate on remote - - scripts/terraform-wrapper.sh plan || exit + - scripts/terraform-wrapper.sh plan - scripts/terraform-wrapper.sh apply # Deploy app when initial deployment diff --git a/outputs.tf b/outputs.tf index 5e99a1b..eb18267 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,5 +1,5 @@ output "ami_web" { - value = "${var.ami_web}" + value = "${aws_launch_configuration.web.image_id}" } output "rds_endpoint" { diff --git a/templates/web_init.sh b/templates/web_init.sh index 02a2741..dd7ea1f 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -13,5 +13,3 @@ ansible-playbook -i inventory --connection=local --sudo --diff \ site.yml /usr/local/bin/update_cert.sh >/var/log/update_cert 2>&1 - -export MY_TEST=hello From 9fc05b7a59bd080d7237d43e27a9a0a086738b7e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 22:56:51 +0700 Subject: [PATCH 011/181] fix travisyml --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a1c31aa..2050f5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ before_script: - export TF_VAR_aws_account_num=${AWS_ACCOUNT_NUMBER} - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: + - echo +after_success: # If branch name was matched with deploy/xxx, continue to apply. - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - if [ -z "${ENV}" ]; then exit ; fi @@ -24,7 +26,7 @@ script: # Enable Terraform Remote - scripts/terraform-enable-remote.sh - # Save current output to compare later + # Save current output to use it later - terraform output | tee out_before # Execute Terraform and create tfstate on remote From 9465c9438a1e4f9ab368364086f90a4c2d77cf5c Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 23:19:12 +0700 Subject: [PATCH 012/181] build.sh --- .travis.yml | 19 +------------------ scripts/build.sh | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 18 deletions(-) create mode 100755 scripts/build.sh diff --git a/.travis.yml b/.travis.yml index 2050f5a..8243433 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,21 +17,4 @@ before_script: - export TF_VAR_aws_account_num=${AWS_ACCOUNT_NUMBER} - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - - echo -after_success: - # If branch name was matched with deploy/xxx, continue to apply. - - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - - if [ -z "${ENV}" ]; then exit ; fi - - # Enable Terraform Remote - - scripts/terraform-enable-remote.sh - - # Save current output to use it later - - terraform output | tee out_before - - # Execute Terraform and create tfstate on remote - - scripts/terraform-wrapper.sh plan - - scripts/terraform-wrapper.sh apply - - # Deploy app when initial deployment - - if [ ! -s out_before ]; then scripts/build-app.sh ; fi + - scripts/build.sh diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..8d93614 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") +if [ -z "${ENV}" ]; then + echo "Branch name does not match with deploy/xxx. Exit now." + exit +fi + +# Enable Terraform Remote +scripts/terraform-enable-remote.sh + +# Save current output to use it later +terraform output | tee out_before + +# Execute Terraform and create tfstate on remote +scripts/terraform-wrapper.sh plan || exit +scripts/terraform-wrapper.sh apply + +# Deploy app when initial deployment +if [ ! -s out_before ]; then scripts/build-app.sh ; fi From 3f78ead9a8b5acdf3fba5df8ebe6cf4a3b4595a5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 12 Apr 2016 23:45:59 +0700 Subject: [PATCH 013/181] test --- scripts/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 8d93614..91796c5 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -13,7 +13,8 @@ scripts/terraform-enable-remote.sh terraform output | tee out_before # Execute Terraform and create tfstate on remote -scripts/terraform-wrapper.sh plan || exit +scripts/terraform-wrapper.sh plan || exit 0 +echo "test" scripts/terraform-wrapper.sh apply # Deploy app when initial deployment From a615209691d00f40c605972762c31c11b06850e5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 08:52:37 +0700 Subject: [PATCH 014/181] update build.sh --- scripts/build.sh | 28 ++++++++++++++++++++++------ scripts/terraform-wrapper.sh | 9 --------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 91796c5..734bc45 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -3,19 +3,35 @@ export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") if [ -z "${ENV}" ]; then echo "Branch name does not match with deploy/xxx. Exit now." - exit + exit 0 fi -# Enable Terraform Remote +echo "Enable Terraform remote." scripts/terraform-enable-remote.sh -# Save current output to use it later +echo "Save current output to use it later" terraform output | tee out_before -# Execute Terraform and create tfstate on remote -scripts/terraform-wrapper.sh plan || exit 0 -echo "test" +echo "Set ami_web." +TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} +if [ -z "${TF_VAR_ami_web}" ]; then + echo "ami_web is empty. Exit now." + exit 0 +fi +export TF_VAR_ami_web +echo "ami_web is ${TF_VAR_ami_web} ." + +scripts/terraform-wrapper.sh plan +if [ $? -ne 0 ]; then + echo "Plan failed." + exit 1 +fi + scripts/terraform-wrapper.sh apply +if [ $? -ne 0 ]; then + echo "Apply failed." + exit 1 +fi # Deploy app when initial deployment if [ ! -s out_before ]; then scripts/build-app.sh ; fi diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index 4a6d306..e4a4dd5 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -2,15 +2,6 @@ . $(dirname $0)/terraform-enable-remote.sh -# If TF_VAR_ami_web is defined already, use it. -# If TF_VAR_ami_web is not defined, get it from tfstate. -TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} -if [ -z "${TF_VAR_ami_web}" ]; then - echo "ami_web is empty" - exit 1 -fi -export TF_VAR_ami_web - terraform $@ \ -state="${ENV}.tfstate" \ -refresh=true \ From 2665a971a3309caf9deb4b08fc99a5957864f835 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 09:20:48 +0700 Subject: [PATCH 015/181] update user data shell --- autoscaling.tf | 3 +++ outputs.tf | 9 --------- templates/web_init.sh | 12 ++++++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 08a049f..bc45a58 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -2,6 +2,9 @@ resource "template_file" "web_init" { template = "${file("templates/web_init.sh")}" vars { logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" + rds_endpoint = "${aws_db_instance.micropost.endpoint}" + // I have only one node... + redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}:${aws_elasticache_cluster.micropost.cache_nodes.0.port}" web_endpoint = "${cloudflare_record.micropost.hostname}" } lifecycle { diff --git a/outputs.tf b/outputs.tf index eb18267..d11aac1 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,12 +1,3 @@ output "ami_web" { value = "${aws_launch_configuration.web.image_id}" } - -output "rds_endpoint" { - value = "${aws_db_instance.micropost.endpoint}" -} - -// I have only one node.. -output "redis_endpoint" { - value = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}:${aws_elasticache_cluster.micropost.cache_nodes.0.port}" -} diff --git a/templates/web_init.sh b/templates/web_init.sh index dd7ea1f..afd0278 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -1,5 +1,15 @@ #!/bin/sh +# Create env file, which will be read from an app. +ENV_FILE=/opt/micropost/env.sh +mkdir -p $(dirname ${ENV_FILE}) +cat << EOF > ${ENV_FILE} +export RDS_ENDPOINT=${rds_endpoint} +export REDIS_ENDPOINT=${redis_endpoint} +EOF + +# Finalize provisioning by using resolved endpoints. +( cd /opt/provisioning cat << EOF > inventory @@ -11,5 +21,7 @@ ansible-playbook -i inventory --connection=local --sudo --diff \ -e "logstash_elasticsearch_host=${logserver_endpoint}" \ -e "letsencrypt_host=${web_endpoint}" \ site.yml +) +# Update SSL cert /usr/local/bin/update_cert.sh >/var/log/update_cert 2>&1 From 02cec2850845ee7ddf896f1258ab29b4f7cc60f3 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 09:25:30 +0700 Subject: [PATCH 016/181] add spring profile active --- autoscaling.tf | 2 +- templates/web_init.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/autoscaling.tf b/autoscaling.tf index bc45a58..c3c222a 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -1,9 +1,9 @@ resource "template_file" "web_init" { template = "${file("templates/web_init.sh")}" vars { + env = "${var.env}" logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" rds_endpoint = "${aws_db_instance.micropost.endpoint}" - // I have only one node... redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}:${aws_elasticache_cluster.micropost.cache_nodes.0.port}" web_endpoint = "${cloudflare_record.micropost.hostname}" } diff --git a/templates/web_init.sh b/templates/web_init.sh index afd0278..0c17baf 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -4,6 +4,7 @@ ENV_FILE=/opt/micropost/env.sh mkdir -p $(dirname ${ENV_FILE}) cat << EOF > ${ENV_FILE} +export SPRING_PROFILES_ACTIVE=${env} export RDS_ENDPOINT=${rds_endpoint} export REDIS_ENDPOINT=${redis_endpoint} EOF From ea227e6087ca00c0b623519405edc85c3c8c1654 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 11:11:34 +0700 Subject: [PATCH 017/181] escape $ to avoid conflicting shell variable and terraform variable --- templates/web_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/web_init.sh b/templates/web_init.sh index 0c17baf..383b3b0 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -2,8 +2,8 @@ # Create env file, which will be read from an app. ENV_FILE=/opt/micropost/env.sh -mkdir -p $(dirname ${ENV_FILE}) -cat << EOF > ${ENV_FILE} +mkdir -p $(dirname $$ENV_FILE) +cat << EOF > $$ENV_FILE export SPRING_PROFILES_ACTIVE=${env} export RDS_ENDPOINT=${rds_endpoint} export REDIS_ENDPOINT=${redis_endpoint} From a24ccf5dd40a260b2e329b504fd26144c1c221d6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 11:22:13 +0700 Subject: [PATCH 018/181] fix build.sh --- scripts/build.sh | 4 ++-- scripts/terraform-wrapper.sh | 4 ++++ templates/web_init.sh | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 734bc45..f423c53 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,13 +23,13 @@ echo "ami_web is ${TF_VAR_ami_web} ." scripts/terraform-wrapper.sh plan if [ $? -ne 0 ]; then - echo "Plan failed." + echo "Plan was failed." exit 1 fi scripts/terraform-wrapper.sh apply if [ $? -ne 0 ]; then - echo "Apply failed." + echo "Apply was failed." exit 1 fi diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index e4a4dd5..aca0d4f 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -6,5 +6,9 @@ terraform $@ \ -state="${ENV}.tfstate" \ -refresh=true \ -var "env=${ENV}" +status=$? +# Need to push state, even if apply was failed. terraform remote push + +exit status diff --git a/templates/web_init.sh b/templates/web_init.sh index 383b3b0..f384194 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -2,8 +2,8 @@ # Create env file, which will be read from an app. ENV_FILE=/opt/micropost/env.sh -mkdir -p $(dirname $$ENV_FILE) -cat << EOF > $$ENV_FILE +mkdir -p $(dirname $${ENV_FILE}) +cat << EOF > $${ENV_FILE} export SPRING_PROFILES_ACTIVE=${env} export RDS_ENDPOINT=${rds_endpoint} export REDIS_ENDPOINT=${redis_endpoint} From 83df00d078aec64869819aca2e2ce119ef982afe Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 11:55:31 +0700 Subject: [PATCH 019/181] fix script --- scripts/terraform-wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index aca0d4f..b3eb9e2 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -11,4 +11,4 @@ status=$? # Need to push state, even if apply was failed. terraform remote push -exit status +exit ${status} From ce96a23bcefa2ff752a16dc49245dd6a613bc018 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 14:07:02 +0700 Subject: [PATCH 020/181] remove unused amis --- .travis.yml | 5 ++- Gemfile | 5 +++ Gemfile.lock | 23 +++++++++++++ scripts/remove_unused_and_rotated_amis.rb | 40 +++++++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 scripts/remove_unused_and_rotated_amis.rb diff --git a/.travis.yml b/.travis.yml index 8243433..b108f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,9 @@ cache: directories: - vendor/bundle -before_install: +install: - gem install travis -v 1.8.2 --no-rdoc --no-ri + - bundle install - wget https://releases.hashicorp.com/terraform/0.6.14/terraform_0.6.14_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip @@ -18,3 +19,5 @@ before_script: - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - scripts/build.sh +after_success: + - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..0cc16a1 --- /dev/null +++ b/Gemfile @@ -0,0 +1,5 @@ +ruby '2.2.3' +source 'http://rubygems.org' + +gem 'aws-sdk' +gem 'awesome_print' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..21cbc48 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,23 @@ +GEM + remote: http://rubygems.org/ + specs: + awesome_print (1.6.1) + aws-sdk (2.2.34) + aws-sdk-resources (= 2.2.34) + aws-sdk-core (2.2.34) + jmespath (~> 1.0) + aws-sdk-resources (2.2.34) + aws-sdk-core (= 2.2.34) + jmespath (1.2.4) + json_pure (>= 1.8.1) + json_pure (1.8.3) + +PLATFORMS + ruby + +DEPENDENCIES + awesome_print + aws-sdk + +BUNDLED WITH + 1.11.2 diff --git a/scripts/remove_unused_and_rotated_amis.rb b/scripts/remove_unused_and_rotated_amis.rb new file mode 100644 index 0000000..513ddef --- /dev/null +++ b/scripts/remove_unused_and_rotated_amis.rb @@ -0,0 +1,40 @@ +require 'aws-sdk' +require 'awesome_print' + +def rotated_images + ec2 = Aws::EC2::Client.new + ec2.describe_images(filters: [ + { + name: 'tag:Rotated', + values: ['true'], + }, + ]).images.map() +end + +def used_image_ids + as = Aws::AutoScaling::Client.new + as.describe_launch_configurations + .launch_configurations + .map(&:image_id) +end + +def unused_rotated_images + rotated_images.reject { |i| used_image_ids.include?(i.image_id) } +end + +images = unused_rotated_images + +puts 'Deregistering amis.' +images.map(&:image_id) + .map { |id| Aws::EC2::Image.new(id) } + .tap { |img| ap img } + .each { |img| img.deregister rescue nil } + +puts 'Removing snapshots related with de-registered ami.' +images.map { |i| i.block_device_mappings.map(&:ebs) } + .flatten + .compact + .map(&:snapshot_id) + .map { |id| Aws::EC2::Snapshot.new(id) } + .tap { |s| ap s } + .each { |s| s.delete rescue nil } From 0136a0d020fc565c9e328dba52f9fdf16ac92efd Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 14:10:51 +0700 Subject: [PATCH 021/181] optimize log --- scripts/remove_unused_and_rotated_amis.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/remove_unused_and_rotated_amis.rb b/scripts/remove_unused_and_rotated_amis.rb index 513ddef..f28ba1b 100644 --- a/scripts/remove_unused_and_rotated_amis.rb +++ b/scripts/remove_unused_and_rotated_amis.rb @@ -26,8 +26,8 @@ def unused_rotated_images puts 'Deregistering amis.' images.map(&:image_id) + .tap { |id| puts id } .map { |id| Aws::EC2::Image.new(id) } - .tap { |img| ap img } .each { |img| img.deregister rescue nil } puts 'Removing snapshots related with de-registered ami.' @@ -35,6 +35,6 @@ def unused_rotated_images .flatten .compact .map(&:snapshot_id) + .tap { |s_id| puts s_id } .map { |id| Aws::EC2::Snapshot.new(id) } - .tap { |s| ap s } .each { |s| s.delete rescue nil } From bfd0452776fc314bb426abc020f55a6a5ffd49af Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 17:16:58 +0700 Subject: [PATCH 022/181] desired capacity --- autoscaling.tf | 2 +- outputs.tf | 4 ++++ variables.tf | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index c3c222a..8040ea1 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -35,7 +35,7 @@ resource "aws_autoscaling_group" "web" { min_size = 1 health_check_grace_period = 300 health_check_type = "ELB" - desired_capacity = 1 + desired_capacity = "${var.web_desired_capacity}" force_delete = true launch_configuration = "${aws_launch_configuration.web.id}" vpc_zone_identifier = [ diff --git a/outputs.tf b/outputs.tf index d11aac1..711d467 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,3 +1,7 @@ output "ami_web" { value = "${aws_launch_configuration.web.image_id}" } + +output "web_desired_capacity" { + value = "${aws_autoscaling_group.web.desired_capacity}" +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 5ae695e..2fd81ba 100644 --- a/variables.tf +++ b/variables.tf @@ -2,14 +2,17 @@ variable "env" { description = "dev, stg, prod and etc." default = "dev" } +variable "cidr" { + type = "map" + default = { + office = "42.116.5.4/32" + } +} variable "aws_account_num" { } - variable "aws_region" { default = "ap-northeast-1" } -variable "ami_web" { -} variable "aws_az_primary" { default = { ap-southeast-1 = "ap-southeast-1a" @@ -23,9 +26,8 @@ variable "aws_az_secondary" { } } -variable "cidr" { - type = "map" - default = { - office = "42.116.5.4/32" - } +variable "ami_web" { +} +variable "web_desired_capacity" { + default = 1 } From 4c2b3eb767d2f1d04749857fdf4e5ca033b68f89 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 18:24:07 +0700 Subject: [PATCH 023/181] travisyml --- .travis.yml | 23 +++++++++++++++++++++- scripts/build.sh | 37 ------------------------------------ scripts/export-tfvars.sh | 14 ++++++++++++++ scripts/terraform-wrapper.sh | 1 + 4 files changed, 37 insertions(+), 38 deletions(-) delete mode 100755 scripts/build.sh create mode 100644 scripts/export-tfvars.sh diff --git a/.travis.yml b/.travis.yml index b108f55..2a18a01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,27 @@ before_script: - export TF_VAR_aws_account_num=${AWS_ACCOUNT_NUMBER} - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - - scripts/build.sh + - echo after_success: + # If branch name matched with deploy/xxx, then continue. + - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") + - test -z "${ENV}" && exit 0 + + # Enable terraform remote + - scripts/terraform-enable-remote.sh + + # Use current output later + - terraform output | tee out_before + + # If tfvars are not enough, exit + - scripts/export-tfvars.sh || exit 0 + + # Plan and apply + - scripts/terraform-wrapper.sh plan || exit 1 + - scripts/terraform-wrapper.sh apply || exit 1 + + # Deploy app when initial deployment + - test -s out_before || scripts/build-app.sh + + # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index f423c53..0000000 --- a/scripts/build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") -if [ -z "${ENV}" ]; then - echo "Branch name does not match with deploy/xxx. Exit now." - exit 0 -fi - -echo "Enable Terraform remote." -scripts/terraform-enable-remote.sh - -echo "Save current output to use it later" -terraform output | tee out_before - -echo "Set ami_web." -TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} -if [ -z "${TF_VAR_ami_web}" ]; then - echo "ami_web is empty. Exit now." - exit 0 -fi -export TF_VAR_ami_web -echo "ami_web is ${TF_VAR_ami_web} ." - -scripts/terraform-wrapper.sh plan -if [ $? -ne 0 ]; then - echo "Plan was failed." - exit 1 -fi - -scripts/terraform-wrapper.sh apply -if [ $? -ne 0 ]; then - echo "Apply was failed." - exit 1 -fi - -# Deploy app when initial deployment -if [ ! -s out_before ]; then scripts/build-app.sh ; fi diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh new file mode 100644 index 0000000..e63d57b --- /dev/null +++ b/scripts/export-tfvars.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} +if [ -n "${TF_VAR_ami_web}" ]; then + export TF_VAR_ami_web +else + echo "ami_web is required. Exit now." + exit 1 +fi + +TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) +if [ -n "${TF_VAR_web_desired_capacity}" ]; then + export TF_VAR_web_desired_capacity +fi diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index b3eb9e2..705c535 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -1,6 +1,7 @@ #!/bin/sh . $(dirname $0)/terraform-enable-remote.sh +. $(dirname $0)/export-tfvars.sh terraform $@ \ -state="${ENV}.tfstate" \ From 2efc34b84934ca26a81aa1187921792556c2a905 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 18:27:29 +0700 Subject: [PATCH 024/181] chmod --- scripts/export-tfvars.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/export-tfvars.sh diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh old mode 100644 new mode 100755 From 363136d522ec3bb4b7dcad7900bc26d7d0d1c46c Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 19:27:18 +0700 Subject: [PATCH 025/181] replace old instances --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2a18a01..a2e2ef7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,3 +42,11 @@ after_success: # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb + + # If ami_web was updated, scale out and in to replace instances. + - lines=$(cat <(terraform output) out_before | grep "^ami_web" | uniq | wc -l) + - capacity=$(terraform output web_desired_capacity) + - > + test $lines -eq 2 \ + && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 2))" \ + && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 1))" From 0bed837fca5e081f2829b61aa564437bc3f01aea Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 20:00:23 +0700 Subject: [PATCH 026/181] travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2e2ef7..ffe395f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,6 @@ after_success: - lines=$(cat <(terraform output) out_before | grep "^ami_web" | uniq | wc -l) - capacity=$(terraform output web_desired_capacity) - > - test $lines -eq 2 \ - && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 2))" \ + test $lines -eq 2 + && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 2))" && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 1))" From d33063a5aec0ab2a70c40fa24985443d8d59f340 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 13 Apr 2016 21:32:56 +0700 Subject: [PATCH 027/181] blue green --- .travis.yml | 10 +--------- autoscaling.tf | 11 ++++++++--- output_before.txt | 0 3 files changed, 9 insertions(+), 12 deletions(-) delete mode 100644 output_before.txt diff --git a/.travis.yml b/.travis.yml index ffe395f..29bd546 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ after_success: # Enable terraform remote - scripts/terraform-enable-remote.sh - # Use current output later + # Save current output to use it later - terraform output | tee out_before # If tfvars are not enough, exit @@ -42,11 +42,3 @@ after_success: # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb - - # If ami_web was updated, scale out and in to replace instances. - - lines=$(cat <(terraform output) out_before | grep "^ami_web" | uniq | wc -l) - - capacity=$(terraform output web_desired_capacity) - - > - test $lines -eq 2 - && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 2))" - && scripts/terraform-wrapper.sh apply -var "web_desired_capacity=$((capacity * 1))" diff --git a/autoscaling.tf b/autoscaling.tf index 8040ea1..9db606b 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -30,19 +30,24 @@ resource "aws_launch_configuration" "web" { } resource "aws_autoscaling_group" "web" { - name = "web-${var.env}" + # https://groups.google.com/forum/m/#!msg/terraform-tool/7Gdhv1OAc80/iNQ93riiLwAJ + name = "web-${var.env}-${aws_launch_configuration.web.name}" + launch_configuration = "${aws_launch_configuration.web.id}" max_size = 4 min_size = 1 + desired_capacity = "${var.web_desired_capacity}" + min_elb_capacity ="${var.web_desired_capacity}" health_check_grace_period = 300 health_check_type = "ELB" - desired_capacity = "${var.web_desired_capacity}" force_delete = true - launch_configuration = "${aws_launch_configuration.web.id}" vpc_zone_identifier = [ "${aws_subnet.public_primary.id}", "${aws_subnet.public_secondary.id}", ] load_balancers = ["${aws_elb.web.name}"] + lifecycle { + create_before_destroy = true + } } resource "aws_autoscaling_policy" "web-scale-out" { diff --git a/output_before.txt b/output_before.txt deleted file mode 100644 index e69de29..0000000 From 9b65320e7a9d5886c7fe521b055ab91fddddf812 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 10:43:48 +0700 Subject: [PATCH 028/181] update autoscaling --- autoscaling.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 9db606b..e5de599 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -31,12 +31,12 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { # https://groups.google.com/forum/m/#!msg/terraform-tool/7Gdhv1OAc80/iNQ93riiLwAJ - name = "web-${var.env}-${aws_launch_configuration.web.name}" + name = "${aws_launch_configuration.web.name}" launch_configuration = "${aws_launch_configuration.web.id}" max_size = 4 min_size = 1 desired_capacity = "${var.web_desired_capacity}" - min_elb_capacity ="${var.web_desired_capacity}" + min_elb_capacity = 0 health_check_grace_period = 300 health_check_type = "ELB" force_delete = true From 41958ec44c4ca2b48e8d7a249a7ef53f086ef56d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 11:26:07 +0700 Subject: [PATCH 029/181] revert https://groups.google.com/forum/m/#!msg/terraform-tool/7Gdhv1OAc80/iNQ93riiLwAJ --- autoscaling.tf | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index e5de599..df021c5 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -30,13 +30,11 @@ resource "aws_launch_configuration" "web" { } resource "aws_autoscaling_group" "web" { - # https://groups.google.com/forum/m/#!msg/terraform-tool/7Gdhv1OAc80/iNQ93riiLwAJ - name = "${aws_launch_configuration.web.name}" + name = "web-${var.env}" launch_configuration = "${aws_launch_configuration.web.id}" max_size = 4 min_size = 1 desired_capacity = "${var.web_desired_capacity}" - min_elb_capacity = 0 health_check_grace_period = 300 health_check_type = "ELB" force_delete = true @@ -45,9 +43,6 @@ resource "aws_autoscaling_group" "web" { "${aws_subnet.public_secondary.id}", ] load_balancers = ["${aws_elb.web.name}"] - lifecycle { - create_before_destroy = true - } } resource "aws_autoscaling_policy" "web-scale-out" { From 5c015396985f3d42b204e9852774491baac25c24 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 17:42:14 +0700 Subject: [PATCH 030/181] replace old instances when ami_web was changed. --- .travis.yml | 6 +++++ outputs.tf | 6 ++++- scripts/replace_old_instances.rb | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 scripts/replace_old_instances.rb diff --git a/.travis.yml b/.travis.yml index 29bd546..505f6c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,3 +42,9 @@ after_success: # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb + + # Replace instances if ami_web was updated. + - terraform output | tee out_after + - lines=$(cat out_before out_after | grep 'ami_web' | uniq | wc -l) + - test $lines -eq 2 && bundle exec ruby replace_old_instances.rb $(terraform output web_autoscaling_group_name) + diff --git a/outputs.tf b/outputs.tf index 711d467..c27edc0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -4,4 +4,8 @@ output "ami_web" { output "web_desired_capacity" { value = "${aws_autoscaling_group.web.desired_capacity}" -} \ No newline at end of file +} + +output "web_autoscaling_group_name" { + value = "${aws_autoscaling_group.web.name}" +} diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb new file mode 100644 index 0000000..f82de0f --- /dev/null +++ b/scripts/replace_old_instances.rb @@ -0,0 +1,41 @@ +require 'aws-sdk' +require 'awesome_print' + +unless ARGV[0] + puts 'Specify auto scaling group name.' + exit 1 +end +ASG_NAME = ARGV[0] + +as = Aws::AutoScaling::Client.new +asg = Aws::AutoScaling::AutoScalingGroup.new(ASG_NAME, client: as) + +old_instances = asg.instances + +if asg.desired_capacity * 2 > asg.max_size + puts "Can not scale out to replace instances in #{ASG_NAME}." + puts 'Consider increasing the max size of it.' + exit 1 +end + +asg.set_desired_capacity( + desired_capacity: asg.desired_capacity * 2, + honor_cooldown: true, +) + +is_all_in_service = lambda do + asg.reload + in_services = asg.instances.select { |i| i.lifecycle_state == 'InService' } + in_services.size >= asg.desired_capacity +end + +puts 'Scaling out to replace old instances.' +until is_all_in_service.call + print '.' + sleep 5 +end +puts '' + +old_instances.each do |i| + i.terminate(should_decrement_desired_capacity: true) +end From 3bd9c152d9bc2b1965d7505d0991f448d7051d52 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 17:44:32 +0700 Subject: [PATCH 031/181] more output --- scripts/replace_old_instances.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb index f82de0f..fd00a40 100644 --- a/scripts/replace_old_instances.rb +++ b/scripts/replace_old_instances.rb @@ -35,7 +35,10 @@ sleep 5 end puts '' +puts 'Completed to scale out.' +puts 'Terminating old instances.' old_instances.each do |i| i.terminate(should_decrement_desired_capacity: true) end +puts 'Done.' From e17a3fdb810b7bccaab21c3bd78556999e9424d7 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 18:16:29 +0700 Subject: [PATCH 032/181] fix load ruby script --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 505f6c7..861111a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,5 +46,6 @@ after_success: # Replace instances if ami_web was updated. - terraform output | tee out_after - lines=$(cat out_before out_after | grep 'ami_web' | uniq | wc -l) - - test $lines -eq 2 && bundle exec ruby replace_old_instances.rb $(terraform output web_autoscaling_group_name) + - asg_name=$(terraform output web_autoscaling_group_name) + - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} From e4793352f9276585739b20ab11003d9efaabeba1 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 21:04:48 +0700 Subject: [PATCH 033/181] update resource name --- autoscaling.tf | 12 ++++++------ codedeploy.tf | 2 +- elasticsearch.tf | 3 ++- iam.tf | 12 ++++++------ security_group.tf | 2 +- variables.tf | 2 +- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index df021c5..cb073c4 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -45,7 +45,7 @@ resource "aws_autoscaling_group" "web" { load_balancers = ["${aws_elb.web.name}"] } -resource "aws_autoscaling_policy" "web-scale-out" { +resource "aws_autoscaling_policy" "web_scale_out" { name = "Instance-ScaleOut-CPU-High" scaling_adjustment = 1 adjustment_type = "ChangeInCapacity" @@ -53,7 +53,7 @@ resource "aws_autoscaling_policy" "web-scale-out" { autoscaling_group_name = "${aws_autoscaling_group.web.name}" } -resource "aws_cloudwatch_metric_alarm" "web-gte-threshold" { +resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { alarm_name = "web-${var.env}-CPU-Utilization-High-30" comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = "1" @@ -65,10 +65,10 @@ resource "aws_cloudwatch_metric_alarm" "web-gte-threshold" { dimensions { AutoScalingGroupName = "${aws_autoscaling_group.web.name}" } - alarm_actions = ["${aws_autoscaling_policy.web-scale-out.arn}"] + alarm_actions = ["${aws_autoscaling_policy.web_scale_out.arn}"] } -resource "aws_autoscaling_policy" "web-scale-in" { +resource "aws_autoscaling_policy" "web_scale_in" { name = "Instance-ScaleIn-CPU-Low" scaling_adjustment = -1 adjustment_type = "ChangeInCapacity" @@ -76,7 +76,7 @@ resource "aws_autoscaling_policy" "web-scale-in" { autoscaling_group_name = "${aws_autoscaling_group.web.name}" } -resource "aws_cloudwatch_metric_alarm" "web-lt-threshold" { +resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { alarm_name = "web-${var.env}-CPU-Utilization-Low-5" comparison_operator = "LessThanThreshold" evaluation_periods = "1" @@ -88,5 +88,5 @@ resource "aws_cloudwatch_metric_alarm" "web-lt-threshold" { dimensions { AutoScalingGroupName = "${aws_autoscaling_group.web.name}" } - alarm_actions = ["${aws_autoscaling_policy.web-scale-in.arn}"] + alarm_actions = ["${aws_autoscaling_policy.web_scale_in.arn}"] } diff --git a/codedeploy.tf b/codedeploy.tf index 30138a7..9527650 100644 --- a/codedeploy.tf +++ b/codedeploy.tf @@ -5,7 +5,7 @@ resource "aws_codedeploy_app" "micropost" { resource "aws_codedeploy_deployment_group" "web" { app_name = "${aws_codedeploy_app.micropost.name}" deployment_group_name = "web" - service_role_arn = "${aws_iam_role.codedeploy-service.arn}" + service_role_arn = "${aws_iam_role.codedeploy_service.arn}" autoscaling_groups = ["${aws_autoscaling_group.web.id}"] deployment_config_name = "CodeDeployDefault.OneAtATime" } diff --git a/elasticsearch.tf b/elasticsearch.tf index 38b87f4..fde5520 100644 --- a/elasticsearch.tf +++ b/elasticsearch.tf @@ -18,13 +18,14 @@ resource "aws_elasticsearch_domain" "logserver" { "Effect": "Allow", "Condition": { "IpAddress": { - "aws:SourceIp": "${var.cidr.office}" + "aws:SourceIp": "${var.segment.office}" } }, "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_num}:domain/micropost-${var.env}/*" } ] } + CONFIG snapshot_options { automated_snapshot_start_hour = 23 diff --git a/iam.tf b/iam.tf index 68678d7..17d7a5a 100644 --- a/iam.tf +++ b/iam.tf @@ -24,7 +24,7 @@ resource "aws_iam_role" "web" { EOF } -resource "aws_iam_role_policy" "es-client" { +resource "aws_iam_role_policy" "es_client" { name = "es-client" role = "${aws_iam_role.web.id}" policy = < Date: Thu, 14 Apr 2016 21:10:04 +0700 Subject: [PATCH 034/181] refactor reading input --- .travis.yml | 5 +++-- scripts/export-tfvars.sh | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 861111a..60f3893 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,9 @@ after_success: # Save current output to use it later - terraform output | tee out_before - # If tfvars are not enough, exit - - scripts/export-tfvars.sh || exit 0 + # If TF_VAR_ami_web is not defined, exit. + - scripts/export-tfvars.sh + - test -z "${TF_VAR_ami_web}" && exit 0 # Plan and apply - scripts/terraform-wrapper.sh plan || exit 1 diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index e63d57b..a71a0d2 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -1,11 +1,10 @@ #!/bin/sh +# Take over current value, if not new value was specified. + TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} if [ -n "${TF_VAR_ami_web}" ]; then export TF_VAR_ami_web -else - echo "ami_web is required. Exit now." - exit 1 fi TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) From 9cc5c067415c351926de480800138a020d8cda9a Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 21:12:39 +0700 Subject: [PATCH 035/181] ami_web -> web_ami --- .travis.yml | 8 ++++---- autoscaling.tf | 2 +- outputs.tf | 2 +- scripts/export-tfvars.sh | 6 +++--- variables.tf | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60f3893..af813f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,9 +30,9 @@ after_success: # Save current output to use it later - terraform output | tee out_before - # If TF_VAR_ami_web is not defined, exit. + # If TF_VAR_web_ami is not defined, exit. - scripts/export-tfvars.sh - - test -z "${TF_VAR_ami_web}" && exit 0 + - test -z "${TF_VAR_web_ami}" && exit 0 # Plan and apply - scripts/terraform-wrapper.sh plan || exit 1 @@ -44,9 +44,9 @@ after_success: # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb - # Replace instances if ami_web was updated. + # Replace instances if web_ami was updated. - terraform output | tee out_after - - lines=$(cat out_before out_after | grep 'ami_web' | uniq | wc -l) + - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(terraform output web_autoscaling_group_name) - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} diff --git a/autoscaling.tf b/autoscaling.tf index cb073c4..ef8962b 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -14,7 +14,7 @@ resource "template_file" "web_init" { resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" - image_id = "${var.ami_web}" + image_id = "${var.web_ami}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", diff --git a/outputs.tf b/outputs.tf index c27edc0..4c8fdbd 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,4 +1,4 @@ -output "ami_web" { +output "web_ami" { value = "${aws_launch_configuration.web.image_id}" } diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index a71a0d2..5858e48 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -2,9 +2,9 @@ # Take over current value, if not new value was specified. -TF_VAR_ami_web=${TF_VAR_ami_web:=$(terraform output ami_web)} -if [ -n "${TF_VAR_ami_web}" ]; then - export TF_VAR_ami_web +TF_VAR_web_ami=${TF_VAR_web_ami:=$(terraform output web_ami)} +if [ -n "${TF_VAR_web_ami}" ]; then + export TF_VAR_web_ami fi TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) diff --git a/variables.tf b/variables.tf index 65fa3de..b392832 100644 --- a/variables.tf +++ b/variables.tf @@ -26,7 +26,7 @@ variable "aws_az_secondary" { } } -variable "ami_web" { +variable "web_ami" { } variable "web_desired_capacity" { default = 1 From 5704dfa746734eb98c5aa38b87d1049fe1930d11 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 14 Apr 2016 22:52:25 +0700 Subject: [PATCH 036/181] separate variales --- constants.tf | 18 ++++++++++++++++++ variables.tf | 22 ++++------------------ 2 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 constants.tf diff --git a/constants.tf b/constants.tf new file mode 100644 index 0000000..9c9a732 --- /dev/null +++ b/constants.tf @@ -0,0 +1,18 @@ +variable "segment" { + default = { + office = "42.116.5.4/32" + } +} + +variable "aws_az_primary" { + default = { + ap-southeast-1 = "ap-southeast-1a" + ap-northeast-1 = "ap-northeast-1a" + } +} +variable "aws_az_secondary" { + default = { + ap-southeast-1 = "ap-southeast-1b" + ap-northeast-1 = "ap-northeast-1c" + } +} diff --git a/variables.tf b/variables.tf index b392832..f6cae54 100644 --- a/variables.tf +++ b/variables.tf @@ -1,30 +1,16 @@ +// stateless variables + variable "env" { description = "dev, stg, prod and etc." default = "dev" } -variable "segment" { - type = "map" - default = { - office = "42.116.5.4/32" - } -} variable "aws_account_num" { } variable "aws_region" { default = "ap-northeast-1" } -variable "aws_az_primary" { - default = { - ap-southeast-1 = "ap-southeast-1a" - ap-northeast-1 = "ap-northeast-1a" - } -} -variable "aws_az_secondary" { - default = { - ap-southeast-1 = "ap-southeast-1b" - ap-northeast-1 = "ap-northeast-1c" - } -} + +// stateful variables variable "web_ami" { } From da06e682b63ca0dc7c4f2a96105c03c5ca2f2d25 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 15 Apr 2016 15:23:19 +0700 Subject: [PATCH 037/181] redis: not include port in endpoint --- autoscaling.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoscaling.tf b/autoscaling.tf index ef8962b..eab9275 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -4,7 +4,7 @@ resource "template_file" "web_init" { env = "${var.env}" logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" rds_endpoint = "${aws_db_instance.micropost.endpoint}" - redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}:${aws_elasticache_cluster.micropost.cache_nodes.0.port}" + redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" web_endpoint = "${cloudflare_record.micropost.hostname}" } lifecycle { From f1f53f446d5b3b240143ea13f126af54f23d2764 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 15 Apr 2016 15:34:54 +0700 Subject: [PATCH 038/181] fix export var --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index af813f4..930d025 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ after_success: - terraform output | tee out_before # If TF_VAR_web_ami is not defined, exit. - - scripts/export-tfvars.sh + - . scripts/export-tfvars.sh - test -z "${TF_VAR_web_ami}" && exit 0 # Plan and apply From 274125b2a86b41cfbc3a8c33c619b50b704d6abc Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 15 Apr 2016 22:48:12 +0700 Subject: [PATCH 039/181] replace test app --- scripts/build-app.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-app.sh b/scripts/build-app.sh index d6725fc..5984e1f 100755 --- a/scripts/build-app.sh +++ b/scripts/build-app.sh @@ -16,4 +16,4 @@ curl -s -X POST \ -H "Travis-API-Version: 3" \ -H "Authorization: token ${token}" \ -d "${body}" \ - https://api.travis-ci.org/repo/akirasosa%2Ftestapp1/requests + https://api.travis-ci.org/repo/springboot-angular2-tutorial%2Fboot-app/requests From ef6d40d913d8d113e7ca1df57ac2367b7b07b820 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 16 Apr 2016 16:46:13 +0700 Subject: [PATCH 040/181] domain name for prod is micropost --- dns.tf | 2 +- scripts/export-tfvars.sh | 7 +++++++ variables.tf | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dns.tf b/dns.tf index 4635e11..ad86b70 100644 --- a/dns.tf +++ b/dns.tf @@ -1,6 +1,6 @@ resource "cloudflare_record" "micropost" { domain = "hana053.com" - name = "micropost-${var.env}" + name = "${var.web_host_name}" value = "${aws_elb.web.dns_name}" type = "CNAME" } \ No newline at end of file diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 5858e48..044e83f 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -11,3 +11,10 @@ TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) if [ -n "${TF_VAR_web_desired_capacity}" ]; then export TF_VAR_web_desired_capacity fi + +if [ "${ENV}" == "prod" ]; then + # prod is special + export TF_VAR_web_host_name="micropost" +else + export TF_VAR_web_host_name="micropost-${ENV}" +fi diff --git a/variables.tf b/variables.tf index f6cae54..e14bf49 100644 --- a/variables.tf +++ b/variables.tf @@ -9,6 +9,8 @@ variable "aws_account_num" { variable "aws_region" { default = "ap-northeast-1" } +variable "web_host_name" { +} // stateful variables From 8646969b93644ad4dc90ed34092b6c5405e153e1 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 16 Apr 2016 18:03:13 +0700 Subject: [PATCH 041/181] fix export var --- scripts/export-tfvars.sh | 2 +- security_group.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 044e83f..7187600 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -12,7 +12,7 @@ if [ -n "${TF_VAR_web_desired_capacity}" ]; then export TF_VAR_web_desired_capacity fi -if [ "${ENV}" == "prod" ]; then +if [ "${ENV}" = "prod" ]; then # prod is special export TF_VAR_web_host_name="micropost" else diff --git a/security_group.tf b/security_group.tf index 093b921..2c11d96 100644 --- a/security_group.tf +++ b/security_group.tf @@ -35,6 +35,7 @@ resource "aws_security_group" "ssh" { protocol = "tcp" cidr_blocks = [ "${var.segment.office}", + "115.74.36.134/32", ] } egress { From 5fc1d97522f6dc574a04c139134d29666e8453f5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 28 Jul 2016 14:58:52 +0700 Subject: [PATCH 042/181] user_data does not allow line break? --- Gemfile.lock | 2 +- templates/web_init.sh | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 21cbc48..61fa8ab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -20,4 +20,4 @@ DEPENDENCIES aws-sdk BUNDLED WITH - 1.11.2 + 1.12.0.rc diff --git a/templates/web_init.sh b/templates/web_init.sh index f384194..981e1bc 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -18,10 +18,7 @@ cat << EOF > inventory localhost EOF -ansible-playbook -i inventory --connection=local --sudo --diff \ - -e "logstash_elasticsearch_host=${logserver_endpoint}" \ - -e "letsencrypt_host=${web_endpoint}" \ - site.yml +ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" site.yml ) # Update SSL cert From 558f4b431572a72c8d2f69583ab6d729b027dbe3 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 5 Aug 2016 14:17:04 +0700 Subject: [PATCH 043/181] terraform 0.7.0 --- .gitignore | 1 + .travis.yml | 11 ++--------- Gemfile | 2 +- Gemfile.lock | 3 +++ autoscaling.tf | 21 ++++++++++++++------- elasticsearch.tf | 2 +- outputs.tf | 2 +- providers.tf | 5 ----- scripts/export-tfvars.sh | 6 ------ scripts/terraform-enable-remote.sh | 2 ++ security_group.tf | 2 +- variables.tf | 2 -- 12 files changed, 26 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index c482008..d22de4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .envrc +.bundle # IDE junk .idea/* diff --git a/.travis.yml b/.travis.yml index 930d025..cfa1041 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: required dist: trusty language: ruby rvm: - - 2.2.3 + - 2.3.0 cache: directories: - vendor/bundle @@ -22,7 +22,7 @@ script: after_success: # If branch name matched with deploy/xxx, then continue. - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - - test -z "${ENV}" && exit 0 + - test -z "${ENV}" && (echo "${TRAVIS_BRANCH} is not a branch to deploy." && exit 0) # Enable terraform remote - scripts/terraform-enable-remote.sh @@ -30,17 +30,10 @@ after_success: # Save current output to use it later - terraform output | tee out_before - # If TF_VAR_web_ami is not defined, exit. - - . scripts/export-tfvars.sh - - test -z "${TF_VAR_web_ami}" && exit 0 - # Plan and apply - scripts/terraform-wrapper.sh plan || exit 1 - scripts/terraform-wrapper.sh apply || exit 1 - # Deploy app when initial deployment - - test -s out_before || scripts/build-app.sh - # Remove unused amis - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb diff --git a/Gemfile b/Gemfile index 0cc16a1..8b0ad27 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,4 @@ -ruby '2.2.3' +ruby '2.3.0' source 'http://rubygems.org' gem 'aws-sdk' diff --git a/Gemfile.lock b/Gemfile.lock index 61fa8ab..333478f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,5 +19,8 @@ DEPENDENCIES awesome_print aws-sdk +RUBY VERSION + ruby 2.3.0p0 + BUNDLED WITH 1.12.0.rc diff --git a/autoscaling.tf b/autoscaling.tf index eab9275..83cfad0 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -1,4 +1,4 @@ -resource "template_file" "web_init" { +data "template_file" "web_init" { template = "${file("templates/web_init.sh")}" vars { env = "${var.env}" @@ -7,14 +7,21 @@ resource "template_file" "web_init" { redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" web_endpoint = "${cloudflare_record.micropost.hostname}" } - lifecycle { - create_before_destroy = true +} + +data "aws_ami" "micropost_web" { + most_recent = true + owners = ["self"] + + filter { + name = "tag:Name" + values = ["micropost-web"] } } resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" - image_id = "${var.web_ami}" + image_id = "${data.aws_ami.micropost_web.id}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", @@ -23,7 +30,7 @@ resource "aws_launch_configuration" "web" { key_name = "id_rsa" associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" - user_data = "${template_file.web_init.rendered}" + user_data = "${data.template_file.web_init.rendered}" lifecycle { create_before_destroy = true } @@ -32,8 +39,8 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web-${var.env}" launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 4 - min_size = 1 + max_size = 0 + min_size = 0 desired_capacity = "${var.web_desired_capacity}" health_check_grace_period = 300 health_check_type = "ELB" diff --git a/elasticsearch.tf b/elasticsearch.tf index fde5520..caf0278 100644 --- a/elasticsearch.tf +++ b/elasticsearch.tf @@ -18,7 +18,7 @@ resource "aws_elasticsearch_domain" "logserver" { "Effect": "Allow", "Condition": { "IpAddress": { - "aws:SourceIp": "${var.segment.office}" + "aws:SourceIp": "${var.segment["office"]}" } }, "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_num}:domain/micropost-${var.env}/*" diff --git a/outputs.tf b/outputs.tf index 4c8fdbd..06e2664 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,5 +1,5 @@ output "web_ami" { - value = "${aws_launch_configuration.web.image_id}" + value = "${data.aws_ami.micropost_web.id}" } output "web_desired_capacity" { diff --git a/providers.tf b/providers.tf index ae660c8..b54eb94 100644 --- a/providers.tf +++ b/providers.tf @@ -1,8 +1,3 @@ provider "aws" { region = "${var.aws_region}" } - -provider "cloudflare" { - email = "" - token = "" -} \ No newline at end of file diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 7187600..80119c0 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -1,12 +1,6 @@ #!/bin/sh # Take over current value, if not new value was specified. - -TF_VAR_web_ami=${TF_VAR_web_ami:=$(terraform output web_ami)} -if [ -n "${TF_VAR_web_ami}" ]; then - export TF_VAR_web_ami -fi - TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) if [ -n "${TF_VAR_web_desired_capacity}" ]; then export TF_VAR_web_desired_capacity diff --git a/scripts/terraform-enable-remote.sh b/scripts/terraform-enable-remote.sh index 1d3a617..2a0c769 100755 --- a/scripts/terraform-enable-remote.sh +++ b/scripts/terraform-enable-remote.sh @@ -5,6 +5,8 @@ if [ -z "${ENV}" ]; then exit 1 fi +rm -rf ./.terraform/ + terraform remote config \ -state="${ENV}.tfstate" \ -backend=S3 \ diff --git a/security_group.tf b/security_group.tf index 2c11d96..e84a365 100644 --- a/security_group.tf +++ b/security_group.tf @@ -34,7 +34,7 @@ resource "aws_security_group" "ssh" { to_port = 22 protocol = "tcp" cidr_blocks = [ - "${var.segment.office}", + "${var.segment["office"]}", "115.74.36.134/32", ] } diff --git a/variables.tf b/variables.tf index e14bf49..2091bb3 100644 --- a/variables.tf +++ b/variables.tf @@ -14,8 +14,6 @@ variable "web_host_name" { // stateful variables -variable "web_ami" { -} variable "web_desired_capacity" { default = 1 } From ec65c850a2b1e6d9463092f68d7b2a721bb98657 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 5 Aug 2016 14:19:52 +0700 Subject: [PATCH 044/181] terraform 0.7.0 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cfa1041..ebd6395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ cache: install: - gem install travis -v 1.8.2 --no-rdoc --no-ri - bundle install - - wget https://releases.hashicorp.com/terraform/0.6.14/terraform_0.6.14_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.0/terraform_0.7.0_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: From 5aaf82f460cb8d8b5e912101540e477696c783ce Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 5 Aug 2016 14:49:19 +0700 Subject: [PATCH 045/181] get aws account_id from sts --- .envrc.example | 2 +- .travis.yml | 2 +- elasticsearch.tf | 6 +++--- scripts/get_account_id.rb | 4 ++++ variables.tf | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 scripts/get_account_id.rb diff --git a/.envrc.example b/.envrc.example index 00fa3f6..2b10c78 100644 --- a/.envrc.example +++ b/.envrc.example @@ -6,7 +6,7 @@ export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_DEFAULT_REGION= -export TF_VAR_aws_account_num= +export TF_VAR_aws_account_id= export TF_VAR_aws_region=${AWS_DEFAULT_REGION} export CLOUDFLARE_EMAIL= diff --git a/.travis.yml b/.travis.yml index ebd6395..2fd7d7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ install: - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: - - export TF_VAR_aws_account_num=${AWS_ACCOUNT_NUMBER} + - export TF_VAR_aws_account_id=$(bundle exec ruby scripts/get_account_id.rb) - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - echo diff --git a/elasticsearch.tf b/elasticsearch.tf index caf0278..a92f6ac 100644 --- a/elasticsearch.tf +++ b/elasticsearch.tf @@ -7,10 +7,10 @@ resource "aws_elasticsearch_domain" "logserver" { { "Action": "es:*", "Principal": { - "AWS": "arn:aws:iam::${var.aws_account_num}:root" + "AWS": "arn:aws:iam::${var.aws_account_id}:root" }, "Effect": "Allow", - "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_num}:domain/micropost-${var.env}/*" + "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_id}:domain/micropost-${var.env}/*" }, { "Action": "es:*", @@ -21,7 +21,7 @@ resource "aws_elasticsearch_domain" "logserver" { "aws:SourceIp": "${var.segment["office"]}" } }, - "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_num}:domain/micropost-${var.env}/*" + "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_id}:domain/micropost-${var.env}/*" } ] } diff --git a/scripts/get_account_id.rb b/scripts/get_account_id.rb new file mode 100644 index 0000000..d21f11a --- /dev/null +++ b/scripts/get_account_id.rb @@ -0,0 +1,4 @@ +require 'aws-sdk' + +sts = Aws::STS::Client.new(region: 'ap-northeast-1') +puts sts.get_caller_identity.account \ No newline at end of file diff --git a/variables.tf b/variables.tf index 2091bb3..6a23ec2 100644 --- a/variables.tf +++ b/variables.tf @@ -4,7 +4,7 @@ variable "env" { description = "dev, stg, prod and etc." default = "dev" } -variable "aws_account_num" { +variable "aws_account_id" { } variable "aws_region" { default = "ap-northeast-1" From fb9de029dfaf0f17a74974680dd792b3d3ecccf9 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 5 Aug 2016 15:15:47 +0700 Subject: [PATCH 046/181] run instance --- autoscaling.tf | 4 ++-- scripts/build-app.sh | 19 ------------------- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100755 scripts/build-app.sh diff --git a/autoscaling.tf b/autoscaling.tf index 83cfad0..662c4b6 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -39,8 +39,8 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web-${var.env}" launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 0 - min_size = 0 + max_size = 4 + min_size = 1 desired_capacity = "${var.web_desired_capacity}" health_check_grace_period = 300 health_check_type = "ELB" diff --git a/scripts/build-app.sh b/scripts/build-app.sh deleted file mode 100755 index 5984e1f..0000000 --- a/scripts/build-app.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -travis login --org --skip-completion-check --github-token ${GH_TOKEN} -token=$(travis token --org) -body=$(cat << EOS -{ - "request": { - "branch":"${TRAVIS_BRANCH}" - } -} -EOS -) -curl -s -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -H "Travis-API-Version: 3" \ - -H "Authorization: token ${token}" \ - -d "${body}" \ - https://api.travis-ci.org/repo/springboot-angular2-tutorial%2Fboot-app/requests From 795cc6fba36eb54823599d0e8cee68794d2c88f0 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 8 Aug 2016 19:02:41 +0700 Subject: [PATCH 047/181] deploy in user data shell --- templates/web_init.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/web_init.sh b/templates/web_init.sh index 981e1bc..dbae1e3 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -16,9 +16,11 @@ cd /opt/provisioning cat << EOF > inventory [web] localhost +[deploy] +localhost EOF -ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" site.yml +ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" site.yml ) # Update SSL cert From 26a2dba38b77f702405b70d4e0563e8f1a37156b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 9 Aug 2016 15:07:05 +0700 Subject: [PATCH 048/181] change bucket name --- s3.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3.tf b/s3.tf index 77c281b..34db618 100644 --- a/s3.tf +++ b/s3.tf @@ -1,4 +1,3 @@ resource "aws_s3_bucket" "web_codedeploy" { - bucket = "web-condedeploy-${var.env}.hana053.com" - force_destroy = true + bucket = "deploy-${var.env}.hana053.com" } From ab1333697ac69360aee9f55633bed290bb4a86c8 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 9 Aug 2016 16:30:25 +0700 Subject: [PATCH 049/181] update bucket for deploy --- autoscaling.tf | 1 + s3.tf | 2 +- templates/web_init.sh | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/autoscaling.tf b/autoscaling.tf index 662c4b6..fb1ef7e 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -6,6 +6,7 @@ data "template_file" "web_init" { rds_endpoint = "${aws_db_instance.micropost.endpoint}" redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" web_endpoint = "${cloudflare_record.micropost.hostname}" + s3_deploy_bucket = "${aws_s3_bucket.deploy.bucket}" } } diff --git a/s3.tf b/s3.tf index 34db618..ffd53ea 100644 --- a/s3.tf +++ b/s3.tf @@ -1,3 +1,3 @@ -resource "aws_s3_bucket" "web_codedeploy" { +resource "aws_s3_bucket" "deploy" { bucket = "deploy-${var.env}.hana053.com" } diff --git a/templates/web_init.sh b/templates/web_init.sh index dbae1e3..82d2f03 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -7,6 +7,7 @@ cat << EOF > $${ENV_FILE} export SPRING_PROFILES_ACTIVE=${env} export RDS_ENDPOINT=${rds_endpoint} export REDIS_ENDPOINT=${redis_endpoint} +export S3_DEPLOY_BUCKET=${s3_deploy_bucket} EOF # Finalize provisioning by using resolved endpoints. @@ -20,7 +21,7 @@ localhost localhost EOF -ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" site.yml +ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml ) # Update SSL cert From 1e5ff1a24f80c677e8a2e0179c767782e281980f Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 9 Aug 2016 16:38:51 +0700 Subject: [PATCH 050/181] force_destroy --- s3.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/s3.tf b/s3.tf index ffd53ea..3abc35f 100644 --- a/s3.tf +++ b/s3.tf @@ -1,3 +1,4 @@ resource "aws_s3_bucket" "deploy" { bucket = "deploy-${var.env}.hana053.com" + force_destroy = true } From 6fc583c8873aa6c15ac80da3d5abb6a5331aa226 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 10 Aug 2016 09:50:24 +0700 Subject: [PATCH 051/181] web_init is logged by syslog --- templates/web_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/web_init.sh b/templates/web_init.sh index 82d2f03..6d1c1e3 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -25,4 +25,4 @@ ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elas ) # Update SSL cert -/usr/local/bin/update_cert.sh >/var/log/update_cert 2>&1 +/usr/local/bin/update_cert.sh From 96ff701c822c553b39adbfc5a492c484bee141de Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 15:02:14 +0700 Subject: [PATCH 052/181] update cloud init script --- elb.tf | 9 +++++---- security_group.tf | 2 +- templates/web_init.sh | 4 +--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/elb.tf b/elb.tf index f5ce953..7421fab 100644 --- a/elb.tf +++ b/elb.tf @@ -9,11 +9,12 @@ resource "aws_elb" "web" { "${aws_security_group.http.id}", "${aws_security_group.https.id}", ] + // use for health check listener { instance_port = 80 - instance_protocol = "tcp" + instance_protocol = "http" lb_port = 80 - lb_protocol = "tcp" + lb_protocol = "http" } listener { instance_port = 443 @@ -25,7 +26,7 @@ resource "aws_elb" "web" { healthy_threshold = 2 unhealthy_threshold = 2 timeout = 5 - target = "HTTP:81/manage/health" + target = "HTTP:80/manage/health" interval = 30 } cross_zone_load_balancing = true @@ -36,5 +37,5 @@ resource "aws_elb" "web" { resource "aws_proxy_protocol_policy" "web" { load_balancer = "${aws_elb.web.name}" - instance_ports = ["80", "443"] + instance_ports = ["443"] } \ No newline at end of file diff --git a/security_group.tf b/security_group.tf index e84a365..1b7b9d5 100644 --- a/security_group.tf +++ b/security_group.tf @@ -35,7 +35,7 @@ resource "aws_security_group" "ssh" { protocol = "tcp" cidr_blocks = [ "${var.segment["office"]}", - "115.74.36.134/32", + "171.232.52.48/32", ] } egress { diff --git a/templates/web_init.sh b/templates/web_init.sh index 6d1c1e3..7249fd6 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -15,9 +15,7 @@ EOF cd /opt/provisioning cat << EOF > inventory -[web] -localhost -[deploy] +[webservers] localhost EOF From b0140e97177f32f92aa56b3b9d4f248aae404cba Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 15:06:13 +0700 Subject: [PATCH 053/181] removed useless option --- templates/web_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/web_init.sh b/templates/web_init.sh index 7249fd6..44206c7 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -19,7 +19,7 @@ cat << EOF > inventory localhost EOF -ansible-playbook -i inventory --connection=local --sudo --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml +ansible-playbook -i inventory --connection=local --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml ) # Update SSL cert From 4b76d0881c1c4b4d9c846dcc5e515b5b119b6b17 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 15:39:10 +0700 Subject: [PATCH 054/181] evict cert on s3 --- s3.tf | 9 +++++++++ scripts/export-tfvars.sh | 6 ------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/s3.tf b/s3.tf index 3abc35f..397b21a 100644 --- a/s3.tf +++ b/s3.tf @@ -1,4 +1,13 @@ resource "aws_s3_bucket" "deploy" { bucket = "deploy-${var.env}.hana053.com" force_destroy = true + // evict letsencrypt cert + lifecycle_rule { + id = "cert" + prefix = "cert/" + enabled = true + expiration { + days = 20 + } + } } diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 80119c0..0b265c8 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -1,11 +1,5 @@ #!/bin/sh -# Take over current value, if not new value was specified. -TF_VAR_web_desired_capacity=$(terraform output web_desired_capacity) -if [ -n "${TF_VAR_web_desired_capacity}" ]; then - export TF_VAR_web_desired_capacity -fi - if [ "${ENV}" = "prod" ]; then # prod is special export TF_VAR_web_host_name="micropost" From 25d87283dd78aa3cb3cb804707bc5bac311fcd66 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 16:32:07 +0700 Subject: [PATCH 055/181] add bastion definition --- bastion.tf | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 bastion.tf diff --git a/bastion.tf b/bastion.tf new file mode 100644 index 0000000..e6888ea --- /dev/null +++ b/bastion.tf @@ -0,0 +1,39 @@ +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = ["*ubuntu-xenial-16.04*"] + } + filter { + name = "architecture" + values = ["x86_64"] + } + filter { + name = "root-device-type" + values = ["ebs"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "block-device-mapping.volume-type" + values = ["gp2"] + } + owners = ["099720109477"] # Canonical +} + +resource "aws_instance" "bastion" { + ami = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" + subnet_id = "${aws_subnet.public_primary.id}" + security_groups = [ + "${aws_security_group.ssh.id}", + "${aws_security_group.internal.id}", + ] + key_name = "id_rsa" + associate_public_ip_address = true + tags { + Name = "Bastion" + } +} \ No newline at end of file From 4abadeb0f172647375a6b2a79a129b21412b3c1b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 16:42:40 +0700 Subject: [PATCH 056/181] autoscaling web does not accept ssh and have no public ip --- ami.tf | 35 +++++++++++++++++++++++++++++++++++ autoscaling.tf | 13 ------------- bastion.tf | 25 ------------------------- 3 files changed, 35 insertions(+), 38 deletions(-) create mode 100644 ami.tf diff --git a/ami.tf b/ami.tf new file mode 100644 index 0000000..2dc1cf4 --- /dev/null +++ b/ami.tf @@ -0,0 +1,35 @@ +data "aws_ami" "micropost_web" { + most_recent = true + owners = ["self"] + + filter { + name = "tag:Name" + values = ["micropost-web"] + } +} + +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = ["*ubuntu-xenial-16.04*"] + } + filter { + name = "architecture" + values = ["x86_64"] + } + filter { + name = "root-device-type" + values = ["ebs"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "block-device-mapping.volume-type" + values = ["gp2"] + } + owners = ["099720109477"] # Canonical +} + diff --git a/autoscaling.tf b/autoscaling.tf index fb1ef7e..c2ae564 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -10,26 +10,13 @@ data "template_file" "web_init" { } } -data "aws_ami" "micropost_web" { - most_recent = true - owners = ["self"] - - filter { - name = "tag:Name" - values = ["micropost-web"] - } -} - resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" image_id = "${data.aws_ami.micropost_web.id}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", - "${aws_security_group.ssh.id}", ] - key_name = "id_rsa" - associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" user_data = "${data.template_file.web_init.rendered}" lifecycle { diff --git a/bastion.tf b/bastion.tf index e6888ea..7371c08 100644 --- a/bastion.tf +++ b/bastion.tf @@ -1,28 +1,3 @@ -data "aws_ami" "ubuntu" { - most_recent = true - filter { - name = "name" - values = ["*ubuntu-xenial-16.04*"] - } - filter { - name = "architecture" - values = ["x86_64"] - } - filter { - name = "root-device-type" - values = ["ebs"] - } - filter { - name = "virtualization-type" - values = ["hvm"] - } - filter { - name = "block-device-mapping.volume-type" - values = ["gp2"] - } - owners = ["099720109477"] # Canonical -} - resource "aws_instance" "bastion" { ami = "${data.aws_ami.ubuntu.id}" instance_type = "t2.micro" From c340fb072229664bc66c872c9d9e6f02bc102900 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 11 Aug 2016 17:49:36 +0700 Subject: [PATCH 057/181] revert public access --- autoscaling.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autoscaling.tf b/autoscaling.tf index c2ae564..34200ba 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -17,6 +17,8 @@ resource "aws_launch_configuration" "web" { security_groups = [ "${aws_security_group.internal.id}", ] + key_name = "id_rsa" + associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" user_data = "${data.template_file.web_init.rendered}" lifecycle { From 9572652c62cfa1542f63c85145bb4ffc04be634d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 12 Aug 2016 09:44:56 +0700 Subject: [PATCH 058/181] added keypair resource --- autoscaling.tf | 2 +- bastion.tf | 2 +- keypair.tf | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 keypair.tf diff --git a/autoscaling.tf b/autoscaling.tf index 34200ba..7fde7a5 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -17,7 +17,7 @@ resource "aws_launch_configuration" "web" { security_groups = [ "${aws_security_group.internal.id}", ] - key_name = "id_rsa" + key_name = "${aws_key_pair.micropost.key_name}" associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" user_data = "${data.template_file.web_init.rendered}" diff --git a/bastion.tf b/bastion.tf index 7371c08..d6d275f 100644 --- a/bastion.tf +++ b/bastion.tf @@ -6,7 +6,7 @@ resource "aws_instance" "bastion" { "${aws_security_group.ssh.id}", "${aws_security_group.internal.id}", ] - key_name = "id_rsa" + key_name = "${aws_key_pair.micropost.key_name}" associate_public_ip_address = true tags { Name = "Bastion" diff --git a/keypair.tf b/keypair.tf new file mode 100644 index 0000000..1e707b1 --- /dev/null +++ b/keypair.tf @@ -0,0 +1,4 @@ +resource "aws_key_pair" "micropost" { + key_name = "micropost" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDbCML308PObiUCSJwkX8JsX434d7TStWTeEiIyOb96JTuT3dxgz4eCGBDcYH3McmSpg4jz4HJAmwxWeeyFhHiAwNoCsND3AaKiG0pzNsfCc7I6IFdFJHubVaAZf+YivoZQUSCzQhZmANMOGF7BPMN3OL5cZMm+7MDdHDJVOtxrM1QhsM9aHxTgWxgn4uKvQoxxDmx/0kbrIC0al3Fi0p+kK8lXbSJ+dIDbWPtX0ntxMK2K8n5lWP8bwBQIh+tSPG4pxgxD++GjZ94PiKcsHLaxhyiI1/H29bQq2+dyO1enHux3JWmPRa0EOXAricyoWxpXskT6UxtEupP4Q+nd6tW/ akira@Macintosh.local" +} \ No newline at end of file From 1664a135d6a17237eac5b9d52f4eb93eaa4647e1 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 12 Aug 2016 14:37:27 +0700 Subject: [PATCH 059/181] updated replacing instance logic watch inserice states of loadbalancer --- scripts/replace_old_instances.rb | 7 ++++++- templates/web_init.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb index fd00a40..1f314fe 100644 --- a/scripts/replace_old_instances.rb +++ b/scripts/replace_old_instances.rb @@ -9,6 +9,7 @@ as = Aws::AutoScaling::Client.new asg = Aws::AutoScaling::AutoScalingGroup.new(ASG_NAME, client: as) +elb = Aws::ElasticLoadBalancing::Client.new old_instances = asg.instances @@ -25,7 +26,11 @@ is_all_in_service = lambda do asg.reload - in_services = asg.instances.select { |i| i.lifecycle_state == 'InService' } + in_services = asg.load_balancers + .map(&:load_balancer_name) + .map { |lb_name| elb.describe_instance_health(load_balancer_name: lb_name).instance_states } + .flatten + .select { |s| s.state == 'InService' } in_services.size >= asg.desired_capacity end diff --git a/templates/web_init.sh b/templates/web_init.sh index 44206c7..357d970 100644 --- a/templates/web_init.sh +++ b/templates/web_init.sh @@ -19,7 +19,7 @@ cat << EOF > inventory localhost EOF -ansible-playbook -i inventory --connection=local --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml +ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml ) # Update SSL cert From df487cc695a65f5ab4437274c5e85971c2f7b55b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 12 Aug 2016 14:42:04 +0700 Subject: [PATCH 060/181] fix bastion to use vpc_security_group_ids --- bastion.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bastion.tf b/bastion.tf index d6d275f..ef973ab 100644 --- a/bastion.tf +++ b/bastion.tf @@ -2,7 +2,7 @@ resource "aws_instance" "bastion" { ami = "${data.aws_ami.ubuntu.id}" instance_type = "t2.micro" subnet_id = "${aws_subnet.public_primary.id}" - security_groups = [ + vpc_security_group_ids = [ "${aws_security_group.ssh.id}", "${aws_security_group.internal.id}", ] From cf0f51c5f1eab54d476c0508eb395b2de840cc69 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 15 Aug 2016 11:05:51 +0700 Subject: [PATCH 061/181] set tags --- autoscaling.tf | 22 +++++++++++++++++++++- bastion.tf | 31 +++++++++++++++++-------------- constants.tf | 18 ------------------ elasticache.tf | 5 +++++ elasticsearch.tf | 9 +++++++-- elb.tf | 8 +++++++- iam.tf | 4 ++-- rds.tf | 6 +++++- s3.tf | 5 +++++ security_group.tf | 40 ++++++++++++++++++++++++---------------- variables.tf | 25 +++++++++++++++++++++++++ vpc.tf | 33 ++++++++++++++++++++++++--------- 12 files changed, 142 insertions(+), 64 deletions(-) delete mode 100644 constants.tf diff --git a/autoscaling.tf b/autoscaling.tf index 7fde7a5..414590e 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -2,7 +2,7 @@ data "template_file" "web_init" { template = "${file("templates/web_init.sh")}" vars { env = "${var.env}" - logserver_endpoint = "${aws_elasticsearch_domain.logserver.endpoint}" + logserver_endpoint = "${aws_elasticsearch_domain.micropost.endpoint}" rds_endpoint = "${aws_db_instance.micropost.endpoint}" redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" web_endpoint = "${cloudflare_record.micropost.hostname}" @@ -40,6 +40,26 @@ resource "aws_autoscaling_group" "web" { "${aws_subnet.public_secondary.id}", ] load_balancers = ["${aws_elb.web.name}"] + tag { + key = "Name" + value = "${var.app}-${var.env}-web" + propagate_at_launch = true + } + tag { + key = "App" + value = "${var.app}" + propagate_at_launch = true + } + tag { + key = "Env" + value = "${var.env}" + propagate_at_launch = true + } + tag { + key = "Role" + value = "web" + propagate_at_launch = true + } } resource "aws_autoscaling_policy" "web_scale_out" { diff --git a/bastion.tf b/bastion.tf index ef973ab..a8e339e 100644 --- a/bastion.tf +++ b/bastion.tf @@ -1,14 +1,17 @@ -resource "aws_instance" "bastion" { - ami = "${data.aws_ami.ubuntu.id}" - instance_type = "t2.micro" - subnet_id = "${aws_subnet.public_primary.id}" - vpc_security_group_ids = [ - "${aws_security_group.ssh.id}", - "${aws_security_group.internal.id}", - ] - key_name = "${aws_key_pair.micropost.key_name}" - associate_public_ip_address = true - tags { - Name = "Bastion" - } -} \ No newline at end of file +//resource "aws_instance" "bastion" { +// ami = "${data.aws_ami.ubuntu.id}" +// instance_type = "t2.micro" +// subnet_id = "${aws_subnet.public_primary.id}" +// vpc_security_group_ids = [ +// "${aws_security_group.ssh.id}", +// "${aws_security_group.internal.id}", +// ] +// key_name = "${aws_key_pair.micropost.key_name}" +// associate_public_ip_address = true +// tags { +// Name = "${var.app}-${var.env}-bastion" +// App = "${var.app}" +// Env = "${var.env}" +// Role = "bastion" +// } +//} \ No newline at end of file diff --git a/constants.tf b/constants.tf deleted file mode 100644 index 9c9a732..0000000 --- a/constants.tf +++ /dev/null @@ -1,18 +0,0 @@ -variable "segment" { - default = { - office = "42.116.5.4/32" - } -} - -variable "aws_az_primary" { - default = { - ap-southeast-1 = "ap-southeast-1a" - ap-northeast-1 = "ap-northeast-1a" - } -} -variable "aws_az_secondary" { - default = { - ap-southeast-1 = "ap-southeast-1b" - ap-northeast-1 = "ap-northeast-1c" - } -} diff --git a/elasticache.tf b/elasticache.tf index 4b3ac60..d584dbc 100644 --- a/elasticache.tf +++ b/elasticache.tf @@ -10,6 +10,11 @@ resource "aws_elasticache_cluster" "micropost" { "${aws_security_group.internal.id}", ] subnet_group_name = "${aws_elasticache_subnet_group.micropost.name}" + tags { + Name = "${var.app}-${var.env}" + App = "${var.app}" + Env = "${var.env}" + } } resource "aws_elasticache_subnet_group" "micropost" { diff --git a/elasticsearch.tf b/elasticsearch.tf index a92f6ac..fddf5f1 100644 --- a/elasticsearch.tf +++ b/elasticsearch.tf @@ -1,5 +1,5 @@ -resource "aws_elasticsearch_domain" "logserver" { - domain_name = "micropost-${var.env}" +resource "aws_elasticsearch_domain" "micropost" { + domain_name = "${var.app}-${var.env}" access_policies = < Date: Wed, 17 Aug 2016 15:46:10 +0700 Subject: [PATCH 062/181] register es snapshot directory automatically --- Gemfile | 2 + Gemfile.lock | 10 ++++ elasticsearch.tf | 65 ++++++++++++++++++++++- s3.tf | 5 ++ scripts/es_register_snapshot_directory.rb | 36 +++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 scripts/es_register_snapshot_directory.rb diff --git a/Gemfile b/Gemfile index 8b0ad27..bd58e0d 100644 --- a/Gemfile +++ b/Gemfile @@ -3,3 +3,5 @@ source 'http://rubygems.org' gem 'aws-sdk' gem 'awesome_print' +gem 'faraday_middleware' +gem 'faraday_middleware-aws-signers-v4' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 333478f..67b0113 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,9 +8,17 @@ GEM jmespath (~> 1.0) aws-sdk-resources (2.2.34) aws-sdk-core (= 2.2.34) + faraday (0.9.2) + multipart-post (>= 1.2, < 3) + faraday_middleware (0.10.0) + faraday (>= 0.7.4, < 0.10) + faraday_middleware-aws-signers-v4 (0.1.5) + aws-sdk (~> 2.1) + faraday (~> 0.9) jmespath (1.2.4) json_pure (>= 1.8.1) json_pure (1.8.3) + multipart-post (2.0.0) PLATFORMS ruby @@ -18,6 +26,8 @@ PLATFORMS DEPENDENCIES awesome_print aws-sdk + faraday_middleware + faraday_middleware-aws-signers-v4 RUBY VERSION ruby 2.3.0p0 diff --git a/elasticsearch.tf b/elasticsearch.tf index fddf5f1..8dbfad3 100644 --- a/elasticsearch.tf +++ b/elasticsearch.tf @@ -1,3 +1,7 @@ +variable "es_backup_repository" { + default = "micropost-log-backups" +} + resource "aws_elasticsearch_domain" "micropost" { domain_name = "${var.app}-${var.env}" access_policies = < /\bjson\b/ + faraday.response :raise_error + faraday.adapter Faraday.default_adapter +end + +res = conn.post do |req| + req.url "/_snapshot/#{option[:repository]}" + req.headers['Content-Type'] = 'application/json' + req.body = { + type: 's3', + settings: { + bucket: option[:bucket], + region: option[:region], + role_arn: option[:role_arn], + base_path: option[:base_path], + } + }.to_json +end + +puts res.body From 27727264f4c1d0dc691fda1dab1318607fa5b64a Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 17 Aug 2016 17:10:46 +0700 Subject: [PATCH 063/181] separate tf file by its functionality --- codedeploy.tf | 55 ++++++++++++- iam.tf | 133 -------------------------------- autoscaling.tf => webservers.tf | 85 ++++++++++++++++++++ 3 files changed, 137 insertions(+), 136 deletions(-) delete mode 100644 iam.tf rename autoscaling.tf => webservers.tf (69%) diff --git a/codedeploy.tf b/codedeploy.tf index 9527650..936328b 100644 --- a/codedeploy.tf +++ b/codedeploy.tf @@ -1,11 +1,60 @@ -resource "aws_codedeploy_app" "micropost" { - name = "micropost-${var.env}" +resource "aws_codedeploy_app" "main" { + name = "${var.app}-${var.env}" } resource "aws_codedeploy_deployment_group" "web" { - app_name = "${aws_codedeploy_app.micropost.name}" + app_name = "${aws_codedeploy_app.main.name}" deployment_group_name = "web" service_role_arn = "${aws_iam_role.codedeploy_service.arn}" autoscaling_groups = ["${aws_autoscaling_group.web.id}"] deployment_config_name = "CodeDeployDefault.OneAtATime" } + +resource "aws_iam_role" "codedeploy_service" { + name = "${var.app}-${var.env}-codedeploy-service" + assume_role_policy = < Date: Thu, 18 Aug 2016 10:50:31 +0700 Subject: [PATCH 064/181] vpc module --- ami.tf | 35 -------------- bastion.tf | 59 ++++++++++++++++------- elasticache.tf | 5 +- elb.tf | 47 ------------------ main.tf | 4 ++ outputs.tf | 2 +- rds.tf | 5 +- scripts/terraform-wrapper.sh | 2 + security_group.tf | 8 ++-- variables.tf | 23 --------- vpc.tf | 92 ------------------------------------ vpc/main.tf | 62 ++++++++++++++++++++++++ vpc/outputs.tf | 11 +++++ vpc/variables.tf | 31 ++++++++++++ webservers.tf | 88 +++++++++++++++++++++++++++++----- 15 files changed, 236 insertions(+), 238 deletions(-) delete mode 100644 ami.tf delete mode 100644 elb.tf create mode 100644 main.tf delete mode 100644 vpc.tf create mode 100644 vpc/main.tf create mode 100644 vpc/outputs.tf create mode 100644 vpc/variables.tf diff --git a/ami.tf b/ami.tf deleted file mode 100644 index 2dc1cf4..0000000 --- a/ami.tf +++ /dev/null @@ -1,35 +0,0 @@ -data "aws_ami" "micropost_web" { - most_recent = true - owners = ["self"] - - filter { - name = "tag:Name" - values = ["micropost-web"] - } -} - -data "aws_ami" "ubuntu" { - most_recent = true - filter { - name = "name" - values = ["*ubuntu-xenial-16.04*"] - } - filter { - name = "architecture" - values = ["x86_64"] - } - filter { - name = "root-device-type" - values = ["ebs"] - } - filter { - name = "virtualization-type" - values = ["hvm"] - } - filter { - name = "block-device-mapping.volume-type" - values = ["gp2"] - } - owners = ["099720109477"] # Canonical -} - diff --git a/bastion.tf b/bastion.tf index a8e339e..fc8c241 100644 --- a/bastion.tf +++ b/bastion.tf @@ -1,17 +1,42 @@ -//resource "aws_instance" "bastion" { -// ami = "${data.aws_ami.ubuntu.id}" -// instance_type = "t2.micro" -// subnet_id = "${aws_subnet.public_primary.id}" -// vpc_security_group_ids = [ -// "${aws_security_group.ssh.id}", -// "${aws_security_group.internal.id}", -// ] -// key_name = "${aws_key_pair.micropost.key_name}" -// associate_public_ip_address = true -// tags { -// Name = "${var.app}-${var.env}-bastion" -// App = "${var.app}" -// Env = "${var.env}" -// Role = "bastion" -// } -//} \ No newline at end of file +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = ["*ubuntu-xenial-16.04*"] + } + filter { + name = "architecture" + values = ["x86_64"] + } + filter { + name = "root-device-type" + values = ["ebs"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "block-device-mapping.volume-type" + values = ["gp2"] + } + owners = ["099720109477"] # Canonical +} + +resource "aws_instance" "bastion" { + ami = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" + subnet_id = "${module.vpc.public_subnets[0]}" + vpc_security_group_ids = [ + "${aws_security_group.ssh.id}", + "${aws_security_group.internal.id}", + ] + key_name = "${aws_key_pair.micropost.key_name}" + associate_public_ip_address = true + tags { + Name = "${var.app}-${var.env}-bastion" + App = "${var.app}" + Env = "${var.env}" + Role = "bastion" + } +} \ No newline at end of file diff --git a/elasticache.tf b/elasticache.tf index d584dbc..4c86b13 100644 --- a/elasticache.tf +++ b/elasticache.tf @@ -20,8 +20,5 @@ resource "aws_elasticache_cluster" "micropost" { resource "aws_elasticache_subnet_group" "micropost" { name = "micropost-${var.env}" description = "main subnet group" - subnet_ids = [ - "${aws_subnet.private_primary.id}", - "${aws_subnet.private_secondary.id}", - ] + subnet_ids = ["${module.vpc.private_subnets}"] } \ No newline at end of file diff --git a/elb.tf b/elb.tf deleted file mode 100644 index 7026ae1..0000000 --- a/elb.tf +++ /dev/null @@ -1,47 +0,0 @@ -resource "aws_elb" "web" { - name = "${var.app}-${var.env}-web" - subnets = [ - "${aws_subnet.public_primary.id}", - "${aws_subnet.public_secondary.id}", - ] - security_groups = [ - "${aws_security_group.internal.id}", - "${aws_security_group.http.id}", - "${aws_security_group.https.id}", - ] - // use for health check - listener { - instance_port = 80 - instance_protocol = "http" - lb_port = 80 - lb_protocol = "http" - } - listener { - instance_port = 443 - instance_protocol = "tcp" - lb_port = 443 - lb_protocol = "tcp" - } - health_check { - healthy_threshold = 2 - unhealthy_threshold = 2 - timeout = 5 - target = "HTTP:80/manage/health" - interval = 30 - } - cross_zone_load_balancing = true - idle_timeout = 400 - connection_draining = true - connection_draining_timeout = 400 - tags { - Name = "${var.app}-${var.env}-web" - App = "${var.app}" - Env = "${var.env}" - Role = "web" - } -} - -resource "aws_proxy_protocol_policy" "web" { - load_balancer = "${aws_elb.web.name}" - instance_ports = ["443"] -} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..b647cb5 --- /dev/null +++ b/main.tf @@ -0,0 +1,4 @@ +module "vpc" { + source = "./vpc" + region = "${var.aws_region}" +} \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index 06e2664..1d4f7e8 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,5 +1,5 @@ output "web_ami" { - value = "${data.aws_ami.micropost_web.id}" + value = "${data.aws_ami.web.id}" } output "web_desired_capacity" { diff --git a/rds.tf b/rds.tf index be50c0a..8e8af4d 100644 --- a/rds.tf +++ b/rds.tf @@ -19,8 +19,5 @@ resource "aws_db_instance" "micropost" { resource "aws_db_subnet_group" "micropost" { name = "micropost-${var.env}" description = "Our main group of subnets" - subnet_ids = [ - "${aws_subnet.private_primary.id}", - "${aws_subnet.private_secondary.id}", - ] + subnet_ids = ["${module.vpc.private_subnets}"] } diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh index 705c535..d551c07 100755 --- a/scripts/terraform-wrapper.sh +++ b/scripts/terraform-wrapper.sh @@ -3,6 +3,8 @@ . $(dirname $0)/terraform-enable-remote.sh . $(dirname $0)/export-tfvars.sh +terraform get + terraform $@ \ -state="${ENV}.tfstate" \ -refresh=true \ diff --git a/security_group.tf b/security_group.tf index b54aa17..90cc618 100644 --- a/security_group.tf +++ b/security_group.tf @@ -1,5 +1,5 @@ resource "aws_security_group" "internal" { - vpc_id = "${aws_vpc.vpc.id}" + vpc_id = "${module.vpc.vpc_id}" name_prefix = "internal-" description = "Allow internal traffic" ingress { @@ -28,7 +28,7 @@ resource "aws_security_group" "internal" { } resource "aws_security_group" "ssh" { - vpc_id = "${aws_vpc.vpc.id}" + vpc_id = "${module.vpc.vpc_id}" name_prefix = "ssh-" description = "Allow ssh inbound traffic" ingress { @@ -60,7 +60,7 @@ resource "aws_security_group" "ssh" { } resource "aws_security_group" "http" { - vpc_id = "${aws_vpc.vpc.id}" + vpc_id = "${module.vpc.vpc_id}" name_prefix = "http-" description = "Allow http inbound traffic" ingress { @@ -91,7 +91,7 @@ resource "aws_security_group" "http" { } resource "aws_security_group" "https" { - vpc_id = "${aws_vpc.vpc.id}" + vpc_id = "${module.vpc.vpc_id}" name_prefix = "https-" description = "Allow https inbound traffic" ingress { diff --git a/variables.tf b/variables.tf index a406655..790d652 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,3 @@ -// stateless variables - variable "env" { description = "dev, stg, prod and etc." default = "dev" @@ -12,14 +10,6 @@ variable "aws_region" { variable "web_host_name" { } -// stateful variables - -variable "web_desired_capacity" { - default = 1 -} - -// constants - variable "app" { default = "micropost" } @@ -29,16 +19,3 @@ variable "segment" { office = "42.116.5.4/32" } } - -variable "aws_az_primary" { - default = { - ap-southeast-1 = "ap-southeast-1a" - ap-northeast-1 = "ap-northeast-1a" - } -} -variable "aws_az_secondary" { - default = { - ap-southeast-1 = "ap-southeast-1b" - ap-northeast-1 = "ap-northeast-1c" - } -} diff --git a/vpc.tf b/vpc.tf deleted file mode 100644 index 5a01bfe..0000000 --- a/vpc.tf +++ /dev/null @@ -1,92 +0,0 @@ -resource "aws_vpc" "vpc" { - cidr_block = "10.1.0.0/16" - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_internet_gateway" "igw" { - vpc_id = "${aws_vpc.vpc.id}" - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_route_table" "public-route" { - vpc_id = "${aws_vpc.vpc.id}" - route { - cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.igw.id}" - } - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } -} - -// -------- public a - -resource "aws_subnet" "public_primary" { - vpc_id = "${aws_vpc.vpc.id}" - cidr_block = "10.1.0.0/24" - availability_zone = "${lookup(var.aws_az_primary, var.aws_region)}" - tags { - Name = "${var.app}-${var.env}-public1" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_route_table_association" "puclic_primary" { - subnet_id = "${aws_subnet.public_primary.id}" - route_table_id = "${aws_route_table.public-route.id}" -} - -// -------- public b - -resource "aws_subnet" "public_secondary" { - vpc_id = "${aws_vpc.vpc.id}" - cidr_block = "10.1.1.0/24" - availability_zone = "${lookup(var.aws_az_secondary, var.aws_region)}" - tags { - Name = "${var.app}-${var.env}-public2" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_route_table_association" "puclic_secondary" { - subnet_id = "${aws_subnet.public_secondary.id}" - route_table_id = "${aws_route_table.public-route.id}" -} - -// -------- private a - -resource "aws_subnet" "private_primary" { - vpc_id = "${aws_vpc.vpc.id}" - cidr_block = "10.1.2.0/24" - availability_zone = "${lookup(var.aws_az_primary, var.aws_region)}" - tags { - Name = "${var.app}-${var.env}-private3" - App = "${var.app}" - Env = "${var.env}" - } -} - -// -------- private b - -resource "aws_subnet" "private_secondary" { - vpc_id = "${aws_vpc.vpc.id}" - cidr_block = "10.1.3.0/24" - availability_zone = "${lookup(var.aws_az_secondary, var.aws_region)}" - tags { - Name = "${var.app}-${var.env}-private4" - App = "${var.app}" - Env = "${var.env}" - } -} diff --git a/vpc/main.tf b/vpc/main.tf new file mode 100644 index 0000000..f014f87 --- /dev/null +++ b/vpc/main.tf @@ -0,0 +1,62 @@ +resource "aws_vpc" "main" { + cidr_block = "${var.cidr}" + // tags { + // Name = "${var.app}-${var.env}" + // App = "${var.app}" + // Env = "${var.env}" + // } +} + +resource "aws_internet_gateway" "main" { + vpc_id = "${aws_vpc.main.id}" + // tags { + // Name = "${var.app}-${var.env}" + // App = "${var.app}" + // Env = "${var.env}" + // } +} + +resource "aws_route_table" "public" { + vpc_id = "${aws_vpc.main.id}" + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.main.id}" + } + // tags { + // Name = "${var.app}-${var.env}" + // App = "${var.app}" + // Env = "${var.env}" + // } +} + +resource "aws_subnet" "public" { + count = "${length(var.public_subnets)}" + vpc_id = "${aws_vpc.main.id}" + cidr_block = "${var.public_subnets[count.index]}" + availability_zone = "${var.azs[count.index]}" + map_public_ip_on_launch = true + // tags { + // Name = "${var.app}-${var.env}-public1" + // App = "${var.app}" + // Env = "${var.env}" + // } +} + +resource "aws_route_table_association" "puclic" { + count = "${length(var.public_subnets)}" + subnet_id = "${element(aws_subnet.public.*.id, count.index)}" + route_table_id = "${aws_route_table.public.id}" +} + +resource "aws_subnet" "private" { + count = "${length(var.public_subnets)}" + vpc_id = "${aws_vpc.main.id}" + cidr_block = "${var.private_subnets[count.index]}" + availability_zone = "${var.azs[count.index]}" + // tags { + // Name = "${var.app}-${var.env}-private3" + // App = "${var.app}" + // Env = "${var.env}" + // } +} + diff --git a/vpc/outputs.tf b/vpc/outputs.tf new file mode 100644 index 0000000..f83e9cc --- /dev/null +++ b/vpc/outputs.tf @@ -0,0 +1,11 @@ +output "private_subnets" { + value = ["${aws_subnet.private.*.id}"] +} + +output "public_subnets" { + value = ["${aws_subnet.public.*.id}"] +} + +output "vpc_id" { + value = "${aws_vpc.main.id}" +} \ No newline at end of file diff --git a/vpc/variables.tf b/vpc/variables.tf new file mode 100644 index 0000000..5e983f3 --- /dev/null +++ b/vpc/variables.tf @@ -0,0 +1,31 @@ +variable "cidr" { + default = "10.1.0.0/16" +} + +variable "public_subnets" { + description = "A list of public subnets inside the VPC." + default = [ + "10.1.0.0/24", + "10.1.1.0/24", + ] +} + +variable "private_subnets" { + description = "A list of private subnets inside the VPC." + default = [ + "10.1.2.0/24", + "10.1.3.0/24", + ] +} + +variable "azs" { + description = "A list of Availability zones in the region" + default = [ + "ap-northeast-1a", + "ap-northeast-1c", + ] +} + +variable "region" { + default = "ap-northeast-1" +} \ No newline at end of file diff --git a/webservers.tf b/webservers.tf index e8003db..0edc6ee 100644 --- a/webservers.tf +++ b/webservers.tf @@ -1,3 +1,23 @@ +variable "web_min_size" { + default = 1 +} + +variable "web_desired_capacity" { + default = 1 +} + +data "aws_ami" "web" { + most_recent = true + owners = [ + "self"] + + filter { + name = "tag:Name" + values = [ + "micropost-web"] + } +} + data "template_file" "web_init" { template = "${file("templates/web_init.sh")}" vars { @@ -12,7 +32,7 @@ data "template_file" "web_init" { resource "aws_launch_configuration" "web" { name_prefix = "web-${var.env}-" - image_id = "${data.aws_ami.micropost_web.id}" + image_id = "${data.aws_ami.web.id}" instance_type = "t2.micro" security_groups = [ "${aws_security_group.internal.id}", @@ -30,16 +50,14 @@ resource "aws_autoscaling_group" "web" { name = "web-${var.env}" launch_configuration = "${aws_launch_configuration.web.id}" max_size = 4 - min_size = 1 + min_size = "${var.web_min_size}" desired_capacity = "${var.web_desired_capacity}" health_check_grace_period = 300 health_check_type = "ELB" force_delete = true - vpc_zone_identifier = [ - "${aws_subnet.public_primary.id}", - "${aws_subnet.public_secondary.id}", - ] - load_balancers = ["${aws_elb.web.name}"] + vpc_zone_identifier = ["${module.vpc.public_subnets}"] + load_balancers = [ + "${aws_elb.web.name}"] tag { key = "Name" value = "${var.app}-${var.env}-web" @@ -82,7 +100,8 @@ resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { dimensions { AutoScalingGroupName = "${aws_autoscaling_group.web.name}" } - alarm_actions = ["${aws_autoscaling_policy.web_scale_out.arn}"] + alarm_actions = [ + "${aws_autoscaling_policy.web_scale_out.arn}"] } resource "aws_autoscaling_policy" "web_scale_in" { @@ -105,7 +124,8 @@ resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { dimensions { AutoScalingGroupName = "${aws_autoscaling_group.web.name}" } - alarm_actions = ["${aws_autoscaling_policy.web_scale_in.arn}"] + alarm_actions = [ + "${aws_autoscaling_policy.web_scale_in.arn}"] } resource "aws_iam_instance_profile" "web" { @@ -166,7 +186,7 @@ resource "aws_iam_role_policy" "letsencrypt_cache_client" { "s3:Put*" ], "Effect": "Allow", - "Resource": "*" + "Resource": "${aws_s3_bucket.deploy.arn}/*" } ] } @@ -186,9 +206,55 @@ resource "aws_iam_role_policy" "codedeploy_client" { "s3:List*" ], "Effect": "Allow", - "Resource": "*" + "Resource": "${aws_s3_bucket.deploy.arn}/*" } ] } EOF } + +resource "aws_elb" "web" { + name = "${var.app}-${var.env}-web" + subnets = ["${module.vpc.public_subnets}"] + security_groups = [ + "${aws_security_group.internal.id}", + "${aws_security_group.http.id}", + "${aws_security_group.https.id}", + ] + // use for health check + listener { + instance_port = 80 + instance_protocol = "http" + lb_port = 80 + lb_protocol = "http" + } + listener { + instance_port = 443 + instance_protocol = "tcp" + lb_port = 443 + lb_protocol = "tcp" + } + health_check { + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 5 + target = "HTTP:80/manage/health" + interval = 30 + } + cross_zone_load_balancing = true + idle_timeout = 400 + connection_draining = true + connection_draining_timeout = 400 + tags { + Name = "${var.app}-${var.env}-web" + App = "${var.app}" + Env = "${var.env}" + Role = "web" + } +} + +resource "aws_proxy_protocol_policy" "web" { + load_balancer = "${aws_elb.web.name}" + instance_ports = [ + "443"] +} From ee54b1e2011b2c649b2dbc03e862510773c67b48 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 15:16:59 +0700 Subject: [PATCH 065/181] webservers module --- .travis.yml | 2 +- codedeploy.tf | 2 +- dns.tf | 2 +- main.tf | 30 +++ outputs.tf | 8 +- providers.tf | 3 - rds.tf | 12 +- webservers.tf | 260 -------------------------- webservers/autoscale.tf | 123 ++++++++++++ webservers/elb.tf | 46 +++++ webservers/iam.tf | 84 +++++++++ webservers/outputs.tf | 19 ++ webservers/variables.tf | 60 ++++++ {templates => webservers}/web_init.sh | 8 +- 14 files changed, 380 insertions(+), 279 deletions(-) delete mode 100644 providers.tf delete mode 100644 webservers.tf create mode 100644 webservers/autoscale.tf create mode 100644 webservers/elb.tf create mode 100644 webservers/iam.tf create mode 100644 webservers/outputs.tf create mode 100644 webservers/variables.tf rename {templates => webservers}/web_init.sh (70%) diff --git a/.travis.yml b/.travis.yml index 2fd7d7c..8263408 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,6 @@ after_success: # Replace instances if web_ami was updated. - terraform output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - - asg_name=$(terraform output web_autoscaling_group_name) + - asg_name=$(terraform output web_asg_name) - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} diff --git a/codedeploy.tf b/codedeploy.tf index 936328b..60c5825 100644 --- a/codedeploy.tf +++ b/codedeploy.tf @@ -6,7 +6,7 @@ resource "aws_codedeploy_deployment_group" "web" { app_name = "${aws_codedeploy_app.main.name}" deployment_group_name = "web" service_role_arn = "${aws_iam_role.codedeploy_service.arn}" - autoscaling_groups = ["${aws_autoscaling_group.web.id}"] + autoscaling_groups = ["${module.webservers.asg_id}"] deployment_config_name = "CodeDeployDefault.OneAtATime" } diff --git a/dns.tf b/dns.tf index ad86b70..10053df 100644 --- a/dns.tf +++ b/dns.tf @@ -1,6 +1,6 @@ resource "cloudflare_record" "micropost" { domain = "hana053.com" name = "${var.web_host_name}" - value = "${aws_elb.web.dns_name}" + value = "${module.webservers.dns_name}" type = "CNAME" } \ No newline at end of file diff --git a/main.tf b/main.tf index b647cb5..1cc7fc6 100644 --- a/main.tf +++ b/main.tf @@ -1,4 +1,34 @@ +provider "aws" { + region = "${var.aws_region}" +} + module "vpc" { source = "./vpc" region = "${var.aws_region}" +} + +module "webservers" { + source = "./webservers" + env = "${var.env}" + hostname = "${cloudflare_record.micropost.hostname}" + logserver_endpoint = "${aws_elasticsearch_domain.micropost.endpoint}" + dbserver_endpoint = "${aws_db_instance.micropost.endpoint}" + cacheserver_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" + deploy_bucket = "${aws_s3_bucket.deploy.bucket}" + deploy_bucket_arn = "${aws_s3_bucket.deploy.arn}" + key_name = "${aws_key_pair.micropost.key_name}" + web_subnets = [ + "${module.vpc.public_subnets}" + ] + web_security_groups = [ + "${aws_security_group.internal.id}", + ] + elb_subnets = [ + "${module.vpc.public_subnets}" + ] + elb_security_groups = [ + "${aws_security_group.internal.id}", + "${aws_security_group.http.id}", + "${aws_security_group.https.id}", + ] } \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index 1d4f7e8..f2b2542 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,11 +1,11 @@ output "web_ami" { - value = "${data.aws_ami.web.id}" + value = "${module.webservers.ami}" } output "web_desired_capacity" { - value = "${aws_autoscaling_group.web.desired_capacity}" + value = "${module.webservers.desired_capacity}" } -output "web_autoscaling_group_name" { - value = "${aws_autoscaling_group.web.name}" +output "web_asg_name" { + value = "${module.webservers.asg_name}" } diff --git a/providers.tf b/providers.tf deleted file mode 100644 index b54eb94..0000000 --- a/providers.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "aws" { - region = "${var.aws_region}" -} diff --git a/rds.tf b/rds.tf index 8e8af4d..154acb3 100644 --- a/rds.tf +++ b/rds.tf @@ -1,9 +1,9 @@ resource "aws_db_instance" "micropost" { snapshot_identifier = "micropost-init" - allocated_storage = 5 - engine = "mysql" - engine_version = "5.7.10" - instance_class = "db.t2.micro" + allocated_storage = 5 + engine = "mysql" + engine_version = "5.7.10" + instance_class = "db.t2.micro" db_subnet_group_name = "${aws_db_subnet_group.micropost.name}" parameter_group_name = "default.mysql5.7" vpc_security_group_ids = [ @@ -19,5 +19,7 @@ resource "aws_db_instance" "micropost" { resource "aws_db_subnet_group" "micropost" { name = "micropost-${var.env}" description = "Our main group of subnets" - subnet_ids = ["${module.vpc.private_subnets}"] + subnet_ids = [ + "${module.vpc.private_subnets}" + ] } diff --git a/webservers.tf b/webservers.tf deleted file mode 100644 index 0edc6ee..0000000 --- a/webservers.tf +++ /dev/null @@ -1,260 +0,0 @@ -variable "web_min_size" { - default = 1 -} - -variable "web_desired_capacity" { - default = 1 -} - -data "aws_ami" "web" { - most_recent = true - owners = [ - "self"] - - filter { - name = "tag:Name" - values = [ - "micropost-web"] - } -} - -data "template_file" "web_init" { - template = "${file("templates/web_init.sh")}" - vars { - env = "${var.env}" - logserver_endpoint = "${aws_elasticsearch_domain.micropost.endpoint}" - rds_endpoint = "${aws_db_instance.micropost.endpoint}" - redis_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" - web_endpoint = "${cloudflare_record.micropost.hostname}" - s3_deploy_bucket = "${aws_s3_bucket.deploy.bucket}" - } -} - -resource "aws_launch_configuration" "web" { - name_prefix = "web-${var.env}-" - image_id = "${data.aws_ami.web.id}" - instance_type = "t2.micro" - security_groups = [ - "${aws_security_group.internal.id}", - ] - key_name = "${aws_key_pair.micropost.key_name}" - associate_public_ip_address = true - iam_instance_profile = "${aws_iam_instance_profile.web.id}" - user_data = "${data.template_file.web_init.rendered}" - lifecycle { - create_before_destroy = true - } -} - -resource "aws_autoscaling_group" "web" { - name = "web-${var.env}" - launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 4 - min_size = "${var.web_min_size}" - desired_capacity = "${var.web_desired_capacity}" - health_check_grace_period = 300 - health_check_type = "ELB" - force_delete = true - vpc_zone_identifier = ["${module.vpc.public_subnets}"] - load_balancers = [ - "${aws_elb.web.name}"] - tag { - key = "Name" - value = "${var.app}-${var.env}-web" - propagate_at_launch = true - } - tag { - key = "App" - value = "${var.app}" - propagate_at_launch = true - } - tag { - key = "Env" - value = "${var.env}" - propagate_at_launch = true - } - tag { - key = "Role" - value = "web" - propagate_at_launch = true - } -} - -resource "aws_autoscaling_policy" "web_scale_out" { - name = "Instance-ScaleOut-CPU-High" - scaling_adjustment = 1 - adjustment_type = "ChangeInCapacity" - cooldown = 300 - autoscaling_group_name = "${aws_autoscaling_group.web.name}" -} - -resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { - alarm_name = "web-${var.env}-CPU-Utilization-High-30" - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "1" - metric_name = "CPUUtilization" - namespace = "AWS/EC2" - period = "120" - statistic = "Average" - threshold = "80" - dimensions { - AutoScalingGroupName = "${aws_autoscaling_group.web.name}" - } - alarm_actions = [ - "${aws_autoscaling_policy.web_scale_out.arn}"] -} - -resource "aws_autoscaling_policy" "web_scale_in" { - name = "Instance-ScaleIn-CPU-Low" - scaling_adjustment = -1 - adjustment_type = "ChangeInCapacity" - cooldown = 300 - autoscaling_group_name = "${aws_autoscaling_group.web.name}" -} - -resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { - alarm_name = "web-${var.env}-CPU-Utilization-Low-5" - comparison_operator = "LessThanThreshold" - evaluation_periods = "1" - metric_name = "CPUUtilization" - namespace = "AWS/EC2" - period = "120" - statistic = "Average" - threshold = "5" - dimensions { - AutoScalingGroupName = "${aws_autoscaling_group.web.name}" - } - alarm_actions = [ - "${aws_autoscaling_policy.web_scale_in.arn}"] -} - -resource "aws_iam_instance_profile" "web" { - name = "${var.app}-${var.env}-web" - roles = [ - "${aws_iam_role.web.name}" - ] -} - -resource "aws_iam_role" "web" { - name = "${var.app}-${var.env}-web" - assume_role_policy = < $${ENV_FILE} export SPRING_PROFILES_ACTIVE=${env} -export RDS_ENDPOINT=${rds_endpoint} -export REDIS_ENDPOINT=${redis_endpoint} -export S3_DEPLOY_BUCKET=${s3_deploy_bucket} +export RDS_ENDPOINT=${dbserver_endpoint} +export REDIS_ENDPOINT=${cacheserver_endpoint} +export S3_DEPLOY_BUCKET=${deploy_bucket} EOF # Finalize provisioning by using resolved endpoints. @@ -19,7 +19,7 @@ cat << EOF > inventory localhost EOF -ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${web_endpoint}" -e "deploy_bucket=${s3_deploy_bucket}" site.yml +ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${hostname}" -e "deploy_bucket=${deploy_bucket}" site.yml ) # Update SSL cert From 8c51e3ac104ec2084b0f017505f7c14c0adf00fb Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 16:06:11 +0700 Subject: [PATCH 066/181] bastion module --- bastion.tf | 42 ------------------------------------ bastion/main.tf | 51 ++++++++++++++++++++++++++++++++++++++++++++ bastion/variables.tf | 12 +++++++++++ main.tf | 12 ++++++++++- 4 files changed, 74 insertions(+), 43 deletions(-) delete mode 100644 bastion.tf create mode 100644 bastion/main.tf create mode 100644 bastion/variables.tf diff --git a/bastion.tf b/bastion.tf deleted file mode 100644 index fc8c241..0000000 --- a/bastion.tf +++ /dev/null @@ -1,42 +0,0 @@ -data "aws_ami" "ubuntu" { - most_recent = true - filter { - name = "name" - values = ["*ubuntu-xenial-16.04*"] - } - filter { - name = "architecture" - values = ["x86_64"] - } - filter { - name = "root-device-type" - values = ["ebs"] - } - filter { - name = "virtualization-type" - values = ["hvm"] - } - filter { - name = "block-device-mapping.volume-type" - values = ["gp2"] - } - owners = ["099720109477"] # Canonical -} - -resource "aws_instance" "bastion" { - ami = "${data.aws_ami.ubuntu.id}" - instance_type = "t2.micro" - subnet_id = "${module.vpc.public_subnets[0]}" - vpc_security_group_ids = [ - "${aws_security_group.ssh.id}", - "${aws_security_group.internal.id}", - ] - key_name = "${aws_key_pair.micropost.key_name}" - associate_public_ip_address = true - tags { - Name = "${var.app}-${var.env}-bastion" - App = "${var.app}" - Env = "${var.env}" - Role = "bastion" - } -} \ No newline at end of file diff --git a/bastion/main.tf b/bastion/main.tf new file mode 100644 index 0000000..a92f840 --- /dev/null +++ b/bastion/main.tf @@ -0,0 +1,51 @@ +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = [ + "*ubuntu-xenial-16.04*" + ] + } + filter { + name = "architecture" + values = [ + "x86_64" + ] + } + filter { + name = "root-device-type" + values = [ + "ebs" + ] + } + filter { + name = "virtualization-type" + values = [ + "hvm" + ] + } + filter { + name = "block-device-mapping.volume-type" + values = [ + "gp2" + ] + } + owners = [ + # Canonical + "099720109477" + ] +} + +resource "aws_instance" "bastion" { + ami = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" + subnet_id = "${var.subnet_id}" + vpc_security_group_ids = [ + "${var.security_groups}" + ] + key_name = "${var.key_name}" + associate_public_ip_address = true + tags { + Name = "bastion" + } +} \ No newline at end of file diff --git a/bastion/variables.tf b/bastion/variables.tf new file mode 100644 index 0000000..3bf4faa --- /dev/null +++ b/bastion/variables.tf @@ -0,0 +1,12 @@ +variable "subnet_id" { + description = "Public subnet for bastion server" +} + +variable "security_groups" { + description = "Security groups for bastion server" + default = [] +} + +variable "key_name" { + description = "SSH key to login to bastion server" +} \ No newline at end of file diff --git a/main.tf b/main.tf index 1cc7fc6..0a09b90 100644 --- a/main.tf +++ b/main.tf @@ -31,4 +31,14 @@ module "webservers" { "${aws_security_group.http.id}", "${aws_security_group.https.id}", ] -} \ No newline at end of file +} + +module "bastion" { + source = "./bastion" + subnet_id = "${module.vpc.public_subnets[0]}" + security_groups = [ + "${aws_security_group.ssh.id}", + "${aws_security_group.internal.id}", + ] + key_name = "${aws_key_pair.micropost.key_name}" +} From 68435205756052c98038ac9879ae313262568c8d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 17:01:15 +0700 Subject: [PATCH 067/181] cacheservers module --- cacheservers/main.tf | 21 +++++++++++++++++++++ cacheservers/outputs.tf | 3 +++ cacheservers/variables.tf | 9 +++++++++ elasticache.tf | 24 ------------------------ main.tf | 12 +++++++++++- terraform.log | 15 +++++++++++++++ 6 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 cacheservers/main.tf create mode 100644 cacheservers/outputs.tf create mode 100644 cacheservers/variables.tf delete mode 100644 elasticache.tf create mode 100644 terraform.log diff --git a/cacheservers/main.tf b/cacheservers/main.tf new file mode 100644 index 0000000..6fb2df4 --- /dev/null +++ b/cacheservers/main.tf @@ -0,0 +1,21 @@ +resource "aws_elasticache_cluster" "cacheservers" { + cluster_id = "cacheservers" + engine = "redis" + engine_version = "2.8.24" + node_type = "cache.t2.micro" + port = 6379 + num_cache_nodes = 1 + parameter_group_name = "default.redis2.8" + security_group_ids = [ + "${var.security_groups}", + ] + subnet_group_name = "${aws_elasticache_subnet_group.cacheservers.name}" +} + +resource "aws_elasticache_subnet_group" "cacheservers" { + name = "cacheservers" + description = "main subnet group" + subnet_ids = [ + "${var.subnets}" + ] +} \ No newline at end of file diff --git a/cacheservers/outputs.tf b/cacheservers/outputs.tf new file mode 100644 index 0000000..023fdf9 --- /dev/null +++ b/cacheservers/outputs.tf @@ -0,0 +1,3 @@ +output "endpoint" { + value = "${aws_elasticache_cluster.cacheservers.cache_nodes.0.address}" +} \ No newline at end of file diff --git a/cacheservers/variables.tf b/cacheservers/variables.tf new file mode 100644 index 0000000..be1ce88 --- /dev/null +++ b/cacheservers/variables.tf @@ -0,0 +1,9 @@ +variable "security_groups" { + description = "Security groups for cache servers" + default = [] +} + +variable "subnets" { + description = "Subnets for cache servers" + default = [] +} diff --git a/elasticache.tf b/elasticache.tf deleted file mode 100644 index 4c86b13..0000000 --- a/elasticache.tf +++ /dev/null @@ -1,24 +0,0 @@ -resource "aws_elasticache_cluster" "micropost" { - cluster_id = "micropost-${var.env}" - engine = "redis" - engine_version = "2.8.24" - node_type = "cache.t2.micro" - port = 6379 - num_cache_nodes = 1 - parameter_group_name = "default.redis2.8" - security_group_ids = [ - "${aws_security_group.internal.id}", - ] - subnet_group_name = "${aws_elasticache_subnet_group.micropost.name}" - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_elasticache_subnet_group" "micropost" { - name = "micropost-${var.env}" - description = "main subnet group" - subnet_ids = ["${module.vpc.private_subnets}"] -} \ No newline at end of file diff --git a/main.tf b/main.tf index 0a09b90..e34d1a1 100644 --- a/main.tf +++ b/main.tf @@ -13,7 +13,7 @@ module "webservers" { hostname = "${cloudflare_record.micropost.hostname}" logserver_endpoint = "${aws_elasticsearch_domain.micropost.endpoint}" dbserver_endpoint = "${aws_db_instance.micropost.endpoint}" - cacheserver_endpoint = "${aws_elasticache_cluster.micropost.cache_nodes.0.address}" + cacheserver_endpoint = "${module.cacheservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" deploy_bucket_arn = "${aws_s3_bucket.deploy.arn}" key_name = "${aws_key_pair.micropost.key_name}" @@ -42,3 +42,13 @@ module "bastion" { ] key_name = "${aws_key_pair.micropost.key_name}" } + +module "cacheservers" { + source = "./cacheservers" + security_groups = [ + "${aws_security_group.internal.id}", + ] + subnets = [ + "${module.vpc.private_subnets}", + ] +} diff --git a/terraform.log b/terraform.log new file mode 100644 index 0000000..c762571 --- /dev/null +++ b/terraform.log @@ -0,0 +1,15 @@ +2016/08/18 17:01:12 [INFO] Terraform version: 0.7.0 +2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira +2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira +2016/08/18 17:01:12 [DEBUG] Attempting to open CLI config file: /Users/akira/.terraformrc +2016/08/18 17:01:12 [DEBUG] File doesn't exist, but doesn't need to. Ignoring. +2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira +2016/08/18 17:01:12 [WARN] Ignoring AWS metadata API endpoint at default location as it doesn't return any instance-id +2016/08/18 17:01:13 [DEBUG] Uploading remote state to S3: { + Body: buffer(0xc82028fbf0), + Bucket: "deploy.hana053.com", + ContentLength: 912, + ContentType: "application/json", + Key: "stg.tfstate" +} +2016/08/18 17:01:14 [DEBUG] plugin: waiting for all plugin processes to complete... From 455cf9797eeca3def7086cdcd6ccfb2cff2ff2c5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 17:26:40 +0700 Subject: [PATCH 068/181] dbservers module --- .gitignore | 1 + cacheservers/main.tf | 10 +++++----- cacheservers/outputs.tf | 2 +- dbservers/outputs.tf | 3 +++ dbservers/rds.tf | 20 ++++++++++++++++++++ dbservers/variables.tf | 13 +++++++++++++ main.tf | 13 ++++++++++++- rds.tf | 25 ------------------------- terraform.log | 15 --------------- 9 files changed, 55 insertions(+), 47 deletions(-) create mode 100644 dbservers/outputs.tf create mode 100644 dbservers/rds.tf create mode 100644 dbservers/variables.tf delete mode 100644 rds.tf delete mode 100644 terraform.log diff --git a/.gitignore b/.gitignore index d22de4f..70d7c51 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ # tfstate is saved on remote .terraform +terraform.log diff --git a/cacheservers/main.tf b/cacheservers/main.tf index 6fb2df4..f3d155a 100644 --- a/cacheservers/main.tf +++ b/cacheservers/main.tf @@ -1,5 +1,5 @@ -resource "aws_elasticache_cluster" "cacheservers" { - cluster_id = "cacheservers" +resource "aws_elasticache_cluster" "main" { + cluster_id = "main" engine = "redis" engine_version = "2.8.24" node_type = "cache.t2.micro" @@ -9,11 +9,11 @@ resource "aws_elasticache_cluster" "cacheservers" { security_group_ids = [ "${var.security_groups}", ] - subnet_group_name = "${aws_elasticache_subnet_group.cacheservers.name}" + subnet_group_name = "${aws_elasticache_subnet_group.main.name}" } -resource "aws_elasticache_subnet_group" "cacheservers" { - name = "cacheservers" +resource "aws_elasticache_subnet_group" "main" { + name = "main" description = "main subnet group" subnet_ids = [ "${var.subnets}" diff --git a/cacheservers/outputs.tf b/cacheservers/outputs.tf index 023fdf9..0d027aa 100644 --- a/cacheservers/outputs.tf +++ b/cacheservers/outputs.tf @@ -1,3 +1,3 @@ output "endpoint" { - value = "${aws_elasticache_cluster.cacheservers.cache_nodes.0.address}" + value = "${aws_elasticache_cluster.main.cache_nodes.0.address}" } \ No newline at end of file diff --git a/dbservers/outputs.tf b/dbservers/outputs.tf new file mode 100644 index 0000000..c385baa --- /dev/null +++ b/dbservers/outputs.tf @@ -0,0 +1,3 @@ +output "endpoint" { + value = "${aws_db_instance.main.endpoint}" +} \ No newline at end of file diff --git a/dbservers/rds.tf b/dbservers/rds.tf new file mode 100644 index 0000000..2798719 --- /dev/null +++ b/dbservers/rds.tf @@ -0,0 +1,20 @@ +resource "aws_db_instance" "main" { + snapshot_identifier = "${var.snapshot_identifier}" + allocated_storage = 5 + engine = "mysql" + engine_version = "5.7.10" + instance_class = "db.t2.micro" + parameter_group_name = "default.mysql5.7" + vpc_security_group_ids = [ + "${var.security_groups}", + ] + db_subnet_group_name = "${aws_db_subnet_group.main.name}" +} + +resource "aws_db_subnet_group" "main" { + name = "main" + description = "main group of subnets" + subnet_ids = [ + "${var.subnets}" + ] +} diff --git a/dbservers/variables.tf b/dbservers/variables.tf new file mode 100644 index 0000000..11eab18 --- /dev/null +++ b/dbservers/variables.tf @@ -0,0 +1,13 @@ +variable "security_groups" { + description = "Security groups for db servers" + default = [] +} + +variable "subnets" { + description = "Subnets for db servers" + default = [] +} + +variable "snapshot_identifier" { + description = "Initial snapshot to restore" +} diff --git a/main.tf b/main.tf index e34d1a1..1286e1b 100644 --- a/main.tf +++ b/main.tf @@ -12,7 +12,7 @@ module "webservers" { env = "${var.env}" hostname = "${cloudflare_record.micropost.hostname}" logserver_endpoint = "${aws_elasticsearch_domain.micropost.endpoint}" - dbserver_endpoint = "${aws_db_instance.micropost.endpoint}" + dbserver_endpoint = "${module.dbservers.endpoint}" cacheserver_endpoint = "${module.cacheservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" deploy_bucket_arn = "${aws_s3_bucket.deploy.arn}" @@ -52,3 +52,14 @@ module "cacheservers" { "${module.vpc.private_subnets}", ] } + +module "dbservers" { + source = "./dbservers" + security_groups = [ + "${aws_security_group.internal.id}", + ] + subnets = [ + "${module.vpc.private_subnets}", + ] + snapshot_identifier = "micropost-init" +} diff --git a/rds.tf b/rds.tf deleted file mode 100644 index 154acb3..0000000 --- a/rds.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "aws_db_instance" "micropost" { - snapshot_identifier = "micropost-init" - allocated_storage = 5 - engine = "mysql" - engine_version = "5.7.10" - instance_class = "db.t2.micro" - db_subnet_group_name = "${aws_db_subnet_group.micropost.name}" - parameter_group_name = "default.mysql5.7" - vpc_security_group_ids = [ - "${aws_security_group.internal.id}", - ] - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } -} - -resource "aws_db_subnet_group" "micropost" { - name = "micropost-${var.env}" - description = "Our main group of subnets" - subnet_ids = [ - "${module.vpc.private_subnets}" - ] -} diff --git a/terraform.log b/terraform.log deleted file mode 100644 index c762571..0000000 --- a/terraform.log +++ /dev/null @@ -1,15 +0,0 @@ -2016/08/18 17:01:12 [INFO] Terraform version: 0.7.0 -2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira -2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira -2016/08/18 17:01:12 [DEBUG] Attempting to open CLI config file: /Users/akira/.terraformrc -2016/08/18 17:01:12 [DEBUG] File doesn't exist, but doesn't need to. Ignoring. -2016/08/18 17:01:12 [DEBUG] Detected home directory from env var: /Users/akira -2016/08/18 17:01:12 [WARN] Ignoring AWS metadata API endpoint at default location as it doesn't return any instance-id -2016/08/18 17:01:13 [DEBUG] Uploading remote state to S3: { - Body: buffer(0xc82028fbf0), - Bucket: "deploy.hana053.com", - ContentLength: 912, - ContentType: "application/json", - Key: "stg.tfstate" -} -2016/08/18 17:01:14 [DEBUG] plugin: waiting for all plugin processes to complete... From 608e577863dcd37d03b461c764e25a3cf93ff253 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 17:28:06 +0700 Subject: [PATCH 069/181] rename file --- dbservers/{rds.tf => main.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dbservers/{rds.tf => main.tf} (100%) diff --git a/dbservers/rds.tf b/dbservers/main.tf similarity index 100% rename from dbservers/rds.tf rename to dbservers/main.tf From 4bb8536c447022fdb1b9bdf7cba99e21b6530850 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 18 Aug 2016 20:42:26 +0700 Subject: [PATCH 070/181] logservers module --- elasticsearch.tf => logservers/main.tf | 43 ++++++++++---------------- logservers/outputs.tf | 3 ++ logservers/variables.tf | 19 ++++++++++++ main.tf | 14 ++++++++- security_group.tf | 3 +- variables.tf | 17 +++++++--- 6 files changed, 66 insertions(+), 33 deletions(-) rename elasticsearch.tf => logservers/main.tf (60%) create mode 100644 logservers/outputs.tf create mode 100644 logservers/variables.tf diff --git a/elasticsearch.tf b/logservers/main.tf similarity index 60% rename from elasticsearch.tf rename to logservers/main.tf index 8dbfad3..dd1c6a9 100644 --- a/elasticsearch.tf +++ b/logservers/main.tf @@ -1,9 +1,5 @@ -variable "es_backup_repository" { - default = "micropost-log-backups" -} - -resource "aws_elasticsearch_domain" "micropost" { - domain_name = "${var.app}-${var.env}" +resource "aws_elasticsearch_domain" "main" { + domain_name = "main" access_policies = < Date: Thu, 18 Aug 2016 22:29:20 +0700 Subject: [PATCH 071/181] codedeploy module --- codedeploy.tf => codedeploy/main.tf | 10 +++++----- codedeploy/variables.tf | 9 +++++++++ main.tf | 7 +++++++ 3 files changed, 21 insertions(+), 5 deletions(-) rename codedeploy.tf => codedeploy/main.tf (85%) create mode 100644 codedeploy/variables.tf diff --git a/codedeploy.tf b/codedeploy/main.tf similarity index 85% rename from codedeploy.tf rename to codedeploy/main.tf index 60c5825..44121b5 100644 --- a/codedeploy.tf +++ b/codedeploy/main.tf @@ -1,17 +1,17 @@ resource "aws_codedeploy_app" "main" { - name = "${var.app}-${var.env}" + name = "${var.name}" } -resource "aws_codedeploy_deployment_group" "web" { +resource "aws_codedeploy_deployment_group" "main" { app_name = "${aws_codedeploy_app.main.name}" - deployment_group_name = "web" + deployment_group_name = "${var.group_name}" service_role_arn = "${aws_iam_role.codedeploy_service.arn}" - autoscaling_groups = ["${module.webservers.asg_id}"] + autoscaling_groups = ["${var.autoscaling_groups}"] deployment_config_name = "CodeDeployDefault.OneAtATime" } resource "aws_iam_role" "codedeploy_service" { - name = "${var.app}-${var.env}-codedeploy-service" + name = "codedeploy-${var.group_name}" assume_role_policy = < Date: Fri, 19 Aug 2016 09:28:02 +0700 Subject: [PATCH 072/181] security_group module --- main.tf | 24 ++++++++----- s3.tf | 5 --- security_group.tf => security_groups/main.tf | 36 +++++++------------- security_groups/outputs.tf | 15 ++++++++ security_groups/variables.tf | 6 ++++ variables.tf | 10 +++--- vpc/variables.tf | 4 --- 7 files changed, 52 insertions(+), 48 deletions(-) rename security_group.tf => security_groups/main.tf (67%) create mode 100644 security_groups/outputs.tf create mode 100644 security_groups/variables.tf diff --git a/main.tf b/main.tf index a52a995..4da598b 100644 --- a/main.tf +++ b/main.tf @@ -4,11 +4,11 @@ provider "aws" { module "vpc" { source = "./vpc" - region = "${var.aws_region}" } module "webservers" { source = "./webservers" + env = "${var.env}" hostname = "${cloudflare_record.micropost.hostname}" logserver_endpoint = "${module.logservers.endpoint}" @@ -21,15 +21,15 @@ module "webservers" { "${module.vpc.public_subnets}" ] web_security_groups = [ - "${aws_security_group.internal.id}", + "${module.security_groups.internal}", ] elb_subnets = [ "${module.vpc.public_subnets}" ] elb_security_groups = [ - "${aws_security_group.internal.id}", - "${aws_security_group.http.id}", - "${aws_security_group.https.id}", + "${module.security_groups.internal}", + "${module.security_groups.internet_in_http}", + "${module.security_groups.internet_in_https}", ] min_scale_size = "${var.web_min_size}" desired_capacity = "${var.web_desired_capacity}" @@ -39,8 +39,8 @@ module "bastion" { source = "./bastion" subnet_id = "${module.vpc.public_subnets[0]}" security_groups = [ - "${aws_security_group.ssh.id}", - "${aws_security_group.internal.id}", + "${module.security_groups.internal}", + "${module.security_groups.internet_in_ssh}", ] key_name = "${aws_key_pair.micropost.key_name}" } @@ -48,7 +48,7 @@ module "bastion" { module "cacheservers" { source = "./cacheservers" security_groups = [ - "${aws_security_group.internal.id}", + "${module.security_groups.internal}", ] subnets = [ "${module.vpc.private_subnets}", @@ -58,7 +58,7 @@ module "cacheservers" { module "dbservers" { source = "./dbservers" security_groups = [ - "${aws_security_group.internal.id}", + "${module.security_groups.internal}", ] subnets = [ "${module.vpc.private_subnets}", @@ -82,3 +82,9 @@ module "web_codedeploy" { group_name = "web" autoscaling_groups = ["${module.webservers.asg_id}"] } + +module "security_groups" { + source = "./security_groups" + vpc_id = "${module.vpc.vpc_id}" + ssh_allowed_segments = ["${var.allowed_segments}"] +} diff --git a/s3.tf b/s3.tf index 9a5b708..2ffa762 100644 --- a/s3.tf +++ b/s3.tf @@ -10,11 +10,6 @@ resource "aws_s3_bucket" "deploy" { days = 20 } } - tags { - Name = "${var.app}-${var.env}" - App = "${var.app}" - Env = "${var.env}" - } } resource "aws_s3_bucket" "backup" { diff --git a/security_group.tf b/security_groups/main.tf similarity index 67% rename from security_group.tf rename to security_groups/main.tf index d5de804..a95eddb 100644 --- a/security_group.tf +++ b/security_groups/main.tf @@ -1,5 +1,5 @@ resource "aws_security_group" "internal" { - vpc_id = "${module.vpc.vpc_id}" + vpc_id = "${var.vpc_id}" name_prefix = "internal-" description = "Allow internal traffic" ingress { @@ -20,15 +20,12 @@ resource "aws_security_group" "internal" { create_before_destroy = true } tags { - Name = "${var.app}-${var.env}-internal" - App = "${var.app}" - Env = "${var.env}" - Role = "internal" + Name = "internal" } } -resource "aws_security_group" "ssh" { - vpc_id = "${module.vpc.vpc_id}" +resource "aws_security_group" "internet_in_ssh" { + vpc_id = "${var.vpc_id}" name_prefix = "ssh-" description = "Allow ssh inbound traffic" ingress { @@ -36,7 +33,7 @@ resource "aws_security_group" "ssh" { to_port = 22 protocol = "tcp" cidr_blocks = [ - "${var.allowed_segments}" + "${var.ssh_allowed_segments}" ] } egress { @@ -51,15 +48,12 @@ resource "aws_security_group" "ssh" { create_before_destroy = true } tags { - Name = "${var.app}-${var.env}-ssh" - App = "${var.app}" - Env = "${var.env}" - Role = "ssh" + Name = "internet-in-ssh" } } -resource "aws_security_group" "http" { - vpc_id = "${module.vpc.vpc_id}" +resource "aws_security_group" "internet_in_http" { + vpc_id = "${var.vpc_id}" name_prefix = "http-" description = "Allow http inbound traffic" ingress { @@ -82,15 +76,12 @@ resource "aws_security_group" "http" { create_before_destroy = true } tags { - Name = "${var.app}-${var.env}-http" - App = "${var.app}" - Env = "${var.env}" - Role = "http" + Name = "internet-in-http" } } -resource "aws_security_group" "https" { - vpc_id = "${module.vpc.vpc_id}" +resource "aws_security_group" "internet_in_https" { + vpc_id = "${var.vpc_id}" name_prefix = "https-" description = "Allow https inbound traffic" ingress { @@ -113,9 +104,6 @@ resource "aws_security_group" "https" { create_before_destroy = true } tags { - Name = "${var.app}-${var.env}-https" - App = "${var.app}" - Env = "${var.env}" - Role = "https" + Name = "internet-in-https" } } \ No newline at end of file diff --git a/security_groups/outputs.tf b/security_groups/outputs.tf new file mode 100644 index 0000000..656f307 --- /dev/null +++ b/security_groups/outputs.tf @@ -0,0 +1,15 @@ +output "internal" { + value = "${aws_security_group.internal.id}" +} + +output "internet_in_ssh" { + value = "${aws_security_group.internet_in_ssh.id}" +} + +output "internet_in_http" { + value = "${aws_security_group.internet_in_http.id}" +} + +output "internet_in_https" { + value = "${aws_security_group.internet_in_https.id}" +} diff --git a/security_groups/variables.tf b/security_groups/variables.tf new file mode 100644 index 0000000..a52ad4e --- /dev/null +++ b/security_groups/variables.tf @@ -0,0 +1,6 @@ +variable "vpc_id" { +} + +variable "ssh_allowed_segments" { + default = [] +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 8fb931c..4a56813 100644 --- a/variables.tf +++ b/variables.tf @@ -2,17 +2,12 @@ variable "env" { description = "dev, stg, prod and etc." default = "dev" } + variable "aws_account_id" { } variable "aws_region" { default = "ap-northeast-1" } -variable "web_host_name" { -} - -variable "app" { - default = "micropost" -} variable "allowed_segments" { default = [ @@ -21,6 +16,9 @@ variable "allowed_segments" { ] } +variable "web_host_name" { +} + variable "web_min_size" { default = 1 } diff --git a/vpc/variables.tf b/vpc/variables.tf index 5e983f3..5ff99c1 100644 --- a/vpc/variables.tf +++ b/vpc/variables.tf @@ -25,7 +25,3 @@ variable "azs" { "ap-northeast-1c", ] } - -variable "region" { - default = "ap-northeast-1" -} \ No newline at end of file From faf7f219d881d25bac56b37405a3ef7688044390 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 09:32:50 +0700 Subject: [PATCH 073/181] domain variable --- dns.tf | 2 +- s3.tf | 4 ++-- variables.tf | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dns.tf b/dns.tf index 10053df..f28e2d9 100644 --- a/dns.tf +++ b/dns.tf @@ -1,5 +1,5 @@ resource "cloudflare_record" "micropost" { - domain = "hana053.com" + domain = "${var.domain}" name = "${var.web_host_name}" value = "${module.webservers.dns_name}" type = "CNAME" diff --git a/s3.tf b/s3.tf index 2ffa762..b6d0193 100644 --- a/s3.tf +++ b/s3.tf @@ -1,5 +1,5 @@ resource "aws_s3_bucket" "deploy" { - bucket = "deploy-${var.env}.hana053.com" + bucket = "deploy-${var.env}.${var.domain}" force_destroy = true // evict letsencrypt cert lifecycle_rule { @@ -13,6 +13,6 @@ resource "aws_s3_bucket" "deploy" { } resource "aws_s3_bucket" "backup" { - bucket = "backup-${var.env}.hana053.com" + bucket = "backup-${var.env}.${var.domain}" force_destroy = true } diff --git a/variables.tf b/variables.tf index 4a56813..4b1a924 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,7 @@ variable "env" { default = "dev" } +// TODO can be acquired from data in Terraform 0.7.1 variable "aws_account_id" { } variable "aws_region" { @@ -16,6 +17,10 @@ variable "allowed_segments" { ] } +variable "domain" { + default = "hana053.com" +} + variable "web_host_name" { } From c8eb335e8461aa3caceb1ccd9767fd80f0ad3ea7 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 10:57:24 +0700 Subject: [PATCH 074/181] respect current desired capacity --- scripts/export-tfvars.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 0b265c8..356b613 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -1,5 +1,13 @@ #!/bin/sh +asg_name=$(terraform output web_asg_name) +if [ -n "${asg_name}" ]; then + desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name web | jq '.AutoScalingGroups[0].DesiredCapacity') + echo "Current desired capacity is ${desired_capacity}." + # respect current desired capacity + export TF_VAR_web_desired_capacity="${desired_capacity}" +fi + if [ "${ENV}" = "prod" ]; then # prod is special export TF_VAR_web_host_name="micropost" From a194484dbd95f0c13fe81c45f231dc2ead32d01e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 10:58:49 +0700 Subject: [PATCH 075/181] fix export-tfvars --- scripts/export-tfvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 356b613..15f9798 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -2,7 +2,7 @@ asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then - desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name web | jq '.AutoScalingGroups[0].DesiredCapacity') + desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') echo "Current desired capacity is ${desired_capacity}." # respect current desired capacity export TF_VAR_web_desired_capacity="${desired_capacity}" From de50cbbf92684ca597f29f69156710960a53f74b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 11:20:33 +0700 Subject: [PATCH 076/181] tag --- main.tf | 3 ++- variables.tf | 1 - vpc/main.tf | 40 +++++++++++++++------------------------- vpc/variables.tf | 3 +++ webservers/autoscale.tf | 2 +- webservers/elb.tf | 6 ------ webservers/variables.tf | 3 +++ 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/main.tf b/main.tf index 4da598b..4c34f36 100644 --- a/main.tf +++ b/main.tf @@ -4,11 +4,11 @@ provider "aws" { module "vpc" { source = "./vpc" + name = "micropost" } module "webservers" { source = "./webservers" - env = "${var.env}" hostname = "${cloudflare_record.micropost.hostname}" logserver_endpoint = "${module.logservers.endpoint}" @@ -17,6 +17,7 @@ module "webservers" { deploy_bucket = "${aws_s3_bucket.deploy.bucket}" deploy_bucket_arn = "${aws_s3_bucket.deploy.arn}" key_name = "${aws_key_pair.micropost.key_name}" + web_ami_tag = "micropost-web" web_subnets = [ "${module.vpc.public_subnets}" ] diff --git a/variables.tf b/variables.tf index 4b1a924..e47107b 100644 --- a/variables.tf +++ b/variables.tf @@ -13,7 +13,6 @@ variable "aws_region" { variable "allowed_segments" { default = [ "42.116.5.4/32", - "171.232.52.48/32", ] } diff --git a/vpc/main.tf b/vpc/main.tf index f014f87..814ef8b 100644 --- a/vpc/main.tf +++ b/vpc/main.tf @@ -1,19 +1,15 @@ resource "aws_vpc" "main" { cidr_block = "${var.cidr}" - // tags { - // Name = "${var.app}-${var.env}" - // App = "${var.app}" - // Env = "${var.env}" - // } + tags { + Name = "${var.name}" + } } resource "aws_internet_gateway" "main" { vpc_id = "${aws_vpc.main.id}" - // tags { - // Name = "${var.app}-${var.env}" - // App = "${var.app}" - // Env = "${var.env}" - // } + tags { + Name = "${var.name}" + } } resource "aws_route_table" "public" { @@ -22,11 +18,9 @@ resource "aws_route_table" "public" { cidr_block = "0.0.0.0/0" gateway_id = "${aws_internet_gateway.main.id}" } - // tags { - // Name = "${var.app}-${var.env}" - // App = "${var.app}" - // Env = "${var.env}" - // } + tags { + Name = "${var.name}" + } } resource "aws_subnet" "public" { @@ -35,11 +29,9 @@ resource "aws_subnet" "public" { cidr_block = "${var.public_subnets[count.index]}" availability_zone = "${var.azs[count.index]}" map_public_ip_on_launch = true - // tags { - // Name = "${var.app}-${var.env}-public1" - // App = "${var.app}" - // Env = "${var.env}" - // } + tags { + Name = "${var.name}-public" + } } resource "aws_route_table_association" "puclic" { @@ -53,10 +45,8 @@ resource "aws_subnet" "private" { vpc_id = "${aws_vpc.main.id}" cidr_block = "${var.private_subnets[count.index]}" availability_zone = "${var.azs[count.index]}" - // tags { - // Name = "${var.app}-${var.env}-private3" - // App = "${var.app}" - // Env = "${var.env}" - // } + tags { + Name = "${var.name}-private" + } } diff --git a/vpc/variables.tf b/vpc/variables.tf index 5ff99c1..f84e850 100644 --- a/vpc/variables.tf +++ b/vpc/variables.tf @@ -1,3 +1,6 @@ +variable "name" { +} + variable "cidr" { default = "10.1.0.0/16" } diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index c01d581..dc308c2 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -6,7 +6,7 @@ data "aws_ami" "web" { filter { name = "tag:Name" values = [ - "micropost-web" + "${var.web_ami_tag}" ] } } diff --git a/webservers/elb.tf b/webservers/elb.tf index 934306b..303479d 100644 --- a/webservers/elb.tf +++ b/webservers/elb.tf @@ -30,12 +30,6 @@ resource "aws_elb" "web" { idle_timeout = 400 connection_draining = true connection_draining_timeout = 400 - // tags { - // Name = "${var.app}-${var.env}-web" - // App = "${var.app}" - // Env = "${var.env}" - // Role = "web" - // } } resource "aws_proxy_protocol_policy" "web" { diff --git a/webservers/variables.tf b/webservers/variables.tf index 759c265..3ad8381 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -38,6 +38,9 @@ variable "key_name" { description = "SSH key name for web servers" } +variable "web_ami_tag" { +} + variable "web_subnets" { description = "Subnets for web servers" default = [] From 3a5e5c3909c8522aa820bd0dba67f5e436f163b9 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 11:30:08 +0700 Subject: [PATCH 077/181] install aws cli --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8263408..326c6c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,11 @@ cache: install: - gem install travis -v 1.8.2 --no-rdoc --no-ri - bundle install + # install aws cli + - sudo apt-get -y install python-pip + - sudo pip install awscli + - aws --version + # install terraform - wget https://releases.hashicorp.com/terraform/0.7.0/terraform_0.7.0_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip From bbcfbfb9ed3a41e7260af10c28661b386343fb09 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 13:43:56 +0700 Subject: [PATCH 078/181] script to switch role --- .envrc.example | 6 +++++- .travis.yml | 3 +++ scripts/switch_production_role.sh | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100755 scripts/switch_production_role.sh diff --git a/.envrc.example b/.envrc.example index 2b10c78..2f365e5 100644 --- a/.envrc.example +++ b/.envrc.example @@ -10,4 +10,8 @@ export TF_VAR_aws_account_id= export TF_VAR_aws_region=${AWS_DEFAULT_REGION} export CLOUDFLARE_EMAIL= -export CLOUDFLARE_TOKEN= \ No newline at end of file +export CLOUDFLARE_TOKEN= + +export PROD_ROLE_ARN= + +unset AWS_SESSION_TOKEN diff --git a/.travis.yml b/.travis.yml index 326c6c5..9d1fb58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,9 @@ after_success: - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - test -z "${ENV}" && (echo "${TRAVIS_BRANCH} is not a branch to deploy." && exit 0) + # Switch AWS Role when ENV is prod + - test "${ENV}" = "prod" && source scripts/switch_production_role.sh + # Enable terraform remote - scripts/terraform-enable-remote.sh diff --git a/scripts/switch_production_role.sh b/scripts/switch_production_role.sh new file mode 100755 index 0000000..28b28ee --- /dev/null +++ b/scripts/switch_production_role.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +credentials=$(aws sts assume-role --role-arn ${PROD_ROLE_ARN} --role-session-name travisci) + +export AWS_ACCESS_KEY_ID=$(echo ${credentials} | jq --raw-output .Credentials.AccessKeyId) +export AWS_SECRET_ACCESS_KEY=$(echo ${credentials} | jq --raw-output .Credentials.SecretAccessKey) +export AWS_SESSION_TOKEN=$(echo ${credentials} | jq --raw-output .Credentials.SessionToken) + +unset credentials \ No newline at end of file From 9f437af34843aefc1ffd416bdb6e38a4da04d464 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 13:44:46 +0700 Subject: [PATCH 079/181] change file name --- .travis.yml | 2 +- .../{switch_production_role.sh => switch-production-role.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{switch_production_role.sh => switch-production-role.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 9d1fb58..92d4428 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ after_success: - test -z "${ENV}" && (echo "${TRAVIS_BRANCH} is not a branch to deploy." && exit 0) # Switch AWS Role when ENV is prod - - test "${ENV}" = "prod" && source scripts/switch_production_role.sh + - test "${ENV}" = "prod" && source scripts/switch-production-role.sh # Enable terraform remote - scripts/terraform-enable-remote.sh diff --git a/scripts/switch_production_role.sh b/scripts/switch-production-role.sh similarity index 100% rename from scripts/switch_production_role.sh rename to scripts/switch-production-role.sh From c4cac95ebe290b21e96876f27d574f4cd312fdaf Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 14:54:26 +0700 Subject: [PATCH 080/181] fix elasticsearch --- logservers/main.tf | 2 +- scripts/export-tfvars.sh | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/logservers/main.tf b/logservers/main.tf index dd1c6a9..6b01123 100644 --- a/logservers/main.tf +++ b/logservers/main.tf @@ -18,7 +18,7 @@ resource "aws_elasticsearch_domain" "main" { "Effect": "Allow", "Condition": { "IpAddress": { - "aws:SourceIp": ${jsonencode(var.allowed_segments)} + "aws:SourceIp": "${var.allowed_segments[0]}" } }, "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_id}:domain/main/*" diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 15f9798..fedc17e 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -8,9 +8,10 @@ if [ -n "${asg_name}" ]; then export TF_VAR_web_desired_capacity="${desired_capacity}" fi -if [ "${ENV}" = "prod" ]; then - # prod is special - export TF_VAR_web_host_name="micropost" -else - export TF_VAR_web_host_name="micropost-${ENV}" -fi +#if [ "${ENV}" = "prod" ]; then +# # prod is special +# export TF_VAR_web_host_name="micropost" +#else +# export TF_VAR_web_host_name="micropost-${ENV}" +#fi +export TF_VAR_web_host_name="micropost-${ENV}" From 0baa47b7ce6fe89b99fff35c9d8f2a01a2a894ee Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 19 Aug 2016 17:50:55 +0700 Subject: [PATCH 081/181] fix es_register_snapshot_directory for switching role --- scripts/es_register_snapshot_directory.rb | 3 ++- scripts/terraform-enable-remote.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/es_register_snapshot_directory.rb b/scripts/es_register_snapshot_directory.rb index 5d46998..d3ae15b 100644 --- a/scripts/es_register_snapshot_directory.rb +++ b/scripts/es_register_snapshot_directory.rb @@ -10,7 +10,8 @@ faraday.request :aws_signers_v4, credentials: Aws::Credentials.new( ENV['AWS_ACCESS_KEY_ID'], - ENV['AWS_SECRET_ACCESS_KEY'] + ENV['AWS_SECRET_ACCESS_KEY'], + ENV['AWS_SESSION_TOKEN'] ), service_name: 'es', region: option[:region] diff --git a/scripts/terraform-enable-remote.sh b/scripts/terraform-enable-remote.sh index 2a0c769..8d16c75 100755 --- a/scripts/terraform-enable-remote.sh +++ b/scripts/terraform-enable-remote.sh @@ -10,7 +10,7 @@ rm -rf ./.terraform/ terraform remote config \ -state="${ENV}.tfstate" \ -backend=S3 \ - -backend-config="region=ap-southeast-1" \ - -backend-config="bucket=deploy.hana053.com" \ + -backend-config="region=ap-northeast-1" \ + -backend-config="bucket=tfstate.hana053.com" \ -backend-config="key=${ENV}.tfstate" terraform remote pull From 7f33f55f1c7764f931a49679c8133e0ebaa92e15 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 20 Aug 2016 11:49:28 +0700 Subject: [PATCH 082/181] [skip ci] reduce max size --- scripts/export-tfvars.sh | 13 ++++++------- webservers/autoscale.tf | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index fedc17e..15f9798 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -8,10 +8,9 @@ if [ -n "${asg_name}" ]; then export TF_VAR_web_desired_capacity="${desired_capacity}" fi -#if [ "${ENV}" = "prod" ]; then -# # prod is special -# export TF_VAR_web_host_name="micropost" -#else -# export TF_VAR_web_host_name="micropost-${ENV}" -#fi -export TF_VAR_web_host_name="micropost-${ENV}" +if [ "${ENV}" = "prod" ]; then + # prod is special + export TF_VAR_web_host_name="micropost" +else + export TF_VAR_web_host_name="micropost-${ENV}" +fi diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index dc308c2..6173000 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -42,7 +42,7 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web" launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 4 + max_size = 2 min_size = "${var.min_scale_size}" desired_capacity = "${var.desired_capacity}" health_check_grace_period = 300 From e30c2a6cd08b9a9fdba931375e79c2a5720400da Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 23 Aug 2016 15:38:03 +0700 Subject: [PATCH 083/181] use elasticsearch 2.3 --- logservers/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/logservers/main.tf b/logservers/main.tf index 6b01123..3422b2c 100644 --- a/logservers/main.tf +++ b/logservers/main.tf @@ -1,5 +1,6 @@ resource "aws_elasticsearch_domain" "main" { domain_name = "main" + elasticsearch_version = "2.3" access_policies = < Date: Tue, 23 Aug 2016 15:46:38 +0700 Subject: [PATCH 084/181] update travis to use terraform 0.7.1 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 92d4428..f27dd51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - sudo pip install awscli - aws --version # install terraform - - wget https://releases.hashicorp.com/terraform/0.7.0/terraform_0.7.0_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.1/terraform_0.7.1_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: From 9b71f5aeed7a73642c1bb1ba1a3fd229179cd570 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 26 Sep 2016 17:49:42 +0700 Subject: [PATCH 085/181] [skip ci] elastic search main is stacking... --- logservers/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logservers/main.tf b/logservers/main.tf index 3422b2c..c2256cf 100644 --- a/logservers/main.tf +++ b/logservers/main.tf @@ -1,5 +1,5 @@ resource "aws_elasticsearch_domain" "main" { - domain_name = "main" + domain_name = "main2" elasticsearch_version = "2.3" access_policies = < Date: Tue, 27 Sep 2016 08:38:05 +0700 Subject: [PATCH 086/181] aws_account_id is available from data resource --- .envrc.example | 1 - .travis.yml | 3 +-- logservers/main.tf | 8 +++++--- logservers/variables.tf | 3 --- main.tf | 1 - variables.tf | 3 --- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.envrc.example b/.envrc.example index 2f365e5..8d76227 100644 --- a/.envrc.example +++ b/.envrc.example @@ -6,7 +6,6 @@ export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_DEFAULT_REGION= -export TF_VAR_aws_account_id= export TF_VAR_aws_region=${AWS_DEFAULT_REGION} export CLOUDFLARE_EMAIL= diff --git a/.travis.yml b/.travis.yml index f27dd51..d674f4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,10 @@ install: - sudo pip install awscli - aws --version # install terraform - - wget https://releases.hashicorp.com/terraform/0.7.1/terraform_0.7.1_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.4/terraform_0.7.4_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: - - export TF_VAR_aws_account_id=$(bundle exec ruby scripts/get_account_id.rb) - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - echo diff --git a/logservers/main.tf b/logservers/main.tf index c2256cf..bc09813 100644 --- a/logservers/main.tf +++ b/logservers/main.tf @@ -1,3 +1,5 @@ +data "aws_caller_identity" "current" {} + resource "aws_elasticsearch_domain" "main" { domain_name = "main2" elasticsearch_version = "2.3" @@ -8,10 +10,10 @@ resource "aws_elasticsearch_domain" "main" { { "Action": "es:*", "Principal": { - "AWS": "arn:aws:iam::${var.aws_account_id}:root" + "AWS": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }, "Effect": "Allow", - "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_id}:domain/main/*" + "Resource": "arn:aws:es:${var.aws_region}:${data.aws_caller_identity.current.account_id}:domain/main/*" }, { "Action": "es:*", @@ -22,7 +24,7 @@ resource "aws_elasticsearch_domain" "main" { "aws:SourceIp": "${var.allowed_segments[0]}" } }, - "Resource": "arn:aws:es:${var.aws_region}:${var.aws_account_id}:domain/main/*" + "Resource": "arn:aws:es:${var.aws_region}:${data.aws_caller_identity.current.account_id}:domain/main/*" } ] } diff --git a/logservers/variables.tf b/logservers/variables.tf index 2cb5709..d6537cf 100644 --- a/logservers/variables.tf +++ b/logservers/variables.tf @@ -1,6 +1,3 @@ -variable "aws_account_id" { -} - variable "aws_region" { } diff --git a/main.tf b/main.tf index 4c34f36..7fd80f7 100644 --- a/main.tf +++ b/main.tf @@ -69,7 +69,6 @@ module "dbservers" { module "logservers" { source = "./logservers" - aws_account_id = "${var.aws_account_id}" aws_region = "${var.aws_region}" allowed_segments = "${var.allowed_segments}" backup_repository = "micropost-log-backups" diff --git a/variables.tf b/variables.tf index e47107b..0123b5f 100644 --- a/variables.tf +++ b/variables.tf @@ -3,9 +3,6 @@ variable "env" { default = "dev" } -// TODO can be acquired from data in Terraform 0.7.1 -variable "aws_account_id" { -} variable "aws_region" { default = "ap-northeast-1" } From 022b352019f3e2e4c78695a926b45fdf17ae05ab Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 27 Sep 2016 10:14:22 +0700 Subject: [PATCH 087/181] replace elb with alb --- main.tf | 6 +++-- variables.tf | 5 ++++ webservers/alb.tf | 57 +++++++++++++++++++++++++++++++++++++++++ webservers/autoscale.tf | 34 ++++++++++++------------ webservers/elb.tf | 40 ----------------------------- webservers/outputs.tf | 2 +- webservers/s3.tf | 4 +++ webservers/variables.tf | 17 +++++++++--- 8 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 webservers/alb.tf delete mode 100644 webservers/elb.tf create mode 100644 webservers/s3.tf diff --git a/main.tf b/main.tf index 7fd80f7..720a1f4 100644 --- a/main.tf +++ b/main.tf @@ -24,16 +24,18 @@ module "webservers" { web_security_groups = [ "${module.security_groups.internal}", ] - elb_subnets = [ + alb_subnets = [ "${module.vpc.public_subnets}" ] - elb_security_groups = [ + alb_security_groups = [ "${module.security_groups.internal}", "${module.security_groups.internet_in_http}", "${module.security_groups.internet_in_https}", ] + alb_certificate_arn = "${var.alb_certificate_arn}" min_scale_size = "${var.web_min_size}" desired_capacity = "${var.web_desired_capacity}" + vpc_id = "${module.vpc.vpc_id}" } module "bastion" { diff --git a/variables.tf b/variables.tf index 0123b5f..c80ab92 100644 --- a/variables.tf +++ b/variables.tf @@ -8,8 +8,10 @@ variable "aws_region" { } variable "allowed_segments" { + type = "list" default = [ "42.116.5.4/32", + "118.69.191.34/32" ] } @@ -17,6 +19,9 @@ variable "domain" { default = "hana053.com" } +variable "alb_certificate_arn" { +} + variable "web_host_name" { } diff --git a/webservers/alb.tf b/webservers/alb.tf new file mode 100644 index 0000000..6a3f263 --- /dev/null +++ b/webservers/alb.tf @@ -0,0 +1,57 @@ +resource "aws_alb" "web" { + name = "web" + internal = false + security_groups = [ + "${var.alb_security_groups}" + ] + subnets = [ + "${var.alb_subnets}" + ] + enable_deletion_protection = true +// access_logs { +// bucket = "${aws_s3_bucket.log.bucket}" +// prefix = "test-alb" +// } +} + +resource "aws_alb_target_group" "web" { + name = "web" + port = 8080 + protocol = "HTTP" + vpc_id = "${var.vpc_id}" + health_check { + interval = 30 + path = "/manage/health" + port = 8080 + protocol = "HTTP" + timeout = 5 + unhealthy_threshold = 2 + } +} + +resource "aws_alb_listener" "web" { + load_balancer_arn = "${aws_alb.web.arn}" + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2015-05" + certificate_arn = "${var.alb_certificate_arn}" + default_action { + target_group_arn = "${aws_alb_target_group.web.arn}" + type = "forward" + } +} + +resource "aws_alb_listener_rule" "web" { + listener_arn = "${aws_alb_listener.web.arn}" + priority = 100 + action { + type = "forward" + target_group_arn = "${aws_alb_target_group.web.arn}" + } + condition { + field = "path-pattern" + values = [ + "*" + ] + } +} \ No newline at end of file diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 6173000..377ffdb 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -51,24 +51,24 @@ resource "aws_autoscaling_group" "web" { vpc_zone_identifier = [ "${var.web_subnets}" ] - load_balancers = [ - "${aws_elb.web.name}" + target_group_arns = [ + "${aws_alb_target_group.web.arn}" ] - tag { - key = "Name" - value = "${var.env}-web" - propagate_at_launch = true - } - tag { - key = "Env" - value = "${var.env}" - propagate_at_launch = true - } - tag { - key = "Role" - value = "web" - propagate_at_launch = true - } + tag { + key = "Name" + value = "${var.env}-web" + propagate_at_launch = true + } + tag { + key = "Env" + value = "${var.env}" + propagate_at_launch = true + } + tag { + key = "Role" + value = "web" + propagate_at_launch = true + } } resource "aws_autoscaling_policy" "web_scale_out" { diff --git a/webservers/elb.tf b/webservers/elb.tf deleted file mode 100644 index 303479d..0000000 --- a/webservers/elb.tf +++ /dev/null @@ -1,40 +0,0 @@ -resource "aws_elb" "web" { - name = "web" - subnets = [ - "${var.elb_subnets}" - ] - security_groups = [ - "${var.elb_security_groups}" - ] - // use for health check - listener { - instance_port = 80 - instance_protocol = "http" - lb_port = 80 - lb_protocol = "http" - } - listener { - instance_port = 443 - instance_protocol = "tcp" - lb_port = 443 - lb_protocol = "tcp" - } - health_check { - healthy_threshold = 2 - unhealthy_threshold = 2 - timeout = 5 - target = "HTTP:80/manage/health" - interval = 30 - } - cross_zone_load_balancing = true - idle_timeout = 400 - connection_draining = true - connection_draining_timeout = 400 -} - -resource "aws_proxy_protocol_policy" "web" { - load_balancer = "${aws_elb.web.name}" - instance_ports = [ - "443" - ] -} diff --git a/webservers/outputs.tf b/webservers/outputs.tf index 036affd..9509cb4 100644 --- a/webservers/outputs.tf +++ b/webservers/outputs.tf @@ -1,5 +1,5 @@ output "dns_name" { - value = "${aws_elb.web.dns_name}" + value = "${aws_alb.web.dns_name}" } output "ami" { diff --git a/webservers/s3.tf b/webservers/s3.tf new file mode 100644 index 0000000..d23d40f --- /dev/null +++ b/webservers/s3.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "log" { + bucket = "log-${var.env}.hana053.com" + force_destroy = true +} diff --git a/webservers/variables.tf b/webservers/variables.tf index 3ad8381..1baef13 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -51,13 +51,22 @@ variable "web_security_groups" { default = [] } -variable "elb_subnets" { - description = "Subnets for elb" +variable "alb_subnets" { + description = "Subnets for alb" default = [] } -variable "elb_security_groups" { - description = "Security groups for elb" +variable "alb_security_groups" { + description = "Security groups for alb" default = [] } +variable "alb_certificate_arn" { + description = "alb certificate arn" +} + +variable "vpc_id" { + description = "vpc id for alb target group" +} + + From 31db8fc12dda1bdf0fe4d654adc1da4be3da8baa Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 27 Sep 2016 11:19:23 +0700 Subject: [PATCH 088/181] enable logging for alb --- webservers/alb.tf | 8 ++++---- webservers/s3.tf | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/webservers/alb.tf b/webservers/alb.tf index 6a3f263..3fcb3bb 100644 --- a/webservers/alb.tf +++ b/webservers/alb.tf @@ -8,10 +8,10 @@ resource "aws_alb" "web" { "${var.alb_subnets}" ] enable_deletion_protection = true -// access_logs { -// bucket = "${aws_s3_bucket.log.bucket}" -// prefix = "test-alb" -// } + access_logs { + bucket = "${aws_s3_bucket.log.bucket}" + prefix = "alb-web" + } } resource "aws_alb_target_group" "web" { diff --git a/webservers/s3.tf b/webservers/s3.tf index d23d40f..9368671 100644 --- a/webservers/s3.tf +++ b/webservers/s3.tf @@ -1,4 +1,25 @@ +data "aws_caller_identity" "current" {} + resource "aws_s3_bucket" "log" { bucket = "log-${var.env}.hana053.com" force_destroy = true + policy = < Date: Tue, 27 Sep 2016 11:56:50 +0700 Subject: [PATCH 089/181] extract s3 log bucket to global --- main.tf | 3 +++ s3.tf | 33 ++++++++++++++++++++++++--------- webservers/alb.tf | 2 +- webservers/s3.tf | 25 ------------------------- webservers/variables.tf | 4 +++- 5 files changed, 31 insertions(+), 36 deletions(-) delete mode 100644 webservers/s3.tf diff --git a/main.tf b/main.tf index 720a1f4..e3754ca 100644 --- a/main.tf +++ b/main.tf @@ -2,6 +2,8 @@ provider "aws" { region = "${var.aws_region}" } +data "aws_caller_identity" "current" {} + module "vpc" { source = "./vpc" name = "micropost" @@ -36,6 +38,7 @@ module "webservers" { min_scale_size = "${var.web_min_size}" desired_capacity = "${var.web_desired_capacity}" vpc_id = "${module.vpc.vpc_id}" + log_bucket = "${aws_s3_bucket.log.bucket}" } module "bastion" { diff --git a/s3.tf b/s3.tf index b6d0193..1053f6a 100644 --- a/s3.tf +++ b/s3.tf @@ -1,18 +1,33 @@ resource "aws_s3_bucket" "deploy" { bucket = "deploy-${var.env}.${var.domain}" force_destroy = true - // evict letsencrypt cert - lifecycle_rule { - id = "cert" - prefix = "cert/" - enabled = true - expiration { - days = 20 - } - } } resource "aws_s3_bucket" "backup" { bucket = "backup-${var.env}.${var.domain}" force_destroy = true } + +resource "aws_s3_bucket" "log" { + bucket = "log-${var.env}.${var.domain}" + force_destroy = true + policy = < Date: Tue, 27 Sep 2016 14:57:43 +0700 Subject: [PATCH 090/181] get certificate arn programatically --- .envrc.example | 5 ++--- .travis.yml | 1 + scripts/export-tfvars.sh | 2 ++ scripts/get_account_id.rb | 4 ---- scripts/get_certificate_arn.rb | 14 ++++++++++++++ webservers/variables.tf | 4 ++++ 6 files changed, 23 insertions(+), 7 deletions(-) delete mode 100644 scripts/get_account_id.rb create mode 100644 scripts/get_certificate_arn.rb diff --git a/.envrc.example b/.envrc.example index 8d76227..b886a4b 100644 --- a/.envrc.example +++ b/.envrc.example @@ -4,9 +4,8 @@ PATH_add scripts export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= -export AWS_DEFAULT_REGION= - -export TF_VAR_aws_region=${AWS_DEFAULT_REGION} +export AWS_DEFAULT_REGION=ap-northeast-1 +export AWS_REGION=ap-northeast-1 export CLOUDFLARE_EMAIL= export CLOUDFLARE_TOKEN= diff --git a/.travis.yml b/.travis.yml index d674f4a..97c113c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ install: - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: + - export AWS_REGION=${AWS_DEFAULT_REGION} - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - echo diff --git a/scripts/export-tfvars.sh b/scripts/export-tfvars.sh index 15f9798..4d20246 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/export-tfvars.sh @@ -1,5 +1,7 @@ #!/bin/sh +export TF_VAR_alb_certificate_arn=$(bundle exec ruby scripts/get_certificate_arn.rb '*.hana053.com') + asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') diff --git a/scripts/get_account_id.rb b/scripts/get_account_id.rb deleted file mode 100644 index d21f11a..0000000 --- a/scripts/get_account_id.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'aws-sdk' - -sts = Aws::STS::Client.new(region: 'ap-northeast-1') -puts sts.get_caller_identity.account \ No newline at end of file diff --git a/scripts/get_certificate_arn.rb b/scripts/get_certificate_arn.rb new file mode 100644 index 0000000..743e0ae --- /dev/null +++ b/scripts/get_certificate_arn.rb @@ -0,0 +1,14 @@ +require 'aws-sdk' + +unless ARGV[0] + puts 'Specify domain name.' + exit 1 +end +DOMAIN_NAME = ARGV[0] + +acm = Aws::ACM::Client.new +puts acm.list_certificates + .certificate_summary_list + .select { |c| c.domain_name == DOMAIN_NAME } + .map(&:certificate_arn) + .first diff --git a/webservers/variables.tf b/webservers/variables.tf index 125fa7b..2db0282 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -42,21 +42,25 @@ variable "web_ami_tag" { } variable "web_subnets" { + type = "list" description = "Subnets for web servers" default = [] } variable "web_security_groups" { + type = "list" description = "Security groups for web servers" default = [] } variable "alb_subnets" { + type = "list" description = "Subnets for alb" default = [] } variable "alb_security_groups" { + type = "list" description = "Security groups for alb" default = [] } From d5356770232209c6dae9c1970257aa9a13daef5b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 27 Sep 2016 15:51:58 +0700 Subject: [PATCH 091/181] don't use replace_old_instances --- .travis.yml | 8 ++++---- Gemfile.lock | 16 +++++++--------- scripts/replace_old_instances.rb | 1 + 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 97c113c..6497e34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,8 +46,8 @@ after_success: - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb # Replace instances if web_ami was updated. - - terraform output | tee out_after - - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - - asg_name=$(terraform output web_asg_name) - - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} +# - terraform output | tee out_after +# - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) +# - asg_name=$(terraform output web_asg_name) +# - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} diff --git a/Gemfile.lock b/Gemfile.lock index 67b0113..cd781cf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,13 +1,13 @@ GEM remote: http://rubygems.org/ specs: - awesome_print (1.6.1) - aws-sdk (2.2.34) - aws-sdk-resources (= 2.2.34) - aws-sdk-core (2.2.34) + awesome_print (1.7.0) + aws-sdk (2.6.3) + aws-sdk-resources (= 2.6.3) + aws-sdk-core (2.6.3) jmespath (~> 1.0) - aws-sdk-resources (2.2.34) - aws-sdk-core (= 2.2.34) + aws-sdk-resources (2.6.3) + aws-sdk-core (= 2.6.3) faraday (0.9.2) multipart-post (>= 1.2, < 3) faraday_middleware (0.10.0) @@ -15,9 +15,7 @@ GEM faraday_middleware-aws-signers-v4 (0.1.5) aws-sdk (~> 2.1) faraday (~> 0.9) - jmespath (1.2.4) - json_pure (>= 1.8.1) - json_pure (1.8.3) + jmespath (1.3.1) multipart-post (2.0.0) PLATFORMS diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb index 1f314fe..0f09f4b 100644 --- a/scripts/replace_old_instances.rb +++ b/scripts/replace_old_instances.rb @@ -24,6 +24,7 @@ honor_cooldown: true, ) +# TODO use target group instead of elb is_all_in_service = lambda do asg.reload in_services = asg.load_balancers From c280eec847f2d74c4ba4f58645708377d44b9e8f Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 27 Sep 2016 15:55:59 +0700 Subject: [PATCH 092/181] use replace_old_instances --- .travis.yml | 8 ++++---- scripts/replace_old_instances.rb | 28 +--------------------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6497e34..97c113c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,8 +46,8 @@ after_success: - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb # Replace instances if web_ami was updated. -# - terraform output | tee out_after -# - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) -# - asg_name=$(terraform output web_asg_name) -# - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} + - terraform output | tee out_after + - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) + - asg_name=$(terraform output web_asg_name) + - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb index 0f09f4b..cbebaaf 100644 --- a/scripts/replace_old_instances.rb +++ b/scripts/replace_old_instances.rb @@ -9,9 +9,6 @@ as = Aws::AutoScaling::Client.new asg = Aws::AutoScaling::AutoScalingGroup.new(ASG_NAME, client: as) -elb = Aws::ElasticLoadBalancing::Client.new - -old_instances = asg.instances if asg.desired_capacity * 2 > asg.max_size puts "Can not scale out to replace instances in #{ASG_NAME}." @@ -24,27 +21,4 @@ honor_cooldown: true, ) -# TODO use target group instead of elb -is_all_in_service = lambda do - asg.reload - in_services = asg.load_balancers - .map(&:load_balancer_name) - .map { |lb_name| elb.describe_instance_health(load_balancer_name: lb_name).instance_states } - .flatten - .select { |s| s.state == 'InService' } - in_services.size >= asg.desired_capacity -end - -puts 'Scaling out to replace old instances.' -until is_all_in_service.call - print '.' - sleep 5 -end -puts '' -puts 'Completed to scale out.' - -puts 'Terminating old instances.' -old_instances.each do |i| - i.terminate(should_decrement_desired_capacity: true) -end -puts 'Done.' +# scaling policy will decrease desired capacity soon. From 76211ae23b35c46dec874052af58d2b71a188e51 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 29 Sep 2016 20:11:45 +0700 Subject: [PATCH 093/181] fix warnings --- bastion/variables.tf | 1 + codedeploy/variables.tf | 1 + dbservers/variables.tf | 2 ++ logservers/variables.tf | 1 + main.tf | 1 - security_groups/variables.tf | 1 + vpc/variables.tf | 3 +++ webservers/iam.tf | 1 + webservers/variables.tf | 4 ---- 9 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bastion/variables.tf b/bastion/variables.tf index 3bf4faa..cd8f0ab 100644 --- a/bastion/variables.tf +++ b/bastion/variables.tf @@ -3,6 +3,7 @@ variable "subnet_id" { } variable "security_groups" { + type = "list" description = "Security groups for bastion server" default = [] } diff --git a/codedeploy/variables.tf b/codedeploy/variables.tf index a8d271c..75a0c45 100644 --- a/codedeploy/variables.tf +++ b/codedeploy/variables.tf @@ -5,5 +5,6 @@ variable "group_name" { } variable "autoscaling_groups" { + type = "list" default = [] } \ No newline at end of file diff --git a/dbservers/variables.tf b/dbservers/variables.tf index 11eab18..bb13128 100644 --- a/dbservers/variables.tf +++ b/dbservers/variables.tf @@ -1,9 +1,11 @@ variable "security_groups" { + type = "list" description = "Security groups for db servers" default = [] } variable "subnets" { + type = "list" description = "Subnets for db servers" default = [] } diff --git a/logservers/variables.tf b/logservers/variables.tf index d6537cf..97391a2 100644 --- a/logservers/variables.tf +++ b/logservers/variables.tf @@ -2,6 +2,7 @@ variable "aws_region" { } variable "allowed_segments" { + type = "list" default = [] } diff --git a/main.tf b/main.tf index e3754ca..f556046 100644 --- a/main.tf +++ b/main.tf @@ -17,7 +17,6 @@ module "webservers" { dbserver_endpoint = "${module.dbservers.endpoint}" cacheserver_endpoint = "${module.cacheservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" - deploy_bucket_arn = "${aws_s3_bucket.deploy.arn}" key_name = "${aws_key_pair.micropost.key_name}" web_ami_tag = "micropost-web" web_subnets = [ diff --git a/security_groups/variables.tf b/security_groups/variables.tf index a52ad4e..cd7fa92 100644 --- a/security_groups/variables.tf +++ b/security_groups/variables.tf @@ -2,5 +2,6 @@ variable "vpc_id" { } variable "ssh_allowed_segments" { + type = "list" default = [] } \ No newline at end of file diff --git a/vpc/variables.tf b/vpc/variables.tf index f84e850..eff193f 100644 --- a/vpc/variables.tf +++ b/vpc/variables.tf @@ -6,6 +6,7 @@ variable "cidr" { } variable "public_subnets" { + type = "list" description = "A list of public subnets inside the VPC." default = [ "10.1.0.0/24", @@ -14,6 +15,7 @@ variable "public_subnets" { } variable "private_subnets" { + type = "list" description = "A list of private subnets inside the VPC." default = [ "10.1.2.0/24", @@ -22,6 +24,7 @@ variable "private_subnets" { } variable "azs" { + type = "list" description = "A list of Availability zones in the region" default = [ "ap-northeast-1a", diff --git a/webservers/iam.tf b/webservers/iam.tf index 5afcd56..bbe791b 100644 --- a/webservers/iam.tf +++ b/webservers/iam.tf @@ -43,6 +43,7 @@ resource "aws_iam_role_policy" "es" { EOF } +// TODO delete resource "aws_iam_role_policy" "letsencrypt_cache" { name = "web-letsencrypt-cache" role = "${aws_iam_role.web.id}" diff --git a/webservers/variables.tf b/webservers/variables.tf index 2db0282..413fcbd 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -30,10 +30,6 @@ variable "deploy_bucket" { description = "Bucket to be used for deployment" } -variable "deploy_bucket_arn" { - description = "Bucket arn to be used for deployment" -} - variable "key_name" { description = "SSH key name for web servers" } From 3124d3a99afdef3933f062c587ce198713a329b8 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 29 Sep 2016 20:45:19 +0700 Subject: [PATCH 094/181] removed unused iam --- main.tf | 18 +++++++++--------- webservers/iam.tf | 21 --------------------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/main.tf b/main.tf index f556046..9af86de 100644 --- a/main.tf +++ b/main.tf @@ -40,15 +40,15 @@ module "webservers" { log_bucket = "${aws_s3_bucket.log.bucket}" } -module "bastion" { - source = "./bastion" - subnet_id = "${module.vpc.public_subnets[0]}" - security_groups = [ - "${module.security_groups.internal}", - "${module.security_groups.internet_in_ssh}", - ] - key_name = "${aws_key_pair.micropost.key_name}" -} +//module "bastion" { +// source = "./bastion" +// subnet_id = "${module.vpc.public_subnets[0]}" +// security_groups = [ +// "${module.security_groups.internal}", +// "${module.security_groups.internet_in_ssh}", +// ] +// key_name = "${aws_key_pair.micropost.key_name}" +//} module "cacheservers" { source = "./cacheservers" diff --git a/webservers/iam.tf b/webservers/iam.tf index bbe791b..e029e70 100644 --- a/webservers/iam.tf +++ b/webservers/iam.tf @@ -43,27 +43,6 @@ resource "aws_iam_role_policy" "es" { EOF } -// TODO delete -resource "aws_iam_role_policy" "letsencrypt_cache" { - name = "web-letsencrypt-cache" - role = "${aws_iam_role.web.id}" - policy = < Date: Wed, 12 Oct 2016 10:46:07 +0700 Subject: [PATCH 095/181] use cloudfront --- .travis.yml | 2 +- cdn/main.tf | 65 ++++++++++++++++++++++++++++++++++ cdn/outputs.tf | 3 ++ cdn/variables.tf | 7 ++++ dns.tf | 9 ++++- main.tf | 7 +++- scripts/export-tfvars.sh | 3 +- scripts/get_certificate_arn.rb | 12 +++---- variables.tf | 5 +++ webservers/autoscale.tf | 1 - webservers/variables.tf | 4 --- webservers/web_init.sh | 5 +-- 12 files changed, 103 insertions(+), 20 deletions(-) create mode 100644 cdn/main.tf create mode 100644 cdn/outputs.tf create mode 100644 cdn/variables.tf diff --git a/.travis.yml b/.travis.yml index 97c113c..d683d8c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - sudo pip install awscli - aws --version # install terraform - - wget https://releases.hashicorp.com/terraform/0.7.4/terraform_0.7.4_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.5/terraform_0.7.5_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: diff --git a/cdn/main.tf b/cdn/main.tf new file mode 100644 index 0000000..61edd0e --- /dev/null +++ b/cdn/main.tf @@ -0,0 +1,65 @@ +resource "aws_s3_bucket" "origin_static" { + bucket = "${var.domain}" + acl = "public-read" + policy = < inventory localhost EOF -ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "letsencrypt_host=${hostname}" -e "deploy_bucket=${deploy_bucket}" site.yml +ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "deploy_bucket=${deploy_bucket}" site.yml ) - -# Update SSL cert -/usr/local/bin/update_cert.sh From de948517b2c12c04c1035390d789c38bd56453cb Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 12 Oct 2016 13:30:32 +0700 Subject: [PATCH 096/181] remove elastcache --- cacheservers/main.tf | 21 --------------------- cacheservers/outputs.tf | 3 --- cacheservers/variables.tf | 9 --------- main.tf | 11 ----------- webservers/autoscale.tf | 1 - webservers/variables.tf | 4 ---- webservers/web_init.sh | 1 - 7 files changed, 50 deletions(-) delete mode 100644 cacheservers/main.tf delete mode 100644 cacheservers/outputs.tf delete mode 100644 cacheservers/variables.tf diff --git a/cacheservers/main.tf b/cacheservers/main.tf deleted file mode 100644 index f3d155a..0000000 --- a/cacheservers/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "aws_elasticache_cluster" "main" { - cluster_id = "main" - engine = "redis" - engine_version = "2.8.24" - node_type = "cache.t2.micro" - port = 6379 - num_cache_nodes = 1 - parameter_group_name = "default.redis2.8" - security_group_ids = [ - "${var.security_groups}", - ] - subnet_group_name = "${aws_elasticache_subnet_group.main.name}" -} - -resource "aws_elasticache_subnet_group" "main" { - name = "main" - description = "main subnet group" - subnet_ids = [ - "${var.subnets}" - ] -} \ No newline at end of file diff --git a/cacheservers/outputs.tf b/cacheservers/outputs.tf deleted file mode 100644 index 0d027aa..0000000 --- a/cacheservers/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "endpoint" { - value = "${aws_elasticache_cluster.main.cache_nodes.0.address}" -} \ No newline at end of file diff --git a/cacheservers/variables.tf b/cacheservers/variables.tf deleted file mode 100644 index be1ce88..0000000 --- a/cacheservers/variables.tf +++ /dev/null @@ -1,9 +0,0 @@ -variable "security_groups" { - description = "Security groups for cache servers" - default = [] -} - -variable "subnets" { - description = "Subnets for cache servers" - default = [] -} diff --git a/main.tf b/main.tf index 0f83d6f..fbf8c03 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,6 @@ module "webservers" { env = "${var.env}" logserver_endpoint = "${module.logservers.endpoint}" dbserver_endpoint = "${module.dbservers.endpoint}" - cacheserver_endpoint = "${module.cacheservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" key_name = "${aws_key_pair.micropost.key_name}" web_ami_tag = "micropost-web" @@ -55,16 +54,6 @@ module "cdn" { // key_name = "${aws_key_pair.micropost.key_name}" //} -module "cacheservers" { - source = "./cacheservers" - security_groups = [ - "${module.security_groups.internal}", - ] - subnets = [ - "${module.vpc.private_subnets}", - ] -} - module "dbservers" { source = "./dbservers" security_groups = [ diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index a7a7f67..40a66c5 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -17,7 +17,6 @@ data "template_file" "web_init" { env = "${var.env}" logserver_endpoint = "${var.logserver_endpoint}" dbserver_endpoint = "${var.dbserver_endpoint}" - cacheserver_endpoint = "${var.cacheserver_endpoint}" deploy_bucket = "${var.deploy_bucket}" } } diff --git a/webservers/variables.tf b/webservers/variables.tf index 1d27663..190a36e 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -18,10 +18,6 @@ variable "dbserver_endpoint" { description = "MySQL endpoint" } -variable "cacheserver_endpoint" { - description = "Redis endpoint" -} - variable "deploy_bucket" { description = "Bucket to be used for deployment" } diff --git a/webservers/web_init.sh b/webservers/web_init.sh index 07ff7a0..5f7198b 100644 --- a/webservers/web_init.sh +++ b/webservers/web_init.sh @@ -6,7 +6,6 @@ mkdir -p $(dirname $${ENV_FILE}) cat << EOF > $${ENV_FILE} export SPRING_PROFILES_ACTIVE=${env} export RDS_ENDPOINT=${dbserver_endpoint} -export REDIS_ENDPOINT=${cacheserver_endpoint} export S3_DEPLOY_BUCKET=${deploy_bucket} EOF From 9261b02f71121dfd32e05256bfaa84d2bad3ddef Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 12 Oct 2016 14:48:44 +0700 Subject: [PATCH 097/181] fix travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d683d8c..f7bd6ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,9 @@ after_success: - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - test -z "${ENV}" && (echo "${TRAVIS_BRANCH} is not a branch to deploy." && exit 0) + # exit unless ENV + - test "${ENV}" = "" && exit + # Switch AWS Role when ENV is prod - test "${ENV}" = "prod" && source scripts/switch-production-role.sh From df58c2e9705a62774524de1829934ab04305e4dd Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 13 Oct 2016 15:56:46 +0700 Subject: [PATCH 098/181] port 80 serve index.html behind alb --- dns.tf | 9 ++++++++- main.tf | 1 + s3.tf | 23 +++++++++++++++++++++ webservers/alb.tf | 44 ++++++++++++++++++++++++++++++++++------- webservers/autoscale.tf | 4 +++- webservers/variables.tf | 4 ++++ webservers/web_init.sh | 2 +- 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/dns.tf b/dns.tf index 808e999..72cfc12 100644 --- a/dns.tf +++ b/dns.tf @@ -10,4 +10,11 @@ resource "cloudflare_record" "backend" { name = "backend-${var.env}" value = "${module.webservers.dns_name}" type = "CNAME" -} \ No newline at end of file +} + +resource "cloudflare_record" "cdn" { + domain = "${var.domain}" + name = "cdn-${var.env}" + value = "${aws_s3_bucket.cdn.bucket}.s3.amazonaws.com" + type = "CNAME" +} diff --git a/main.tf b/main.tf index fbf8c03..5f7a615 100644 --- a/main.tf +++ b/main.tf @@ -15,6 +15,7 @@ module "webservers" { logserver_endpoint = "${module.logservers.endpoint}" dbserver_endpoint = "${module.dbservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" + nginx_cdn_bucket = "${aws_s3_bucket.cdn.bucket}" key_name = "${aws_key_pair.micropost.key_name}" web_ami_tag = "micropost-web" web_subnets = [ diff --git a/s3.tf b/s3.tf index 1053f6a..5a5c752 100644 --- a/s3.tf +++ b/s3.tf @@ -8,6 +8,29 @@ resource "aws_s3_bucket" "backup" { force_destroy = true } +resource "aws_s3_bucket" "cdn" { + bucket = "cdn-${var.env}.${var.domain}" + acl = "public-read" + policy = < inventory localhost EOF -ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "deploy_bucket=${deploy_bucket}" site.yml +ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "deploy_bucket=${deploy_bucket}" -e "nginx_cdn_bucket=${nginx_cdn_bucket}" site.yml ) From 6f783ff0a46eaec63b1cb1d6d271e9699225ff00 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 13 Oct 2016 16:02:27 +0700 Subject: [PATCH 099/181] remove cloudfront --- cdn/main.tf | 65 ---------------------------------------- cdn/outputs.tf | 3 -- cdn/variables.tf | 7 ----- dns.tf | 9 +----- main.tf | 6 ---- scripts/export-tfvars.sh | 1 - variables.tf | 4 --- 7 files changed, 1 insertion(+), 94 deletions(-) delete mode 100644 cdn/main.tf delete mode 100644 cdn/outputs.tf delete mode 100644 cdn/variables.tf diff --git a/cdn/main.tf b/cdn/main.tf deleted file mode 100644 index 61edd0e..0000000 --- a/cdn/main.tf +++ /dev/null @@ -1,65 +0,0 @@ -resource "aws_s3_bucket" "origin_static" { - bucket = "${var.domain}" - acl = "public-read" - policy = < Date: Thu, 13 Oct 2016 16:44:49 +0700 Subject: [PATCH 100/181] fix cloudflare setting --- dns.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/dns.tf b/dns.tf index 3c04611..3151a14 100644 --- a/dns.tf +++ b/dns.tf @@ -9,5 +9,6 @@ resource "cloudflare_record" "cdn" { domain = "${var.domain}" name = "cdn-${var.env}" value = "${aws_s3_bucket.cdn.bucket}.s3.amazonaws.com" + proxied = true type = "CNAME" } From 4ae901138fb1359e63318383b124a2313f33fc87 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 14 Oct 2016 21:06:41 +0700 Subject: [PATCH 101/181] use logentries --- webservers/web_init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webservers/web_init.sh b/webservers/web_init.sh index 3688f72..5c90cd3 100644 --- a/webservers/web_init.sh +++ b/webservers/web_init.sh @@ -18,5 +18,6 @@ cat << EOF > inventory localhost EOF -ansible-playbook -i inventory -c local --tags configuration,deploy --diff -e "logstash_elasticsearch_host=${logserver_endpoint}" -e "deploy_bucket=${deploy_bucket}" -e "nginx_cdn_bucket=${nginx_cdn_bucket}" site.yml +le_key=$(cat /etc/le/config | grep user-key | cut -d" " -f3) +ansible-playbook -i inventory -c local --diff -e "deploy_bucket=${deploy_bucket}" -e "nginx_cdn_bucket=${nginx_cdn_bucket}" -e "logentries_account_key=$${le_key}" site.yml ) From 9b531fa95526d44ae65ac84ece99d6ac2ca32090 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 14 Oct 2016 22:32:55 +0700 Subject: [PATCH 102/181] remove elasticsearch --- Gemfile | 2 - Gemfile.lock | 12 +-- logservers/main.tf | 103 ---------------------- logservers/outputs.tf | 3 - logservers/variables.tf | 17 ---- main.tf | 10 --- s3.tf | 5 -- scripts/es_register_snapshot_directory.rb | 37 -------- webservers/autoscale.tf | 1 - webservers/iam.tf | 19 ---- webservers/variables.tf | 4 - 11 files changed, 1 insertion(+), 212 deletions(-) delete mode 100644 logservers/main.tf delete mode 100644 logservers/outputs.tf delete mode 100644 logservers/variables.tf delete mode 100644 scripts/es_register_snapshot_directory.rb diff --git a/Gemfile b/Gemfile index bd58e0d..8b0ad27 100644 --- a/Gemfile +++ b/Gemfile @@ -3,5 +3,3 @@ source 'http://rubygems.org' gem 'aws-sdk' gem 'awesome_print' -gem 'faraday_middleware' -gem 'faraday_middleware-aws-signers-v4' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index cd781cf..10b74ef 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,15 +8,7 @@ GEM jmespath (~> 1.0) aws-sdk-resources (2.6.3) aws-sdk-core (= 2.6.3) - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - faraday_middleware (0.10.0) - faraday (>= 0.7.4, < 0.10) - faraday_middleware-aws-signers-v4 (0.1.5) - aws-sdk (~> 2.1) - faraday (~> 0.9) jmespath (1.3.1) - multipart-post (2.0.0) PLATFORMS ruby @@ -24,11 +16,9 @@ PLATFORMS DEPENDENCIES awesome_print aws-sdk - faraday_middleware - faraday_middleware-aws-signers-v4 RUBY VERSION ruby 2.3.0p0 BUNDLED WITH - 1.12.0.rc + 1.13.2 diff --git a/logservers/main.tf b/logservers/main.tf deleted file mode 100644 index bc09813..0000000 --- a/logservers/main.tf +++ /dev/null @@ -1,103 +0,0 @@ -data "aws_caller_identity" "current" {} - -resource "aws_elasticsearch_domain" "main" { - domain_name = "main2" - elasticsearch_version = "2.3" - access_policies = < /\bjson\b/ - faraday.response :raise_error - faraday.adapter Faraday.default_adapter -end - -res = conn.post do |req| - req.url "/_snapshot/#{option[:repository]}" - req.headers['Content-Type'] = 'application/json' - req.body = { - type: 's3', - settings: { - bucket: option[:bucket], - region: option[:region], - role_arn: option[:role_arn], - base_path: option[:base_path], - } - }.to_json -end - -puts res.body diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 6151368..d6a6e71 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -15,7 +15,6 @@ data "template_file" "web_init" { template = "${file("./webservers/web_init.sh")}" vars { env = "${var.env}" - logserver_endpoint = "${var.logserver_endpoint}" dbserver_endpoint = "${var.dbserver_endpoint}" deploy_bucket = "${var.deploy_bucket}" nginx_cdn_bucket = "${var.nginx_cdn_bucket}" diff --git a/webservers/iam.tf b/webservers/iam.tf index e029e70..fe3cc84 100644 --- a/webservers/iam.tf +++ b/webservers/iam.tf @@ -24,25 +24,6 @@ resource "aws_iam_role" "web" { EOF } -resource "aws_iam_role_policy" "es" { - name = "web-es" - role = "${aws_iam_role.web.id}" - policy = < Date: Sat, 15 Oct 2016 10:54:45 +0700 Subject: [PATCH 103/181] apply changing ami From 34deb5ca4e521ce5a7bf4d7da54f630a8462b31d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 17 Oct 2016 22:58:46 +0700 Subject: [PATCH 104/181] update terraform --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f7bd6ad..2b549b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - sudo pip install awscli - aws --version # install terraform - - wget https://releases.hashicorp.com/terraform/0.7.5/terraform_0.7.5_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.6/terraform_0.7.6_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: From 95520b474d2f4602ac03c810089bfaa8c180fb88 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 18 Oct 2016 14:26:59 +0700 Subject: [PATCH 105/181] apply changing ami From 2c00c5abc9ec2c3c54726b650b4a912f93805a5e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 18 Oct 2016 15:01:15 +0700 Subject: [PATCH 106/181] rotate ami From 29e3003138aafe3f25c2149268168c429a3df565 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 19 Oct 2016 17:01:00 +0700 Subject: [PATCH 107/181] rotate ami From 31a6ddec734cb2912a12d91fa368c7500d3a0a1e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 19 Oct 2016 22:14:28 +0700 Subject: [PATCH 108/181] refactor the way to execute terraform --- .travis.yml | 43 ++++++----- README.md | 31 ++++++++ env/prod/vars | 2 + env/stg/vars | 3 + scripts/post.sh | 8 +++ scripts/{export-tfvars.sh => pre.sh} | 15 ++-- scripts/switch-production-role.sh | 4 +- scripts/terraform-enable-remote.sh | 16 ----- scripts/terraform-wrapper.sh | 17 ----- terraform-env.sh | 103 +++++++++++++++++++++++++++ terraform.cfg | 6 ++ variables.tf | 1 - 12 files changed, 184 insertions(+), 65 deletions(-) create mode 100644 README.md create mode 100644 env/prod/vars create mode 100644 env/stg/vars create mode 100755 scripts/post.sh rename scripts/{export-tfvars.sh => pre.sh} (69%) delete mode 100755 scripts/terraform-enable-remote.sh delete mode 100755 scripts/terraform-wrapper.sh create mode 100755 terraform-env.sh create mode 100644 terraform.cfg diff --git a/.travis.yml b/.travis.yml index 2b549b7..833b303 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,34 +23,31 @@ before_script: - export AWS_REGION=${AWS_DEFAULT_REGION} - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} script: - - echo + # If branch name matched with deploy/xxx, then continue. + - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") + - | + if [ -z "${ENV}" ]; then + echo "${TRAVIS_BRANCH} is not a branch to deploy." + else + ./terraform-env.sh ${ENV} get + ./terraform-env.sh ${ENV} plan + fi + +# TODO I want to make it deploy. But my ruby and travis's ruby make conflicts... after_success: # If branch name matched with deploy/xxx, then continue. - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - - test -z "${ENV}" && (echo "${TRAVIS_BRANCH} is not a branch to deploy." && exit 0) - - # exit unless ENV - - test "${ENV}" = "" && exit - - # Switch AWS Role when ENV is prod - - test "${ENV}" = "prod" && source scripts/switch-production-role.sh - - # Enable terraform remote - - scripts/terraform-enable-remote.sh - + - if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy."; exit 0; fi # Save current output to use it later - - terraform output | tee out_before - - # Plan and apply - - scripts/terraform-wrapper.sh plan || exit 1 - - scripts/terraform-wrapper.sh apply || exit 1 - + - ./terraform-env.sh ${ENV} output | tee out_before + # Apply + - ./terraform-env.sh ${ENV} get + - ./terraform-env.sh ${ENV} apply # Remove unused amis - - bundle exec ruby scripts/remove_unused_and_rotated_amis.rb - + - ruby scripts/remove_unused_and_rotated_amis.rb # Replace instances if web_ami was updated. - - terraform output | tee out_after + - ./terraform-env.sh ${ENV} output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - - asg_name=$(terraform output web_asg_name) - - test ${lines} -eq 2 && bundle exec ruby scripts/replace_old_instances.rb ${asg_name} + - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') + - test ${lines} -eq 2 && ruby scripts/replace_old_instances.rb ${asg_name} diff --git a/README.md b/README.md new file mode 100644 index 0000000..cfcb15b --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# micropost-formation + +## Dependencies + +* Terraform +* direnv +* Ruby +* AWS CLI + +## Getting started + +Configure settings. + +``` +$ cp .envrc.example .envrc +$ vi .envrc +$ direnv allow +``` + +Install gems. + +``` +$ bundle install +``` + +Plan and Apply + +``` +$ ./terraform-env.sh stg plan +$ ./terraform-env.sh stg apply +``` diff --git a/env/prod/vars b/env/prod/vars new file mode 100644 index 0000000..2d7f80c --- /dev/null +++ b/env/prod/vars @@ -0,0 +1,2 @@ +env = "prod" +web_host_name = "micropost" diff --git a/env/stg/vars b/env/stg/vars new file mode 100644 index 0000000..aaa64e0 --- /dev/null +++ b/env/stg/vars @@ -0,0 +1,3 @@ +env = "stg" +web_host_name = "micropost-stg" + diff --git a/scripts/post.sh b/scripts/post.sh new file mode 100755 index 0000000..c8046f8 --- /dev/null +++ b/scripts/post.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ "${ENVIRONMENT}" = "prod" ]; then + # reset current role if exists + test ! -v AWS_SESSION_TOKEN && direnv reload > /dev/null 2>&1 +fi + +exit 0 diff --git a/scripts/export-tfvars.sh b/scripts/pre.sh similarity index 69% rename from scripts/export-tfvars.sh rename to scripts/pre.sh index c8b4f0e..220a644 100755 --- a/scripts/export-tfvars.sh +++ b/scripts/pre.sh @@ -1,4 +1,11 @@ -#!/bin/sh +#!/usr/bin/env bash + +if [ "${ENVIRONMENT}" = "prod" ]; then + # reset current role if exists + test ! -v AWS_SESSION_TOKEN && direnv reload > /dev/null 2>&1 + # switch to production role + source scripts/switch-production-role.sh +fi export TF_VAR_alb_certificate_arn=$(bundle exec ruby scripts/get_certificate_arn.rb --domain "*.hana053.com" --region ${AWS_DEFAULT_REGION}) @@ -10,9 +17,3 @@ if [ -n "${asg_name}" ]; then export TF_VAR_web_desired_capacity="${desired_capacity}" fi -if [ "${ENV}" = "prod" ]; then - # prod is special - export TF_VAR_web_host_name="micropost" -else - export TF_VAR_web_host_name="micropost-${ENV}" -fi diff --git a/scripts/switch-production-role.sh b/scripts/switch-production-role.sh index 28b28ee..f0e3b51 100755 --- a/scripts/switch-production-role.sh +++ b/scripts/switch-production-role.sh @@ -6,4 +6,6 @@ export AWS_ACCESS_KEY_ID=$(echo ${credentials} | jq --raw-output .Credentials.Ac export AWS_SECRET_ACCESS_KEY=$(echo ${credentials} | jq --raw-output .Credentials.SecretAccessKey) export AWS_SESSION_TOKEN=$(echo ${credentials} | jq --raw-output .Credentials.SessionToken) -unset credentials \ No newline at end of file +unset credentials + +echo "Switched to production role." diff --git a/scripts/terraform-enable-remote.sh b/scripts/terraform-enable-remote.sh deleted file mode 100755 index 8d16c75..0000000 --- a/scripts/terraform-enable-remote.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ -z "${ENV}" ]; then - echo "ENV is required." - exit 1 -fi - -rm -rf ./.terraform/ - -terraform remote config \ - -state="${ENV}.tfstate" \ - -backend=S3 \ - -backend-config="region=ap-northeast-1" \ - -backend-config="bucket=tfstate.hana053.com" \ - -backend-config="key=${ENV}.tfstate" -terraform remote pull diff --git a/scripts/terraform-wrapper.sh b/scripts/terraform-wrapper.sh deleted file mode 100755 index d551c07..0000000 --- a/scripts/terraform-wrapper.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -. $(dirname $0)/terraform-enable-remote.sh -. $(dirname $0)/export-tfvars.sh - -terraform get - -terraform $@ \ - -state="${ENV}.tfstate" \ - -refresh=true \ - -var "env=${ENV}" -status=$? - -# Need to push state, even if apply was failed. -terraform remote push - -exit ${status} diff --git a/terraform-env.sh b/terraform-env.sh new file mode 100755 index 0000000..1e9a0b5 --- /dev/null +++ b/terraform-env.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +set -o pipefail +set -u + +function help { + echo "usage: ${0} []" + exit 1 +} + +function contains_element () { + local i + for i in "${@:2}"; do + [[ "$i" == "$1" ]] && return 0 + done + return 1 +} + +function files_exist(){ + ls ${1} 1> /dev/null 2>&1 +} + +#All of the args are mandatory. +if [ $# -lt 1 ]; then + help +fi + + +# Let's set up our environment +export ENVIRONMENT=$1 +export ACTION=$2 +ADDTL_PARAMS=${*:3} + +CONFIG_FILE=./terraform.cfg + +# Let's check the existence of the config file +if [ ! -f $CONFIG_FILE ]; then + echo "Error: $CONFIG_FILE does not exist. You'll need to create a config file so we know where to set up the remote config." + exit 1 +fi + +source ${CONFIG_FILE} + +# Let's set up our variables +ALLOWS_VARFILE=(apply plan push refresh destroy) +ENV_FILE=.terraform/environment +ENV_DIR=env/$ENVIRONMENT +VARS_FILE=${ENV_DIR}/vars +VARS_FILE_FLAG= +BUCKET_KEY=$bucket_prefix/state/$ENVIRONMENT +PREVIOUS_ENVIRONMENT=$([ -f $ENV_FILE ] && echo "$(<$ENV_FILE)" || echo "previous") +EXTRA_ARGS=${extra_args:-''} +PRE_CMD=${pre_command:-''} +POST_CMD=${post_command:-''} + +# Let's check to see if a vars file exists for the requested environment before proceeding +if [ ! -f $VARS_FILE ]; then + echo "Error: $VARS_FILE does not exist. You'll need to create a vars file for the requested environment before continuing." + exit 1 +fi + +# Checks if current action allows a varfile to be passed +contains_element "$ACTION" "${ALLOWS_VARFILE[@]}" +if [ $? -eq 0 ]; then + VARS_FILE_FLAG="-var-file=$VARS_FILE" +fi + +# Let's check if the requested environment is different from the previous environment +if [ $PREVIOUS_ENVIRONMENT != $ENVIRONMENT ] || [ ! -f '.terraform/terraform.tfstate' ] +then + + # Move current state out of the way to make room for the new state + mv -f .terraform/terraform.tfstate .terraform/terraform.tfstate.$PREVIOUS_ENVIRONMENT > /dev/null 2>&1 + mv -f .terraform/terraform.tfstate.backup .terraform/terraform.tfstate.backup.$PREVIOUS_ENVIRONMENT > /dev/null 2>&1 + + # Let's log the new environment for later + echo $ENVIRONMENT > $ENV_FILE + + # Set up remote configuration and pull latest version + terraform remote config -backend S3 -backend-config="bucket=$bucket" -backend-config="key=$BUCKET_KEY" -backend-config="region=$region" + +fi + +# Let's run the PRE_CMD hook if it's defined +. ${PRE_CMD} + +# let's copy environment specific configuration to the root of the directory +if files_exist ${ENV_DIR}/*.tf; then + cd env/${ENVIRONMENT} + pax -wrs'/\.tf$/\.env\.tf/' *.tf ../../ + cd ../../ +fi + +# Let's do work! +terraform $ACTION $VARS_FILE_FLAG $ADDTL_PARAMS ${EXTRA_ARGS} + +# Let's remove those environment-specific configuration files we copied earlier +if files_exist *.env.tf; then + rm *.env.tf +fi + +# Let's run the POST_CMD hook if it's defined +eval ${POST_CMD} diff --git a/terraform.cfg b/terraform.cfg new file mode 100644 index 0000000..7b24fa5 --- /dev/null +++ b/terraform.cfg @@ -0,0 +1,6 @@ +bucket=tfstate.hana053.com +bucket_prefix= +region=ap-northeast-1 + +pre_command='scripts/pre.sh' +post_command='scripts/post.sh' \ No newline at end of file diff --git a/variables.tf b/variables.tf index e9644d9..78f2caf 100644 --- a/variables.tf +++ b/variables.tf @@ -1,6 +1,5 @@ variable "env" { description = "dev, stg, prod and etc." - default = "dev" } variable "aws_region" { From 29127be2975f0e0a0e95a59de6467894b65bf9c6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 19 Oct 2016 23:57:29 +0700 Subject: [PATCH 109/181] remove unused AWS_REGION --- .envrc.example | 1 - .travis.yml | 6 +----- scripts/pre.sh | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.envrc.example b/.envrc.example index b886a4b..5035f91 100644 --- a/.envrc.example +++ b/.envrc.example @@ -5,7 +5,6 @@ PATH_add scripts export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_DEFAULT_REGION=ap-northeast-1 -export AWS_REGION=ap-northeast-1 export CLOUDFLARE_EMAIL= export CLOUDFLARE_TOKEN= diff --git a/.travis.yml b/.travis.yml index 833b303..62dba1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,8 @@ install: - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: - - export AWS_REGION=${AWS_DEFAULT_REGION} - - export TF_VAR_aws_region=${AWS_DEFAULT_REGION} -script: - # If branch name matched with deploy/xxx, then continue. - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") +script: - | if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy." @@ -36,7 +33,6 @@ script: # TODO I want to make it deploy. But my ruby and travis's ruby make conflicts... after_success: # If branch name matched with deploy/xxx, then continue. - - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy."; exit 0; fi # Save current output to use it later - ./terraform-env.sh ${ENV} output | tee out_before diff --git a/scripts/pre.sh b/scripts/pre.sh index 220a644..49c0d13 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -7,6 +7,7 @@ if [ "${ENVIRONMENT}" = "prod" ]; then source scripts/switch-production-role.sh fi +export TF_VAR_aws_region=${AWS_DEFAULT_REGION} export TF_VAR_alb_certificate_arn=$(bundle exec ruby scripts/get_certificate_arn.rb --domain "*.hana053.com" --region ${AWS_DEFAULT_REGION}) asg_name=$(terraform output web_asg_name) From 8b2107290159e087a5991408437a1115dab6e6b6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 00:00:34 +0700 Subject: [PATCH 110/181] ruby aws sdk needs AWS_REGION --- .envrc.example | 1 + .travis.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.envrc.example b/.envrc.example index 5035f91..957c1d4 100644 --- a/.envrc.example +++ b/.envrc.example @@ -5,6 +5,7 @@ PATH_add scripts export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_DEFAULT_REGION=ap-northeast-1 +export AWS_REGION=${AWS_DEFAULT_REGION} export CLOUDFLARE_EMAIL= export CLOUDFLARE_TOKEN= diff --git a/.travis.yml b/.travis.yml index 62dba1d..262d508 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ install: - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: + - export AWS_REGION=${AWS_DEFAULT_REGION} - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") script: - | From 046aaf50afe2eb5c85fb29e79a6474ee9e58c659 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 15:57:55 +0700 Subject: [PATCH 111/181] use node.js instead of ruby --- .editorconfig | 13 +++ .travis.yml | 19 ++-- Gemfile | 5 -- Gemfile.lock | 24 ----- README.md | 5 +- package.json | 19 ++++ scripts/get_certificate_arn.js | 26 ++++++ scripts/get_certificate_arn.rb | 12 --- scripts/pre.sh | 2 +- scripts/remove_unused_and_rotated_amis.js | 72 +++++++++++++++ scripts/remove_unused_and_rotated_amis.rb | 40 --------- scripts/replace_old_instances.js | 51 +++++++++++ scripts/replace_old_instances.rb | 24 ----- yarn.lock | 105 ++++++++++++++++++++++ 14 files changed, 300 insertions(+), 117 deletions(-) create mode 100644 .editorconfig delete mode 100644 Gemfile delete mode 100644 Gemfile.lock create mode 100644 package.json create mode 100644 scripts/get_certificate_arn.js delete mode 100644 scripts/get_certificate_arn.rb create mode 100644 scripts/remove_unused_and_rotated_amis.js delete mode 100644 scripts/remove_unused_and_rotated_amis.rb create mode 100644 scripts/replace_old_instances.js delete mode 100644 scripts/replace_old_instances.rb create mode 100644 yarn.lock diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..271822f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +insert_final_newline = false +trim_trailing_whitespace = false diff --git a/.travis.yml b/.travis.yml index 262d508..8c1c853 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,18 @@ --- sudo: required dist: trusty -language: ruby -rvm: - - 2.3.0 +language: node_js cache: directories: - - vendor/bundle + - $HOME/.npm + - $HOME/.yarn-cache + - node_modules +node_js: +- '6.9.1' install: - - gem install travis -v 1.8.2 --no-rdoc --no-ri - - bundle install + - npm install -g yarn@">=0.16.0" --cache-min 999999999 + - yarn install # install aws cli - sudo apt-get -y install python-pip - sudo pip install awscli @@ -31,7 +33,6 @@ script: ./terraform-env.sh ${ENV} plan fi -# TODO I want to make it deploy. But my ruby and travis's ruby make conflicts... after_success: # If branch name matched with deploy/xxx, then continue. - if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy."; exit 0; fi @@ -41,10 +42,10 @@ after_success: - ./terraform-env.sh ${ENV} get - ./terraform-env.sh ${ENV} apply # Remove unused amis - - ruby scripts/remove_unused_and_rotated_amis.rb + - node scripts/remove_unused_and_rotated_amis.js # Replace instances if web_ami was updated. - ./terraform-env.sh ${ENV} output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - - test ${lines} -eq 2 && ruby scripts/replace_old_instances.rb ${asg_name} + - test ${lines} -eq 2 && node scripts/replace_old_instances.js -n ${asg_name} diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 8b0ad27..0000000 --- a/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -ruby '2.3.0' -source 'http://rubygems.org' - -gem 'aws-sdk' -gem 'awesome_print' diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 10b74ef..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,24 +0,0 @@ -GEM - remote: http://rubygems.org/ - specs: - awesome_print (1.7.0) - aws-sdk (2.6.3) - aws-sdk-resources (= 2.6.3) - aws-sdk-core (2.6.3) - jmespath (~> 1.0) - aws-sdk-resources (2.6.3) - aws-sdk-core (= 2.6.3) - jmespath (1.3.1) - -PLATFORMS - ruby - -DEPENDENCIES - awesome_print - aws-sdk - -RUBY VERSION - ruby 2.3.0p0 - -BUNDLED WITH - 1.13.2 diff --git a/README.md b/README.md index cfcb15b..0229e65 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ ## Dependencies * Terraform -* direnv -* Ruby +* Node.js * AWS CLI +* direnv ## Getting started @@ -26,6 +26,7 @@ $ bundle install Plan and Apply ``` +$ ./terraform-env.sh stg get $ ./terraform-env.sh stg plan $ ./terraform-env.sh stg apply ``` diff --git a/package.json b/package.json new file mode 100644 index 0000000..60dea39 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "micropost-formation", + "version": "1.0.0", + "engines": { + "npm": "~6.7.0" + }, + "repository": { + "url": "ssh://git@github.com/springboot-angular2-tutorial/micropost-formation.git", + "type": "git" + }, + "author": "Akira Sosa ", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.9", + "aws-sdk": "^2.6.10", + "bluebird": "^3.4.6", + "lodash": "^4.16.4" + } +} diff --git a/scripts/get_certificate_arn.js b/scripts/get_certificate_arn.js new file mode 100644 index 0000000..bc27fca --- /dev/null +++ b/scripts/get_certificate_arn.js @@ -0,0 +1,26 @@ +const AWS = require('aws-sdk'); +const ArgumentParser = require('argparse').ArgumentParser; + +AWS.config.region = process.env.AWS_DEFAULT_REGION; + +function parseArguments() { + const parser = new ArgumentParser({ + description: 'This script find an ACM arn by domain name', + }); + parser.addArgument(['-d', '--domain'], { + help: 'domain name of certificate to search', + required: true, + }); + return parser.parseArgs(); +} + +const args = parseArguments(); +const ACM = new AWS.ACM(); + +ACM.listCertificates((err, data) => { + const arn = data.CertificateSummaryList + .filter(s => s.DomainName === args.domain) + .map(s => s.CertificateArn); + console.log(arn[0]); +}); + diff --git a/scripts/get_certificate_arn.rb b/scripts/get_certificate_arn.rb deleted file mode 100644 index 912bf18..0000000 --- a/scripts/get_certificate_arn.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'aws-sdk' -require 'optparse' - -option = ARGV.getopts(nil, 'domain:', 'region:') - .inject({}) { |hash, (k, v)| hash[k.to_sym] = v; hash } - -acm = Aws::ACM::Client.new(region: option[:region]) -puts acm.list_certificates - .certificate_summary_list - .select { |c| c.domain_name == option[:domain] } - .map(&:certificate_arn) - .first diff --git a/scripts/pre.sh b/scripts/pre.sh index 49c0d13..fc40c53 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -8,7 +8,7 @@ if [ "${ENVIRONMENT}" = "prod" ]; then fi export TF_VAR_aws_region=${AWS_DEFAULT_REGION} -export TF_VAR_alb_certificate_arn=$(bundle exec ruby scripts/get_certificate_arn.rb --domain "*.hana053.com" --region ${AWS_DEFAULT_REGION}) +export TF_VAR_alb_certificate_arn=$(node scripts/get_certificate_arn.js -d "*.hana053.com") asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then diff --git a/scripts/remove_unused_and_rotated_amis.js b/scripts/remove_unused_and_rotated_amis.js new file mode 100644 index 0000000..485c6df --- /dev/null +++ b/scripts/remove_unused_and_rotated_amis.js @@ -0,0 +1,72 @@ +const AWS = require('aws-sdk'); +const _ = require('lodash'); + +AWS.config.setPromisesDependency(require('bluebird')); +AWS.config.region = process.env.AWS_DEFAULT_REGION; + +const EC2 = new AWS.EC2(); +const AutoScaling = new AWS.AutoScaling(); + +function rotatedImages() { + return EC2.describeImages({ + Filters: [ + { + Name: 'tag:Rotated', + Values: ['true'], + }, + ], + }).promise() + .then(data => data.Images); +} + +function unusedImageIds() { + return AutoScaling.describeLaunchConfigurations({}).promise() + .then(data => { + return data.LaunchConfigurations.map(lc => lc.ImageId); + }); +} + +function deregisterImage(image) { + return EC2.deregisterImage({ + ImageId: image.ImageId, + }).promise() + .then(() => { + console.log(`deregistered ${image.ImageId}`); + return image; + }) + // ignore error and continue + .catch(() => image) +} + +function deleteSnapShot(image) { + const snapShotId = _.chain(image.BlockDeviceMappings) + .flatten() + .map(m => m.Ebs) + .compact() + .map(m => m.SnapshotId) + .first() + .value(); + return EC2.deleteSnapshot({ + SnapshotId: snapShotId, + }).promise() + .then(() => { + console.log(`deleted ${snapShotId}`); + return image; + }) + .catch(() => image) +} + +Promise.all([rotatedImages(), unusedImageIds()]) + .then(results => { + const [rotatedImages, unusedImageIds] = results; + return rotatedImages.filter(image => { + return !unusedImageIds.includes(image.ImageId) + }) + }) + .then(images => { + const promises = images.map(image => { + return deregisterImage(image).then(deleteSnapShot) + }); + return Promise.all(promises); + }) +; diff --git a/scripts/remove_unused_and_rotated_amis.rb b/scripts/remove_unused_and_rotated_amis.rb deleted file mode 100644 index f28ba1b..0000000 --- a/scripts/remove_unused_and_rotated_amis.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'aws-sdk' -require 'awesome_print' - -def rotated_images - ec2 = Aws::EC2::Client.new - ec2.describe_images(filters: [ - { - name: 'tag:Rotated', - values: ['true'], - }, - ]).images.map() -end - -def used_image_ids - as = Aws::AutoScaling::Client.new - as.describe_launch_configurations - .launch_configurations - .map(&:image_id) -end - -def unused_rotated_images - rotated_images.reject { |i| used_image_ids.include?(i.image_id) } -end - -images = unused_rotated_images - -puts 'Deregistering amis.' -images.map(&:image_id) - .tap { |id| puts id } - .map { |id| Aws::EC2::Image.new(id) } - .each { |img| img.deregister rescue nil } - -puts 'Removing snapshots related with de-registered ami.' -images.map { |i| i.block_device_mappings.map(&:ebs) } - .flatten - .compact - .map(&:snapshot_id) - .tap { |s_id| puts s_id } - .map { |id| Aws::EC2::Snapshot.new(id) } - .each { |s| s.delete rescue nil } diff --git a/scripts/replace_old_instances.js b/scripts/replace_old_instances.js new file mode 100644 index 0000000..075b6f5 --- /dev/null +++ b/scripts/replace_old_instances.js @@ -0,0 +1,51 @@ +const AWS = require('aws-sdk'); +const ArgumentParser = require('argparse').ArgumentParser; + +AWS.config.setPromisesDependency(require('bluebird')); +AWS.config.region = process.env.AWS_DEFAULT_REGION; + +const AutoScaling = new AWS.AutoScaling(); + +function parseArguments() { + const parser = new ArgumentParser({ + description: 'This script scale out specified auto scaling group to flip out old instances.', + }); + parser.addArgument(['-n', '--name'], { + help: 'name of a auto scaling group', + required: true, + }); + return parser.parseArgs(); +} + +function validateAsgCapacity(asg) { + if (!asg) { + console.error('This autoscaling group does not exist...'); + process.exit(1); + } + if (asg.DesiredCapacity * 2 > asg.MaxSize) { + console.error(`Can not scale out to replace instances. Consider increasing the max capacity.`); + process.exit(1); + } + console.log(`Current desired capacity is ${asg.DesiredCapacity}`); + + return asg; +} + +function multipleDesiredCapacity(asg) { + console.log(`Scale out to ${asg.DesiredCapacity * 2}.`); + + return AutoScaling.setDesiredCapacity({ + AutoScalingGroupName: asg.AutoScalingGroupName, + DesiredCapacity: asg.DesiredCapacity * 2, + HonorCooldown: true, + }).promise(); +} + +const asgName = parseArguments().name; + +AutoScaling.describeAutoScalingGroups({ + AutoScalingGroupNames: [asgName], +}).promise() + .then(data => data.AutoScalingGroups[0]) + .then(validateAsgCapacity) + .then(multipleDesiredCapacity); diff --git a/scripts/replace_old_instances.rb b/scripts/replace_old_instances.rb deleted file mode 100644 index cbebaaf..0000000 --- a/scripts/replace_old_instances.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'aws-sdk' -require 'awesome_print' - -unless ARGV[0] - puts 'Specify auto scaling group name.' - exit 1 -end -ASG_NAME = ARGV[0] - -as = Aws::AutoScaling::Client.new -asg = Aws::AutoScaling::AutoScalingGroup.new(ASG_NAME, client: as) - -if asg.desired_capacity * 2 > asg.max_size - puts "Can not scale out to replace instances in #{ASG_NAME}." - puts 'Consider increasing the max size of it.' - exit 1 -end - -asg.set_desired_capacity( - desired_capacity: asg.desired_capacity * 2, - honor_cooldown: true, -) - -# scaling policy will decrease desired capacity soon. diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..c5b6b43 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,105 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 +argparse: + version "1.0.9" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" + dependencies: + sprintf-js "~1.0.2" + +aws-sdk: + version "2.6.10" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.10.tgz#ae3be91a855fbe01426886e844c38e4d64a5c4c7" + dependencies: + buffer "4.9.1" + crypto-browserify "1.0.9" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.1.5" + url "0.10.3" + xml2js "0.4.15" + xmlbuilder "2.6.2" + +base64-js@^1.0.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" + +bluebird: + version "3.4.6" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.6.tgz#01da8d821d87813d158967e743d5fe6c62cf8c0f" + +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + +crypto-browserify@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" + +ieee754@^1.1.4: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + +lodash: + version "4.16.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.4.tgz#01ce306b9bad1319f2a5528674f88297aeb70127" + +lodash@~3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" + +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + +sax@>=0.6.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +xml2js@0.4.15: + version "0.4.15" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" + dependencies: + sax ">=0.6.0" + xmlbuilder ">=2.4.6" + +xmlbuilder@>=2.4.6: + version "8.2.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" + +xmlbuilder@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" + dependencies: + lodash "~3.5.0" + From f490a839fbedfb43fe603aacf63cbb9b7a172540 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 16:09:02 +0700 Subject: [PATCH 112/181] update travis yml --- .travis.yml | 18 +++++++++++------- scripts/pre.sh | 1 - 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c1c853..e243911 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ node_js: - '6.9.1' install: + # install npm packages - npm install -g yarn@">=0.16.0" --cache-min 999999999 - yarn install # install aws cli @@ -33,14 +34,18 @@ script: ./terraform-env.sh ${ENV} plan fi -after_success: - # If branch name matched with deploy/xxx, then continue. - - if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy."; exit 0; fi +before_deploy: # Save current output to use it later - ./terraform-env.sh ${ENV} output | tee out_before - # Apply - - ./terraform-env.sh ${ENV} get - - ./terraform-env.sh ${ENV} apply + +deploy: + - provider: script + script: ./terraform-env.sh ${ENV} apply + skip_cleanup: true + on: + branch: deploy/* + +after_deploy: # Remove unused amis - node scripts/remove_unused_and_rotated_amis.js # Replace instances if web_ami was updated. @@ -48,4 +53,3 @@ after_success: - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - test ${lines} -eq 2 && node scripts/replace_old_instances.js -n ${asg_name} - diff --git a/scripts/pre.sh b/scripts/pre.sh index fc40c53..f99dcec 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -13,7 +13,6 @@ export TF_VAR_alb_certificate_arn=$(node scripts/get_certificate_arn.js -d "*.ha asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') - echo "Current desired capacity is ${desired_capacity}." # respect current desired capacity export TF_VAR_web_desired_capacity="${desired_capacity}" fi From 9659cef8c8d5909127b919b15737e50435ee5061 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 17:15:16 +0700 Subject: [PATCH 113/181] no need to use bluebird --- package.json | 1 - scripts/remove_unused_and_rotated_amis.js | 1 - scripts/replace_old_instances.js | 1 - yarn.lock | 4 ---- 4 files changed, 7 deletions(-) diff --git a/package.json b/package.json index 60dea39..7239fb6 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "dependencies": { "argparse": "^1.0.9", "aws-sdk": "^2.6.10", - "bluebird": "^3.4.6", "lodash": "^4.16.4" } } diff --git a/scripts/remove_unused_and_rotated_amis.js b/scripts/remove_unused_and_rotated_amis.js index 485c6df..44a7465 100644 --- a/scripts/remove_unused_and_rotated_amis.js +++ b/scripts/remove_unused_and_rotated_amis.js @@ -1,7 +1,6 @@ const AWS = require('aws-sdk'); const _ = require('lodash'); -AWS.config.setPromisesDependency(require('bluebird')); AWS.config.region = process.env.AWS_DEFAULT_REGION; const EC2 = new AWS.EC2(); diff --git a/scripts/replace_old_instances.js b/scripts/replace_old_instances.js index 075b6f5..f87d68d 100644 --- a/scripts/replace_old_instances.js +++ b/scripts/replace_old_instances.js @@ -1,7 +1,6 @@ const AWS = require('aws-sdk'); const ArgumentParser = require('argparse').ArgumentParser; -AWS.config.setPromisesDependency(require('bluebird')); AWS.config.region = process.env.AWS_DEFAULT_REGION; const AutoScaling = new AWS.AutoScaling(); diff --git a/yarn.lock b/yarn.lock index c5b6b43..3d1fed5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,10 +23,6 @@ base64-js@^1.0.2: version "1.2.0" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" -bluebird: - version "3.4.6" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.6.tgz#01da8d821d87813d158967e743d5fe6c62cf8c0f" - buffer@4.9.1: version "4.9.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" From efad27bdd568d7ba927e7bdeb517880d797cb282 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 18:33:11 +0700 Subject: [PATCH 114/181] fix readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0229e65..909f920 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ $ direnv allow Install gems. ``` -$ bundle install +$ npm install -g yarn@">=0.16.0" +$ yarn install ``` Plan and Apply From 5a4cf009918c03e27f79d694efb8c740b8cb678c Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 20 Oct 2016 18:34:20 +0700 Subject: [PATCH 115/181] fix readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 909f920..b43671a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ $ vi .envrc $ direnv allow ``` -Install gems. +Install npm packages. ``` $ npm install -g yarn@">=0.16.0" From 489513af9debc408dc2f00ab8dfc1d51cadf3a8e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 18:07:26 +0700 Subject: [PATCH 116/181] added lambda replace_autoscaling_instances --- .gitignore | 2 + functions/replace_autoscaling_instances.tf | 76 +++++++++++++++++++ .../replace_autoscaling_instances/index.js | 44 +++++++++++ functions/variables.tf | 5 ++ main.tf | 6 ++ scripts/post.sh | 3 + scripts/pre.sh | 10 +++ scripts/replace_old_instances.js | 50 ------------ sns.tf | 7 ++ 9 files changed, 153 insertions(+), 50 deletions(-) create mode 100644 functions/replace_autoscaling_instances.tf create mode 100644 functions/replace_autoscaling_instances/index.js create mode 100644 functions/variables.tf delete mode 100644 scripts/replace_old_instances.js create mode 100644 sns.tf diff --git a/.gitignore b/.gitignore index 70d7c51..0c3f99f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ # tfstate is saved on remote .terraform terraform.log + +functions/*.zip diff --git a/functions/replace_autoscaling_instances.tf b/functions/replace_autoscaling_instances.tf new file mode 100644 index 0000000..bdf9e75 --- /dev/null +++ b/functions/replace_autoscaling_instances.tf @@ -0,0 +1,76 @@ +resource "aws_lambda_function" "replace_autoscaling_instances" { + filename = "functions/replace_autoscaling_instances.zip" + function_name = "replace_autoscaling_instances" + role = "${aws_iam_role.replace_autoscaling_instances.arn}" + handler = "index.handle" + runtime = "nodejs4.3" + source_code_hash = "${base64sha256(file("functions/replace_autoscaling_instances.zip"))}" +} + +resource "aws_iam_role" "replace_autoscaling_instances" { + name = "replace_autoscaling_instances" + assume_role_policy = < { + console.log('processing event: %j', event); + + const message = JSON.parse(event.Records[0].Sns.Message); + console.log('ASG name:', message.asgName); + + AutoScaling.describeAutoScalingGroups({ + AutoScalingGroupNames: [message.asgName], + }).promise() + .then(data => data.AutoScalingGroups[0]) + .then(validateAsgCapacity) + .then(multipleDesiredCapacity) + .then(() => cb(null, {message: 'success'})) + ; +}; + +function validateAsgCapacity(asg) { + if (!asg) { + throw 'The autoscaling group name does not exist...'; + } + if (asg.DesiredCapacity * 2 > asg.MaxSize) { + throw 'Max capacity is too small so that it can not replace instances.'; + } + console.log(`Current desired capacity is ${asg.DesiredCapacity}`); + + return asg; +} + +function multipleDesiredCapacity(asg) { + console.log(`Scale out to ${asg.DesiredCapacity * 2}.`); + + return AutoScaling.setDesiredCapacity({ + AutoScalingGroupName: asg.AutoScalingGroupName, + DesiredCapacity: asg.DesiredCapacity * 2, + HonorCooldown: true, + }).promise(); +} + diff --git a/functions/variables.tf b/functions/variables.tf new file mode 100644 index 0000000..f32241b --- /dev/null +++ b/functions/variables.tf @@ -0,0 +1,5 @@ +variable "sns_topic_arn_backend_app_updated" { +} + +variable "sns_topic_arn_asg_image_updated" { +} diff --git a/main.tf b/main.tf index 7941c22..595fc51 100644 --- a/main.tf +++ b/main.tf @@ -71,3 +71,9 @@ module "security_groups" { vpc_id = "${module.vpc.vpc_id}" ssh_allowed_segments = ["${var.allowed_segments}"] } + +module "functions" { + source = "./functions" + sns_topic_arn_backend_app_updated = "${aws_sns_topic.backend_app_updated.arn}" + sns_topic_arn_asg_image_updated = "${aws_sns_topic.asg_image_updated.arn}" +} diff --git a/scripts/post.sh b/scripts/post.sh index c8046f8..ed8d0b1 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -5,4 +5,7 @@ if [ "${ENVIRONMENT}" = "prod" ]; then test ! -v AWS_SESSION_TOKEN && direnv reload > /dev/null 2>&1 fi +# remove archived lambda functions +find functions -depth 1 -name "*.zip" | xargs rm -rf + exit 0 diff --git a/scripts/pre.sh b/scripts/pre.sh index f99dcec..3dd21c3 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -17,3 +17,13 @@ if [ -n "${asg_name}" ]; then export TF_VAR_web_desired_capacity="${desired_capacity}" fi +# zip lambda functions + +dirs=$(find functions -depth 1 -type d ) +for dir in ${dirs} +do + ( + cd ${dir} + zip -X -r ../$(echo ${dir} | cut -d"/" -f2).zip * + ) +done diff --git a/scripts/replace_old_instances.js b/scripts/replace_old_instances.js deleted file mode 100644 index f87d68d..0000000 --- a/scripts/replace_old_instances.js +++ /dev/null @@ -1,50 +0,0 @@ -const AWS = require('aws-sdk'); -const ArgumentParser = require('argparse').ArgumentParser; - -AWS.config.region = process.env.AWS_DEFAULT_REGION; - -const AutoScaling = new AWS.AutoScaling(); - -function parseArguments() { - const parser = new ArgumentParser({ - description: 'This script scale out specified auto scaling group to flip out old instances.', - }); - parser.addArgument(['-n', '--name'], { - help: 'name of a auto scaling group', - required: true, - }); - return parser.parseArgs(); -} - -function validateAsgCapacity(asg) { - if (!asg) { - console.error('This autoscaling group does not exist...'); - process.exit(1); - } - if (asg.DesiredCapacity * 2 > asg.MaxSize) { - console.error(`Can not scale out to replace instances. Consider increasing the max capacity.`); - process.exit(1); - } - console.log(`Current desired capacity is ${asg.DesiredCapacity}`); - - return asg; -} - -function multipleDesiredCapacity(asg) { - console.log(`Scale out to ${asg.DesiredCapacity * 2}.`); - - return AutoScaling.setDesiredCapacity({ - AutoScalingGroupName: asg.AutoScalingGroupName, - DesiredCapacity: asg.DesiredCapacity * 2, - HonorCooldown: true, - }).promise(); -} - -const asgName = parseArguments().name; - -AutoScaling.describeAutoScalingGroups({ - AutoScalingGroupNames: [asgName], -}).promise() - .then(data => data.AutoScalingGroups[0]) - .then(validateAsgCapacity) - .then(multipleDesiredCapacity); diff --git a/sns.tf b/sns.tf new file mode 100644 index 0000000..26c9e68 --- /dev/null +++ b/sns.tf @@ -0,0 +1,7 @@ +resource "aws_sns_topic" "backend_app_updated" { + name = "backend_app_updated" +} + +resource "aws_sns_topic" "asg_image_updated" { + name = "asg_image_updated" +} From 6d144b10a9d9667d2548e380d44504f5264c3025 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 21:45:39 +0700 Subject: [PATCH 117/181] add remove_unused_and_rotated_amis as lambda --- functions/remove_unused_and_rotated_amis.tf | 64 +++++++++++++ .../remove_unused_and_rotated_amis/index.js | 36 ++++---- .../package.json | 6 ++ .../remove_unused_and_rotated_amis/yarn.lock | 91 +++++++++++++++++++ .../package.json | 5 + .../replace_autoscaling_instances/yarn.lock | 87 ++++++++++++++++++ scripts/pre.sh | 3 +- 7 files changed, 274 insertions(+), 18 deletions(-) create mode 100644 functions/remove_unused_and_rotated_amis.tf rename scripts/remove_unused_and_rotated_amis.js => functions/remove_unused_and_rotated_amis/index.js (68%) create mode 100644 functions/remove_unused_and_rotated_amis/package.json create mode 100644 functions/remove_unused_and_rotated_amis/yarn.lock create mode 100644 functions/replace_autoscaling_instances/package.json create mode 100644 functions/replace_autoscaling_instances/yarn.lock diff --git a/functions/remove_unused_and_rotated_amis.tf b/functions/remove_unused_and_rotated_amis.tf new file mode 100644 index 0000000..782c6ae --- /dev/null +++ b/functions/remove_unused_and_rotated_amis.tf @@ -0,0 +1,64 @@ +resource "aws_lambda_function" "remove_unused_and_rotated_amis" { + filename = "functions/remove_unused_and_rotated_amis.zip" + function_name = "remove_unused_and_rotated_amis" + role = "${aws_iam_role.remove_unused_and_rotated_amis.arn}" + handler = "index.handle" + runtime = "nodejs4.3" + source_code_hash = "${base64sha256(file("functions/remove_unused_and_rotated_amis.zip"))}" +} + +resource "aws_iam_role" "remove_unused_and_rotated_amis" { + name = "remove_unused_and_rotated_amis" + assume_role_policy = < { + console.log('processing event: %j', event); + + Promise.all([rotatedImages(), unusedImageIds()]) + .then(_.spread((rotatedImages, unusedImageIds) => { + return rotatedImages.filter(image => { + return !unusedImageIds.includes(image.ImageId) + }) + })) + .then(images => { + const promises = images.map(image => { + return deregisterImage(image) + .then(deleteSnapShot) + }); + return Promise.all(promises); + }) + .then(() => cb(null, {message: 'success'})); +}; + function rotatedImages() { return EC2.describeImages({ Filters: [ @@ -54,18 +71,3 @@ function deleteSnapShot(image) { }) .catch(() => image) } - -Promise.all([rotatedImages(), unusedImageIds()]) - .then(results => { - const [rotatedImages, unusedImageIds] = results; - return rotatedImages.filter(image => { - return !unusedImageIds.includes(image.ImageId) - }) - }) - .then(images => { - const promises = images.map(image => { - return deregisterImage(image).then(deleteSnapShot) - }); - return Promise.all(promises); - }) -; diff --git a/functions/remove_unused_and_rotated_amis/package.json b/functions/remove_unused_and_rotated_amis/package.json new file mode 100644 index 0000000..666f25e --- /dev/null +++ b/functions/remove_unused_and_rotated_amis/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "aws-sdk": "^2.6.11", + "lodash": "^4.16.4" + } +} diff --git a/functions/remove_unused_and_rotated_amis/yarn.lock b/functions/remove_unused_and_rotated_amis/yarn.lock new file mode 100644 index 0000000..51dbb61 --- /dev/null +++ b/functions/remove_unused_and_rotated_amis/yarn.lock @@ -0,0 +1,91 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 +aws-sdk: + version "2.6.11" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.11.tgz#686ec162c9aad0334c3732acab492b815bbc11b4" + dependencies: + buffer "4.9.1" + crypto-browserify "1.0.9" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.1.5" + url "0.10.3" + xml2js "0.4.15" + xmlbuilder "2.6.2" + +base64-js@^1.0.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" + +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + +crypto-browserify@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" + +ieee754@^1.1.4: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + +lodash: + version "4.16.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.4.tgz#01ce306b9bad1319f2a5528674f88297aeb70127" + +lodash@~3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" + +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + +sax@>=0.6.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" + +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +xml2js@0.4.15: + version "0.4.15" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" + dependencies: + sax ">=0.6.0" + xmlbuilder ">=2.4.6" + +xmlbuilder@>=2.4.6: + version "8.2.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" + +xmlbuilder@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" + dependencies: + lodash "~3.5.0" + diff --git a/functions/replace_autoscaling_instances/package.json b/functions/replace_autoscaling_instances/package.json new file mode 100644 index 0000000..2b951b2 --- /dev/null +++ b/functions/replace_autoscaling_instances/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "aws-sdk": "^2.6.11" + } +} diff --git a/functions/replace_autoscaling_instances/yarn.lock b/functions/replace_autoscaling_instances/yarn.lock new file mode 100644 index 0000000..2df65ff --- /dev/null +++ b/functions/replace_autoscaling_instances/yarn.lock @@ -0,0 +1,87 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 +aws-sdk: + version "2.6.11" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.11.tgz#686ec162c9aad0334c3732acab492b815bbc11b4" + dependencies: + buffer "4.9.1" + crypto-browserify "1.0.9" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.1.5" + url "0.10.3" + xml2js "0.4.15" + xmlbuilder "2.6.2" + +base64-js@^1.0.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" + +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + +crypto-browserify@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" + +ieee754@^1.1.4: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + +lodash@~3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" + +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + +sax@>=0.6.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" + +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +xml2js@0.4.15: + version "0.4.15" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" + dependencies: + sax ">=0.6.0" + xmlbuilder ">=2.4.6" + +xmlbuilder@>=2.4.6: + version "8.2.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" + +xmlbuilder@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" + dependencies: + lodash "~3.5.0" + diff --git a/scripts/pre.sh b/scripts/pre.sh index 3dd21c3..83e3e1b 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -24,6 +24,7 @@ for dir in ${dirs} do ( cd ${dir} - zip -X -r ../$(echo ${dir} | cut -d"/" -f2).zip * + yarn install + zip -X -r -q ../$(echo ${dir} | cut -d"/" -f2).zip * ) done From aa96d28f4fa5e17967267a54256e7f44039a2f59 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 22:12:18 +0700 Subject: [PATCH 118/181] update travis to publish to sns topic --- .travis.yml | 16 +++--- package.json | 18 ------ scripts/get_certificate_arn.js | 26 --------- scripts/pre.sh | 4 +- yarn.lock | 101 --------------------------------- 5 files changed, 12 insertions(+), 153 deletions(-) delete mode 100644 package.json delete mode 100644 scripts/get_certificate_arn.js delete mode 100644 yarn.lock diff --git a/.travis.yml b/.travis.yml index e243911..0a83e5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,11 @@ cache: - $HOME/.yarn-cache - node_modules node_js: -- '6.9.1' +- '4.3.2' install: - # install npm packages + # install yarn to create archives for lambda functions - npm install -g yarn@">=0.16.0" --cache-min 999999999 - - yarn install # install aws cli - sudo apt-get -y install python-pip - sudo pip install awscli @@ -46,10 +45,13 @@ deploy: branch: deploy/* after_deploy: - # Remove unused amis - - node scripts/remove_unused_and_rotated_amis.js - # Replace instances if web_ami was updated. + # Publish to SNS Topic if web_ami was updated. - ./terraform-env.sh ${ENV} output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - - test ${lines} -eq 2 && node scripts/replace_old_instances.js -n ${asg_name} + - account_number=$(aws sts get-caller-identity --output text --query 'Account') + - | + if [ ${lines} -eq 2 ]; then + aws sns publish --topic-arn "arn:aws:sns:${AWS_REGION}:${account_number}:asg_image_updated" \ + --message "{\"asgName\": \"${asg_name}\"}" + fi diff --git a/package.json b/package.json deleted file mode 100644 index 7239fb6..0000000 --- a/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "micropost-formation", - "version": "1.0.0", - "engines": { - "npm": "~6.7.0" - }, - "repository": { - "url": "ssh://git@github.com/springboot-angular2-tutorial/micropost-formation.git", - "type": "git" - }, - "author": "Akira Sosa ", - "license": "MIT", - "dependencies": { - "argparse": "^1.0.9", - "aws-sdk": "^2.6.10", - "lodash": "^4.16.4" - } -} diff --git a/scripts/get_certificate_arn.js b/scripts/get_certificate_arn.js deleted file mode 100644 index bc27fca..0000000 --- a/scripts/get_certificate_arn.js +++ /dev/null @@ -1,26 +0,0 @@ -const AWS = require('aws-sdk'); -const ArgumentParser = require('argparse').ArgumentParser; - -AWS.config.region = process.env.AWS_DEFAULT_REGION; - -function parseArguments() { - const parser = new ArgumentParser({ - description: 'This script find an ACM arn by domain name', - }); - parser.addArgument(['-d', '--domain'], { - help: 'domain name of certificate to search', - required: true, - }); - return parser.parseArgs(); -} - -const args = parseArguments(); -const ACM = new AWS.ACM(); - -ACM.listCertificates((err, data) => { - const arn = data.CertificateSummaryList - .filter(s => s.DomainName === args.domain) - .map(s => s.CertificateArn); - console.log(arn[0]); -}); - diff --git a/scripts/pre.sh b/scripts/pre.sh index 83e3e1b..06e9350 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -7,8 +7,10 @@ if [ "${ENVIRONMENT}" = "prod" ]; then source scripts/switch-production-role.sh fi +certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') + export TF_VAR_aws_region=${AWS_DEFAULT_REGION} -export TF_VAR_alb_certificate_arn=$(node scripts/get_certificate_arn.js -d "*.hana053.com") +export TF_VAR_alb_certificate_arn=${certificate_arn} asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index 3d1fed5..0000000 --- a/yarn.lock +++ /dev/null @@ -1,101 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 -argparse: - version "1.0.9" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" - dependencies: - sprintf-js "~1.0.2" - -aws-sdk: - version "2.6.10" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.10.tgz#ae3be91a855fbe01426886e844c38e4d64a5c4c7" - dependencies: - buffer "4.9.1" - crypto-browserify "1.0.9" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.1.5" - url "0.10.3" - xml2js "0.4.15" - xmlbuilder "2.6.2" - -base64-js@^1.0.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" - -buffer@4.9.1: - version "4.9.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - isarray "^1.0.0" - -crypto-browserify@1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" - -ieee754@^1.1.4: - version "1.1.8" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" - -isarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - -lodash: - version "4.16.4" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.4.tgz#01ce306b9bad1319f2a5528674f88297aeb70127" - -lodash@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" - -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - -sax@>=0.6.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - -sax@1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" - -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - dependencies: - punycode "1.3.2" - querystring "0.2.0" - -xml2js@0.4.15: - version "0.4.15" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" - dependencies: - sax ">=0.6.0" - xmlbuilder ">=2.4.6" - -xmlbuilder@>=2.4.6: - version "8.2.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" - -xmlbuilder@2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" - dependencies: - lodash "~3.5.0" - From 93edfe916a31bb0e72f77fdeaad2e5399597adc1 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 23:05:54 +0700 Subject: [PATCH 119/181] fix script for ubuntu linux --- scripts/post.sh | 2 +- scripts/pre.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/post.sh b/scripts/post.sh index ed8d0b1..7f37b1e 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -6,6 +6,6 @@ if [ "${ENVIRONMENT}" = "prod" ]; then fi # remove archived lambda functions -find functions -depth 1 -name "*.zip" | xargs rm -rf +find functions -mindepth 1 -maxdepth 1 -name "*.zip" | xargs rm -rf exit 0 diff --git a/scripts/pre.sh b/scripts/pre.sh index 06e9350..83af7f5 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -21,7 +21,7 @@ fi # zip lambda functions -dirs=$(find functions -depth 1 -type d ) +dirs=$(find functions -mindepth 1 -maxdepth 1 -type d) for dir in ${dirs} do ( From ca2fb036c7d6d0795b9db55338682556b9410fea Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 23:25:12 +0700 Subject: [PATCH 120/181] fix readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b43671a..e2e61cf 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Install npm packages. ``` $ npm install -g yarn@">=0.16.0" -$ yarn install ``` Plan and Apply From 5aed3e5a5988c83933458db843a97ba93bfc786d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 23:33:06 +0700 Subject: [PATCH 121/181] hi travis? From 3bdd0a9f0390ff8807121641e5da15061f966f85 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 23:53:49 +0700 Subject: [PATCH 122/181] hi travis? From 31edfa72b842e6b56e26d76ba26fae9d07cfed9e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 21 Oct 2016 23:53:59 +0700 Subject: [PATCH 123/181] silence pre and post for terraform output --- scripts/post.sh | 2 +- scripts/pre.sh | 2 +- terraform-env.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/post.sh b/scripts/post.sh index 7f37b1e..3a12bc9 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -2,7 +2,7 @@ if [ "${ENVIRONMENT}" = "prod" ]; then # reset current role if exists - test ! -v AWS_SESSION_TOKEN && direnv reload > /dev/null 2>&1 + test ! -v AWS_SESSION_TOKEN && direnv reload fi # remove archived lambda functions diff --git a/scripts/pre.sh b/scripts/pre.sh index 83af7f5..d2824a9 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -2,7 +2,7 @@ if [ "${ENVIRONMENT}" = "prod" ]; then # reset current role if exists - test ! -v AWS_SESSION_TOKEN && direnv reload > /dev/null 2>&1 + test ! -v AWS_SESSION_TOKEN && direnv reload # switch to production role source scripts/switch-production-role.sh fi diff --git a/terraform-env.sh b/terraform-env.sh index 1e9a0b5..415487b 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -82,7 +82,7 @@ then fi # Let's run the PRE_CMD hook if it's defined -. ${PRE_CMD} +. ${PRE_CMD} > /dev/null 2>&1 # let's copy environment specific configuration to the root of the directory if files_exist ${ENV_DIR}/*.tf; then @@ -100,4 +100,4 @@ if files_exist *.env.tf; then fi # Let's run the POST_CMD hook if it's defined -eval ${POST_CMD} +. ${POST_CMD} > /dev/null 2>&1 From 1015f1ba9cdcaf4015edd000ef009a28b45075de Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 22 Oct 2016 09:28:49 +0700 Subject: [PATCH 124/181] hi travis? From 5245b416286947574cc7db1538d215db33b0efa4 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 22 Oct 2016 09:29:03 +0700 Subject: [PATCH 125/181] update shebang --- .envrc.example | 2 +- scripts/post.sh | 2 -- scripts/switch-production-role.sh | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.envrc.example b/.envrc.example index 957c1d4..7eb5e9f 100644 --- a/.envrc.example +++ b/.envrc.example @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash PATH_add scripts diff --git a/scripts/post.sh b/scripts/post.sh index 3a12bc9..b9bfe1a 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -7,5 +7,3 @@ fi # remove archived lambda functions find functions -mindepth 1 -maxdepth 1 -name "*.zip" | xargs rm -rf - -exit 0 diff --git a/scripts/switch-production-role.sh b/scripts/switch-production-role.sh index f0e3b51..544ea9d 100755 --- a/scripts/switch-production-role.sh +++ b/scripts/switch-production-role.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash credentials=$(aws sts assume-role --role-arn ${PROD_ROLE_ARN} --role-session-name travisci) From 7b047f5ea1c9dc5b2e3248735c699579f310ffbc Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 22 Oct 2016 10:36:15 +0700 Subject: [PATCH 126/181] do not use codedeploy --- codedeploy/main.tf | 60 ----------------------------------------- codedeploy/variables.tf | 10 ------- main.tf | 7 ----- webservers/iam.tf | 4 +-- webservers/outputs.tf | 4 --- 5 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 codedeploy/main.tf delete mode 100644 codedeploy/variables.tf diff --git a/codedeploy/main.tf b/codedeploy/main.tf deleted file mode 100644 index 44121b5..0000000 --- a/codedeploy/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -resource "aws_codedeploy_app" "main" { - name = "${var.name}" -} - -resource "aws_codedeploy_deployment_group" "main" { - app_name = "${aws_codedeploy_app.main.name}" - deployment_group_name = "${var.group_name}" - service_role_arn = "${aws_iam_role.codedeploy_service.arn}" - autoscaling_groups = ["${var.autoscaling_groups}"] - deployment_config_name = "CodeDeployDefault.OneAtATime" -} - -resource "aws_iam_role" "codedeploy_service" { - name = "codedeploy-${var.group_name}" - assume_role_policy = < Date: Sat, 22 Oct 2016 10:41:35 +0700 Subject: [PATCH 127/181] cache pip --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0a83e5f..48db97c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ cache: - $HOME/.npm - $HOME/.yarn-cache - node_modules + - $HOME/.cache/pip node_js: - '4.3.2' From c86a7dad7cb8e477b75ad358b568d0210157d124 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 22 Oct 2016 13:59:01 +0700 Subject: [PATCH 128/181] fix lambda function remove_unused_and_rotated_amis Node.js is 4.7 not 6 --- functions/remove_unused_and_rotated_amis/index.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/functions/remove_unused_and_rotated_amis/index.js b/functions/remove_unused_and_rotated_amis/index.js index d407303..d7d637d 100644 --- a/functions/remove_unused_and_rotated_amis/index.js +++ b/functions/remove_unused_and_rotated_amis/index.js @@ -7,11 +7,11 @@ const AutoScaling = new AWS.AutoScaling(); exports.handle = (event, ctx, cb) => { console.log('processing event: %j', event); - Promise.all([rotatedImages(), unusedImageIds()]) - .then(_.spread((rotatedImages, unusedImageIds) => { - return rotatedImages.filter(image => { - return !unusedImageIds.includes(image.ImageId) - }) + Promise.all([rotatedImages(), usedImageIds()]) + .then(_.spread((rotatedImages, usedImageIds) => { + return _.reject(rotatedImages, image => { + return _.includes(usedImageIds, image.ImageId); + }); })) .then(images => { const promises = images.map(image => { @@ -35,7 +35,7 @@ function rotatedImages() { .then(data => data.Images); } -function unusedImageIds() { +function usedImageIds() { return AutoScaling.describeLaunchConfigurations({}).promise() .then(data => { return data.LaunchConfigurations.map(lc => lc.ImageId); @@ -62,6 +62,7 @@ function deleteSnapShot(image) { .map(m => m.SnapshotId) .first() .value(); + return EC2.deleteSnapshot({ SnapshotId: snapShotId, }).promise() @@ -71,3 +72,4 @@ function deleteSnapShot(image) { }) .catch(() => image) } + From bc0c7c54a06e05a877bd455fec0776e0aba76450 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 27 Oct 2016 21:02:24 +0700 Subject: [PATCH 129/181] decrease scale in period --- webservers/autoscale.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index d6a6e71..ce789ba 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -109,7 +109,7 @@ resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { evaluation_periods = "1" metric_name = "CPUUtilization" namespace = "AWS/EC2" - period = "120" + period = "60" statistic = "Average" threshold = "5" dimensions { From 13d23b12f588f3e5ec52b8ab7fa84107e57808b6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sat, 29 Oct 2016 15:16:38 +0700 Subject: [PATCH 130/181] use codedeploy to deploy frontend --- codedeploy/main.tf | 60 +++++++++++++++++++++++++++++++++++++++++ codedeploy/variables.tf | 10 +++++++ main.tf | 7 +++++ webservers/outputs.tf | 4 +++ 4 files changed, 81 insertions(+) create mode 100644 codedeploy/main.tf create mode 100644 codedeploy/variables.tf diff --git a/codedeploy/main.tf b/codedeploy/main.tf new file mode 100644 index 0000000..44121b5 --- /dev/null +++ b/codedeploy/main.tf @@ -0,0 +1,60 @@ +resource "aws_codedeploy_app" "main" { + name = "${var.name}" +} + +resource "aws_codedeploy_deployment_group" "main" { + app_name = "${aws_codedeploy_app.main.name}" + deployment_group_name = "${var.group_name}" + service_role_arn = "${aws_iam_role.codedeploy_service.arn}" + autoscaling_groups = ["${var.autoscaling_groups}"] + deployment_config_name = "CodeDeployDefault.OneAtATime" +} + +resource "aws_iam_role" "codedeploy_service" { + name = "codedeploy-${var.group_name}" + assume_role_policy = < Date: Sat, 29 Oct 2016 15:52:24 +0700 Subject: [PATCH 131/181] rotate ami From 4203fec6f263b72f16e52de27a28ab1eb9d923c8 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 30 Oct 2016 10:34:12 +0700 Subject: [PATCH 132/181] Removed lambda functions. They are managed by serverless. --- .travis.yml | 19 +--- functions/remove_unused_and_rotated_amis.tf | 64 ------------- .../remove_unused_and_rotated_amis/index.js | 75 --------------- .../package.json | 6 -- .../remove_unused_and_rotated_amis/yarn.lock | 91 ------------------- functions/replace_autoscaling_instances.tf | 76 ---------------- .../replace_autoscaling_instances/index.js | 44 --------- .../package.json | 5 - .../replace_autoscaling_instances/yarn.lock | 87 ------------------ functions/variables.tf | 5 - main.tf | 6 -- scripts/post.sh | 3 - scripts/pre.sh | 12 --- sns.tf | 7 -- 14 files changed, 5 insertions(+), 495 deletions(-) delete mode 100644 functions/remove_unused_and_rotated_amis.tf delete mode 100644 functions/remove_unused_and_rotated_amis/index.js delete mode 100644 functions/remove_unused_and_rotated_amis/package.json delete mode 100644 functions/remove_unused_and_rotated_amis/yarn.lock delete mode 100644 functions/replace_autoscaling_instances.tf delete mode 100644 functions/replace_autoscaling_instances/index.js delete mode 100644 functions/replace_autoscaling_instances/package.json delete mode 100644 functions/replace_autoscaling_instances/yarn.lock delete mode 100644 functions/variables.tf delete mode 100644 sns.tf diff --git a/.travis.yml b/.travis.yml index 48db97c..df3676b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,13 @@ --- sudo: required dist: trusty -language: node_js -cache: - directories: - - $HOME/.npm - - $HOME/.yarn-cache - - node_modules - - $HOME/.cache/pip -node_js: -- '4.3.2' +language: python +cache: pip +python: +- '2.7' install: - # install yarn to create archives for lambda functions - - npm install -g yarn@">=0.16.0" --cache-min 999999999 - # install aws cli - - sudo apt-get -y install python-pip - - sudo pip install awscli + - pip install awscli - aws --version # install terraform - wget https://releases.hashicorp.com/terraform/0.7.6/terraform_0.7.6_linux_amd64.zip -O /tmp/terraform.zip diff --git a/functions/remove_unused_and_rotated_amis.tf b/functions/remove_unused_and_rotated_amis.tf deleted file mode 100644 index 782c6ae..0000000 --- a/functions/remove_unused_and_rotated_amis.tf +++ /dev/null @@ -1,64 +0,0 @@ -resource "aws_lambda_function" "remove_unused_and_rotated_amis" { - filename = "functions/remove_unused_and_rotated_amis.zip" - function_name = "remove_unused_and_rotated_amis" - role = "${aws_iam_role.remove_unused_and_rotated_amis.arn}" - handler = "index.handle" - runtime = "nodejs4.3" - source_code_hash = "${base64sha256(file("functions/remove_unused_and_rotated_amis.zip"))}" -} - -resource "aws_iam_role" "remove_unused_and_rotated_amis" { - name = "remove_unused_and_rotated_amis" - assume_role_policy = < { - console.log('processing event: %j', event); - - Promise.all([rotatedImages(), usedImageIds()]) - .then(_.spread((rotatedImages, usedImageIds) => { - return _.reject(rotatedImages, image => { - return _.includes(usedImageIds, image.ImageId); - }); - })) - .then(images => { - const promises = images.map(image => { - return deregisterImage(image) - .then(deleteSnapShot) - }); - return Promise.all(promises); - }) - .then(() => cb(null, {message: 'success'})); -}; - -function rotatedImages() { - return EC2.describeImages({ - Filters: [ - { - Name: 'tag:Rotated', - Values: ['true'], - }, - ], - }).promise() - .then(data => data.Images); -} - -function usedImageIds() { - return AutoScaling.describeLaunchConfigurations({}).promise() - .then(data => { - return data.LaunchConfigurations.map(lc => lc.ImageId); - }); -} - -function deregisterImage(image) { - return EC2.deregisterImage({ - ImageId: image.ImageId, - }).promise() - .then(() => { - console.log(`deregistered ${image.ImageId}`); - return image; - }) - // ignore error and continue - .catch(() => image) -} - -function deleteSnapShot(image) { - const snapShotId = _.chain(image.BlockDeviceMappings) - .flatten() - .map(m => m.Ebs) - .compact() - .map(m => m.SnapshotId) - .first() - .value(); - - return EC2.deleteSnapshot({ - SnapshotId: snapShotId, - }).promise() - .then(() => { - console.log(`deleted ${snapShotId}`); - return image; - }) - .catch(() => image) -} - diff --git a/functions/remove_unused_and_rotated_amis/package.json b/functions/remove_unused_and_rotated_amis/package.json deleted file mode 100644 index 666f25e..0000000 --- a/functions/remove_unused_and_rotated_amis/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "dependencies": { - "aws-sdk": "^2.6.11", - "lodash": "^4.16.4" - } -} diff --git a/functions/remove_unused_and_rotated_amis/yarn.lock b/functions/remove_unused_and_rotated_amis/yarn.lock deleted file mode 100644 index 51dbb61..0000000 --- a/functions/remove_unused_and_rotated_amis/yarn.lock +++ /dev/null @@ -1,91 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 -aws-sdk: - version "2.6.11" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.11.tgz#686ec162c9aad0334c3732acab492b815bbc11b4" - dependencies: - buffer "4.9.1" - crypto-browserify "1.0.9" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.1.5" - url "0.10.3" - xml2js "0.4.15" - xmlbuilder "2.6.2" - -base64-js@^1.0.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" - -buffer@4.9.1: - version "4.9.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - isarray "^1.0.0" - -crypto-browserify@1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" - -ieee754@^1.1.4: - version "1.1.8" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" - -isarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - -lodash: - version "4.16.4" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.4.tgz#01ce306b9bad1319f2a5528674f88297aeb70127" - -lodash@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" - -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - -sax@>=0.6.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - -sax@1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" - -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - dependencies: - punycode "1.3.2" - querystring "0.2.0" - -xml2js@0.4.15: - version "0.4.15" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" - dependencies: - sax ">=0.6.0" - xmlbuilder ">=2.4.6" - -xmlbuilder@>=2.4.6: - version "8.2.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" - -xmlbuilder@2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" - dependencies: - lodash "~3.5.0" - diff --git a/functions/replace_autoscaling_instances.tf b/functions/replace_autoscaling_instances.tf deleted file mode 100644 index bdf9e75..0000000 --- a/functions/replace_autoscaling_instances.tf +++ /dev/null @@ -1,76 +0,0 @@ -resource "aws_lambda_function" "replace_autoscaling_instances" { - filename = "functions/replace_autoscaling_instances.zip" - function_name = "replace_autoscaling_instances" - role = "${aws_iam_role.replace_autoscaling_instances.arn}" - handler = "index.handle" - runtime = "nodejs4.3" - source_code_hash = "${base64sha256(file("functions/replace_autoscaling_instances.zip"))}" -} - -resource "aws_iam_role" "replace_autoscaling_instances" { - name = "replace_autoscaling_instances" - assume_role_policy = < { - console.log('processing event: %j', event); - - const message = JSON.parse(event.Records[0].Sns.Message); - console.log('ASG name:', message.asgName); - - AutoScaling.describeAutoScalingGroups({ - AutoScalingGroupNames: [message.asgName], - }).promise() - .then(data => data.AutoScalingGroups[0]) - .then(validateAsgCapacity) - .then(multipleDesiredCapacity) - .then(() => cb(null, {message: 'success'})) - ; -}; - -function validateAsgCapacity(asg) { - if (!asg) { - throw 'The autoscaling group name does not exist...'; - } - if (asg.DesiredCapacity * 2 > asg.MaxSize) { - throw 'Max capacity is too small so that it can not replace instances.'; - } - console.log(`Current desired capacity is ${asg.DesiredCapacity}`); - - return asg; -} - -function multipleDesiredCapacity(asg) { - console.log(`Scale out to ${asg.DesiredCapacity * 2}.`); - - return AutoScaling.setDesiredCapacity({ - AutoScalingGroupName: asg.AutoScalingGroupName, - DesiredCapacity: asg.DesiredCapacity * 2, - HonorCooldown: true, - }).promise(); -} - diff --git a/functions/replace_autoscaling_instances/package.json b/functions/replace_autoscaling_instances/package.json deleted file mode 100644 index 2b951b2..0000000 --- a/functions/replace_autoscaling_instances/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "aws-sdk": "^2.6.11" - } -} diff --git a/functions/replace_autoscaling_instances/yarn.lock b/functions/replace_autoscaling_instances/yarn.lock deleted file mode 100644 index 2df65ff..0000000 --- a/functions/replace_autoscaling_instances/yarn.lock +++ /dev/null @@ -1,87 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 -aws-sdk: - version "2.6.11" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.6.11.tgz#686ec162c9aad0334c3732acab492b815bbc11b4" - dependencies: - buffer "4.9.1" - crypto-browserify "1.0.9" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.1.5" - url "0.10.3" - xml2js "0.4.15" - xmlbuilder "2.6.2" - -base64-js@^1.0.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1" - -buffer@4.9.1: - version "4.9.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - isarray "^1.0.0" - -crypto-browserify@1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" - -ieee754@^1.1.4: - version "1.1.8" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" - -isarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - -lodash@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.5.0.tgz#19bb3f4d51278f0b8c818ed145c74ecf9fe40e6d" - -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - -sax@>=0.6.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - -sax@1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.5.tgz#1da50a8d00cdecd59405659f5ff85349fe773743" - -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - dependencies: - punycode "1.3.2" - querystring "0.2.0" - -xml2js@0.4.15: - version "0.4.15" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.15.tgz#95cd03ff2dd144ec28bc6273bf2b2890c581ad0c" - dependencies: - sax ">=0.6.0" - xmlbuilder ">=2.4.6" - -xmlbuilder@>=2.4.6: - version "8.2.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-8.2.2.tgz#69248673410b4ba42e1a6136551d2922335aa773" - -xmlbuilder@2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-2.6.2.tgz#f916f6d10d45dc171b1be2e6e673fb6e0cc35d0a" - dependencies: - lodash "~3.5.0" - diff --git a/functions/variables.tf b/functions/variables.tf deleted file mode 100644 index f32241b..0000000 --- a/functions/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "sns_topic_arn_backend_app_updated" { -} - -variable "sns_topic_arn_asg_image_updated" { -} diff --git a/main.tf b/main.tf index 684c4c0..8e98df3 100644 --- a/main.tf +++ b/main.tf @@ -71,9 +71,3 @@ module "security_groups" { vpc_id = "${module.vpc.vpc_id}" ssh_allowed_segments = ["${var.allowed_segments}"] } - -module "functions" { - source = "./functions" - sns_topic_arn_backend_app_updated = "${aws_sns_topic.backend_app_updated.arn}" - sns_topic_arn_asg_image_updated = "${aws_sns_topic.asg_image_updated.arn}" -} diff --git a/scripts/post.sh b/scripts/post.sh index b9bfe1a..4e2233c 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -4,6 +4,3 @@ if [ "${ENVIRONMENT}" = "prod" ]; then # reset current role if exists test ! -v AWS_SESSION_TOKEN && direnv reload fi - -# remove archived lambda functions -find functions -mindepth 1 -maxdepth 1 -name "*.zip" | xargs rm -rf diff --git a/scripts/pre.sh b/scripts/pre.sh index d2824a9..4dc3e23 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -18,15 +18,3 @@ if [ -n "${asg_name}" ]; then # respect current desired capacity export TF_VAR_web_desired_capacity="${desired_capacity}" fi - -# zip lambda functions - -dirs=$(find functions -mindepth 1 -maxdepth 1 -type d) -for dir in ${dirs} -do - ( - cd ${dir} - yarn install - zip -X -r -q ../$(echo ${dir} | cut -d"/" -f2).zip * - ) -done diff --git a/sns.tf b/sns.tf deleted file mode 100644 index 26c9e68..0000000 --- a/sns.tf +++ /dev/null @@ -1,7 +0,0 @@ -resource "aws_sns_topic" "backend_app_updated" { - name = "backend_app_updated" -} - -resource "aws_sns_topic" "asg_image_updated" { - name = "asg_image_updated" -} From 1746d34c582091bde1e0f129db99cdfd183745d6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 30 Oct 2016 17:44:50 +0700 Subject: [PATCH 133/181] updated sns topic name in travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df3676b..c2eb320 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,6 @@ after_deploy: - account_number=$(aws sts get-caller-identity --output text --query 'Account') - | if [ ${lines} -eq 2 ]; then - aws sns publish --topic-arn "arn:aws:sns:${AWS_REGION}:${account_number}:asg_image_updated" \ + aws sns publish --topic-arn "arn:aws:sns:${AWS_REGION}:${account_number}:web_image_updated" \ --message "{\"asgName\": \"${asg_name}\"}" fi From f8aa955c86cbecf39f5744acf45ccefe60460798 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 30 Oct 2016 18:20:27 +0700 Subject: [PATCH 134/181] integrate with sns --- codedeploy/main.tf | 14 ++++++++++++++ webservers/autoscale.tf | 17 +++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/codedeploy/main.tf b/codedeploy/main.tf index 44121b5..696d62a 100644 --- a/codedeploy/main.tf +++ b/codedeploy/main.tf @@ -1,3 +1,9 @@ +data "aws_caller_identity" "current" {} + +data "aws_region" "current" { + current = true +} + resource "aws_codedeploy_app" "main" { name = "${var.name}" } @@ -8,6 +14,14 @@ resource "aws_codedeploy_deployment_group" "main" { service_role_arn = "${aws_iam_role.codedeploy_service.arn}" autoscaling_groups = ["${var.autoscaling_groups}"] deployment_config_name = "CodeDeployDefault.OneAtATime" + trigger_configuration { + trigger_events = [ + "DeploymentSuccess", + "DeploymentFailure", + ] + trigger_name = "frontend_deployed" + trigger_target_arn = "arn:aws:sns:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:frontend_deployed" + } } resource "aws_iam_role" "codedeploy_service" { diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index ce789ba..717c676 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -1,3 +1,9 @@ +data "aws_caller_identity" "current" {} + +data "aws_region" "current" { + current = true +} + data "aws_ami" "web" { most_recent = true owners = [ @@ -120,3 +126,14 @@ resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { ] } +resource "aws_autoscaling_notification" "web" { + group_names = [ + "${aws_autoscaling_group.web.name}", + ] + notifications = [ + "autoscaling:EC2_INSTANCE_LAUNCH", + "autoscaling:EC2_INSTANCE_TERMINATE", + "autoscaling:EC2_INSTANCE_LAUNCH_ERROR" + ] + topic_arn = "arn:aws:sns:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:web_autoscaled" +} From 1b84c1ae23911f691efeb28e72b6eeac6df399ea Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 30 Oct 2016 18:30:55 +0700 Subject: [PATCH 135/181] add sns publishing role to codedeploy service --- codedeploy/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codedeploy/main.tf b/codedeploy/main.tf index 696d62a..c31751b 100644 --- a/codedeploy/main.tf +++ b/codedeploy/main.tf @@ -64,7 +64,8 @@ resource "aws_iam_role_policy" "codedeploy_service_policy" { "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "tag:GetTags", - "tag:GetResources" + "tag:GetResources", + "sns:Publish" ], "Resource": "*" } From 8ce7ddd8a8c07abf1781dd9f6fd4bd3652194558 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 30 Oct 2016 19:01:08 +0700 Subject: [PATCH 136/181] fix deploy script --- scripts/post.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/post.sh b/scripts/post.sh index 4e2233c..abf4f40 100755 --- a/scripts/post.sh +++ b/scripts/post.sh @@ -4,3 +4,5 @@ if [ "${ENVIRONMENT}" = "prod" ]; then # reset current role if exists test ! -v AWS_SESSION_TOKEN && direnv reload fi + +exit 0 From a5b10796a560dec875598e3447ede49571592849 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 31 Oct 2016 17:58:23 +0700 Subject: [PATCH 137/181] use terraform 0.7.7 --- .travis.yml | 2 +- README.md | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2eb320..ba4d025 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip install awscli - aws --version # install terraform - - wget https://releases.hashicorp.com/terraform/0.7.6/terraform_0.7.6_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.7/terraform_0.7.7_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: diff --git a/README.md b/README.md index e2e61cf..be8fee8 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ $ direnv allow Install npm packages. ``` -$ npm install -g yarn@">=0.16.0" +$ npm install -g yarn ``` Plan and Apply @@ -30,3 +30,11 @@ $ ./terraform-env.sh stg get $ ./terraform-env.sh stg plan $ ./terraform-env.sh stg apply ``` + +## Related Projects + +* [Angular2 app](https://github.com/springboot-angular2-tutorial/angular2-app) +* [Spring Boot app](https://github.com/springboot-angular2-tutorial/boot-app) +* [Android app](https://github.com/springboot-angular2-tutorial/android-app) +* [Server provisioning by Ansible and Packer](https://github.com/springboot-angular2-tutorial/micropost-provisionings) +* [Lambda functions by Serverless](https://github.com/springboot-angular2-tutorial/micropost-functions) From 84461e47ffd931a9f119eae8f72099b265ff479c Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 09:21:13 +0700 Subject: [PATCH 138/181] customize termination policy I use scale out and in when deploy new app. So, OldestInstance must be prior to ClosestToNextInstanceHour. --- webservers/autoscale.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 717c676..f44f314 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -59,6 +59,12 @@ resource "aws_autoscaling_group" "web" { "${aws_alb_target_group.api.arn}", "${aws_alb_target_group.index.arn}" ] + termination_policies = [ + 'OldestLaunchConfiguration', + 'OldestInstance', + 'ClosestToNextInstanceHour', + 'Default', + ] tag { key = "Name" value = "${var.env}-web" From b8ed639a88c342288f22379481fdd2207abff163 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:13:27 +0700 Subject: [PATCH 139/181] use aws profile --- .travis.yml | 12 ++++++++++-- scripts/post.sh | 8 -------- scripts/pre.sh | 16 +++++++--------- scripts/prepare-aws-profile.sh | 31 +++++++++++++++++++++++++++++++ scripts/switch-production-role.sh | 11 ----------- 5 files changed, 48 insertions(+), 30 deletions(-) delete mode 100755 scripts/post.sh create mode 100755 scripts/prepare-aws-profile.sh delete mode 100755 scripts/switch-production-role.sh diff --git a/.travis.yml b/.travis.yml index ba4d025..6dd777d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,15 +7,20 @@ python: - '2.7' install: + # Install AWS CLI - pip install awscli - aws --version - # install terraform + # Install Terraform - wget https://releases.hashicorp.com/terraform/0.7.7/terraform_0.7.7_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: - - export AWS_REGION=${AWS_DEFAULT_REGION} + # Determine environment - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") + # Create AWS profile + - ./scripts/prepare-aws-profile.sh + # Set current profile + - export AWS_PROFILE="micropost-${ENV}" script: - | if [ -z "${ENV}" ]; then @@ -47,3 +52,6 @@ after_deploy: aws sns publish --topic-arn "arn:aws:sns:${AWS_REGION}:${account_number}:web_image_updated" \ --message "{\"asgName\": \"${asg_name}\"}" fi + +after_script: + - rm -rf ${HOME}/.aws diff --git a/scripts/post.sh b/scripts/post.sh deleted file mode 100755 index abf4f40..0000000 --- a/scripts/post.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [ "${ENVIRONMENT}" = "prod" ]; then - # reset current role if exists - test ! -v AWS_SESSION_TOKEN && direnv reload -fi - -exit 0 diff --git a/scripts/pre.sh b/scripts/pre.sh index 4dc3e23..88ae180 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -1,17 +1,15 @@ #!/usr/bin/env bash -if [ "${ENVIRONMENT}" = "prod" ]; then - # reset current role if exists - test ! -v AWS_SESSION_TOKEN && direnv reload - # switch to production role - source scripts/switch-production-role.sh -fi +# Ensure AWS profile +export AWS_PROFILE="micropost-${ENVIRONMENT}" -certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') +# Set AWS region. It's required. +export TF_VAR_aws_region=$(aws configure get region) -export TF_VAR_aws_region=${AWS_DEFAULT_REGION} -export TF_VAR_alb_certificate_arn=${certificate_arn} +# Set ALB certificate ARN +export TF_VAR_alb_certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') +# Set ASG desired capacity, if it exists asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') diff --git a/scripts/prepare-aws-profile.sh b/scripts/prepare-aws-profile.sh new file mode 100755 index 0000000..0c48c2f --- /dev/null +++ b/scripts/prepare-aws-profile.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -u + +cat << EOS > ${HOME}/.aws/credentials +[micropost-stg] +aws_access_key_id = ${AWS_ACCESS_KEY_ID} +aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} + +[micropost-prod] +role_arn = ${PROD_ROLE_ARN} +source_profile = micropost-stg +EOS + +cat << EOS > ${HOME}/.aws/config +[profile micropost-stg] +region = ${AWS_DEFAULT_REGION} +output = json + +[profile micropost-prod] +region = ${AWS_DEFAULT_REGION} +output = json +EOS + +chmod 600 ${HOME}/.aws/credentials +chmod 600 ${HOME}/.aws/config + +unset AWS_ACCESS_KEY_ID +unset AWS_SECRET_ACCESS_KEY +unset AWS_DEFAULT_REGION +unset PROD_ROLE_ARN diff --git a/scripts/switch-production-role.sh b/scripts/switch-production-role.sh deleted file mode 100755 index 544ea9d..0000000 --- a/scripts/switch-production-role.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -credentials=$(aws sts assume-role --role-arn ${PROD_ROLE_ARN} --role-session-name travisci) - -export AWS_ACCESS_KEY_ID=$(echo ${credentials} | jq --raw-output .Credentials.AccessKeyId) -export AWS_SECRET_ACCESS_KEY=$(echo ${credentials} | jq --raw-output .Credentials.SecretAccessKey) -export AWS_SESSION_TOKEN=$(echo ${credentials} | jq --raw-output .Credentials.SessionToken) - -unset credentials - -echo "Switched to production role." From c9b1485b2ebc6953ff85a79fecb0d5b4fa762c80 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:16:34 +0700 Subject: [PATCH 140/181] sudo --- scripts/prepare-aws-profile.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/prepare-aws-profile.sh b/scripts/prepare-aws-profile.sh index 0c48c2f..9c15782 100755 --- a/scripts/prepare-aws-profile.sh +++ b/scripts/prepare-aws-profile.sh @@ -2,7 +2,7 @@ set -u -cat << EOS > ${HOME}/.aws/credentials +sudo cat << EOS > ${HOME}/.aws/credentials [micropost-stg] aws_access_key_id = ${AWS_ACCESS_KEY_ID} aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} @@ -12,7 +12,7 @@ role_arn = ${PROD_ROLE_ARN} source_profile = micropost-stg EOS -cat << EOS > ${HOME}/.aws/config +sudo cat << EOS > ${HOME}/.aws/config [profile micropost-stg] region = ${AWS_DEFAULT_REGION} output = json @@ -22,8 +22,8 @@ region = ${AWS_DEFAULT_REGION} output = json EOS -chmod 600 ${HOME}/.aws/credentials -chmod 600 ${HOME}/.aws/config +sudo chmod 600 ${HOME}/.aws/credentials +sudo chmod 600 ${HOME}/.aws/config unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY From 6653b441e921462b3b2558d2304ead63072d2c68 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:20:25 +0700 Subject: [PATCH 141/181] mkdir --- scripts/prepare-aws-profile.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/prepare-aws-profile.sh b/scripts/prepare-aws-profile.sh index 9c15782..da90de6 100755 --- a/scripts/prepare-aws-profile.sh +++ b/scripts/prepare-aws-profile.sh @@ -2,7 +2,9 @@ set -u -sudo cat << EOS > ${HOME}/.aws/credentials +mkdir ${HOME}/.aws + +cat << EOS > ${HOME}/.aws/credentials [micropost-stg] aws_access_key_id = ${AWS_ACCESS_KEY_ID} aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} @@ -12,7 +14,7 @@ role_arn = ${PROD_ROLE_ARN} source_profile = micropost-stg EOS -sudo cat << EOS > ${HOME}/.aws/config +cat << EOS > ${HOME}/.aws/config [profile micropost-stg] region = ${AWS_DEFAULT_REGION} output = json @@ -22,8 +24,8 @@ region = ${AWS_DEFAULT_REGION} output = json EOS -sudo chmod 600 ${HOME}/.aws/credentials -sudo chmod 600 ${HOME}/.aws/config +chmod 600 ${HOME}/.aws/credentials +chmod 600 ${HOME}/.aws/config unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY From dad2d966c5465f750cce42641d483036b1350426 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:24:54 +0700 Subject: [PATCH 142/181] single quote to double quote in tf --- webservers/autoscale.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index f44f314..3baae3c 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -60,10 +60,10 @@ resource "aws_autoscaling_group" "web" { "${aws_alb_target_group.index.arn}" ] termination_policies = [ - 'OldestLaunchConfiguration', - 'OldestInstance', - 'ClosestToNextInstanceHour', - 'Default', + "OldestLaunchConfiguration", + "OldestInstance", + "ClosestToNextInstanceHour", + "Default", ] tag { key = "Name" From db3cecf1d83af0f2eae1d511d1b1e7dee65a90b0 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:46:48 +0700 Subject: [PATCH 143/181] fix exit status of terraform-env --- terraform-env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform-env.sh b/terraform-env.sh index 415487b..cd5d503 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -93,6 +93,7 @@ fi # Let's do work! terraform $ACTION $VARS_FILE_FLAG $ADDTL_PARAMS ${EXTRA_ARGS} +terraform_status=$? # Let's remove those environment-specific configuration files we copied earlier if files_exist *.env.tf; then @@ -101,3 +102,5 @@ fi # Let's run the POST_CMD hook if it's defined . ${POST_CMD} > /dev/null 2>&1 + +exit ${terraform_status} From 76488ac1f8bcbe892051847145bbce7396eff848 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 10:54:00 +0700 Subject: [PATCH 144/181] fix travis --- .envrc.example | 11 ----------- .travis.yml | 3 ++- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.envrc.example b/.envrc.example index 7eb5e9f..95745ff 100644 --- a/.envrc.example +++ b/.envrc.example @@ -1,15 +1,4 @@ #!/usr/bin/env bash -PATH_add scripts - -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -export AWS_DEFAULT_REGION=ap-northeast-1 -export AWS_REGION=${AWS_DEFAULT_REGION} - export CLOUDFLARE_EMAIL= export CLOUDFLARE_TOKEN= - -export PROD_ROLE_ARN= - -unset AWS_SESSION_TOKEN diff --git a/.travis.yml b/.travis.yml index 6dd777d..4ebe3be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,10 +46,11 @@ after_deploy: - ./terraform-env.sh ${ENV} output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') + - region=$(aws configure get region) - account_number=$(aws sts get-caller-identity --output text --query 'Account') - | if [ ${lines} -eq 2 ]; then - aws sns publish --topic-arn "arn:aws:sns:${AWS_REGION}:${account_number}:web_image_updated" \ + aws sns publish --topic-arn "arn:aws:sns:${region}:${account_number}:web_image_updated" \ --message "{\"asgName\": \"${asg_name}\"}" fi From d8b461260e5f4ad7e6dfd8bfb306bbf0f914f20e Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 16:44:09 +0700 Subject: [PATCH 145/181] use role_arn in terraform --- .travis.yml | 3 +-- main.tf | 3 +++ scripts/pre.sh | 12 +++++++----- scripts/prepare-aws-profile.sh | 16 +++++++++++----- terraform-env.sh | 6 ++++-- terraform.cfg | 4 ++-- variables.tf | 4 ++++ 7 files changed, 32 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ebe3be..775197e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,6 @@ before_script: - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") # Create AWS profile - ./scripts/prepare-aws-profile.sh - # Set current profile - - export AWS_PROFILE="micropost-${ENV}" script: - | if [ -z "${ENV}" ]; then @@ -44,6 +42,7 @@ deploy: after_deploy: # Publish to SNS Topic if web_ami was updated. - ./terraform-env.sh ${ENV} output | tee out_after + - export AWS_PROFILE="micropost-${ENV}" - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - region=$(aws configure get region) diff --git a/main.tf b/main.tf index 8e98df3..3209046 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,8 @@ provider "aws" { region = "${var.aws_region}" + assume_role { + role_arn = "${var.aws_role_arn}" + } } data "aws_caller_identity" "current" {} diff --git a/scripts/pre.sh b/scripts/pre.sh index 88ae180..0999249 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -1,18 +1,20 @@ #!/usr/bin/env bash -# Ensure AWS profile -export AWS_PROFILE="micropost-${ENVIRONMENT}" +AWS_PROFILE="micropost-${ENVIRONMENT}" # Set AWS region. It's required. -export TF_VAR_aws_region=$(aws configure get region) +export TF_VAR_aws_region=$(aws configure get region --profile ${AWS_PROFILE}) # Set ALB certificate ARN -export TF_VAR_alb_certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') +export TF_VAR_alb_certificate_arn=$(aws acm list-certificates --profile ${AWS_PROFILE} | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') + +# Set role_arn +export TF_VAR_aws_role_arn=$(aws configure get role_arn --profile ${AWS_PROFILE}) # Set ASG desired capacity, if it exists asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then - desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') + desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} --profile ${AWS_PROFILE} | jq '.AutoScalingGroups[0].DesiredCapacity') # respect current desired capacity export TF_VAR_web_desired_capacity="${desired_capacity}" fi diff --git a/scripts/prepare-aws-profile.sh b/scripts/prepare-aws-profile.sh index da90de6..620ba00 100755 --- a/scripts/prepare-aws-profile.sh +++ b/scripts/prepare-aws-profile.sh @@ -5,23 +5,28 @@ set -u mkdir ${HOME}/.aws cat << EOS > ${HOME}/.aws/credentials -[micropost-stg] +[default] aws_access_key_id = ${AWS_ACCESS_KEY_ID} aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} - -[micropost-prod] -role_arn = ${PROD_ROLE_ARN} -source_profile = micropost-stg EOS cat << EOS > ${HOME}/.aws/config +[default] +region = ${AWS_DEFAULT_REGION} +output = json + [profile micropost-stg] +role_arn = ${STG_ROLE_ARN} +source_profile = default region = ${AWS_DEFAULT_REGION} output = json [profile micropost-prod] +role_arn = ${PROD_ROLE_ARN} +source_profile = default region = ${AWS_DEFAULT_REGION} output = json + EOS chmod 600 ${HOME}/.aws/credentials @@ -30,4 +35,5 @@ chmod 600 ${HOME}/.aws/config unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY unset AWS_DEFAULT_REGION +unset STG_ROLE_ARN unset PROD_ROLE_ARN diff --git a/terraform-env.sh b/terraform-env.sh index cd5d503..07b76e0 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -77,8 +77,10 @@ then echo $ENVIRONMENT > $ENV_FILE # Set up remote configuration and pull latest version - terraform remote config -backend S3 -backend-config="bucket=$bucket" -backend-config="key=$BUCKET_KEY" -backend-config="region=$region" - + terraform remote config -backend S3 \ + -backend-config="bucket=$bucket" \ + -backend-config="key=$BUCKET_KEY" \ + -backend-config="region=$region" fi # Let's run the PRE_CMD hook if it's defined diff --git a/terraform.cfg b/terraform.cfg index 7b24fa5..bb4cda9 100644 --- a/terraform.cfg +++ b/terraform.cfg @@ -1,6 +1,6 @@ -bucket=tfstate.hana053.com +bucket=state.hana053.com bucket_prefix= region=ap-northeast-1 pre_command='scripts/pre.sh' -post_command='scripts/post.sh' \ No newline at end of file +post_command='scripts/post.sh' diff --git a/variables.tf b/variables.tf index 78f2caf..d3fe7de 100644 --- a/variables.tf +++ b/variables.tf @@ -6,6 +6,10 @@ variable "aws_region" { default = "ap-northeast-1" } +variable "aws_role_arn" { + default = "" +} + variable "allowed_segments" { type = "list" default = [ From 523d95c0a5870e257cb18f82b787841534948d03 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 1 Nov 2016 17:00:29 +0700 Subject: [PATCH 146/181] ? --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 775197e..e1d93a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,15 +42,16 @@ deploy: after_deploy: # Publish to SNS Topic if web_ami was updated. - ./terraform-env.sh ${ENV} output | tee out_after - - export AWS_PROFILE="micropost-${ENV}" + - AWS_PROFILE="micropost-${ENV}" - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - - region=$(aws configure get region) - - account_number=$(aws sts get-caller-identity --output text --query 'Account') + - region=$(aws configure get region --profile $AWS_PROFILE) + - account_number=$(aws sts get-caller-identity --output text --query 'Account' --profile $AWS_PROFILE) - | if [ ${lines} -eq 2 ]; then aws sns publish --topic-arn "arn:aws:sns:${region}:${account_number}:web_image_updated" \ - --message "{\"asgName\": \"${asg_name}\"}" + --message "{\"asgName\": \"${asg_name}\"}" \ + --profile $AWS_PROFILE fi after_script: From ed4cce1bd31ce72e536acf4077b76dc196c14320 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 10:24:54 +0700 Subject: [PATCH 147/181] do not use AWS_PROFILE because Terraform does not handle it well. Terraform (remote) has a problem with AWS_PROFILE with role_arn. --- .envrc.example | 3 +++ .travis.yml | 10 +++------ main.tf | 5 +---- scripts/pre.sh | 14 +++--------- scripts/prepare-aws-profile.sh | 39 ---------------------------------- scripts/switch-role.sh | 12 +++++++++++ terraform-env.sh | 4 ++-- terraform.cfg | 4 ++++ variables.tf | 8 ------- 9 files changed, 28 insertions(+), 71 deletions(-) delete mode 100755 scripts/prepare-aws-profile.sh create mode 100755 scripts/switch-role.sh diff --git a/.envrc.example b/.envrc.example index 95745ff..2f21ee9 100644 --- a/.envrc.example +++ b/.envrc.example @@ -2,3 +2,6 @@ export CLOUDFLARE_EMAIL= export CLOUDFLARE_TOKEN= + +export ROLE_ARN_stg= +export ROLE_ARN_prod= diff --git a/.travis.yml b/.travis.yml index e1d93a6..3af238d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,12 @@ install: before_script: # Determine environment - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - # Create AWS profile - - ./scripts/prepare-aws-profile.sh script: - | if [ -z "${ENV}" ]; then echo "${TRAVIS_BRANCH} is not a branch to deploy." else + source ./scripts/switch-role.sh ./terraform-env.sh ${ENV} get ./terraform-env.sh ${ENV} plan fi @@ -42,16 +41,13 @@ deploy: after_deploy: # Publish to SNS Topic if web_ami was updated. - ./terraform-env.sh ${ENV} output | tee out_after - - AWS_PROFILE="micropost-${ENV}" - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - - region=$(aws configure get region --profile $AWS_PROFILE) - account_number=$(aws sts get-caller-identity --output text --query 'Account' --profile $AWS_PROFILE) - | if [ ${lines} -eq 2 ]; then - aws sns publish --topic-arn "arn:aws:sns:${region}:${account_number}:web_image_updated" \ - --message "{\"asgName\": \"${asg_name}\"}" \ - --profile $AWS_PROFILE + aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" \ + --message "{\"asgName\": \"${asg_name}\"}" fi after_script: diff --git a/main.tf b/main.tf index 3209046..02e9593 100644 --- a/main.tf +++ b/main.tf @@ -1,8 +1,5 @@ provider "aws" { - region = "${var.aws_region}" - assume_role { - role_arn = "${var.aws_role_arn}" - } + region = "ap-northeast-1" } data "aws_caller_identity" "current" {} diff --git a/scripts/pre.sh b/scripts/pre.sh index 0999249..9fbd85d 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -1,20 +1,12 @@ #!/usr/bin/env bash -AWS_PROFILE="micropost-${ENVIRONMENT}" - -# Set AWS region. It's required. -export TF_VAR_aws_region=$(aws configure get region --profile ${AWS_PROFILE}) - # Set ALB certificate ARN -export TF_VAR_alb_certificate_arn=$(aws acm list-certificates --profile ${AWS_PROFILE} | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') - -# Set role_arn -export TF_VAR_aws_role_arn=$(aws configure get role_arn --profile ${AWS_PROFILE}) +export TF_VAR_alb_certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') # Set ASG desired capacity, if it exists asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then - desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} --profile ${AWS_PROFILE} | jq '.AutoScalingGroups[0].DesiredCapacity') - # respect current desired capacity + desired_capacity=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name ${asg_name} | jq '.AutoScalingGroups[0].DesiredCapacity') + # Respect current desired capacity export TF_VAR_web_desired_capacity="${desired_capacity}" fi diff --git a/scripts/prepare-aws-profile.sh b/scripts/prepare-aws-profile.sh deleted file mode 100755 index 620ba00..0000000 --- a/scripts/prepare-aws-profile.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -u - -mkdir ${HOME}/.aws - -cat << EOS > ${HOME}/.aws/credentials -[default] -aws_access_key_id = ${AWS_ACCESS_KEY_ID} -aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} -EOS - -cat << EOS > ${HOME}/.aws/config -[default] -region = ${AWS_DEFAULT_REGION} -output = json - -[profile micropost-stg] -role_arn = ${STG_ROLE_ARN} -source_profile = default -region = ${AWS_DEFAULT_REGION} -output = json - -[profile micropost-prod] -role_arn = ${PROD_ROLE_ARN} -source_profile = default -region = ${AWS_DEFAULT_REGION} -output = json - -EOS - -chmod 600 ${HOME}/.aws/credentials -chmod 600 ${HOME}/.aws/config - -unset AWS_ACCESS_KEY_ID -unset AWS_SECRET_ACCESS_KEY -unset AWS_DEFAULT_REGION -unset STG_ROLE_ARN -unset PROD_ROLE_ARN diff --git a/scripts/switch-role.sh b/scripts/switch-role.sh new file mode 100755 index 0000000..05bdb4d --- /dev/null +++ b/scripts/switch-role.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -u + +# Parse variable such as ROLE_ARN_stg, ROLE_ARN_prod and etc. +ROLE_ARN=$(eval echo '$ROLE_ARN_'${ENV}) + +CREDENTIALS=$(aws sts assume-role --role-arn ${ROLE_ARN} --role-session-name travisci) + +export AWS_ACCESS_KEY_ID=$(echo ${CREDENTIALS} | jq --raw-output .Credentials.AccessKeyId) +export AWS_SECRET_ACCESS_KEY=$(echo ${CREDENTIALS} | jq --raw-output .Credentials.SecretAccessKey) +export AWS_SESSION_TOKEN=$(echo ${CREDENTIALS} | jq --raw-output .Credentials.SessionToken) diff --git a/terraform-env.sh b/terraform-env.sh index 07b76e0..ff3252f 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -84,7 +84,7 @@ then fi # Let's run the PRE_CMD hook if it's defined -. ${PRE_CMD} > /dev/null 2>&1 +source ${PRE_CMD} > /dev/null 2>&1 # let's copy environment specific configuration to the root of the directory if files_exist ${ENV_DIR}/*.tf; then @@ -103,6 +103,6 @@ if files_exist *.env.tf; then fi # Let's run the POST_CMD hook if it's defined -. ${POST_CMD} > /dev/null 2>&1 +source ${POST_CMD} > /dev/null 2>&1 exit ${terraform_status} diff --git a/terraform.cfg b/terraform.cfg index bb4cda9..fe68cd0 100644 --- a/terraform.cfg +++ b/terraform.cfg @@ -4,3 +4,7 @@ region=ap-northeast-1 pre_command='scripts/pre.sh' post_command='scripts/post.sh' + +if [ ! -v AWS_SESSION_TOKEN ]; then + ENV=${ENVIRONMENT} source ./scripts/switch-role.sh +fi diff --git a/variables.tf b/variables.tf index d3fe7de..61154f0 100644 --- a/variables.tf +++ b/variables.tf @@ -2,14 +2,6 @@ variable "env" { description = "dev, stg, prod and etc." } -variable "aws_region" { - default = "ap-northeast-1" -} - -variable "aws_role_arn" { - default = "" -} - variable "allowed_segments" { type = "list" default = [ From 1310c3d4a0a634165239c5f39f8202225d8a2465 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 10:36:12 +0700 Subject: [PATCH 148/181] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3af238d..4319faa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ after_deploy: - ./terraform-env.sh ${ENV} output | tee out_after - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') - - account_number=$(aws sts get-caller-identity --output text --query 'Account' --profile $AWS_PROFILE) + - account_number=$(aws sts get-caller-identity --output text --query 'Account') - | if [ ${lines} -eq 2 ]; then aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" \ From 6a017420eed6f0dce50894acdd90dc22348e8340 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 10:41:09 +0700 Subject: [PATCH 149/181] debug --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4319faa..5b533a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: before_script: # Determine environment - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") + - sed -n 55p /home/travis/build.sh script: - | if [ -z "${ENV}" ]; then From 0cdd643786dc29c863d6e8e3db0f87a2c61394ea Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 10:44:52 +0700 Subject: [PATCH 150/181] test --- .travis.yml | 1 - scripts/switch-role.sh | 2 -- 2 files changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b533a5..4319faa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ install: before_script: # Determine environment - export ENV=$(echo "${TRAVIS_BRANCH}" | perl -ne "print $& if /(?<=deploy\/).*/") - - sed -n 55p /home/travis/build.sh script: - | if [ -z "${ENV}" ]; then diff --git a/scripts/switch-role.sh b/scripts/switch-role.sh index 05bdb4d..023abbd 100755 --- a/scripts/switch-role.sh +++ b/scripts/switch-role.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -u - # Parse variable such as ROLE_ARN_stg, ROLE_ARN_prod and etc. ROLE_ARN=$(eval echo '$ROLE_ARN_'${ENV}) From 536a3f994deb247b6d2441cfae3211a6fd6ac245 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 10:52:26 +0700 Subject: [PATCH 151/181] refactor travis --- .travis.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4319faa..94d3878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,8 +28,8 @@ script: fi before_deploy: - # Save current output to use it later - - ./terraform-env.sh ${ENV} output | tee out_before + # Save current web_ami to use it later + - web_ami_before=$(./terraform-env.sh ${ENV} output web_ami) deploy: - provider: script @@ -40,15 +40,9 @@ deploy: after_deploy: # Publish to SNS Topic if web_ami was updated. - - ./terraform-env.sh ${ENV} output | tee out_after - - lines=$(cat out_before out_after | grep 'web_ami' | uniq | wc -l) - - asg_name=$(grep "web_asg_name" out_after | awk -F ' = ' '{print $2}') + - web_ami_after=$(./terraform-env.sh ${ENV} output web_ami) - account_number=$(aws sts get-caller-identity --output text --query 'Account') - | - if [ ${lines} -eq 2 ]; then - aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" \ - --message "{\"asgName\": \"${asg_name}\"}" + if [ ${web_ami_after} != ${web_ami_before} ]; then + aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" fi - -after_script: - - rm -rf ${HOME}/.aws From 11d24f631c423db14a266651e3d3d08d8be2cb26 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 11:33:58 +0700 Subject: [PATCH 152/181] rotate ami From b7c0b3bd173df068cb74f5ca02411bd67a49df31 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 2 Nov 2016 11:39:09 +0700 Subject: [PATCH 153/181] fix sns command --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94d3878..f41d57d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,5 +44,6 @@ after_deploy: - account_number=$(aws sts get-caller-identity --output text --query 'Account') - | if [ ${web_ami_after} != ${web_ami_before} ]; then - aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" + aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" \ + --message "${ENV}: ${TRAVIS_COMMIT}" fi From 9faf857eec3b19d84b7190ad51fb61fbce2799f0 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 10 Nov 2016 19:59:18 +0700 Subject: [PATCH 154/181] use aws_acm_certificate resource --- .travis.yml | 2 +- main.tf | 1 - scripts/pre.sh | 3 --- variables.tf | 4 ---- webservers/alb.tf | 9 +++++++-- webservers/variables.tf | 4 ---- 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index f41d57d..6cf8b82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install awscli - aws --version # Install Terraform - - wget https://releases.hashicorp.com/terraform/0.7.7/terraform_0.7.7_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.10/terraform_0.7.10_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: diff --git a/main.tf b/main.tf index 02e9593..18a2928 100644 --- a/main.tf +++ b/main.tf @@ -31,7 +31,6 @@ module "webservers" { "${module.security_groups.internet_in_http}", "${module.security_groups.internet_in_https}", ] - alb_certificate_arn = "${var.alb_certificate_arn}" min_scale_size = "${var.web_min_size}" desired_capacity = "${var.web_desired_capacity}" vpc_id = "${module.vpc.vpc_id}" diff --git a/scripts/pre.sh b/scripts/pre.sh index 9fbd85d..9d5088e 100755 --- a/scripts/pre.sh +++ b/scripts/pre.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -# Set ALB certificate ARN -export TF_VAR_alb_certificate_arn=$(aws acm list-certificates | jq --raw-output '.CertificateSummaryList[] | select(.DomainName == "*.hana053.com") | .CertificateArn') - # Set ASG desired capacity, if it exists asg_name=$(terraform output web_asg_name) if [ -n "${asg_name}" ]; then diff --git a/variables.tf b/variables.tf index 61154f0..0afcec4 100644 --- a/variables.tf +++ b/variables.tf @@ -14,10 +14,6 @@ variable "domain" { default = "hana053.com" } -variable "alb_certificate_arn" { - description = "ACM certificate arn for ALB" -} - variable "web_host_name" { } diff --git a/webservers/alb.tf b/webservers/alb.tf index 08c90c6..f852c70 100644 --- a/webservers/alb.tf +++ b/webservers/alb.tf @@ -1,3 +1,8 @@ +data "aws_acm_certificate" "main" { + domain = "*.hana053.com" + statuses = ["ISSUED"] +} + resource "aws_alb" "web" { name = "web" internal = false @@ -49,7 +54,7 @@ resource "aws_alb_listener" "https" { port = "443" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2015-05" - certificate_arn = "${var.alb_certificate_arn}" + certificate_arn = "${data.aws_acm_certificate.main.arn}" default_action { target_group_arn = "${aws_alb_target_group.index.arn}" type = "forward" @@ -84,4 +89,4 @@ resource "aws_alb_listener_rule" "index" { "*" ] } -} \ No newline at end of file +} diff --git a/webservers/variables.tf b/webservers/variables.tf index f8f051d..af64bf6 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -53,10 +53,6 @@ variable "alb_security_groups" { default = [] } -variable "alb_certificate_arn" { - description = "alb certificate arn" -} - variable "vpc_id" { description = "vpc id for alb target group" } From 62a475685861d871dd2bcfef69de9dce3b3fa3cb Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Wed, 7 Dec 2016 15:23:26 +0700 Subject: [PATCH 155/181] remove nginx_cdn_bucket --- main.tf | 1 - webservers/autoscale.tf | 1 - webservers/web_init.sh | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 18a2928..8de6ce7 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,6 @@ module "webservers" { env = "${var.env}" dbserver_endpoint = "${module.dbservers.endpoint}" deploy_bucket = "${aws_s3_bucket.deploy.bucket}" - nginx_cdn_bucket = "${aws_s3_bucket.cdn.bucket}" key_name = "${aws_key_pair.micropost.key_name}" web_ami_tag = "micropost-web" web_subnets = [ diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 3baae3c..8512e17 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -23,7 +23,6 @@ data "template_file" "web_init" { env = "${var.env}" dbserver_endpoint = "${var.dbserver_endpoint}" deploy_bucket = "${var.deploy_bucket}" - nginx_cdn_bucket = "${var.nginx_cdn_bucket}" } } diff --git a/webservers/web_init.sh b/webservers/web_init.sh index 5c90cd3..8a8420d 100644 --- a/webservers/web_init.sh +++ b/webservers/web_init.sh @@ -19,5 +19,5 @@ localhost EOF le_key=$(cat /etc/le/config | grep user-key | cut -d" " -f3) -ansible-playbook -i inventory -c local --diff -e "deploy_bucket=${deploy_bucket}" -e "nginx_cdn_bucket=${nginx_cdn_bucket}" -e "logentries_account_key=$${le_key}" site.yml +ansible-playbook -i inventory -c local --diff -e "deploy_bucket=${deploy_bucket}" -e "logentries_account_key=$${le_key}" site.yml ) From 0b53d57bbdd3b78ac54ffba2f296d1b47056ca9f Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 9 Dec 2016 23:23:36 +0700 Subject: [PATCH 156/181] ecs for frontend --- codedeploy/main.tf | 75 ------------- codedeploy/variables.tf | 10 -- dns.tf | 2 +- main.tf | 10 -- outputs.tf | 8 -- s3.tf | 23 ---- scripts/pre.sh | 9 -- variables.tf | 10 +- webservers/alb.tf | 92 ++++++++-------- webservers/autoscale.tf | 123 ++++++++++++++-------- webservers/current.tf | 5 + webservers/ecs.tf | 87 +++++++++++++++ webservers/iam.tf | 45 -------- webservers/outputs.tf | 12 --- webservers/task-definitions/frontend.json | 15 +++ webservers/variables.tf | 15 --- 16 files changed, 232 insertions(+), 309 deletions(-) delete mode 100644 codedeploy/main.tf delete mode 100644 codedeploy/variables.tf delete mode 100755 scripts/pre.sh create mode 100644 webservers/current.tf create mode 100644 webservers/ecs.tf delete mode 100644 webservers/iam.tf create mode 100644 webservers/task-definitions/frontend.json diff --git a/codedeploy/main.tf b/codedeploy/main.tf deleted file mode 100644 index c31751b..0000000 --- a/codedeploy/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -data "aws_caller_identity" "current" {} - -data "aws_region" "current" { - current = true -} - -resource "aws_codedeploy_app" "main" { - name = "${var.name}" -} - -resource "aws_codedeploy_deployment_group" "main" { - app_name = "${aws_codedeploy_app.main.name}" - deployment_group_name = "${var.group_name}" - service_role_arn = "${aws_iam_role.codedeploy_service.arn}" - autoscaling_groups = ["${var.autoscaling_groups}"] - deployment_config_name = "CodeDeployDefault.OneAtATime" - trigger_configuration { - trigger_events = [ - "DeploymentSuccess", - "DeploymentFailure", - ] - trigger_name = "frontend_deployed" - trigger_target_arn = "arn:aws:sns:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:frontend_deployed" - } -} - -resource "aws_iam_role" "codedeploy_service" { - name = "codedeploy-${var.group_name}" - assume_role_policy = <> /etc/ecs/ecs.config +EOF lifecycle { create_before_destroy = true } @@ -45,18 +29,16 @@ resource "aws_launch_configuration" "web" { resource "aws_autoscaling_group" "web" { name = "web" launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 2 - min_size = "${var.min_scale_size}" - desired_capacity = "${var.desired_capacity}" - health_check_grace_period = 300 + max_size = 4 + min_size = 1 health_check_type = "ELB" force_delete = true vpc_zone_identifier = [ "${var.web_subnets}" ] target_group_arns = [ - "${aws_alb_target_group.api.arn}", - "${aws_alb_target_group.index.arn}" +// "${aws_alb_target_group.api.arn}", + "${aws_alb_target_group.frontend.arn}" ] termination_policies = [ "OldestLaunchConfiguration", @@ -64,23 +46,10 @@ resource "aws_autoscaling_group" "web" { "ClosestToNextInstanceHour", "Default", ] - tag { - key = "Name" - value = "${var.env}-web" - propagate_at_launch = true - } - tag { - key = "Env" - value = "${var.env}" - propagate_at_launch = true - } - tag { - key = "Role" - value = "web" - propagate_at_launch = true - } } +// ------ scale out ------- + resource "aws_autoscaling_policy" "web_scale_out" { name = "web-Instance-ScaleOut-CPU-High" scaling_adjustment = 1 @@ -106,6 +75,8 @@ resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { ] } +// ------ scale in ------- + resource "aws_autoscaling_policy" "web_scale_in" { name = "web-Instance-ScaleIn-CPU-Low" scaling_adjustment = -1 @@ -131,6 +102,8 @@ resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { ] } +// ------ notification ------- + resource "aws_autoscaling_notification" "web" { group_names = [ "${aws_autoscaling_group.web.name}", @@ -142,3 +115,61 @@ resource "aws_autoscaling_notification" "web" { ] topic_arn = "arn:aws:sns:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:web_autoscaled" } + +// ------ iam ------- + +resource "aws_iam_instance_profile" "web" { + name = "web" + roles = [ + "${aws_iam_role.web.name}" + ] +} + +resource "aws_iam_role" "web" { + name = "web" + assume_role_policy = < Date: Sun, 11 Dec 2016 09:10:32 +0700 Subject: [PATCH 157/181] docker for backend --- .envrc.example | 3 + .travis.yml | 16 +---- main.tf | 3 +- outputs.tf | 3 - s3.tf | 5 -- variables.tf | 8 +++ webservers/alb.tf | 66 +++++++++---------- webservers/autoscale.tf | 17 +++-- webservers/ecs.tf | 40 +++++++++-- webservers/outputs.tf | 4 -- webservers/task_definitions/backend.json | 33 ++++++++++ .../frontend.json | 2 +- webservers/user_data.sh | 11 ++++ webservers/variables.tf | 8 ++- webservers/web_init.sh | 23 ------- 15 files changed, 143 insertions(+), 99 deletions(-) delete mode 100644 outputs.tf create mode 100644 webservers/task_definitions/backend.json rename webservers/{task-definitions => task_definitions}/frontend.json (90%) create mode 100644 webservers/user_data.sh delete mode 100644 webservers/web_init.sh diff --git a/.envrc.example b/.envrc.example index 2f21ee9..beb08ec 100644 --- a/.envrc.example +++ b/.envrc.example @@ -5,3 +5,6 @@ export CLOUDFLARE_TOKEN= export ROLE_ARN_stg= export ROLE_ARN_prod= + +export TF_VAR_app_encryption_password= +export TF_VAR_newrelic_license_key= diff --git a/.travis.yml b/.travis.yml index 6cf8b82..e18a27f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install awscli - aws --version # Install Terraform - - wget https://releases.hashicorp.com/terraform/0.7.10/terraform_0.7.10_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.7.13/terraform_0.7.13_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: @@ -27,23 +27,9 @@ script: ./terraform-env.sh ${ENV} plan fi -before_deploy: - # Save current web_ami to use it later - - web_ami_before=$(./terraform-env.sh ${ENV} output web_ami) - deploy: - provider: script script: ./terraform-env.sh ${ENV} apply skip_cleanup: true on: branch: deploy/* - -after_deploy: - # Publish to SNS Topic if web_ami was updated. - - web_ami_after=$(./terraform-env.sh ${ENV} output web_ami) - - account_number=$(aws sts get-caller-identity --output text --query 'Account') - - | - if [ ${web_ami_after} != ${web_ami_before} ]; then - aws sns publish --topic-arn "arn:aws:sns:${AWS_DEFAULT_REGION}:${account_number}:web_image_updated" \ - --message "${ENV}: ${TRAVIS_COMMIT}" - fi diff --git a/main.tf b/main.tf index afb04d1..b52e001 100644 --- a/main.tf +++ b/main.tf @@ -13,7 +13,8 @@ module "webservers" { source = "./webservers" env = "${var.env}" dbserver_endpoint = "${module.dbservers.endpoint}" - deploy_bucket = "${aws_s3_bucket.deploy.bucket}" + app_encryption_password = "${var.app_encryption_password}" + newrelic_license_key = "${var.newrelic_license_key}" key_name = "${aws_key_pair.micropost.key_name}" web_subnets = [ "${module.vpc.public_subnets}" diff --git a/outputs.tf b/outputs.tf deleted file mode 100644 index 24a1310..0000000 --- a/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "web_asg_name" { - value = "${module.webservers.asg_name}" -} diff --git a/s3.tf b/s3.tf index e805482..a34e414 100644 --- a/s3.tf +++ b/s3.tf @@ -1,8 +1,3 @@ -resource "aws_s3_bucket" "deploy" { - bucket = "deploy-${var.env}.${var.domain}" - force_destroy = true -} - resource "aws_s3_bucket" "log" { bucket = "log-${var.env}.${var.domain}" force_destroy = true diff --git a/variables.tf b/variables.tf index 59c91c1..171b211 100644 --- a/variables.tf +++ b/variables.tf @@ -16,3 +16,11 @@ variable "domain" { variable "web_host_name" { } + +variable "app_encryption_password" { + description = "Applicatoin password to decrypt secret properties" +} + +variable "newrelic_license_key" { + description = "New Relic licence key" +} diff --git a/webservers/alb.tf b/webservers/alb.tf index 39f9f22..bb5aad5 100644 --- a/webservers/alb.tf +++ b/webservers/alb.tf @@ -35,6 +35,25 @@ resource "aws_alb_target_group" "frontend" { } } +// --------- backend ----------- + +resource "aws_alb_target_group" "backend" { + name = "backend" + port = 8080 + protocol = "HTTP" + vpc_id = "${var.vpc_id}" + health_check { + interval = 30 + path = "/manage/health" + port = 8080 + protocol = "HTTP" + timeout = 5 + unhealthy_threshold = 2 + } +} + +// --------- listeners ----------- + resource "aws_alb_listener" "http" { load_balancer_arn = "${aws_alb.web.arn}" port = "80" @@ -57,36 +76,17 @@ resource "aws_alb_listener" "https" { } } -// --------- backend ----------- - -//resource "aws_alb_target_group" "api" { -// name = "api" -// port = 8080 -// protocol = "HTTP" -// vpc_id = "${var.vpc_id}" -// health_check { -// interval = 30 -// path = "/manage/health" -// port = 8080 -// protocol = "HTTP" -// timeout = 5 -// unhealthy_threshold = 2 -// } -//} - - -//resource "aws_alb_listener_rule" "api" { -// listener_arn = "${aws_alb_listener.https.arn}" -// priority = 100 -// action { -// type = "forward" -// target_group_arn = "${aws_alb_target_group.api.arn}" -// } -// condition { -// field = "path-pattern" -// values = [ -// "/api/*" -// ] -// } -//} - +resource "aws_alb_listener_rule" "backend" { + listener_arn = "${aws_alb_listener.https.arn}" + priority = 100 + condition { + field = "path-pattern" + values = [ + "/api/*" + ] + } + action { + type = "forward" + target_group_arn = "${aws_alb_target_group.backend.arn}" + } +} diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 4f8d5f1..aeda34c 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -7,6 +7,14 @@ data "aws_ami" "ecs" { } } +data "template_file" "user_data" { + template = "${file("./webservers/user_data.sh")}" + vars { + cluster_name = "${aws_ecs_cluster.main.name}" + newrelic_license_key = "${var.newrelic_license_key}" + } +} + resource "aws_launch_configuration" "web" { name_prefix = "web-" image_id = "${data.aws_ami.ecs.id}" @@ -17,10 +25,7 @@ resource "aws_launch_configuration" "web" { key_name = "${var.key_name}" associate_public_ip_address = true iam_instance_profile = "${aws_iam_instance_profile.web.id}" - user_data = <> /etc/ecs/ecs.config -EOF + user_data = "${data.template_file.user_data.rendered}" lifecycle { create_before_destroy = true } @@ -29,7 +34,7 @@ EOF resource "aws_autoscaling_group" "web" { name = "web" launch_configuration = "${aws_launch_configuration.web.id}" - max_size = 4 + max_size = 2 min_size = 1 health_check_type = "ELB" force_delete = true @@ -37,7 +42,7 @@ resource "aws_autoscaling_group" "web" { "${var.web_subnets}" ] target_group_arns = [ -// "${aws_alb_target_group.api.arn}", + "${aws_alb_target_group.backend.arn}", "${aws_alb_target_group.frontend.arn}" ] termination_policies = [ diff --git a/webservers/ecs.tf b/webservers/ecs.tf index 5f3e7f2..24d0894 100644 --- a/webservers/ecs.tf +++ b/webservers/ecs.tf @@ -4,10 +4,6 @@ resource "aws_ecs_cluster" "main" { // ------ frontend ------- -resource "aws_ecr_repository" "frontend" { - name = "micropost/frontend" -} - resource "aws_ecs_service" "frontend" { name = "frontend" cluster = "${aws_ecs_cluster.main.id}" @@ -25,11 +21,10 @@ resource "aws_ecs_service" "frontend" { } data "template_file" "frontend_task_definition" { - template = "${file("./webservers/task-definitions/frontend.json")}" + template = "${file("./webservers/task_definitions/frontend.json")}" vars { account_number = "${data.aws_caller_identity.current.account_id}" region = "${data.aws_region.current.id}" - image_id = "${aws_ecr_repository.frontend.id}" } } @@ -40,6 +35,39 @@ resource "aws_ecs_task_definition" "frontend" { // ------ backend ------- +resource "aws_ecs_service" "backend" { + name = "backend" + cluster = "${aws_ecs_cluster.main.id}" + task_definition = "${aws_ecs_task_definition.backend.arn}" + desired_count = 1 + iam_role = "${aws_iam_role.ecs_service.arn}" + depends_on = [ + "aws_iam_role_policy.ecs_service_role_policy"] + + load_balancer { + target_group_arn = "${aws_alb_target_group.backend.id}" + container_name = "backend" + container_port = "8080" + } +} + +data "template_file" "backend_task_definition" { + template = "${file("./webservers/task_definitions/backend.json")}" + vars { + account_number = "${data.aws_caller_identity.current.account_id}" + region = "${data.aws_region.current.id}" + env = "${var.env}" + dbserver_endpoint = "${var.dbserver_endpoint}" + app_encryption_password = "${var.app_encryption_password}" + newrelic_license_key = "${var.newrelic_license_key}" + } +} + +resource "aws_ecs_task_definition" "backend" { + family = "micropost-backend" + container_definitions = "${data.template_file.backend_task_definition.rendered}" +} + // ------ iam ------- resource "aws_iam_role" "ecs_service" { diff --git a/webservers/outputs.tf b/webservers/outputs.tf index cd10f5b..f524578 100644 --- a/webservers/outputs.tf +++ b/webservers/outputs.tf @@ -1,7 +1,3 @@ output "dns_name" { value = "${aws_alb.web.dns_name}" } - -output "asg_name" { - value = "${aws_autoscaling_group.web.name}" -} diff --git a/webservers/task_definitions/backend.json b/webservers/task_definitions/backend.json new file mode 100644 index 0000000..32a2081 --- /dev/null +++ b/webservers/task_definitions/backend.json @@ -0,0 +1,33 @@ +[ + { + "cpu": 10, + "essential": true, + "image": "${account_number}.dkr.ecr.${region}.amazonaws.com/micropost/backend:latest", + "memory": 400, + "name": "backend", + "environment": [ + { + "name": "SPRING_PROFILES_ACTIVE", + "value": "${env}" + }, + { + "name": "MYSQL_ENDPOINT", + "value": "${dbserver_endpoint}" + }, + { + "name": "JASYPT_ENCRYPTOR_PASSWORD", + "value": "${app_encryption_password}" + }, + { + "name": "NEW_RELIC_LICENSE_KEY", + "value": "${newrelic_license_key}" + } + ], + "portMappings": [ + { + "containerPort": 8080, + "hostPort": 8080 + } + ] + } +] diff --git a/webservers/task-definitions/frontend.json b/webservers/task_definitions/frontend.json similarity index 90% rename from webservers/task-definitions/frontend.json rename to webservers/task_definitions/frontend.json index 048b075..1dac257 100644 --- a/webservers/task-definitions/frontend.json +++ b/webservers/task_definitions/frontend.json @@ -2,7 +2,7 @@ { "cpu": 10, "essential": true, - "image": "${account_number}.dkr.ecr.${region}.amazonaws.com/${image_id}:latest", + "image": "${account_number}.dkr.ecr.${region}.amazonaws.com/micropost/frontend:latest", "memory": 300, "name": "frontend", "portMappings": [ diff --git a/webservers/user_data.sh b/webservers/user_data.sh new file mode 100644 index 0000000..d283e47 --- /dev/null +++ b/webservers/user_data.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Install New Relic +rpm -Uvh https://yum.newrelic.com/pub/newrelic/el5/x86_64/newrelic-repo-5-3.noarch.rpm +yum install -y newrelic-sysmond +nrsysmond-config --set license_key=${newrelic_license_key} +usermod -a -G docker newrelic +/etc/init.d/newrelic-sysmond start + +# Register ecs cluster +echo "ECS_CLUSTER=${cluster_name}" >> /etc/ecs/ecs.config diff --git a/webservers/variables.tf b/webservers/variables.tf index 42c624b..6e53419 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -6,8 +6,12 @@ variable "dbserver_endpoint" { description = "MySQL endpoint" } -variable "deploy_bucket" { - description = "Bucket to be used for deployment" +variable "app_encryption_password" { + description = "Applicatoin password to decrypt secret properties" +} + +variable "newrelic_license_key" { + description = "New Relic licence key" } variable "key_name" { diff --git a/webservers/web_init.sh b/webservers/web_init.sh deleted file mode 100644 index 8a8420d..0000000 --- a/webservers/web_init.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# Create env file, which will be read from an app. -ENV_FILE=/opt/micropost/env.sh -mkdir -p $(dirname $${ENV_FILE}) -cat << EOF > $${ENV_FILE} -export SPRING_PROFILES_ACTIVE=${env} -export RDS_ENDPOINT=${dbserver_endpoint} -export S3_DEPLOY_BUCKET=${deploy_bucket} -EOF - -# Finalize provisioning by using resolved endpoints. -( -cd /opt/provisioning - -cat << EOF > inventory -[webservers] -localhost -EOF - -le_key=$(cat /etc/le/config | grep user-key | cut -d" " -f3) -ansible-playbook -i inventory -c local --diff -e "deploy_bucket=${deploy_bucket}" -e "logentries_account_key=$${le_key}" site.yml -) From 8e632ca76a45609ebae7ba96af6f1b31ae00dc25 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 11 Dec 2016 09:22:57 +0700 Subject: [PATCH 158/181] update readme --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index be8fee8..8d5da57 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ ## Dependencies * Terraform -* Node.js * AWS CLI * direnv @@ -17,12 +16,6 @@ $ vi .envrc $ direnv allow ``` -Install npm packages. - -``` -$ npm install -g yarn -``` - Plan and Apply ``` From e6fe43d19ddb104be6de19a31b2259ffe2514f3b Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Sun, 11 Dec 2016 17:31:33 +0700 Subject: [PATCH 159/181] Enable ecs metric and awslogs --- webservers/autoscale.tf | 8 ++++++-- webservers/ecs.tf | 10 ++++++++++ webservers/task_definitions/backend.json | 9 ++++++++- webservers/task_definitions/frontend.json | 9 ++++++++- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index aeda34c..68197df 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -160,15 +160,19 @@ resource "aws_iam_role_policy" "ecs_host" { "Sid": "ecsInstanceRole", "Effect": "Allow", "Action": [ + "ecs:CreateCluster", "ecs:DeregisterContainerInstance", "ecs:DiscoverPollEndpoint", "ecs:Poll", "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", "ecs:Submit*", + "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer", - "ecr:GetAuthorizationToken" + "ecr:BatchGetImage", + "logs:CreateLogStream", + "logs:PutLogEvents" ], "Resource": [ "*" diff --git a/webservers/ecs.tf b/webservers/ecs.tf index 24d0894..e3d9230 100644 --- a/webservers/ecs.tf +++ b/webservers/ecs.tf @@ -25,6 +25,7 @@ data "template_file" "frontend_task_definition" { vars { account_number = "${data.aws_caller_identity.current.account_id}" region = "${data.aws_region.current.id}" + log_group_name = "${aws_cloudwatch_log_group.frontend.name}" } } @@ -33,6 +34,10 @@ resource "aws_ecs_task_definition" "frontend" { container_definitions = "${data.template_file.frontend_task_definition.rendered}" } +resource "aws_cloudwatch_log_group" "frontend" { + name = "frontend" +} + // ------ backend ------- resource "aws_ecs_service" "backend" { @@ -60,6 +65,7 @@ data "template_file" "backend_task_definition" { dbserver_endpoint = "${var.dbserver_endpoint}" app_encryption_password = "${var.app_encryption_password}" newrelic_license_key = "${var.newrelic_license_key}" + log_group_name = "${aws_cloudwatch_log_group.backend.name}" } } @@ -68,6 +74,10 @@ resource "aws_ecs_task_definition" "backend" { container_definitions = "${data.template_file.backend_task_definition.rendered}" } +resource "aws_cloudwatch_log_group" "backend" { + name = "backend" +} + // ------ iam ------- resource "aws_iam_role" "ecs_service" { diff --git a/webservers/task_definitions/backend.json b/webservers/task_definitions/backend.json index 32a2081..6db334a 100644 --- a/webservers/task_definitions/backend.json +++ b/webservers/task_definitions/backend.json @@ -28,6 +28,13 @@ "containerPort": 8080, "hostPort": 8080 } - ] + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${log_group_name}", + "awslogs-region": "${region}" + } + } } ] diff --git a/webservers/task_definitions/frontend.json b/webservers/task_definitions/frontend.json index 1dac257..c66ca44 100644 --- a/webservers/task_definitions/frontend.json +++ b/webservers/task_definitions/frontend.json @@ -10,6 +10,13 @@ "containerPort": 80, "hostPort": 80 } - ] + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${log_group_name}", + "awslogs-region": "${region}" + } + } } ] From b261337c2382a3ad1f4fe5847b68469c859be046 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 12 Dec 2016 11:08:55 +0700 Subject: [PATCH 160/181] use dynamic port --- webservers/alb.tf | 2 -- webservers/task_definitions/backend.json | 2 +- webservers/task_definitions/frontend.json | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/webservers/alb.tf b/webservers/alb.tf index bb5aad5..b9bd17c 100644 --- a/webservers/alb.tf +++ b/webservers/alb.tf @@ -28,7 +28,6 @@ resource "aws_alb_target_group" "frontend" { health_check { interval = 30 path = "/" - port = 80 protocol = "HTTP" timeout = 5 unhealthy_threshold = 2 @@ -45,7 +44,6 @@ resource "aws_alb_target_group" "backend" { health_check { interval = 30 path = "/manage/health" - port = 8080 protocol = "HTTP" timeout = 5 unhealthy_threshold = 2 diff --git a/webservers/task_definitions/backend.json b/webservers/task_definitions/backend.json index 6db334a..1d61d2a 100644 --- a/webservers/task_definitions/backend.json +++ b/webservers/task_definitions/backend.json @@ -26,7 +26,7 @@ "portMappings": [ { "containerPort": 8080, - "hostPort": 8080 + "hostPort": 0 } ], "logConfiguration": { diff --git a/webservers/task_definitions/frontend.json b/webservers/task_definitions/frontend.json index c66ca44..f29ce4c 100644 --- a/webservers/task_definitions/frontend.json +++ b/webservers/task_definitions/frontend.json @@ -3,12 +3,12 @@ "cpu": 10, "essential": true, "image": "${account_number}.dkr.ecr.${region}.amazonaws.com/micropost/frontend:latest", - "memory": 300, + "memory": 50, "name": "frontend", "portMappings": [ { "containerPort": 80, - "hostPort": 80 + "hostPort": 0 } ], "logConfiguration": { From 537e0dcdca1fb4de8b2920e567248d6540994855 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 12 Dec 2016 16:34:21 +0700 Subject: [PATCH 161/181] autoscale for ecs --- webservers/autoscale.tf | 8 +- webservers/ecs.tf | 106 ++++++---------------- webservers/ecs_backend.tf | 108 +++++++++++++++++++++++ webservers/ecs_frontend.tf | 104 ++++++++++++++++++++++ webservers/task_definitions/backend.json | 2 +- 5 files changed, 246 insertions(+), 82 deletions(-) create mode 100644 webservers/ecs_backend.tf create mode 100644 webservers/ecs_frontend.tf diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 68197df..466ddaa 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -68,12 +68,12 @@ resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = "1" metric_name = "CPUUtilization" - namespace = "AWS/EC2" + namespace = "AWS/ECS" period = "120" statistic = "Average" threshold = "80" dimensions { - AutoScalingGroupName = "${aws_autoscaling_group.web.name}" + ClusterName = "${aws_ecs_cluster.main.name}" } alarm_actions = [ "${aws_autoscaling_policy.web_scale_out.arn}" @@ -95,12 +95,12 @@ resource "aws_cloudwatch_metric_alarm" "web_lt_threshold" { comparison_operator = "LessThanThreshold" evaluation_periods = "1" metric_name = "CPUUtilization" - namespace = "AWS/EC2" + namespace = "AWS/ECS" period = "60" statistic = "Average" threshold = "5" dimensions { - AutoScalingGroupName = "${aws_autoscaling_group.web.name}" + ClusterName = "${aws_ecs_cluster.main.name}" } alarm_actions = [ "${aws_autoscaling_policy.web_scale_in.arn}" diff --git a/webservers/ecs.tf b/webservers/ecs.tf index e3d9230..087ce5f 100644 --- a/webservers/ecs.tf +++ b/webservers/ecs.tf @@ -1,84 +1,10 @@ +// ------ cluster ------- + resource "aws_ecs_cluster" "main" { name = "micropost" } -// ------ frontend ------- - -resource "aws_ecs_service" "frontend" { - name = "frontend" - cluster = "${aws_ecs_cluster.main.id}" - task_definition = "${aws_ecs_task_definition.frontend.arn}" - desired_count = 1 - iam_role = "${aws_iam_role.ecs_service.arn}" - depends_on = [ - "aws_iam_role_policy.ecs_service_role_policy"] - - load_balancer { - target_group_arn = "${aws_alb_target_group.frontend.id}" - container_name = "frontend" - container_port = "80" - } -} - -data "template_file" "frontend_task_definition" { - template = "${file("./webservers/task_definitions/frontend.json")}" - vars { - account_number = "${data.aws_caller_identity.current.account_id}" - region = "${data.aws_region.current.id}" - log_group_name = "${aws_cloudwatch_log_group.frontend.name}" - } -} - -resource "aws_ecs_task_definition" "frontend" { - family = "micropost-frontend" - container_definitions = "${data.template_file.frontend_task_definition.rendered}" -} - -resource "aws_cloudwatch_log_group" "frontend" { - name = "frontend" -} - -// ------ backend ------- - -resource "aws_ecs_service" "backend" { - name = "backend" - cluster = "${aws_ecs_cluster.main.id}" - task_definition = "${aws_ecs_task_definition.backend.arn}" - desired_count = 1 - iam_role = "${aws_iam_role.ecs_service.arn}" - depends_on = [ - "aws_iam_role_policy.ecs_service_role_policy"] - - load_balancer { - target_group_arn = "${aws_alb_target_group.backend.id}" - container_name = "backend" - container_port = "8080" - } -} - -data "template_file" "backend_task_definition" { - template = "${file("./webservers/task_definitions/backend.json")}" - vars { - account_number = "${data.aws_caller_identity.current.account_id}" - region = "${data.aws_region.current.id}" - env = "${var.env}" - dbserver_endpoint = "${var.dbserver_endpoint}" - app_encryption_password = "${var.app_encryption_password}" - newrelic_license_key = "${var.newrelic_license_key}" - log_group_name = "${aws_cloudwatch_log_group.backend.name}" - } -} - -resource "aws_ecs_task_definition" "backend" { - family = "micropost-backend" - container_definitions = "${data.template_file.backend_task_definition.rendered}" -} - -resource "aws_cloudwatch_log_group" "backend" { - name = "backend" -} - -// ------ iam ------- +// ------ iam for ecs service ------- resource "aws_iam_role" "ecs_service" { name = "ecs-service" @@ -123,3 +49,29 @@ resource "aws_iam_role_policy" "ecs_service_role_policy" { } EOF } + +// ------ iam for application autoscaling ------- + +resource "aws_iam_role" "ecs_autoscale" { + name = "ecs-autoscale" + assume_role_policy = < Date: Mon, 12 Dec 2016 16:53:19 +0700 Subject: [PATCH 162/181] use aws managed role --- webservers/autoscale.tf | 41 ++++------------------------------------- webservers/ecs.tf | 28 +++++----------------------- 2 files changed, 9 insertions(+), 60 deletions(-) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 466ddaa..4452818 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -45,12 +45,6 @@ resource "aws_autoscaling_group" "web" { "${aws_alb_target_group.backend.arn}", "${aws_alb_target_group.frontend.arn}" ] - termination_policies = [ - "OldestLaunchConfiguration", - "OldestInstance", - "ClosestToNextInstanceHour", - "Default", - ] } // ------ scale out ------- @@ -149,36 +143,9 @@ resource "aws_iam_role" "web" { EOF } -resource "aws_iam_role_policy" "ecs_host" { +resource "aws_iam_policy_attachment" "ecs_host" { name = "ecs-host" - role = "${aws_iam_role.web.id}" - policy = < Date: Mon, 12 Dec 2016 17:12:40 +0700 Subject: [PATCH 163/181] wait more to scale up cluster --- webservers/autoscale.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webservers/autoscale.tf b/webservers/autoscale.tf index 4452818..cc25b11 100644 --- a/webservers/autoscale.tf +++ b/webservers/autoscale.tf @@ -63,7 +63,7 @@ resource "aws_cloudwatch_metric_alarm" "web_gte_threshold" { evaluation_periods = "1" metric_name = "CPUUtilization" namespace = "AWS/ECS" - period = "120" + period = "300" statistic = "Average" threshold = "80" dimensions { From 0181ebafa5f542873923ae630c4dc4184e3437ff Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 09:17:25 +0700 Subject: [PATCH 164/181] format --- webservers/ecs_backend.tf | 11 ++++++++--- webservers/ecs_frontend.tf | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/webservers/ecs_backend.tf b/webservers/ecs_backend.tf index 906b7c7..695cc9e 100644 --- a/webservers/ecs_backend.tf +++ b/webservers/ecs_backend.tf @@ -7,7 +7,8 @@ resource "aws_ecs_service" "backend" { desired_count = 1 iam_role = "${aws_iam_role.ecs_service.arn}" depends_on = [ - "aws_iam_role_policy.ecs_service_role_policy"] + "aws_iam_policy_attachment.ecs_service" + ] load_balancer { target_group_arn = "${aws_alb_target_group.backend.id}" @@ -76,7 +77,9 @@ resource "aws_cloudwatch_metric_alarm" "backend_cpu_high" { ClusterName = "${aws_ecs_cluster.main.name}" ServiceName = "${aws_ecs_service.backend.name}" } - alarm_actions = ["${aws_appautoscaling_policy.backend_scale_out.arn}"] + alarm_actions = [ + "${aws_appautoscaling_policy.backend_scale_out.arn}" + ] } resource "aws_appautoscaling_policy" "backend_scale_in" { @@ -104,5 +107,7 @@ resource "aws_cloudwatch_metric_alarm" "backend_cpu_low" { ClusterName = "${aws_ecs_cluster.main.name}" ServiceName = "${aws_ecs_service.backend.name}" } - alarm_actions = ["${aws_appautoscaling_policy.backend_scale_in.arn}"] + alarm_actions = [ + "${aws_appautoscaling_policy.backend_scale_in.arn}" + ] } diff --git a/webservers/ecs_frontend.tf b/webservers/ecs_frontend.tf index a1cc422..236defa 100644 --- a/webservers/ecs_frontend.tf +++ b/webservers/ecs_frontend.tf @@ -7,7 +7,8 @@ resource "aws_ecs_service" "frontend" { desired_count = 1 iam_role = "${aws_iam_role.ecs_service.arn}" depends_on = [ - "aws_iam_role_policy.ecs_service_role_policy"] + "aws_iam_policy_attachment.ecs_service" + ] load_balancer { target_group_arn = "${aws_alb_target_group.frontend.id}" @@ -72,7 +73,9 @@ resource "aws_cloudwatch_metric_alarm" "frontend_cpu_high" { ClusterName = "${aws_ecs_cluster.main.name}" ServiceName = "${aws_ecs_service.frontend.name}" } - alarm_actions = ["${aws_appautoscaling_policy.frontend_scale_out.arn}"] + alarm_actions = [ + "${aws_appautoscaling_policy.frontend_scale_out.arn}" + ] } resource "aws_appautoscaling_policy" "frontend_scale_in" { @@ -100,5 +103,7 @@ resource "aws_cloudwatch_metric_alarm" "frontend_cpu_low" { ClusterName = "${aws_ecs_cluster.main.name}" ServiceName = "${aws_ecs_service.frontend.name}" } - alarm_actions = ["${aws_appautoscaling_policy.frontend_scale_in.arn}"] + alarm_actions = [ + "${aws_appautoscaling_policy.frontend_scale_in.arn}" + ] } From 19421a3d17165d1bdba10c535e045baee9d60f6c Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 09:24:40 +0700 Subject: [PATCH 165/181] rename files --- webservers/{task_definitions => }/backend.json | 0 webservers/{ecs_backend.tf => backend.tf} | 2 +- webservers/{task_definitions => }/frontend.json | 0 webservers/{ecs_frontend.tf => frontend.tf} | 2 +- webservers/{autoscale.tf => host_instances.tf} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename webservers/{task_definitions => }/backend.json (100%) rename webservers/{ecs_backend.tf => backend.tf} (97%) rename webservers/{task_definitions => }/frontend.json (100%) rename webservers/{ecs_frontend.tf => frontend.tf} (97%) rename webservers/{autoscale.tf => host_instances.tf} (100%) diff --git a/webservers/task_definitions/backend.json b/webservers/backend.json similarity index 100% rename from webservers/task_definitions/backend.json rename to webservers/backend.json diff --git a/webservers/ecs_backend.tf b/webservers/backend.tf similarity index 97% rename from webservers/ecs_backend.tf rename to webservers/backend.tf index 695cc9e..65361ec 100644 --- a/webservers/ecs_backend.tf +++ b/webservers/backend.tf @@ -20,7 +20,7 @@ resource "aws_ecs_service" "backend" { // ------ task ------- data "template_file" "backend_task_definition" { - template = "${file("./webservers/task_definitions/backend.json")}" + template = "${file("./webservers/backend.json")}" vars { account_number = "${data.aws_caller_identity.current.account_id}" region = "${data.aws_region.current.id}" diff --git a/webservers/task_definitions/frontend.json b/webservers/frontend.json similarity index 100% rename from webservers/task_definitions/frontend.json rename to webservers/frontend.json diff --git a/webservers/ecs_frontend.tf b/webservers/frontend.tf similarity index 97% rename from webservers/ecs_frontend.tf rename to webservers/frontend.tf index 236defa..52cb6a7 100644 --- a/webservers/ecs_frontend.tf +++ b/webservers/frontend.tf @@ -20,7 +20,7 @@ resource "aws_ecs_service" "frontend" { // ------ task ------- data "template_file" "frontend_task_definition" { - template = "${file("./webservers/task_definitions/frontend.json")}" + template = "${file("./webservers/frontend.json")}" vars { account_number = "${data.aws_caller_identity.current.account_id}" region = "${data.aws_region.current.id}" diff --git a/webservers/autoscale.tf b/webservers/host_instances.tf similarity index 100% rename from webservers/autoscale.tf rename to webservers/host_instances.tf From b845fd9871ed4d1fc47ca14442e4d810eed8cc27 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 10:23:09 +0700 Subject: [PATCH 166/181] remove app_encryption_password --- .envrc.example | 1 - main.tf | 1 - variables.tf | 4 ---- webservers/backend.json | 4 ---- webservers/backend.tf | 1 - webservers/variables.tf | 4 ---- 6 files changed, 15 deletions(-) diff --git a/.envrc.example b/.envrc.example index beb08ec..c983c85 100644 --- a/.envrc.example +++ b/.envrc.example @@ -6,5 +6,4 @@ export CLOUDFLARE_TOKEN= export ROLE_ARN_stg= export ROLE_ARN_prod= -export TF_VAR_app_encryption_password= export TF_VAR_newrelic_license_key= diff --git a/main.tf b/main.tf index b52e001..417f103 100644 --- a/main.tf +++ b/main.tf @@ -13,7 +13,6 @@ module "webservers" { source = "./webservers" env = "${var.env}" dbserver_endpoint = "${module.dbservers.endpoint}" - app_encryption_password = "${var.app_encryption_password}" newrelic_license_key = "${var.newrelic_license_key}" key_name = "${aws_key_pair.micropost.key_name}" web_subnets = [ diff --git a/variables.tf b/variables.tf index 171b211..7954a90 100644 --- a/variables.tf +++ b/variables.tf @@ -17,10 +17,6 @@ variable "domain" { variable "web_host_name" { } -variable "app_encryption_password" { - description = "Applicatoin password to decrypt secret properties" -} - variable "newrelic_license_key" { description = "New Relic licence key" } diff --git a/webservers/backend.json b/webservers/backend.json index 14ecce8..54404b7 100644 --- a/webservers/backend.json +++ b/webservers/backend.json @@ -14,10 +14,6 @@ "name": "MYSQL_ENDPOINT", "value": "${dbserver_endpoint}" }, - { - "name": "JASYPT_ENCRYPTOR_PASSWORD", - "value": "${app_encryption_password}" - }, { "name": "NEW_RELIC_LICENSE_KEY", "value": "${newrelic_license_key}" diff --git a/webservers/backend.tf b/webservers/backend.tf index 65361ec..9489db3 100644 --- a/webservers/backend.tf +++ b/webservers/backend.tf @@ -26,7 +26,6 @@ data "template_file" "backend_task_definition" { region = "${data.aws_region.current.id}" env = "${var.env}" dbserver_endpoint = "${var.dbserver_endpoint}" - app_encryption_password = "${var.app_encryption_password}" newrelic_license_key = "${var.newrelic_license_key}" log_group_name = "${aws_cloudwatch_log_group.backend.name}" } diff --git a/webservers/variables.tf b/webservers/variables.tf index 6e53419..6257660 100644 --- a/webservers/variables.tf +++ b/webservers/variables.tf @@ -6,10 +6,6 @@ variable "dbserver_endpoint" { description = "MySQL endpoint" } -variable "app_encryption_password" { - description = "Applicatoin password to decrypt secret properties" -} - variable "newrelic_license_key" { description = "New Relic licence key" } From 27d841722484b37d8df4dd8c4878d893362a6c88 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 10:38:15 +0700 Subject: [PATCH 167/181] assign more cpu for backend --- webservers/backend.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webservers/backend.json b/webservers/backend.json index 54404b7..d17c7b0 100644 --- a/webservers/backend.json +++ b/webservers/backend.json @@ -1,6 +1,6 @@ [ { - "cpu": 256, + "cpu": 400, "essential": true, "image": "${account_number}.dkr.ecr.${region}.amazonaws.com/micropost/backend:latest", "memory": 400, From bd5a540d05460167e515be42b449b937c5440db5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 16:30:22 +0700 Subject: [PATCH 168/181] aws_autoscaling_group must not have target_group_arns. It's very important... I don't want to terminate instance when health check was failed, but just want to deregister ecs task. --- webservers/host_instances.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/webservers/host_instances.tf b/webservers/host_instances.tf index cc25b11..f3ab22a 100644 --- a/webservers/host_instances.tf +++ b/webservers/host_instances.tf @@ -41,10 +41,6 @@ resource "aws_autoscaling_group" "web" { vpc_zone_identifier = [ "${var.web_subnets}" ] - target_group_arns = [ - "${aws_alb_target_group.backend.arn}", - "${aws_alb_target_group.frontend.arn}" - ] } // ------ scale out ------- From 77a698f6bbfc8f6b9d9720ef5762818f15d6b1b5 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Tue, 13 Dec 2016 16:37:04 +0700 Subject: [PATCH 169/181] updated readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d5da57..34fd0e4 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,7 @@ $ ./terraform-env.sh stg apply ## Related Projects -* [Angular2 app](https://github.com/springboot-angular2-tutorial/angular2-app) +* [Angular 2 app](https://github.com/springboot-angular2-tutorial/angular2-app) * [Spring Boot app](https://github.com/springboot-angular2-tutorial/boot-app) * [Android app](https://github.com/springboot-angular2-tutorial/android-app) -* [Server provisioning by Ansible and Packer](https://github.com/springboot-angular2-tutorial/micropost-provisionings) * [Lambda functions by Serverless](https://github.com/springboot-angular2-tutorial/micropost-functions) From 7c29ae7e33bc3d6f174471750835a52f11c1734a Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 16 Dec 2016 08:52:38 +0700 Subject: [PATCH 170/181] Updated Readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 34fd0e4..f6a2701 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # micropost-formation +## Core components + +* AWS VPC +* AWS ECS with ALB and AutoScaling +* AWS ECR +* AWS RDS +* Cloudflare DNS and CDN + ## Dependencies * Terraform From 5a72752704e798fdac1654eaec1d4dfd1898eea2 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 16 Dec 2016 08:53:50 +0700 Subject: [PATCH 171/181] Updated Readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f6a2701..2c3c858 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ * AWS VPC * AWS ECS with ALB and AutoScaling -* AWS ECR * AWS RDS * Cloudflare DNS and CDN From 8138e300267194ab37bd4e9232b9c5178ecf3f21 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 2 Jan 2017 13:31:56 +0700 Subject: [PATCH 172/181] use terraform 0.8.1 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e18a27f..a638ba6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install awscli - aws --version # Install Terraform - - wget https://releases.hashicorp.com/terraform/0.7.13/terraform_0.7.13_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.8.1/terraform_0.8.1_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: From 1dc102c00855cc403ed7a3188224ec7cd718819d Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 12 Jan 2017 14:39:24 +0700 Subject: [PATCH 173/181] use terraform 0.8.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a638ba6..a643771 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install awscli - aws --version # Install Terraform - - wget https://releases.hashicorp.com/terraform/0.8.1/terraform_0.8.1_linux_amd64.zip -O /tmp/terraform.zip + - wget https://releases.hashicorp.com/terraform/0.8.4/terraform_0.8.4_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: From fdbf2f9efb5b3c939320b88cda08a95282bb72d6 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 2 Mar 2017 17:14:38 +0700 Subject: [PATCH 174/181] use custom db parameter --- dbservers/main.tf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dbservers/main.tf b/dbservers/main.tf index 2798719..7576b59 100644 --- a/dbservers/main.tf +++ b/dbservers/main.tf @@ -9,6 +9,9 @@ resource "aws_db_instance" "main" { "${var.security_groups}", ] db_subnet_group_name = "${aws_db_subnet_group.main.name}" + db_parameter_groups = [ + "${aws_db_parameter_group.main}}" + ] } resource "aws_db_subnet_group" "main" { @@ -18,3 +21,14 @@ resource "aws_db_subnet_group" "main" { "${var.subnets}" ] } + +resource "aws_db_parameter_group" "main" { + name = "main" + family = "mysql5.7" + + // prevent password expiration + parameter { + name = "default_password_lifetime" + value = "0" + } +} From 7d32f805772c465964b7bd6e75776096008191f8 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Thu, 2 Mar 2017 17:30:55 +0700 Subject: [PATCH 175/181] custome db parameter --- dbservers/main.tf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dbservers/main.tf b/dbservers/main.tf index 7576b59..37785e5 100644 --- a/dbservers/main.tf +++ b/dbservers/main.tf @@ -4,14 +4,11 @@ resource "aws_db_instance" "main" { engine = "mysql" engine_version = "5.7.10" instance_class = "db.t2.micro" - parameter_group_name = "default.mysql5.7" + parameter_group_name = "${aws_db_parameter_group.main.name}" vpc_security_group_ids = [ "${var.security_groups}", ] db_subnet_group_name = "${aws_db_subnet_group.main.name}" - db_parameter_groups = [ - "${aws_db_parameter_group.main}}" - ] } resource "aws_db_subnet_group" "main" { From 3d8abb37ebe468a4e982e488f931bce40dc884bb Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Fri, 3 Mar 2017 09:40:12 +0700 Subject: [PATCH 176/181] update terraform to 0.8.8 --- .travis.yml | 3 ++- webservers/backend.tf | 6 +++++- webservers/frontend.tf | 6 +++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a643771..d7e114f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ install: - pip install awscli - aws --version # Install Terraform - - wget https://releases.hashicorp.com/terraform/0.8.4/terraform_0.8.4_linux_amd64.zip -O /tmp/terraform.zip + - TERRAFORM_VERSION="0.8.8" + - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip before_script: diff --git a/webservers/backend.tf b/webservers/backend.tf index 9489db3..1553058 100644 --- a/webservers/backend.tf +++ b/webservers/backend.tf @@ -44,8 +44,8 @@ resource "aws_cloudwatch_log_group" "backend" { resource "aws_appautoscaling_target" "backend" { service_namespace = "ecs" - resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.backend.name}" scalable_dimension = "ecs:service:DesiredCount" + resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.backend.name}" role_arn = "${aws_iam_role.ecs_autoscale.arn}" min_capacity = 1 max_capacity = 2 @@ -61,6 +61,8 @@ resource "aws_appautoscaling_policy" "backend_scale_out" { metric_interval_lower_bound = 0 scaling_adjustment = 1 } + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" } resource "aws_cloudwatch_metric_alarm" "backend_cpu_high" { @@ -91,6 +93,8 @@ resource "aws_appautoscaling_policy" "backend_scale_in" { metric_interval_upper_bound = 0 scaling_adjustment = -1 } + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" } resource "aws_cloudwatch_metric_alarm" "backend_cpu_low" { diff --git a/webservers/frontend.tf b/webservers/frontend.tf index 52cb6a7..a4edfc6 100644 --- a/webservers/frontend.tf +++ b/webservers/frontend.tf @@ -41,8 +41,8 @@ resource "aws_cloudwatch_log_group" "frontend" { resource "aws_appautoscaling_target" "frontend" { service_namespace = "ecs" - resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.frontend.name}" scalable_dimension = "ecs:service:DesiredCount" + resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.frontend.name}" role_arn = "${aws_iam_role.ecs_autoscale.arn}" min_capacity = 1 max_capacity = 2 @@ -58,6 +58,8 @@ resource "aws_appautoscaling_policy" "frontend_scale_out" { metric_interval_lower_bound = 0 scaling_adjustment = 1 } + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" } resource "aws_cloudwatch_metric_alarm" "frontend_cpu_high" { @@ -88,6 +90,8 @@ resource "aws_appautoscaling_policy" "frontend_scale_in" { metric_interval_upper_bound = 0 scaling_adjustment = -1 } + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" } resource "aws_cloudwatch_metric_alarm" "frontend_cpu_low" { From e2006a1b7a9b787af9423034721254bdd52606e2 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 21 Aug 2017 16:44:19 +0700 Subject: [PATCH 177/181] terraform 0.10 --- .gitignore | 1 + .travis.yml | 10 ++-- dbservers/main.tf | 10 +--- dns.tf | 13 ++++- env/prod/vars | 2 - env/stg/vars | 3 - main.tf | 2 +- remote.tf | 7 +++ s3.tf | 4 +- terraform-env.sh | 108 ----------------------------------- terraform.cfg | 10 ---- variables.tf | 7 +-- webservers/host_instances.tf | 5 +- 13 files changed, 34 insertions(+), 148 deletions(-) delete mode 100644 env/prod/vars delete mode 100644 env/stg/vars create mode 100644 remote.tf delete mode 100755 terraform-env.sh delete mode 100644 terraform.cfg diff --git a/.gitignore b/.gitignore index 0c3f99f..28de410 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ # tfstate is saved on remote .terraform +terraform.tfstate.d terraform.log functions/*.zip diff --git a/.travis.yml b/.travis.yml index d7e114f..fccb6ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install awscli - aws --version # Install Terraform - - TERRAFORM_VERSION="0.8.8" + - TERRAFORM_VERSION="0.10.2" - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -O /tmp/terraform.zip - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip @@ -24,13 +24,15 @@ script: echo "${TRAVIS_BRANCH} is not a branch to deploy." else source ./scripts/switch-role.sh - ./terraform-env.sh ${ENV} get - ./terraform-env.sh ${ENV} plan + terraform get + terraform init + terraform workspace new ${ENV} + terraform plan fi deploy: - provider: script - script: ./terraform-env.sh ${ENV} apply + script: terraform plan skip_cleanup: true on: branch: deploy/* diff --git a/dbservers/main.tf b/dbservers/main.tf index 37785e5..4f0de4d 100644 --- a/dbservers/main.tf +++ b/dbservers/main.tf @@ -1,8 +1,9 @@ resource "aws_db_instance" "main" { snapshot_identifier = "${var.snapshot_identifier}" + skip_final_snapshot = true allocated_storage = 5 engine = "mysql" - engine_version = "5.7.10" + engine_version = "5.7.17" instance_class = "db.t2.micro" parameter_group_name = "${aws_db_parameter_group.main.name}" vpc_security_group_ids = [ @@ -19,13 +20,8 @@ resource "aws_db_subnet_group" "main" { ] } +# It's not used. resource "aws_db_parameter_group" "main" { name = "main" family = "mysql5.7" - - // prevent password expiration - parameter { - name = "default_password_lifetime" - value = "0" - } } diff --git a/dns.tf b/dns.tf index 65d7b1d..f88b62c 100644 --- a/dns.tf +++ b/dns.tf @@ -1,4 +1,12 @@ resource "cloudflare_record" "main" { + domain = "${var.domain}" + name = "${var.web_host_name}-${terraform.workspace}" + value = "${module.webservers.dns_name}" + type = "CNAME" +} + +resource "cloudflare_record" "main_prod" { + count = "${terraform.workspace == "prod" ? 1 : 0}" domain = "${var.domain}" name = "${var.web_host_name}" value = "${module.webservers.dns_name}" @@ -7,8 +15,9 @@ resource "cloudflare_record" "main" { resource "cloudflare_record" "cdn" { domain = "${var.domain}" - name = "cdn-${var.env}" - value = "${var.web_host_name}.hana053.com" + name = "cdn-${terraform.workspace}" + value = "${cloudflare_record.main.name}.hana053.com" proxied = true type = "CNAME" } + diff --git a/env/prod/vars b/env/prod/vars deleted file mode 100644 index 2d7f80c..0000000 --- a/env/prod/vars +++ /dev/null @@ -1,2 +0,0 @@ -env = "prod" -web_host_name = "micropost" diff --git a/env/stg/vars b/env/stg/vars deleted file mode 100644 index aaa64e0..0000000 --- a/env/stg/vars +++ /dev/null @@ -1,3 +0,0 @@ -env = "stg" -web_host_name = "micropost-stg" - diff --git a/main.tf b/main.tf index 417f103..cfd9f78 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,7 @@ module "vpc" { module "webservers" { source = "./webservers" - env = "${var.env}" + env = "${terraform.workspace}" dbserver_endpoint = "${module.dbservers.endpoint}" newrelic_license_key = "${var.newrelic_license_key}" key_name = "${aws_key_pair.micropost.key_name}" diff --git a/remote.tf b/remote.tf new file mode 100644 index 0000000..7266947 --- /dev/null +++ b/remote.tf @@ -0,0 +1,7 @@ +terraform { + backend "s3" { + bucket = "state.hana053.com" + key = "micropost" + region = "ap-northeast-1" + } +} diff --git a/s3.tf b/s3.tf index a34e414..8255352 100644 --- a/s3.tf +++ b/s3.tf @@ -1,5 +1,5 @@ resource "aws_s3_bucket" "log" { - bucket = "log-${var.env}.${var.domain}" + bucket = "log-${terraform.workspace}.${var.domain}" force_destroy = true policy = < []" - exit 1 -} - -function contains_element () { - local i - for i in "${@:2}"; do - [[ "$i" == "$1" ]] && return 0 - done - return 1 -} - -function files_exist(){ - ls ${1} 1> /dev/null 2>&1 -} - -#All of the args are mandatory. -if [ $# -lt 1 ]; then - help -fi - - -# Let's set up our environment -export ENVIRONMENT=$1 -export ACTION=$2 -ADDTL_PARAMS=${*:3} - -CONFIG_FILE=./terraform.cfg - -# Let's check the existence of the config file -if [ ! -f $CONFIG_FILE ]; then - echo "Error: $CONFIG_FILE does not exist. You'll need to create a config file so we know where to set up the remote config." - exit 1 -fi - -source ${CONFIG_FILE} - -# Let's set up our variables -ALLOWS_VARFILE=(apply plan push refresh destroy) -ENV_FILE=.terraform/environment -ENV_DIR=env/$ENVIRONMENT -VARS_FILE=${ENV_DIR}/vars -VARS_FILE_FLAG= -BUCKET_KEY=$bucket_prefix/state/$ENVIRONMENT -PREVIOUS_ENVIRONMENT=$([ -f $ENV_FILE ] && echo "$(<$ENV_FILE)" || echo "previous") -EXTRA_ARGS=${extra_args:-''} -PRE_CMD=${pre_command:-''} -POST_CMD=${post_command:-''} - -# Let's check to see if a vars file exists for the requested environment before proceeding -if [ ! -f $VARS_FILE ]; then - echo "Error: $VARS_FILE does not exist. You'll need to create a vars file for the requested environment before continuing." - exit 1 -fi - -# Checks if current action allows a varfile to be passed -contains_element "$ACTION" "${ALLOWS_VARFILE[@]}" -if [ $? -eq 0 ]; then - VARS_FILE_FLAG="-var-file=$VARS_FILE" -fi - -# Let's check if the requested environment is different from the previous environment -if [ $PREVIOUS_ENVIRONMENT != $ENVIRONMENT ] || [ ! -f '.terraform/terraform.tfstate' ] -then - - # Move current state out of the way to make room for the new state - mv -f .terraform/terraform.tfstate .terraform/terraform.tfstate.$PREVIOUS_ENVIRONMENT > /dev/null 2>&1 - mv -f .terraform/terraform.tfstate.backup .terraform/terraform.tfstate.backup.$PREVIOUS_ENVIRONMENT > /dev/null 2>&1 - - # Let's log the new environment for later - echo $ENVIRONMENT > $ENV_FILE - - # Set up remote configuration and pull latest version - terraform remote config -backend S3 \ - -backend-config="bucket=$bucket" \ - -backend-config="key=$BUCKET_KEY" \ - -backend-config="region=$region" -fi - -# Let's run the PRE_CMD hook if it's defined -source ${PRE_CMD} > /dev/null 2>&1 - -# let's copy environment specific configuration to the root of the directory -if files_exist ${ENV_DIR}/*.tf; then - cd env/${ENVIRONMENT} - pax -wrs'/\.tf$/\.env\.tf/' *.tf ../../ - cd ../../ -fi - -# Let's do work! -terraform $ACTION $VARS_FILE_FLAG $ADDTL_PARAMS ${EXTRA_ARGS} -terraform_status=$? - -# Let's remove those environment-specific configuration files we copied earlier -if files_exist *.env.tf; then - rm *.env.tf -fi - -# Let's run the POST_CMD hook if it's defined -source ${POST_CMD} > /dev/null 2>&1 - -exit ${terraform_status} diff --git a/terraform.cfg b/terraform.cfg deleted file mode 100644 index fe68cd0..0000000 --- a/terraform.cfg +++ /dev/null @@ -1,10 +0,0 @@ -bucket=state.hana053.com -bucket_prefix= -region=ap-northeast-1 - -pre_command='scripts/pre.sh' -post_command='scripts/post.sh' - -if [ ! -v AWS_SESSION_TOKEN ]; then - ENV=${ENVIRONMENT} source ./scripts/switch-role.sh -fi diff --git a/variables.tf b/variables.tf index 7954a90..30a9025 100644 --- a/variables.tf +++ b/variables.tf @@ -1,12 +1,6 @@ -variable "env" { - description = "dev, stg, prod and etc." -} - variable "allowed_segments" { type = "list" default = [ - "42.116.5.4/32", - "119.15.185.149/32", ] } @@ -15,6 +9,7 @@ variable "domain" { } variable "web_host_name" { + default = "micropost" } variable "newrelic_license_key" { diff --git a/webservers/host_instances.tf b/webservers/host_instances.tf index f3ab22a..1dc43c8 100644 --- a/webservers/host_instances.tf +++ b/webservers/host_instances.tf @@ -115,9 +115,8 @@ resource "aws_autoscaling_notification" "web" { resource "aws_iam_instance_profile" "web" { name = "web" - roles = [ - "${aws_iam_role.web.name}" - ] + role = "${aws_iam_role.web.name}" + } resource "aws_iam_role" "web" { From 5e961e2bbb4aa9449e8617b169405a953c3bc101 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 21 Aug 2017 16:48:44 +0700 Subject: [PATCH 178/181] init before get --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fccb6ca..11dca2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,8 @@ script: echo "${TRAVIS_BRANCH} is not a branch to deploy." else source ./scripts/switch-role.sh - terraform get terraform init + terraform get terraform workspace new ${ENV} terraform plan fi From 9f6541681aef4c170362d2f54d2af79b781972b8 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 21 Aug 2017 16:51:52 +0700 Subject: [PATCH 179/181] select workspace --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11dca2c..1476cfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: source ./scripts/switch-role.sh terraform init terraform get - terraform workspace new ${ENV} + terraform workspace select ${ENV} terraform plan fi From b171808f1070630e8eeaff86e646ebd539ca0963 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 21 Aug 2017 16:58:27 +0700 Subject: [PATCH 180/181] add allowed segment --- README.md | 10 +++++++++- variables.tf | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c3c858..24afa34 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,18 @@ $ vi .envrc $ direnv allow ``` +Init +``` +$ export ENV=stg +$ source ./scripts/switch-role.sh +$ terraform init +$ terraform get +$ terraform workspace select ${ENV} +``` + Plan and Apply ``` -$ ./terraform-env.sh stg get $ ./terraform-env.sh stg plan $ ./terraform-env.sh stg apply ``` diff --git a/variables.tf b/variables.tf index 30a9025..2666e66 100644 --- a/variables.tf +++ b/variables.tf @@ -1,6 +1,7 @@ variable "allowed_segments" { type = "list" default = [ + "101.53.23.34/32", ] } From b558cea91b307f56337a8ec3b182a029082c16b2 Mon Sep 17 00:00:00 2001 From: Akira Sosa Date: Mon, 21 Aug 2017 17:00:38 +0700 Subject: [PATCH 181/181] plan to apply --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1476cfd..03d0a32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ script: deploy: - provider: script - script: terraform plan + script: terraform apply skip_cleanup: true on: branch: deploy/*