You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/07/15 08:34:21 UTC
[incubator-dlab] 01/01: [DLAB-901][DLAB-902]: added creation k8s in
multiple subnets; added creation of ALB for K8S
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-836
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3537483ece2196629bb47e2dbf1d22f042661ae1
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Mon Jul 15 11:34:09 2019 +0300
[DLAB-901][DLAB-902]: added creation k8s in multiple subnets; added creation of ALB for K8S
---
.../terraform/aws/main/main.tf | 49 +++++++-------
.../terraform/aws/main/variables.tf | 13 +++-
.../aws/modules/ssn-k8s/auto_scaling_groups.tf | 15 +++--
.../aws/modules/ssn-k8s/files/masters-user-data.sh | 6 +-
.../terraform/aws/modules/ssn-k8s/lb.tf | 55 ++++++++++++----
.../aws/modules/ssn-k8s/security_groups.tf | 30 ++++++---
.../terraform/aws/modules/ssn-k8s/variables.tf | 10 ++-
.../terraform/aws/modules/ssn-k8s/vpc.tf | 74 ++++++++++++++++++----
.../terraform/bin/terraform-cli.py | 18 ++++--
9 files changed, 197 insertions(+), 73 deletions(-)
diff --git a/infrastructure-provisioning/terraform/aws/main/main.tf b/infrastructure-provisioning/terraform/aws/main/main.tf
index 2a45d7e..10d3ad3 100644
--- a/infrastructure-provisioning/terraform/aws/main/main.tf
+++ b/infrastructure-provisioning/terraform/aws/main/main.tf
@@ -26,24 +26,27 @@ provider "aws" {
}
module "ssn-k8s" {
- source = "../modules/ssn-k8s"
- service_base_name = var.service_base_name
- vpc_id = var.vpc_id
- vpc_cidr = var.vpc_cidr
- subnet_id = var.subnet_id
- env_os = var.env_os
- ami = var.ami
- key_name = var.key_name
- region = var.region
- zone = var.zone
- ssn_k8s_masters_count = var.ssn_k8s_masters_count
- ssn_k8s_workers_count = var.ssn_k8s_workers_count
- ssn_root_volume_size = var.ssn_root_volume_size
- allowed_cidrs = var.allowed_cidrs
- subnet_cidr = var.subnet_cidr
- ssn_k8s_masters_shape = var.ssn_k8s_masters_shape
- ssn_k8s_workers_shape = var.ssn_k8s_workers_shape
- os_user = var.os_user
+ source = "../modules/ssn-k8s"
+ service_base_name = var.service_base_name
+ vpc_id = var.vpc_id
+ vpc_cidr = var.vpc_cidr
+ subnet_id_a = var.subnet_id_a
+ subnet_id_b = var.subnet_id_b
+ env_os = var.env_os
+ ami = var.ami
+ key_name = var.key_name
+ region = var.region
+ zone = var.zone
+ ssn_k8s_masters_count = var.ssn_k8s_masters_count
+ ssn_k8s_workers_count = var.ssn_k8s_workers_count
+ ssn_root_volume_size = var.ssn_root_volume_size
+ allowed_cidrs = var.allowed_cidrs
+ subnet_cidr_a = var.subnet_cidr_a
+ subnet_cidr_b = var.subnet_cidr_b
+ subnet_cidr_c = var.subnet_cidr_c
+ ssn_k8s_masters_shape = var.ssn_k8s_masters_shape
+ ssn_k8s_workers_shape = var.ssn_k8s_workers_shape
+ os_user = var.os_user
}
module "common" {
@@ -69,7 +72,7 @@ module "notebook" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -85,7 +88,7 @@ module "data_engine" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -104,7 +107,7 @@ module "emr" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -131,12 +134,12 @@ module "endpoint" {
region = var.region
zone = var.zone
product = var.product_name
- subnet_cidr = var.subnet_cidr
+ subnet_cidr = var.subnet_cidr_a
endpoint_instance_shape = var.endpoint_instance_shape
key_name = var.key_name
ami = var.ami
vpc_id = var.vpc_id
- subnet_id = var.subnet_id
+ subnet_id = var.subnet_id_a
network_type = var.network_type
vpc_cidr = var.vpc_cidr
endpoint_volume_size = var.endpoint_volume_size
diff --git a/infrastructure-provisioning/terraform/aws/main/variables.tf b/infrastructure-provisioning/terraform/aws/main/variables.tf
index 111a68f..fc50cf2 100644
--- a/infrastructure-provisioning/terraform/aws/main/variables.tf
+++ b/infrastructure-provisioning/terraform/aws/main/variables.tf
@@ -62,12 +62,21 @@ variable "vpc_id" {
variable "vpc_cidr" {
default = "172.31.0.0/16"
}
-variable "subnet_id" {
+variable "subnet_id_a" {
default = ""
}
-variable "subnet_cidr" {
+variable "subnet_id_b" {
+ default = ""
+}
+variable "subnet_cidr_a" {
default = "172.31.0.0/24"
}
+variable "subnet_cidr_b" {
+ default = "172.31.1.0/24"
+}
+variable "subnet_cidr_c" {
+ default = "172.31.2.0/24"
+}
variable "ami" {
default = "ami-07b4f3c02c7f83d59"
}
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
index 6aa3e42..0ee3f35 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
@@ -19,14 +19,18 @@
#
# ******************************************************************************
+locals {
+ subnet_c_id = data.aws_subnet.k8s-subnet-c-data == [] ? "" : data.aws_subnet.k8s-subnet-c-data.0.id
+}
+
data "template_file" "ssn_k8s_masters_user_data" {
template = file("../modules/ssn-k8s/files/masters-user-data.sh")
vars = {
k8s-asg = "${var.service_base_name}-ssn-masters"
k8s-region = var.region
k8s-bucket-name = aws_s3_bucket.ssn_k8s_bucket.id
- k8s-eip = aws_eip.k8s-lb-eip.public_ip
- k8s-tg-arn = aws_lb_target_group.ssn_k8s_lb_target_group.arn
+ k8s-nlb-dns-name = aws_lb.ssn_k8s_nlb.dns_name #aws_eip.k8s-lb-eip.public_ip
+ k8s-tg-arn = aws_lb_target_group.ssn_k8s_nlb_target_group.arn
k8s_os_user = var.os_user
}
}
@@ -82,8 +86,9 @@ resource "aws_autoscaling_group" "ssn_k8s_autoscaling_group_masters" {
launch_configuration = aws_launch_configuration.ssn_k8s_launch_conf_masters.name
min_size = var.ssn_k8s_masters_count
max_size = var.ssn_k8s_masters_count
- vpc_zone_identifier = [data.aws_subnet.k8s-subnet-data.id]
- target_group_arns = [aws_lb_target_group.ssn_k8s_lb_target_group.arn]
+ vpc_zone_identifier = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+ target_group_arns = [aws_lb_target_group.ssn_k8s_nlb_target_group.arn,
+ aws_lb_target_group.ssn_k8s_alb_target_group.arn]
lifecycle {
create_before_destroy = true
@@ -102,7 +107,7 @@ resource "aws_autoscaling_group" "ssn_k8s_autoscaling_group_workers" {
launch_configuration = aws_launch_configuration.ssn_k8s_launch_conf_workers.name
min_size = var.ssn_k8s_workers_count
max_size = var.ssn_k8s_workers_count
- vpc_zone_identifier = [data.aws_subnet.k8s-subnet-data.id]
+ vpc_zone_identifier = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
lifecycle {
create_before_destroy = true
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
index 2091b89..8a8ab96 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
@@ -74,15 +74,15 @@ apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: stable
apiServerCertSANs:
- - ${k8s-eip}
-controlPlaneEndpoint: "${k8s-eip}:6443"
+ - ${k8s-nlb-dns-name}
+controlPlaneEndpoint: "${k8s-nlb-dns-name}:6443"
EOF
sudo kubeadm init --config=/tmp/kubeadm-config.yaml --upload-certs
while check_elb_status
do
if [[ $RUN == "false" ]];
then
- echo "Waiting for LB healthy status..."
+ echo "Waiting for NLB healthy status..."
else
echo "LB status is healthy!"
break
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
index 18afc73..552481f 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
@@ -19,36 +19,65 @@
#
# ******************************************************************************
-resource "aws_lb" "ssn_k8s_lb" {
- name = "${var.service_base_name}-ssn-lb"
+resource "aws_lb" "ssn_k8s_nlb" {
+ name = "${var.service_base_name}-ssn-nlb"
load_balancer_type = "network"
-
- subnet_mapping {
- subnet_id = data.aws_subnet.k8s-subnet-data.id
- allocation_id = aws_eip.k8s-lb-eip.id
+ subnets = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+ tags = {
+ Name = "${var.service_base_name}-ssn-nlb"
}
+}
+
+resource "aws_lb" "ssn_k8s_alb" {
+ name = "${var.service_base_name}-ssn-alb"
+ internal = false
+ load_balancer_type = "application"
+ security_groups = [aws_security_group.ssn_k8s_sg.id]
+ subnets = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+
tags = {
- Name = "${var.service_base_name}-ssn-lb"
+ Name = "${var.service_base_name}-ssn-alb"
}
}
-resource "aws_lb_target_group" "ssn_k8s_lb_target_group" {
- name = "${var.service_base_name}-ssn-lb-target-group"
+resource "aws_lb_target_group" "ssn_k8s_nlb_target_group" {
+ name = "${var.service_base_name}-ssn-nlb-target-group"
port = 6443
protocol = "TCP"
vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
tags = {
- Name = "${var.service_base_name}-ssn-lb-target-group"
+ Name = "${var.service_base_name}-ssn-nlb-target-group"
+ }
+}
+
+resource "aws_lb_target_group" "ssn_k8s_alb_target_group" {
+ name = "${var.service_base_name}-ssn-alb-target-group"
+ port = 31080
+ protocol = "HTTP"
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ tags = {
+ Name = "${var.service_base_name}-ssn-alb-target-group"
+ }
+}
+
+resource "aws_lb_listener" "ssn_k8s_alb_listener" {
+ load_balancer_arn = aws_lb.ssn_k8s_alb.arn
+ port = "80"
+ protocol = "HTTP"
+
+ default_action {
+ type = "forward"
+ target_group_arn = aws_lb_target_group.ssn_k8s_alb_target_group.arn
}
}
-resource "aws_lb_listener" "ssn_k8s_lb_listener" {
- load_balancer_arn = aws_lb.ssn_k8s_lb.arn
+resource "aws_lb_listener" "ssn_k8s_nlb_listener" {
+ load_balancer_arn = aws_lb.ssn_k8s_nlb.arn
port = "6443"
protocol = "TCP"
default_action {
type = "forward"
- target_group_arn = aws_lb_target_group.ssn_k8s_lb_target_group.arn
+ target_group_arn = aws_lb_target_group.ssn_k8s_nlb_target_group.arn
}
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
index 95881fb..70fb6e4 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
@@ -19,10 +19,20 @@
#
# ******************************************************************************
-data "aws_eip" "ssn_k8s_lb_eip" {
- id = aws_eip.k8s-lb-eip.id
- depends_on = [aws_lb_listener.ssn_k8s_lb_listener]
-}
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-a.id
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
+//
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-b.id # Need to be refactored
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
+//
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-a.id
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
resource "aws_security_group" "ssn_k8s_sg" {
name = "${var.service_base_name}-ssn-sg"
@@ -48,12 +58,12 @@ resource "aws_security_group" "ssn_k8s_sg" {
cidr_blocks = ["0.0.0.0/0"]
description = "Need to be changed in the future"
}
- ingress {
- from_port = 0
- to_port = 0
- protocol = -1
- cidr_blocks = ["${data.aws_eip.ssn_k8s_lb_eip.public_ip}/32", "${data.aws_eip.ssn_k8s_lb_eip.private_ip}/32"]
- }
+// ingress {
+// from_port = 0
+// to_port = 0 # Need to be refactored
+// protocol = -1
+// cidr_blocks = ["${data.aws_eip.ssn_k8s_lb_eip.public_ip}/32", "${data.aws_eip.ssn_k8s_lb_eip.private_ip}/32"]
+// }
egress {
from_port = 0
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
index 7660088..a9ef123 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
@@ -25,9 +25,15 @@ variable "vpc_id" {}
variable "vpc_cidr" {}
-variable "subnet_id" {}
+variable "subnet_id_a" {}
-variable "subnet_cidr" {}
+variable "subnet_id_b" {}
+
+variable "subnet_cidr_a" {}
+
+variable "subnet_cidr_b" {}
+
+variable "subnet_cidr_c" {}
variable "env_os" {}
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
index 4c50323..78e26b6 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
@@ -51,25 +51,77 @@ data "aws_vpc" "ssn_k8s_vpc_data" {
id = var.vpc_id == "" ? aws_vpc.ssn_k8s_vpc.0.id : var.vpc_id
}
-resource "aws_subnet" "ssn_k8s_subnet" {
- count = var.subnet_id == "" ? 1 : 0
+resource "aws_subnet" "ssn_k8s_subnet_a" {
+ count = var.subnet_id_a == "" ? 1 : 0
vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
- availability_zone = "${var.region}${var.zone}"
- cidr_block = var.subnet_cidr
+ availability_zone = "${var.region}a"
+ cidr_block = var.subnet_cidr_a
map_public_ip_on_launch = true
tags = {
- Name = "${var.service_base_name}-ssn-subnet"
+ Name = "${var.service_base_name}-ssn-subnet-az-a"
}
}
-data "aws_subnet" "k8s-subnet-data" {
- id = var.subnet_id == "" ? aws_subnet.ssn_k8s_subnet.0.id : var.subnet_id
+resource "aws_subnet" "ssn_k8s_subnet_b" {
+ count = var.subnet_id_b == "" ? 1 : 0
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ availability_zone = "${var.region}b"
+ cidr_block = var.subnet_cidr_b
+ map_public_ip_on_launch = true
+
+ tags = {
+ Name = "${var.service_base_name}-ssn-subnet-az-b"
+ }
}
-resource "aws_eip" "k8s-lb-eip" {
- vpc = true
+resource "aws_subnet" "ssn_k8s_subnet_c" {
+ count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ availability_zone = "${var.region}c"
+ cidr_block = var.subnet_cidr_c
+ map_public_ip_on_launch = true
+
tags = {
- Name = "${var.service_base_name}-ssn-eip"
+ Name = "${var.service_base_name}-ssn-subnet-az-c"
}
-}
\ No newline at end of file
+}
+
+data "aws_subnet" "k8s-subnet-a-data" {
+ id = var.subnet_id_a == "" ? aws_subnet.ssn_k8s_subnet_a.0.id : var.subnet_id_a
+}
+
+data "aws_subnet" "k8s-subnet-b-data" {
+ id = var.subnet_id_b == "" ? aws_subnet.ssn_k8s_subnet_b.0.id : var.subnet_id_b
+}
+
+data "aws_subnet" "k8s-subnet-c-data" {
+ count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+ id = aws_subnet.ssn_k8s_subnet_c.0.id
+}
+
+//resource "aws_eip" "k8s-lb-eip-a" {
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-a"
+// }
+//}
+//
+//resource "aws_eip" "k8s-lb-eip-b" {
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-b"
+// }
+//}
+//
+//resource "aws_eip" "k8s-lb-eip-c" {
+// count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-c"
+// }
+//}
+//
+//data "aws_eip" "k8s-lb-eip-c-data" {
+// id = aws_eip.k8s-lb-eip-c.0.id
+//}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/bin/terraform-cli.py b/infrastructure-provisioning/terraform/bin/terraform-cli.py
index 845ff1c..f62e852 100755
--- a/infrastructure-provisioning/terraform/bin/terraform-cli.py
+++ b/infrastructure-provisioning/terraform/bin/terraform-cli.py
@@ -359,11 +359,21 @@ class AWSSourceBuilder(AbstractDeployBuilder):
default='t2.medium')
.add_int('--ssn_root_volume_size', 'Size of root volume in GB.',
default=30)
- .add_str('--subnet_cidr',
- 'CIDR for Subnet creation. Conflicts with subnet_id.',
+ .add_str('--subnet_cidr_a',
+ 'CIDR for Subnet creation in zone a. Conflicts with subnet_id_a.',
default='172.31.0.0/24')
- .add_str('--subnet_id',
- 'ID of AWS Subnet if you already have subnet created.')
+ .add_str('--subnet_cidr_b',
+ 'CIDR for Subnet creation in zone b. Conflicts with subnet_id_b.',
+ default='172.31.1.0/24')
+ .add_str('--subnet_cidr_c',
+ 'CIDR for Subnet creation in zone c. Conflicts with subnet_id_c.',
+ default='172.31.2.0/24')
+ .add_str('--subnet_id_a',
+ 'ID of AWS Subnet in zone a if you already have subnet created.')
+ .add_str('--subnet_id_b',
+ 'ID of AWS Subnet in zone b if you already have subnet created.')
+ .add_str('--subnet_id_c',
+ 'ID of AWS Subnet in zone c if you already have subnet created.')
.add_str('--vpc_cidr', 'CIDR for VPC creation. Conflicts with vpc_id',
default='172.31.0.0/16')
.add_str('--vpc_id', 'ID of AWS VPC if you already have VPC created.')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org