You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/07/15 08:34:20 UTC
[incubator-dlab] branch DLAB-836 created (now 3537483)
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a change to branch DLAB-836
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 3537483 [DLAB-901][DLAB-902]: added creation k8s in multiple subnets; added creation of ALB for K8S
This branch includes the following new commits:
new 3537483 [DLAB-901][DLAB-902]: added creation k8s in multiple subnets; added creation of ALB for K8S
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-901][DLAB-902]: added creation k8s in
multiple subnets; added creation of ALB for K8S
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-836
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3537483ece2196629bb47e2dbf1d22f042661ae1
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Mon Jul 15 11:34:09 2019 +0300
[DLAB-901][DLAB-902]: added creation k8s in multiple subnets; added creation of ALB for K8S
---
.../terraform/aws/main/main.tf | 49 +++++++-------
.../terraform/aws/main/variables.tf | 13 +++-
.../aws/modules/ssn-k8s/auto_scaling_groups.tf | 15 +++--
.../aws/modules/ssn-k8s/files/masters-user-data.sh | 6 +-
.../terraform/aws/modules/ssn-k8s/lb.tf | 55 ++++++++++++----
.../aws/modules/ssn-k8s/security_groups.tf | 30 ++++++---
.../terraform/aws/modules/ssn-k8s/variables.tf | 10 ++-
.../terraform/aws/modules/ssn-k8s/vpc.tf | 74 ++++++++++++++++++----
.../terraform/bin/terraform-cli.py | 18 ++++--
9 files changed, 197 insertions(+), 73 deletions(-)
diff --git a/infrastructure-provisioning/terraform/aws/main/main.tf b/infrastructure-provisioning/terraform/aws/main/main.tf
index 2a45d7e..10d3ad3 100644
--- a/infrastructure-provisioning/terraform/aws/main/main.tf
+++ b/infrastructure-provisioning/terraform/aws/main/main.tf
@@ -26,24 +26,27 @@ provider "aws" {
}
module "ssn-k8s" {
- source = "../modules/ssn-k8s"
- service_base_name = var.service_base_name
- vpc_id = var.vpc_id
- vpc_cidr = var.vpc_cidr
- subnet_id = var.subnet_id
- env_os = var.env_os
- ami = var.ami
- key_name = var.key_name
- region = var.region
- zone = var.zone
- ssn_k8s_masters_count = var.ssn_k8s_masters_count
- ssn_k8s_workers_count = var.ssn_k8s_workers_count
- ssn_root_volume_size = var.ssn_root_volume_size
- allowed_cidrs = var.allowed_cidrs
- subnet_cidr = var.subnet_cidr
- ssn_k8s_masters_shape = var.ssn_k8s_masters_shape
- ssn_k8s_workers_shape = var.ssn_k8s_workers_shape
- os_user = var.os_user
+ source = "../modules/ssn-k8s"
+ service_base_name = var.service_base_name
+ vpc_id = var.vpc_id
+ vpc_cidr = var.vpc_cidr
+ subnet_id_a = var.subnet_id_a
+ subnet_id_b = var.subnet_id_b
+ env_os = var.env_os
+ ami = var.ami
+ key_name = var.key_name
+ region = var.region
+ zone = var.zone
+ ssn_k8s_masters_count = var.ssn_k8s_masters_count
+ ssn_k8s_workers_count = var.ssn_k8s_workers_count
+ ssn_root_volume_size = var.ssn_root_volume_size
+ allowed_cidrs = var.allowed_cidrs
+ subnet_cidr_a = var.subnet_cidr_a
+ subnet_cidr_b = var.subnet_cidr_b
+ subnet_cidr_c = var.subnet_cidr_c
+ ssn_k8s_masters_shape = var.ssn_k8s_masters_shape
+ ssn_k8s_workers_shape = var.ssn_k8s_workers_shape
+ os_user = var.os_user
}
module "common" {
@@ -69,7 +72,7 @@ module "notebook" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -85,7 +88,7 @@ module "data_engine" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -104,7 +107,7 @@ module "emr" {
user_tag = "${var.user_tag}"
custom_tag = "${var.custom_tag}"
notebook_name = "${var.notebook_name}"
- subnet_id = "${var.subnet_id}"
+ subnet_id = "${var.subnet_id_a}"
nb-sg_id = "${var.nb-sg_id}"
note_profile_name = "${var.note_profile_name}"
product = "${var.product_name}"
@@ -131,12 +134,12 @@ module "endpoint" {
region = var.region
zone = var.zone
product = var.product_name
- subnet_cidr = var.subnet_cidr
+ subnet_cidr = var.subnet_cidr_a
endpoint_instance_shape = var.endpoint_instance_shape
key_name = var.key_name
ami = var.ami
vpc_id = var.vpc_id
- subnet_id = var.subnet_id
+ subnet_id = var.subnet_id_a
network_type = var.network_type
vpc_cidr = var.vpc_cidr
endpoint_volume_size = var.endpoint_volume_size
diff --git a/infrastructure-provisioning/terraform/aws/main/variables.tf b/infrastructure-provisioning/terraform/aws/main/variables.tf
index 111a68f..fc50cf2 100644
--- a/infrastructure-provisioning/terraform/aws/main/variables.tf
+++ b/infrastructure-provisioning/terraform/aws/main/variables.tf
@@ -62,12 +62,21 @@ variable "vpc_id" {
variable "vpc_cidr" {
default = "172.31.0.0/16"
}
-variable "subnet_id" {
+variable "subnet_id_a" {
default = ""
}
-variable "subnet_cidr" {
+variable "subnet_id_b" {
+ default = ""
+}
+variable "subnet_cidr_a" {
default = "172.31.0.0/24"
}
+variable "subnet_cidr_b" {
+ default = "172.31.1.0/24"
+}
+variable "subnet_cidr_c" {
+ default = "172.31.2.0/24"
+}
variable "ami" {
default = "ami-07b4f3c02c7f83d59"
}
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
index 6aa3e42..0ee3f35 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/auto_scaling_groups.tf
@@ -19,14 +19,18 @@
#
# ******************************************************************************
+locals {
+ subnet_c_id = data.aws_subnet.k8s-subnet-c-data == [] ? "" : data.aws_subnet.k8s-subnet-c-data.0.id
+}
+
data "template_file" "ssn_k8s_masters_user_data" {
template = file("../modules/ssn-k8s/files/masters-user-data.sh")
vars = {
k8s-asg = "${var.service_base_name}-ssn-masters"
k8s-region = var.region
k8s-bucket-name = aws_s3_bucket.ssn_k8s_bucket.id
- k8s-eip = aws_eip.k8s-lb-eip.public_ip
- k8s-tg-arn = aws_lb_target_group.ssn_k8s_lb_target_group.arn
+ k8s-nlb-dns-name = aws_lb.ssn_k8s_nlb.dns_name #aws_eip.k8s-lb-eip.public_ip
+ k8s-tg-arn = aws_lb_target_group.ssn_k8s_nlb_target_group.arn
k8s_os_user = var.os_user
}
}
@@ -82,8 +86,9 @@ resource "aws_autoscaling_group" "ssn_k8s_autoscaling_group_masters" {
launch_configuration = aws_launch_configuration.ssn_k8s_launch_conf_masters.name
min_size = var.ssn_k8s_masters_count
max_size = var.ssn_k8s_masters_count
- vpc_zone_identifier = [data.aws_subnet.k8s-subnet-data.id]
- target_group_arns = [aws_lb_target_group.ssn_k8s_lb_target_group.arn]
+ vpc_zone_identifier = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+ target_group_arns = [aws_lb_target_group.ssn_k8s_nlb_target_group.arn,
+ aws_lb_target_group.ssn_k8s_alb_target_group.arn]
lifecycle {
create_before_destroy = true
@@ -102,7 +107,7 @@ resource "aws_autoscaling_group" "ssn_k8s_autoscaling_group_workers" {
launch_configuration = aws_launch_configuration.ssn_k8s_launch_conf_workers.name
min_size = var.ssn_k8s_workers_count
max_size = var.ssn_k8s_workers_count
- vpc_zone_identifier = [data.aws_subnet.k8s-subnet-data.id]
+ vpc_zone_identifier = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
lifecycle {
create_before_destroy = true
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
index 2091b89..8a8ab96 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/files/masters-user-data.sh
@@ -74,15 +74,15 @@ apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: stable
apiServerCertSANs:
- - ${k8s-eip}
-controlPlaneEndpoint: "${k8s-eip}:6443"
+ - ${k8s-nlb-dns-name}
+controlPlaneEndpoint: "${k8s-nlb-dns-name}:6443"
EOF
sudo kubeadm init --config=/tmp/kubeadm-config.yaml --upload-certs
while check_elb_status
do
if [[ $RUN == "false" ]];
then
- echo "Waiting for LB healthy status..."
+ echo "Waiting for NLB healthy status..."
else
echo "LB status is healthy!"
break
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
index 18afc73..552481f 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/lb.tf
@@ -19,36 +19,65 @@
#
# ******************************************************************************
-resource "aws_lb" "ssn_k8s_lb" {
- name = "${var.service_base_name}-ssn-lb"
+resource "aws_lb" "ssn_k8s_nlb" {
+ name = "${var.service_base_name}-ssn-nlb"
load_balancer_type = "network"
-
- subnet_mapping {
- subnet_id = data.aws_subnet.k8s-subnet-data.id
- allocation_id = aws_eip.k8s-lb-eip.id
+ subnets = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+ tags = {
+ Name = "${var.service_base_name}-ssn-nlb"
}
+}
+
+resource "aws_lb" "ssn_k8s_alb" {
+ name = "${var.service_base_name}-ssn-alb"
+ internal = false
+ load_balancer_type = "application"
+ security_groups = [aws_security_group.ssn_k8s_sg.id]
+ subnets = compact([data.aws_subnet.k8s-subnet-a-data.id, data.aws_subnet.k8s-subnet-b-data.id, local.subnet_c_id])
+
tags = {
- Name = "${var.service_base_name}-ssn-lb"
+ Name = "${var.service_base_name}-ssn-alb"
}
}
-resource "aws_lb_target_group" "ssn_k8s_lb_target_group" {
- name = "${var.service_base_name}-ssn-lb-target-group"
+resource "aws_lb_target_group" "ssn_k8s_nlb_target_group" {
+ name = "${var.service_base_name}-ssn-nlb-target-group"
port = 6443
protocol = "TCP"
vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
tags = {
- Name = "${var.service_base_name}-ssn-lb-target-group"
+ Name = "${var.service_base_name}-ssn-nlb-target-group"
+ }
+}
+
+resource "aws_lb_target_group" "ssn_k8s_alb_target_group" {
+ name = "${var.service_base_name}-ssn-alb-target-group"
+ port = 31080
+ protocol = "HTTP"
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ tags = {
+ Name = "${var.service_base_name}-ssn-alb-target-group"
+ }
+}
+
+resource "aws_lb_listener" "ssn_k8s_alb_listener" {
+ load_balancer_arn = aws_lb.ssn_k8s_alb.arn
+ port = "80"
+ protocol = "HTTP"
+
+ default_action {
+ type = "forward"
+ target_group_arn = aws_lb_target_group.ssn_k8s_alb_target_group.arn
}
}
-resource "aws_lb_listener" "ssn_k8s_lb_listener" {
- load_balancer_arn = aws_lb.ssn_k8s_lb.arn
+resource "aws_lb_listener" "ssn_k8s_nlb_listener" {
+ load_balancer_arn = aws_lb.ssn_k8s_nlb.arn
port = "6443"
protocol = "TCP"
default_action {
type = "forward"
- target_group_arn = aws_lb_target_group.ssn_k8s_lb_target_group.arn
+ target_group_arn = aws_lb_target_group.ssn_k8s_nlb_target_group.arn
}
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
index 95881fb..70fb6e4 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/security_groups.tf
@@ -19,10 +19,20 @@
#
# ******************************************************************************
-data "aws_eip" "ssn_k8s_lb_eip" {
- id = aws_eip.k8s-lb-eip.id
- depends_on = [aws_lb_listener.ssn_k8s_lb_listener]
-}
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-a.id
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
+//
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-b.id # Need to be refactored
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
+//
+//data "aws_eip" "ssn_k8s_lb_eip_a" {
+// id = aws_eip.k8s-lb-eip-a.id
+// depends_on = [aws_lb_listener.ssn_k8s_nlb_listener]
+//}
resource "aws_security_group" "ssn_k8s_sg" {
name = "${var.service_base_name}-ssn-sg"
@@ -48,12 +58,12 @@ resource "aws_security_group" "ssn_k8s_sg" {
cidr_blocks = ["0.0.0.0/0"]
description = "Need to be changed in the future"
}
- ingress {
- from_port = 0
- to_port = 0
- protocol = -1
- cidr_blocks = ["${data.aws_eip.ssn_k8s_lb_eip.public_ip}/32", "${data.aws_eip.ssn_k8s_lb_eip.private_ip}/32"]
- }
+// ingress {
+// from_port = 0
+// to_port = 0 # Need to be refactored
+// protocol = -1
+// cidr_blocks = ["${data.aws_eip.ssn_k8s_lb_eip.public_ip}/32", "${data.aws_eip.ssn_k8s_lb_eip.private_ip}/32"]
+// }
egress {
from_port = 0
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
index 7660088..a9ef123 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/variables.tf
@@ -25,9 +25,15 @@ variable "vpc_id" {}
variable "vpc_cidr" {}
-variable "subnet_id" {}
+variable "subnet_id_a" {}
-variable "subnet_cidr" {}
+variable "subnet_id_b" {}
+
+variable "subnet_cidr_a" {}
+
+variable "subnet_cidr_b" {}
+
+variable "subnet_cidr_c" {}
variable "env_os" {}
diff --git a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
index 4c50323..78e26b6 100644
--- a/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
+++ b/infrastructure-provisioning/terraform/aws/modules/ssn-k8s/vpc.tf
@@ -51,25 +51,77 @@ data "aws_vpc" "ssn_k8s_vpc_data" {
id = var.vpc_id == "" ? aws_vpc.ssn_k8s_vpc.0.id : var.vpc_id
}
-resource "aws_subnet" "ssn_k8s_subnet" {
- count = var.subnet_id == "" ? 1 : 0
+resource "aws_subnet" "ssn_k8s_subnet_a" {
+ count = var.subnet_id_a == "" ? 1 : 0
vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
- availability_zone = "${var.region}${var.zone}"
- cidr_block = var.subnet_cidr
+ availability_zone = "${var.region}a"
+ cidr_block = var.subnet_cidr_a
map_public_ip_on_launch = true
tags = {
- Name = "${var.service_base_name}-ssn-subnet"
+ Name = "${var.service_base_name}-ssn-subnet-az-a"
}
}
-data "aws_subnet" "k8s-subnet-data" {
- id = var.subnet_id == "" ? aws_subnet.ssn_k8s_subnet.0.id : var.subnet_id
+resource "aws_subnet" "ssn_k8s_subnet_b" {
+ count = var.subnet_id_b == "" ? 1 : 0
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ availability_zone = "${var.region}b"
+ cidr_block = var.subnet_cidr_b
+ map_public_ip_on_launch = true
+
+ tags = {
+ Name = "${var.service_base_name}-ssn-subnet-az-b"
+ }
}
-resource "aws_eip" "k8s-lb-eip" {
- vpc = true
+resource "aws_subnet" "ssn_k8s_subnet_c" {
+ count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+ vpc_id = data.aws_vpc.ssn_k8s_vpc_data.id
+ availability_zone = "${var.region}c"
+ cidr_block = var.subnet_cidr_c
+ map_public_ip_on_launch = true
+
tags = {
- Name = "${var.service_base_name}-ssn-eip"
+ Name = "${var.service_base_name}-ssn-subnet-az-c"
}
-}
\ No newline at end of file
+}
+
+data "aws_subnet" "k8s-subnet-a-data" {
+ id = var.subnet_id_a == "" ? aws_subnet.ssn_k8s_subnet_a.0.id : var.subnet_id_a
+}
+
+data "aws_subnet" "k8s-subnet-b-data" {
+ id = var.subnet_id_b == "" ? aws_subnet.ssn_k8s_subnet_b.0.id : var.subnet_id_b
+}
+
+data "aws_subnet" "k8s-subnet-c-data" {
+ count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+ id = aws_subnet.ssn_k8s_subnet_c.0.id
+}
+
+//resource "aws_eip" "k8s-lb-eip-a" {
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-a"
+// }
+//}
+//
+//resource "aws_eip" "k8s-lb-eip-b" {
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-b"
+// }
+//}
+//
+//resource "aws_eip" "k8s-lb-eip-c" {
+// count = var.ssn_k8s_masters_count > 2 ? 1 : 0
+// vpc = true
+// tags = {
+// Name = "${var.service_base_name}-ssn-eip-c"
+// }
+//}
+//
+//data "aws_eip" "k8s-lb-eip-c-data" {
+// id = aws_eip.k8s-lb-eip-c.0.id
+//}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/bin/terraform-cli.py b/infrastructure-provisioning/terraform/bin/terraform-cli.py
index 845ff1c..f62e852 100755
--- a/infrastructure-provisioning/terraform/bin/terraform-cli.py
+++ b/infrastructure-provisioning/terraform/bin/terraform-cli.py
@@ -359,11 +359,21 @@ class AWSSourceBuilder(AbstractDeployBuilder):
default='t2.medium')
.add_int('--ssn_root_volume_size', 'Size of root volume in GB.',
default=30)
- .add_str('--subnet_cidr',
- 'CIDR for Subnet creation. Conflicts with subnet_id.',
+ .add_str('--subnet_cidr_a',
+ 'CIDR for Subnet creation in zone a. Conflicts with subnet_id_a.',
default='172.31.0.0/24')
- .add_str('--subnet_id',
- 'ID of AWS Subnet if you already have subnet created.')
+ .add_str('--subnet_cidr_b',
+ 'CIDR for Subnet creation in zone b. Conflicts with subnet_id_b.',
+ default='172.31.1.0/24')
+ .add_str('--subnet_cidr_c',
+ 'CIDR for Subnet creation in zone c. Conflicts with subnet_id_c.',
+ default='172.31.2.0/24')
+ .add_str('--subnet_id_a',
+ 'ID of AWS Subnet in zone a if you already have subnet created.')
+ .add_str('--subnet_id_b',
+ 'ID of AWS Subnet in zone b if you already have subnet created.')
+ .add_str('--subnet_id_c',
+ 'ID of AWS Subnet in zone c if you already have subnet created.')
.add_str('--vpc_cidr', 'CIDR for VPC creation. Conflicts with vpc_id',
default='172.31.0.0/16')
.add_str('--vpc_id', 'ID of AWS VPC if you already have VPC created.')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org