You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/07/01 15:45:55 UTC
[incubator-dlab] branch DLAB-terraform updated: [DLAB-667] Added
terraform scripts for notebooks and computational resources
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-terraform by this push:
new 4d966a2 [DLAB-667] Added terraform scripts for notebooks and computational resources
4d966a2 is described below
commit 4d966a278fa10319f73d9443131bcd10a8f3b179
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Mon Jul 1 18:45:42 2019 +0300
[DLAB-667] Added terraform scripts for notebooks and computational resources
---
.../terraform/aws/main/main.tf | 82 +++++++++++++++++++-
.../terraform/aws/main/variables.tf | 48 +++++++++++-
.../terraform/aws/modules/ami/ami.tf | 4 +
.../terraform/aws/modules/ami/variables.tf | 5 ++
.../terraform/aws/modules/common/iam.tf | 90 ++++++++++++++++++++++
.../terraform/aws/modules/common/network.tf | 46 +++++++++++
.../terraform/aws/modules/common/variables.tf | 23 ++++++
.../terraform/aws/modules/data_engine/instance.tf | 44 +++++++++++
.../terraform/aws/modules/data_engine/variables.tf | 27 +++++++
.../terraform/aws/modules/emr/instance.tf | 59 ++++++++++++++
.../terraform/aws/modules/emr/variables.tf | 34 ++++++++
.../terraform/aws/modules/notebook/instance.tf | 21 +++++
.../terraform/aws/modules/notebook/variables.tf | 23 ++++++
13 files changed, 504 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/terraform/aws/main/main.tf b/infrastructure-provisioning/terraform/aws/main/main.tf
index 881b333..3d77b3c 100644
--- a/infrastructure-provisioning/terraform/aws/main/main.tf
+++ b/infrastructure-provisioning/terraform/aws/main/main.tf
@@ -1,5 +1,7 @@
provider "aws" {
- region = var.region
+ region = var.region
+ access_key = var.access_key_var
+ secret_key = var.secret_key_var
}
module "ssn-k8s" {
@@ -22,3 +24,81 @@ module "ssn-k8s" {
workers_shape = var.workers_shape
os-user = var.os-user
}
+
+module "common" {
+ source = "../modules/common"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ region = "${var.region}"
+ zone = "${var.zone}"
+ product = "${var.product_name}"
+ vpc = "${var.vpc_id}"
+ cidr_range = "${var.note_cidr_range}"
+ traefik_cidr = "${var.traefik_cidr}"
+ instance_type = "${var.instance_type}"
+}
+
+module "notebook" {
+ source = "../modules/notebook"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ note_profile_name = "${var.note_profile_name}"
+ product = "${var.product_name}"
+ note_ami = "${var.note_ami}"
+ instance_type = "${var.instance_type}"
+ key_name = "${var.key_name}"
+}
+
+module "data_engine" {
+ source = "../modules/data_engine"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ note_profile_name = "${var.note_profile_name}"
+ product = "${var.product_name}"
+ note_ami = "${var.note_ami}"
+ instance_type = "${var.instance_type}"
+ key_name = "${var.key_name}"
+ cluster_name = "${var.cluster_name}"
+ slave_count = "${var.slave_count}"
+}
+
+module "emr" {
+ source = "../modules/emr"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ note_profile_name = "${var.note_profile_name}"
+ product = "${var.product_name}"
+ note_ami = "${var.note_ami}"
+ emr_template = "${var.emr_template}"
+ master_shape = "${var.master_shape}"
+ slave_shape = "${var.slave_shape}"
+ key_name = "${var.key_name}"
+ cluster_name = "${var.cluster_name}"
+ instance_count = "${var.instance_count}"
+ bid_price = "${var.bid_price}"
+}
+
+module "ami" {
+ source = "../modules/ami"
+ source_instance_id = "${var.source_instance_id}"
+ project_tag = "${var.project_tag}"
+ notebook_name = "${var.notebook_name}"
+}
diff --git a/infrastructure-provisioning/terraform/aws/main/variables.tf b/infrastructure-provisioning/terraform/aws/main/variables.tf
index 6f86c42..fb1c3c5 100644
--- a/infrastructure-provisioning/terraform/aws/main/variables.tf
+++ b/infrastructure-provisioning/terraform/aws/main/variables.tf
@@ -1,3 +1,7 @@
+variable "access_key_var" {}
+
+variable "secret_key_var" {}
+
variable "region" {
default = "us-west-2"
}
@@ -68,4 +72,46 @@ variable "workers_shape" {
variable "os-user" {
default = "dlab-user"
-}
\ No newline at end of file
+}
+
+variable "project_tag" {
+ default = ""
+}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product_name" {}
+
+variable "nb-sg_id" {}
+
+variable "note_profile_name" {}
+
+variable "note_cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "note_ami" {}
+
+variable "instance_type" {}
+
+variable "cluster_name" {}
+
+variable "slave_count" {}
+
+variable "emr_template" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "instance_count" {}
+
+variable "bid_price" {}
+
+variable "source_instance_id" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/ami/ami.tf b/infrastructure-provisioning/terraform/aws/modules/ami/ami.tf
new file mode 100644
index 0000000..e0e29b7
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/ami/ami.tf
@@ -0,0 +1,4 @@
+resource "aws_ami_from_instance" "ami" {
+ name = "${var.project_tag}-${var.notebook_name}-ami"
+ source_instance_id = "${var.source_instance_id}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/ami/variables.tf b/infrastructure-provisioning/terraform/aws/modules/ami/variables.tf
new file mode 100644
index 0000000..4507958
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/ami/variables.tf
@@ -0,0 +1,5 @@
+variable "source_instance_id" {}
+
+variable "project_tag" {}
+
+variable "notebook_name" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/common/iam.tf b/infrastructure-provisioning/terraform/aws/modules/common/iam.tf
new file mode 100644
index 0000000..377e64d
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/common/iam.tf
@@ -0,0 +1,90 @@
+locals {
+ role_name = "${var.project_tag}-nb-de-Role"
+ role_profile = "${var.project_tag}-nb-Profile"
+ policy_name = "${var.project_tag}-strict_to_S3-Policy"
+}
+
+resource "aws_iam_role" "nb_de_role" {
+ name = "${local.role_name}"
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+EOF
+
+ tags = {
+ product = "${var.product}"
+ Name = "${local.role_name}"
+ environment_tag = "${var.project_tag}"
+ }
+}
+
+resource "aws_iam_instance_profile" "nb_profile" {
+ name = "${local.role_profile}"
+ role = "${aws_iam_role.nb_de_role.name}"
+}
+
+resource "aws_iam_policy" "strict_S3_policy" {
+ name = "${local.policy_name}"
+ description = "Strict Bucket only policy"
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "s3:ListAllMyBuckets",
+ "Resource": "arn:aws:s3:::*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket",
+ "s3:GetBucketLocation",
+ "s3:PutBucketPolicy",
+ "s3:PutEncryptionConfiguration"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject",
+ "s3:HeadObject"
+ ],
+ "Resource": "arn:aws:s3:::${var.project_tag}-ssn-bucket/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:HeadObject",
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}-bucket/*",
+ "arn:aws:s3:::${var.project_tag}-shared-bucket/*"
+ ]
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "strict_S3_policy-attach" {
+ role = "${aws_iam_role.nb_de_role.name}"
+ policy_arn = "${aws_iam_policy.strict_S3_policy.arn}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/common/network.tf b/infrastructure-provisioning/terraform/aws/modules/common/network.tf
new file mode 100644
index 0000000..df9dd10
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/common/network.tf
@@ -0,0 +1,46 @@
+locals {
+ subnet_name = "${var.project_tag}-subnet"
+ sg_name = "${var.project_tag}-nb-sg" #sg - security group
+}
+
+resource "aws_subnet" "subnet" {
+ vpc_id = "${var.vpc}"
+ cidr_block = "${var.cidr_range}"
+
+ tags = {
+ Name = "${local.subnet_name}"
+ Env-Tag = "${local.subnet_name}"
+ product = "${var.product}"
+ }
+}
+
+resource "aws_security_group" "nb-sg" {
+ name = "${local.sg_name}"
+ vpc_id = "${var.vpc}"
+
+ ingress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["${var.cidr_range}", "${var.traefik_cidr}"]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ egress {
+ from_port = 443
+ to_port = 443
+ protocol = "TCP"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ tags = {
+ Name = "${local.sg_name}"
+ product = "${var.product}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/common/variables.tf b/infrastructure-provisioning/terraform/aws/modules/common/variables.tf
new file mode 100644
index 0000000..0cd76b1
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/common/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region" {}
+
+variable "zone" {}
+
+variable "product" {}
+
+variable "vpc" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "instance_type" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/aws/modules/data_engine/instance.tf
new file mode 100644
index 0000000..a0d9b2c
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/data_engine/instance.tf
@@ -0,0 +1,44 @@
+locals {
+ cluster_name = "${var.project_tag}-de-${var.notebook_name}-${var.cluster_name}"
+ notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "master" {
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ key_name = "${var.key_name}"
+ subnet_id = "${var.subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.cluster_name}-m"
+ Type = "master"
+ dataengine_notebook_name = "${local.notebook_name}"
+ "${var.project_tag}-Tag" = "${local.cluster_name}-m"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.cluster_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
+
+
+resource "aws_instance" "slave" {
+ count = "${var.slave_count}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ key_name = "${var.key_name}"
+ subnet_id = "${var.subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.cluster_name}-s${count.index + 1}"
+ Type = "slave"
+ dataengine_notebook_name = "${local.notebook_name}"
+ "${var.project_tag}-Tag" = "${local.cluster_name}-s${count.index + 1}"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.cluster_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/aws/modules/data_engine/variables.tf
new file mode 100644
index 0000000..b2e94fe
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/data_engine/variables.tf
@@ -0,0 +1,27 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "note_ami" {}
+
+variable "instance_type" {}
+
+variable "key_name" {}
+
+variable "subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "note_profile_name" {}
+
+variable "cluster_name" {}
+
+variable "slave_count" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/emr/instance.tf b/infrastructure-provisioning/terraform/aws/modules/emr/instance.tf
new file mode 100644
index 0000000..0e8a79f
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/emr/instance.tf
@@ -0,0 +1,59 @@
+locals {
+ cluster_name = "${var.project_tag}-des-${var.notebook_name}-${var.cluster_name}"
+ notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_emr_cluster" "cluster" {
+ name = "${local.cluster_name}"
+ release_label = "${var.emr_template}"
+ applications = ["Spark"]
+
+ termination_protection = false
+ keep_job_flow_alive_when_no_steps = true
+
+ ec2_attributes {
+ subnet_id = "${var.subnet_id}"
+ emr_managed_master_security_group = "${var.nb-sg_id}"
+ emr_managed_slave_security_group = "${var.nb-sg_id}"
+ instance_profile = "arn:aws:iam::203753054073:instance-profile/EMR_EC2_DefaultRole"
+ }
+
+ master_instance_group {
+ instance_type = "${var.master_shape}"
+ }
+
+ core_instance_group {
+ instance_type = "${var.slave_shape}"
+ instance_count = "${var.instance_count - 1}"
+
+ ebs_config {
+ size = "40"
+ type = "gp2"
+ volumes_per_instance = 1
+ }
+
+ bid_price = "0.${var.bid_price}"
+ }
+
+ ebs_root_volume_size = 100
+
+ tags = {
+ ComputationalName = "${var.cluster_name}"
+ Name = "${local.cluster_name}"
+ Notebook = "${local.notebook_name}"
+ product = "${var.product}"
+ "${var.project_tag}-Tag" = "${local.cluster_name}"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.cluster_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+
+ bootstrap_action {
+ path = "s3://elasticmapreduce/bootstrap-actions/run-if"
+ name = "runif"
+ args = ["instance.isMaster=true", "echo running on master node"]
+ }
+
+ service_role = "arn:aws:iam::203753054073:role/EMR_DefaultRole"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/emr/variables.tf b/infrastructure-provisioning/terraform/aws/modules/emr/variables.tf
new file mode 100644
index 0000000..6049f69
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/emr/variables.tf
@@ -0,0 +1,34 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "note_ami" {}
+
+variable "emr_template" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "key_name" {}
+
+variable "subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "note_profile_name" {}
+
+variable "cluster_name" {}
+
+variable "instance_count" {}
+
+variable "bid_price" {}
+
diff --git a/infrastructure-provisioning/terraform/aws/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/aws/modules/notebook/instance.tf
new file mode 100644
index 0000000..5b07c94
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/notebook/instance.tf
@@ -0,0 +1,21 @@
+locals {
+ node_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "notebook" {
+ ami = "${var.note_ami}"
+ instance_type = "${var.instance_type}"
+ key_name = "${var.key_name}"
+ subnet_id = "${var.subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ product = "${var.product}"
+ User_Tag = "${var.user_tag}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/aws/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/aws/modules/notebook/variables.tf
new file mode 100644
index 0000000..b572871
--- /dev/null
+++ b/infrastructure-provisioning/terraform/aws/modules/notebook/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "note_ami" {}
+
+variable "instance_type" {}
+
+variable "key_name" {}
+
+variable "subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "note_profile_name" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org