You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/06/21 14:34:40 UTC
[incubator-dlab] 01/01: [DLAB-667] Created Terraform templates for
notebooks and computational resources
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-667
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit db76975024738abe0debd5645de829cc43458749
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Fri Jun 21 17:34:19 2019 +0300
[DLAB-667] Created Terraform templates for notebooks and computational resources
---
infrastructure-provisioning/terraform/main/main.tf | 59 ++++++++++++++
.../terraform/main/variables.tf | 39 ++++++++++
.../terraform/modules/ami/ami.tf | 4 +
.../terraform/modules/ami/variables.tf | 5 ++
.../terraform/modules/common/iam.tf | 90 ++++++++++++++++++++++
.../terraform/modules/common/network.tf | 46 +++++++++++
.../terraform/modules/common/variables.tf | 23 ++++++
.../terraform/modules/data_engine/instance.tf | 42 ++++++++++
.../terraform/modules/data_engine/variables.tf | 23 ++++++
.../terraform/modules/notebook/instance.tf | 19 +++++
.../terraform/modules/notebook/variables.tf | 21 +++++
11 files changed, 371 insertions(+)
diff --git a/infrastructure-provisioning/terraform/main/main.tf b/infrastructure-provisioning/terraform/main/main.tf
new file mode 100644
index 0000000..11abc1e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/main/main.tf
@@ -0,0 +1,59 @@
+provider "aws" {
+ access_key = "${var.access_key_var}"
+ secret_key = "${var.secret_key_var}"
+ region = "${var.region_var}"
+}
+
+module "common" {
+ source = "../modules/common"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ region = "${var.region_var}"
+ zone = "${var.zone_var}"
+ product = "${var.product_name}"
+ vpc = "${var.vpc_id}"
+ cidr_range = "${var.cidr_range}"
+ traefik_cidr = "${var.traefik_cidr}"
+ instance_type = "${var.instance_type}"
+}
+
+module "notebook" {
+ source = "../modules/notebook"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ aws_subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ iam_profile_name = "${var.iam_profile_name}"
+ product = "${var.product_name}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+}
+
+module "data_engine" {
+ source = "../modules/data_engine"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ aws_subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ iam_profile_name = "${var.iam_profile_name}"
+ product = "${var.product_name}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ slave_count = "${var.slave_count}"
+}
+
+module "ami" {
+ source = "../modules/ami"
+ source_instance_id = "${var.source_instance_id}"
+ project_tag = "${var.project_tag}"
+ notebook_name = "${var.notebook_name}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/main/variables.tf b/infrastructure-provisioning/terraform/main/variables.tf
new file mode 100644
index 0000000..9783e9e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/main/variables.tf
@@ -0,0 +1,39 @@
+variable "access_key_var" {}
+
+variable "secret_key_var" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region_var" {}
+
+variable "zone_var" {}
+
+variable "product_name" {}
+
+variable "vpc_id" {}
+
+variable "subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "slave_count" {}
+
+variable "source_instance_id" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/ami/ami.tf b/infrastructure-provisioning/terraform/modules/ami/ami.tf
new file mode 100644
index 0000000..e0e29b7
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/ami/ami.tf
@@ -0,0 +1,4 @@
+resource "aws_ami_from_instance" "ami" {
+ name = "${var.project_tag}-${var.notebook_name}-ami"
+ source_instance_id = "${var.source_instance_id}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/ami/variables.tf b/infrastructure-provisioning/terraform/modules/ami/variables.tf
new file mode 100644
index 0000000..4507958
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/ami/variables.tf
@@ -0,0 +1,5 @@
+variable "source_instance_id" {}
+
+variable "project_tag" {}
+
+variable "notebook_name" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/iam.tf b/infrastructure-provisioning/terraform/modules/common/iam.tf
new file mode 100644
index 0000000..377e64d
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/iam.tf
@@ -0,0 +1,90 @@
+locals {
+ role_name = "${var.project_tag}-nb-de-Role"
+ role_profile = "${var.project_tag}-nb-Profile"
+ policy_name = "${var.project_tag}-strict_to_S3-Policy"
+}
+
+resource "aws_iam_role" "nb_de_role" {
+ name = "${local.role_name}"
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+EOF
+
+ tags = {
+ product = "${var.product}"
+ Name = "${local.role_name}"
+ environment_tag = "${var.project_tag}"
+ }
+}
+
+resource "aws_iam_instance_profile" "nb_profile" {
+ name = "${local.role_profile}"
+ role = "${aws_iam_role.nb_de_role.name}"
+}
+
+resource "aws_iam_policy" "strict_S3_policy" {
+ name = "${local.policy_name}"
+ description = "Strict Bucket only policy"
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "s3:ListAllMyBuckets",
+ "Resource": "arn:aws:s3:::*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket",
+ "s3:GetBucketLocation",
+ "s3:PutBucketPolicy",
+ "s3:PutEncryptionConfiguration"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject",
+ "s3:HeadObject"
+ ],
+ "Resource": "arn:aws:s3:::${var.project_tag}-ssn-bucket/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:HeadObject",
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}-bucket/*",
+ "arn:aws:s3:::${var.project_tag}-shared-bucket/*"
+ ]
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "strict_S3_policy-attach" {
+ role = "${aws_iam_role.nb_de_role.name}"
+ policy_arn = "${aws_iam_policy.strict_S3_policy.arn}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/network.tf b/infrastructure-provisioning/terraform/modules/common/network.tf
new file mode 100644
index 0000000..df9dd10
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/network.tf
@@ -0,0 +1,46 @@
+locals {
+ subnet_name = "${var.project_tag}-subnet"
+ sg_name = "${var.project_tag}-nb-sg" #sg - security group
+}
+
+resource "aws_subnet" "subnet" {
+ vpc_id = "${var.vpc}"
+ cidr_block = "${var.cidr_range}"
+
+ tags = {
+ Name = "${local.subnet_name}"
+ Env-Tag = "${local.subnet_name}"
+ product = "${var.product}"
+ }
+}
+
+resource "aws_security_group" "nb-sg" {
+ name = "${local.sg_name}"
+ vpc_id = "${var.vpc}"
+
+ ingress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["${var.cidr_range}", "${var.traefik_cidr}"]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ egress {
+ from_port = 443
+ to_port = 443
+ protocol = "TCP"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ tags = {
+ Name = "${local.sg_name}"
+ product = "${var.product}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/variables.tf b/infrastructure-provisioning/terraform/modules/common/variables.tf
new file mode 100644
index 0000000..0cd76b1
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region" {}
+
+variable "zone" {}
+
+variable "product" {}
+
+variable "vpc" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "instance_type" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/modules/data_engine/instance.tf
new file mode 100644
index 0000000..6f2e55e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/data_engine/instance.tf
@@ -0,0 +1,42 @@
+locals {
+ node_name = "${var.project_tag}-de-${var.notebook_name}-Spark"
+ dataengine_notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "master" {
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}-m"
+ Type = "master"
+ dataengine_notebook_name = "${local.dataengine_notebook_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}-m"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
+
+
+resource "aws_instance" "slave" {
+ count = "${var.slave_count}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}-s${count.index + 1}"
+ Type = "slave"
+ dataengine_notebook_name = "${local.dataengine_notebook_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}-s${count.index + 1}"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/modules/data_engine/variables.tf
new file mode 100644
index 0000000..b90e326
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/data_engine/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "aws_subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
+
+variable "slave_count" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/modules/notebook/instance.tf
new file mode 100644
index 0000000..e06ad1f
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/notebook/instance.tf
@@ -0,0 +1,19 @@
+locals {
+ node_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "notebook" {
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ User_Tag = "${var.user_tag}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/modules/notebook/variables.tf
new file mode 100644
index 0000000..d308075
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/notebook/variables.tf
@@ -0,0 +1,21 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "aws_subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org