You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/06/21 14:34:39 UTC
[incubator-dlab] branch DLAB-667 created (now db76975)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a change to branch DLAB-667
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at db76975 [DLAB-667] Created Terraform templates for notebooks and computational resources
This branch includes the following new commits:
new db76975 [DLAB-667] Created Terraform templates for notebooks and computational resources
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-667] Created Terraform templates for
notebooks and computational resources
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-667
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit db76975024738abe0debd5645de829cc43458749
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Fri Jun 21 17:34:19 2019 +0300
[DLAB-667] Created Terraform templates for notebooks and computational resources
---
infrastructure-provisioning/terraform/main/main.tf | 59 ++++++++++++++
.../terraform/main/variables.tf | 39 ++++++++++
.../terraform/modules/ami/ami.tf | 4 +
.../terraform/modules/ami/variables.tf | 5 ++
.../terraform/modules/common/iam.tf | 90 ++++++++++++++++++++++
.../terraform/modules/common/network.tf | 46 +++++++++++
.../terraform/modules/common/variables.tf | 23 ++++++
.../terraform/modules/data_engine/instance.tf | 42 ++++++++++
.../terraform/modules/data_engine/variables.tf | 23 ++++++
.../terraform/modules/notebook/instance.tf | 19 +++++
.../terraform/modules/notebook/variables.tf | 21 +++++
11 files changed, 371 insertions(+)
diff --git a/infrastructure-provisioning/terraform/main/main.tf b/infrastructure-provisioning/terraform/main/main.tf
new file mode 100644
index 0000000..11abc1e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/main/main.tf
@@ -0,0 +1,59 @@
+provider "aws" {
+ access_key = "${var.access_key_var}"
+ secret_key = "${var.secret_key_var}"
+ region = "${var.region_var}"
+}
+
+module "common" {
+ source = "../modules/common"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ region = "${var.region_var}"
+ zone = "${var.zone_var}"
+ product = "${var.product_name}"
+ vpc = "${var.vpc_id}"
+ cidr_range = "${var.cidr_range}"
+ traefik_cidr = "${var.traefik_cidr}"
+ instance_type = "${var.instance_type}"
+}
+
+module "notebook" {
+ source = "../modules/notebook"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ aws_subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ iam_profile_name = "${var.iam_profile_name}"
+ product = "${var.product_name}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+}
+
+module "data_engine" {
+ source = "../modules/data_engine"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ notebook_name = "${var.notebook_name}"
+ aws_subnet_id = "${var.subnet_id}"
+ nb-sg_id = "${var.nb-sg_id}"
+ iam_profile_name = "${var.iam_profile_name}"
+ product = "${var.product_name}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ slave_count = "${var.slave_count}"
+}
+
+module "ami" {
+ source = "../modules/ami"
+ source_instance_id = "${var.source_instance_id}"
+ project_tag = "${var.project_tag}"
+ notebook_name = "${var.notebook_name}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/main/variables.tf b/infrastructure-provisioning/terraform/main/variables.tf
new file mode 100644
index 0000000..9783e9e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/main/variables.tf
@@ -0,0 +1,39 @@
+variable "access_key_var" {}
+
+variable "secret_key_var" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region_var" {}
+
+variable "zone_var" {}
+
+variable "product_name" {}
+
+variable "vpc_id" {}
+
+variable "subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "slave_count" {}
+
+variable "source_instance_id" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/ami/ami.tf b/infrastructure-provisioning/terraform/modules/ami/ami.tf
new file mode 100644
index 0000000..e0e29b7
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/ami/ami.tf
@@ -0,0 +1,4 @@
+resource "aws_ami_from_instance" "ami" {
+ name = "${var.project_tag}-${var.notebook_name}-ami"
+ source_instance_id = "${var.source_instance_id}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/ami/variables.tf b/infrastructure-provisioning/terraform/modules/ami/variables.tf
new file mode 100644
index 0000000..4507958
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/ami/variables.tf
@@ -0,0 +1,5 @@
+variable "source_instance_id" {}
+
+variable "project_tag" {}
+
+variable "notebook_name" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/iam.tf b/infrastructure-provisioning/terraform/modules/common/iam.tf
new file mode 100644
index 0000000..377e64d
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/iam.tf
@@ -0,0 +1,90 @@
+locals {
+ role_name = "${var.project_tag}-nb-de-Role"
+ role_profile = "${var.project_tag}-nb-Profile"
+ policy_name = "${var.project_tag}-strict_to_S3-Policy"
+}
+
+resource "aws_iam_role" "nb_de_role" {
+ name = "${local.role_name}"
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+EOF
+
+ tags = {
+ product = "${var.product}"
+ Name = "${local.role_name}"
+ environment_tag = "${var.project_tag}"
+ }
+}
+
+resource "aws_iam_instance_profile" "nb_profile" {
+ name = "${local.role_profile}"
+ role = "${aws_iam_role.nb_de_role.name}"
+}
+
+resource "aws_iam_policy" "strict_S3_policy" {
+ name = "${local.policy_name}"
+ description = "Strict Bucket only policy"
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "s3:ListAllMyBuckets",
+ "Resource": "arn:aws:s3:::*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket",
+ "s3:GetBucketLocation",
+ "s3:PutBucketPolicy",
+ "s3:PutEncryptionConfiguration"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject",
+ "s3:HeadObject"
+ ],
+ "Resource": "arn:aws:s3:::${var.project_tag}-ssn-bucket/*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:HeadObject",
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.project_tag}-bucket/*",
+ "arn:aws:s3:::${var.project_tag}-shared-bucket/*"
+ ]
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "strict_S3_policy-attach" {
+ role = "${aws_iam_role.nb_de_role.name}"
+ policy_arn = "${aws_iam_policy.strict_S3_policy.arn}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/network.tf b/infrastructure-provisioning/terraform/modules/common/network.tf
new file mode 100644
index 0000000..df9dd10
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/network.tf
@@ -0,0 +1,46 @@
+locals {
+ subnet_name = "${var.project_tag}-subnet"
+ sg_name = "${var.project_tag}-nb-sg" #sg - security group
+}
+
+resource "aws_subnet" "subnet" {
+ vpc_id = "${var.vpc}"
+ cidr_block = "${var.cidr_range}"
+
+ tags = {
+ Name = "${local.subnet_name}"
+ Env-Tag = "${local.subnet_name}"
+ product = "${var.product}"
+ }
+}
+
+resource "aws_security_group" "nb-sg" {
+ name = "${local.sg_name}"
+ vpc_id = "${var.vpc}"
+
+ ingress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["${var.cidr_range}", "${var.traefik_cidr}"]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ egress {
+ from_port = 443
+ to_port = 443
+ protocol = "TCP"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ tags = {
+ Name = "${local.sg_name}"
+ product = "${var.product}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/common/variables.tf b/infrastructure-provisioning/terraform/modules/common/variables.tf
new file mode 100644
index 0000000..0cd76b1
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/common/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region" {}
+
+variable "zone" {}
+
+variable "product" {}
+
+variable "vpc" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "instance_type" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/modules/data_engine/instance.tf
new file mode 100644
index 0000000..6f2e55e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/data_engine/instance.tf
@@ -0,0 +1,42 @@
+locals {
+ node_name = "${var.project_tag}-de-${var.notebook_name}-Spark"
+ dataengine_notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "master" {
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}-m"
+ Type = "master"
+ dataengine_notebook_name = "${local.dataengine_notebook_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}-m"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
+
+
+resource "aws_instance" "slave" {
+ count = "${var.slave_count}"
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}-s${count.index + 1}"
+ Type = "slave"
+ dataengine_notebook_name = "${local.dataengine_notebook_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}-s${count.index + 1}"
+ User_tag = "${var.user_tag}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/modules/data_engine/variables.tf
new file mode 100644
index 0000000..b90e326
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/data_engine/variables.tf
@@ -0,0 +1,23 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "aws_subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
+
+variable "slave_count" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/modules/notebook/instance.tf
new file mode 100644
index 0000000..e06ad1f
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/notebook/instance.tf
@@ -0,0 +1,19 @@
+locals {
+ node_name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "aws_instance" "notebook" {
+ ami = "${var.ami}"
+ instance_type = "${var.instance_type}"
+ subnet_id = "${var.aws_subnet_id}"
+ security_groups = ["${var.nb-sg_id}"]
+ iam_instance_profile = "${var.iam_profile_name}"
+ tags = {
+ Name = "${local.node_name}"
+ "${var.project_tag}-Tag" = "${local.node_name}"
+ Endpoint_Tag = "${var.endpoint_tag}"
+ "user:tag" = "${var.project_tag}:${local.node_name}"
+ User_Tag = "${var.user_tag}"
+ Custom_Tag = "${var.custom_tag}"
+ }
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/modules/notebook/variables.tf
new file mode 100644
index 0000000..d308075
--- /dev/null
+++ b/infrastructure-provisioning/terraform/modules/notebook/variables.tf
@@ -0,0 +1,21 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "product" {}
+
+variable "ami" {}
+
+variable "instance_type" {}
+
+variable "aws_subnet_id" {}
+
+variable "nb-sg_id" {}
+
+variable "iam_profile_name" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org