You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/08/16 09:35:42 UTC
[incubator-dlab] 02/03: [DLAB-669] Created terraform scripts for
exploratoty and computatoonal resources on Azure[2]
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-669
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 1edbbdedd80210c1a38cef1c677527b2694d14d8
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Fri Aug 16 12:34:52 2019 +0300
[DLAB-669] Created terraform scripts for exploratoty and computatoonal resources on Azure[2]
---
.../azure/computational_resources/main/main.tf | 70 ++++++++--------------
.../computational_resources/main/variables.tf | 24 ++++----
.../modules/common/network.tf | 57 ++----------------
.../modules/common/variables.tf | 4 +-
.../modules/data_engine/instance.tf | 29 +++++----
.../modules/data_engine/variables.tf | 6 +-
.../modules/notebook/instance.tf | 60 +++++++++++++++++--
.../modules/notebook/variables.tf | 42 ++++++++++++-
8 files changed, 164 insertions(+), 128 deletions(-)
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/main/main.tf b/infrastructure-provisioning/terraform/azure/computational_resources/main/main.tf
index 5eb3bc4..26ac8ac 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/main/main.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/main/main.tf
@@ -1,35 +1,26 @@
provider "azurerm" {
- credentials = var.credentials
- region = var.region
+ subscription_id = var.subscription_id
+ client_id = var.client_id
+ client_secret = var.client_secret
+ tenant_id = var.tenant_id
}
-resource "azurerm_resource_group" "env_rg" {
- name = var.service_base_name
- location = var.region
-
- tags = {
- Name = var.service_base_name
- }
-}
-
-
module "common" {
- source = "../modules/common"
- sbn = var.service_base_name
- project_name = var.project_name
- project_tag = var.project_tag
- endpoint_tag = var.endpoint_tag
- user_tag = var.user_tag
- custom_tag = var.custom_tag
- notebook_name = var.notebook_name
- region = var.region
- zone = var.zone
- product = var.product_name
- vpc = var.vpc_id
- cidr_range = var.cidr_range
- traefik_cidr = var.traefik_cidr
- instance_type = var.instance_type
- env_rg = azurerm_resource_group.env_rg.name
+ source = "../modules/common"
+ sbn = var.service_base_name
+ project_name = var.project_name
+ project_tag = var.project_tag
+ endpoint_tag = var.endpoint_tag
+ user_tag = var.user_tag
+ custom_tag = var.custom_tag
+ notebook_name = var.notebook_name
+ region = var.region
+ product = var.product_name
+ vpc = var.vpc_id
+ cidr_range = var.cidr_range
+ traefik_cidr = var.traefik_cidr
+ instance_type = var.instance_type
+ resource_group = var.resource_group
}
module "notebook" {
@@ -40,16 +31,18 @@ module "notebook" {
endpoint_tag = var.endpoint_tag
user_tag = var.user_tag
custom_tag = var.custom_tag
+ os_env = var.os_env
notebook_name = var.notebook_name
region = var.region
subnet_id = var.subnet_id
nb-sg_id = var.nb-sg_id
product = var.product_name
ami = var.ami
+ custom_ami = var.custom_ami
instance_type = var.instance_type
ssh_key = var.ssh_key
initial_user = var.initial_user
- env_rg = azurerm_resource_group.env_rg.name
+ resource_group = var.resource_group
}
module "data_engine" {
@@ -66,24 +59,11 @@ module "data_engine" {
nb-sg_id = var.nb-sg_id
product = var.product_name
ami = var.ami
- instance_type = var.instance_type
+ master_shape = var.master_shape
+ slave_shape = var.slave_shape
ssh_key = var.ssh_key
initial_user = var.initial_user
cluster_name = var.cluster_name
slave_count = var.slave_count
- env_rg = azurerm_resource_group.env_rg.name
-}
-
-module "ami" {
- source = "../modules/ami"
- sbn = var.service_base_name
- project_name = var.project_name
- source_instance_id = var.source_instance_id
- project_tag = var.project_tag
- notebook_name = var.notebook_name
- product = var.product_name
- endpoint_tag = var.endpoint_tag
- user_tag = var.user_tag
- custom_tag = var.custom_tag
- env_rg = azurerm_resource_group.env_rg.name
+ resource_group = var.resource_group
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/main/variables.tf b/infrastructure-provisioning/terraform/azure/computational_resources/main/variables.tf
index 309556e..3171558 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/main/variables.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/main/variables.tf
@@ -1,7 +1,15 @@
-variable "credentials" {}
+variable "subscription_id" {}
+
+variable "client_id" {}
+
+variable "client_secret" {}
+
+variable "tenant_id" {}
variable "service_base_name" {}
+variable "resource_group" {}
+
variable "project_name" {}
variable "project_tag" {}
@@ -12,12 +20,12 @@ variable "user_tag" {}
variable "custom_tag" {}
+variable "os_env" {}
+
variable "notebook_name" {}
variable "region" {}
-variable "zone" {}
-
variable "product_name" {}
variable "vpc_id" {}
@@ -26,14 +34,14 @@ variable "subnet_id" {}
variable "nb-sg_id" {}
-variable "iam_profile_name" {}
-
variable "cidr_range" {}
variable "traefik_cidr" {}
variable "ami" {}
+variable "custom_ami" {}
+
variable "instance_type" {}
variable "ssh_key" {}
@@ -44,14 +52,8 @@ variable "cluster_name" {}
variable "slave_count" {}
-variable "emr_template" {}
-
variable "master_shape" {}
variable "slave_shape" {}
-variable "instance_count" {}
-
-variable "bid_price" {}
-
variable "source_instance_id" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/network.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/network.tf
index c00cc51..2ba6da5 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/network.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/network.tf
@@ -26,7 +26,7 @@ locals {
resource "azurerm_subnet" "subnet" {
name = local.subnet_name
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
virtual_network_name = var.vpc
address_prefix = var.cidr_range
}
@@ -34,7 +34,7 @@ resource "azurerm_subnet" "subnet" {
resource "azurerm_network_security_group" "nb-sg" {
name = local.sg_name
location = var.region
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
security_rule {
name = "in-1"
@@ -108,55 +108,10 @@ resource "azurerm_network_security_group" "nb-sg" {
}
}
-resource "aws_subnet" "subnet" {
- vpc_id = var.vpc
- cidr_block = var.cidr_range
-
- tags = {
- Name = local.subnet_name
- SBN = var.sbn
- Product = var.product
- Project_name = var.project_name
- Project_tag = var.project_tag
- Endpoint_tag = var.endpoint_tag
- User_tag = var.user_tag
- Custom_tag = var.custom_tag
- }
+output "subnet_id" {
+ value = azurerm_subnet
}
-resource "aws_security_group" "nb-sg" {
- name = local.sg_name
- vpc_id = var.vpc
-
- ingress {
- from_port = 0
- to_port = 0
- protocol = "-1"
- cidr_blocks = ["${var.cidr_range}", "${var.traefik_cidr}"]
- }
-
- egress {
- from_port = 0
- to_port = 0
- protocol = "-1"
- cidr_blocks = ["0.0.0.0/0"]
- }
-
- egress {
- from_port = 443
- to_port = 443
- protocol = "TCP"
- cidr_blocks = ["0.0.0.0/0"]
- }
-
- tags = {
- Name = local.sg_name
- SBN = var.sbn
- Product = var.product
- Project_name = var.project_name
- Project_tag = var.project_tag
- Endpoint_tag = var.endpoint_tag
- User_tag = var.user_tag
- Custom_tag = var.custom_tag
- }
+output "nb-sg_id" {
+ value = azurerm_network_security_group
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/variables.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/variables.tf
index 4393491..4b1b398 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/variables.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/common/variables.tf
@@ -35,8 +35,6 @@ variable "notebook_name" {}
variable "region" {}
-variable "zone" {}
-
variable "product" {}
variable "vpc" {}
@@ -47,4 +45,4 @@ variable "traefik_cidr" {}
variable "instance_type" {}
-variable "env_rg" {}
\ No newline at end of file
+variable "resource_group" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/instance.tf
index d4253aa..470e474 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/instance.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/instance.tf
@@ -28,13 +28,13 @@ locals {
resource "azurerm_network_interface" "master-nic" {
name = "${local.nic}-m"
location = var.region
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
network_security_group_id = var.nb-sg_id
ip_configuration {
name = "${local.nic}-m-IPconigurations"
subnet_id = var.subnet_id
- private_ip_address_allocation = "Static"
+ private_ip_address_allocation = "Dynamic"
}
tags = {
@@ -52,16 +52,19 @@ resource "azurerm_network_interface" "master-nic" {
resource "azurerm_virtual_machine" "master" {
name = "${local.cluster_name}-m"
location = var.region
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
network_interface_ids = ["${azurerm_network_interface.master-nic.id}"]
- vm_size = var.instance_type
+ vm_size = var.master_shape
storage_os_disk {
name = "${local.cluster_name}-m-disk0"
caching = "ReadWrite"
create_option = "FromImage"
managed_disk_type = "Premium_LRS"
- image_uri = var.ami #publisherName:offer:skus:version
+ }
+
+ storage_image_reference {
+ id = var.ami
}
os_profile {
@@ -96,13 +99,13 @@ resource "azurerm_network_interface" "slave-nic" {
count = var.slave_count
name = "${local.nic}-s-${count.index + 1}"
location = var.region
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
network_security_group_id = var.nb-sg_id
ip_configuration {
name = "${local.nic}-s-${count.index + 1}-IPconigurations"
subnet_id = var.subnet_id
- private_ip_address_allocation = "Static"
+ private_ip_address_allocation = "Dynamic"
}
tags = {
@@ -118,18 +121,22 @@ resource "azurerm_network_interface" "slave-nic" {
}
resource "azurerm_virtual_machine" "slave" {
+ count = var.slave_count
name = "${local.cluster_name}-s-${count.index + 1}"
location = var.region
- resource_group_name = var.sbn
- network_interface_ids = ["${azurerm_network_interface.slave-nic.id}"]
- vm_size = var.instance_type
+ resource_group_name = var.resource_group
+ network_interface_ids = ["${azurerm_network_interface.slave-nic[count.index].id}"]
+ vm_size = var.slave_shape
storage_os_disk {
name = "${local.notebook_name}-s-${count.index + 1}-disk0"
caching = "ReadWrite"
create_option = "FromImage"
managed_disk_type = "Premium_LRS"
- image_uri = var.ami #publisherName:offer:skus:version
+ }
+
+ storage_image_reference {
+ id = var.ami
}
os_profile {
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/variables.tf
index 7c4d38d..e18e812 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/variables.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/data_engine/variables.tf
@@ -39,7 +39,9 @@ variable "product" {}
variable "ami" {}
-variable "instance_type" {}
+variable "master_shape" {}
+
+variable "slave_shape" {}
variable "ssh_key" {}
@@ -53,4 +55,4 @@ variable "cluster_name" {}
variable "slave_count" {}
-variable "env_rg" {}
\ No newline at end of file
+variable "resource_group" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/instance.tf
index 88d50d9..0283038 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/instance.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/instance.tf
@@ -27,13 +27,13 @@ locals {
resource "azurerm_network_interface" "nic" {
name = local.nic
location = var.region
- resource_group_name = var.env_rg
+ resource_group_name = var.resource_group
network_security_group_id = var.nb-sg_id
ip_configuration {
name = "${local.nic}-IPconigurations"
subnet_id = var.subnet_id
- private_ip_address_allocation = "Static"
+ private_ip_address_allocation = "dynamic"
}
tags = {
@@ -50,9 +50,10 @@ resource "azurerm_network_interface" "nic" {
}
resource "azurerm_virtual_machine" "instance" {
+ count = var.custom_ami == true ? 0 : 1
name = local.node_name
location = var.region
- resource_group_name = var.sbn
+ resource_group_name = var.resource_group
network_interface_ids = ["${azurerm_network_interface.nic.id}"]
vm_size = var.instance_type
@@ -61,7 +62,58 @@ resource "azurerm_virtual_machine" "instance" {
caching = "ReadWrite"
create_option = "FromImage"
managed_disk_type = "Premium_LRS"
- image_uri = var.ami #publisherName:offer:skus:version
+ }
+
+ storage_image_reference {
+ publisher = var.ami_publisher[var.os_env]
+ offer = var.ami_offer[var.os_env]
+ sku = var.ami_sku[var.os_env]
+ version = var.ami_version[var.os_env]
+ }
+
+ os_profile {
+ computer_name = local.node_name
+ admin_username = var.initial_user
+ }
+
+ os_profile_linux_config {
+ disable_password_authentication = true
+ ssh_keys {
+ path = "/home/${var.initial_user}/.ssh/authorized_keys"
+ key_data = "${file("${var.ssh_key}")}"
+ }
+ }
+
+ tags = {
+ Exploratory = var.notebook_name
+ SBN = var.sbn
+ Name = local.node_name
+ Project_name = var.project_name
+ Project_tag = var.project_tag
+ Endpoint_Tag = var.endpoint_tag
+ Product = var.product
+ User_Tag = var.user_tag
+ Custom_Tag = var.custom_tag
+ }
+}
+
+resource "azurerm_virtual_machine" "instance_custom" {
+ count = var.custom_ami == true ? 1 : 0
+ name = local.node_name
+ location = var.region
+ resource_group_name = var.resource_group
+ network_interface_ids = ["${azurerm_network_interface.nic.id}"]
+ vm_size = var.instance_type
+
+ storage_os_disk {
+ name = "${local.node_name}-disk0"
+ caching = "ReadWrite"
+ create_option = "FromImage"
+ managed_disk_type = "Premium_LRS"
+ }
+
+ storage_image_reference {
+ id = var.ami
}
os_profile {
diff --git a/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/variables.tf
index cd9d897..547dce0 100644
--- a/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/variables.tf
+++ b/infrastructure-provisioning/terraform/azure/computational_resources/modules/notebook/variables.tf
@@ -49,4 +49,44 @@ variable "subnet_id" {}
variable "nb-sg_id" {}
-variable "env_rg" {}
+variable "resource_group" {}
+
+variable "ami_publisher" {
+ type = "map"
+ default = {
+ debian = "Canonical"
+ redhat = "RedHat"
+ custom = ""
+ }
+}
+
+variable "ami_offer" {
+ type = "map"
+ default = {
+ debian = "UbuntuServer"
+ redhat = "RHEL"
+ custom = ""
+ }
+}
+
+variable "ami_sku" {
+ type = "map"
+ default = {
+ debian = "16.04-LTS"
+ redhat = "7.3"
+ custom = ""
+ }
+}
+
+variable "ami_version" {
+ type = "map"
+ default = {
+ debian = "16.04.201907290"
+ redhat = "7.3.2017090800"
+ custom = ""
+ }
+}
+
+variable "custom_ami" {}
+
+variable "os_env" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org