You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/07/02 08:19:21 UTC
[incubator-dlab] 01/01: [DLAB-668] Added terraform scripts for
notebooks and computational resources
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-668
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e31002d1d3b9208a17be85deab706fbdc5c15e0e
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Tue Jul 2 11:19:07 2019 +0300
[DLAB-668] Added terraform scripts for notebooks and computational resources
---
.../terraform/gcp/main/main.tf | 88 ++++++++++++++++++
.../terraform/gcp/main/variables.tf | 55 +++++++++++
.../terraform/gcp/modules/common/iam.tf | 36 ++++++++
.../terraform/gcp/modules/common/network.tf | 39 ++++++++
.../terraform/gcp/modules/common/variables.tf | 39 ++++++++
.../terraform/gcp/modules/data_engine/instance.tf | 101 +++++++++++++++++++++
.../terraform/gcp/modules/data_engine/variables.tf | 35 +++++++
.../terraform/gcp/modules/dataproc/instance.tf | 42 +++++++++
.../terraform/gcp/modules/dataproc/variables.tf | 39 ++++++++
.../terraform/gcp/modules/notebook/instance.tf | 65 +++++++++++++
.../terraform/gcp/modules/notebook/variables.tf | 29 ++++++
11 files changed, 568 insertions(+)
diff --git a/infrastructure-provisioning/terraform/gcp/main/main.tf b/infrastructure-provisioning/terraform/gcp/main/main.tf
new file mode 100644
index 0000000..3cbca78
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/main/main.tf
@@ -0,0 +1,88 @@
+provider "google" {
+ credentials = "${var.credentials}"
+ project = "${var.project_name}"
+ region = "${var.region_var}"
+ zone = "${var.zone_var}"
+}
+
+module "common" {
+ source = "../modules/common"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ region = "${var.region_var}"
+ vpc_name = "${var.vpc_name}"
+ fw_ingress = "${var.fw_ingress}"
+ fw_egress_public = "${var.fw_egress_public}"
+ fw_egress_private = "${var.fw_egress_private}"
+ network_tag = "${var.network_tag}"
+ cidr_range = "${var.cidr_range}"
+ traefik_cidr = "${var.traefik_cidr}"
+}
+
+module "notebook" {
+ source = "../modules/notebook"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ machine_type = "${var.machine_type}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+}
+
+module "data_engine" {
+ source = "../modules/data_engine"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+ cluster_name = "${var.cluster_name}"
+ total_count = "${var.total_count}"
+ master_shape = "${var.master_shape}"
+ slave_shape = "${var.slave_shape}"
+}
+
+module "dataproc" {
+ source = "../modules/dataproc"
+ region = "${var.region_var}"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+ cluster_name = "${var.cluster_name}"
+ total_count = "${var.total_count}"
+ master_shape = "${var.master_shape}"
+ slave_shape = "${var.slave_shape}"
+ preemptible_count = "${var.preemptible_count}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/main/variables.tf b/infrastructure-provisioning/terraform/gcp/main/variables.tf
new file mode 100644
index 0000000..2b328ee
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/main/variables.tf
@@ -0,0 +1,55 @@
+variable "credentials" {}
+
+variable "project_name" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region_var" {}
+
+variable "zone_var" {}
+
+variable "product_name" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "fw_ingress" {}
+
+variable "fw_egress_public" {}
+
+variable "fw_egress_private" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ami" {}
+
+variable "machine_type" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "preemptible_count" {}
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf
new file mode 100644
index 0000000..6931f60
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf
@@ -0,0 +1,36 @@
+locals {
+ service_name = "${var.project_tag}-ps-sa"
+ role_name = "${var.project_tag}-ps-role"
+}
+
+resource "google_service_account" "ps_sa" {
+ #Create service account for notebooks and computational resources
+ account_id = "${var.project_tag}-ps-sa"
+ display_name = "${var.project_tag}-ps-sa"
+}
+
+resource "google_service_account_key" "ps_sa_key" {
+ #Create service account key
+ depends_on = ["google_project_iam_member.iam"]
+ service_account_id = google_service_account.ps_sa.name
+}
+
+resource "google_project_iam_custom_role" "ps-custom-role" {
+ #Create custom role for ps_sa
+ role_id = "${replace("${var.project_tag}-ps-role", "-", "_")}"
+ title = "${var.project_tag}-ps-role"
+ permissions = "${var.ps_policy}"
+}
+
+resource "google_project_iam_member" "role_for_member" {
+ #Grant the custom role for the ps_sa
+ member = "serviceAccount:${google_service_account.ps_sa.email}"
+ role = "${google_project_iam_custom_role.ps-custom-role.id}"
+}
+
+resource "google_project_iam_member" "iam" {
+ #Grant other roles for the ps_sa
+ count = "${length(var.ps_roles)}"
+ member = "serviceAccount:${google_service_account.ps_sa.email}"
+ role = "${element(var.ps_roles, count.index)}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/network.tf b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf
new file mode 100644
index 0000000..8d34249
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf
@@ -0,0 +1,39 @@
+resource "google_compute_subnetwork" "subnet" {
+ name = "${var.project_tag}-subnet"
+ ip_cidr_range = "${var.cidr_range}"
+ region = "${var.region}"
+ network = "${var.vpc_name}"
+}
+
+resource "google_compute_firewall" "fw_ingress" {
+ name = "${var.fw_ingress}"
+ network = "${var.vpc_name}"
+ allow {
+ protocol = "all"
+ }
+ target_tags = ["${var.network_tag}"]
+ source_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"]
+}
+
+resource "google_compute_firewall" "fw_egress_public" {
+ name = "${var.fw_egress_public}"
+ network = "${var.vpc_name}"
+ direction = "EGRESS"
+ allow {
+ protocol = "tcp"
+ ports = ["443"]
+ }
+ target_tags = ["${var.network_tag}"]
+ destination_ranges = ["0.0.0.0/0"]
+}
+
+resource "google_compute_firewall" "fw_egress_private" {
+ name = "${var.fw_egress_private}"
+ network = "${var.vpc_name}"
+ direction = "EGRESS"
+ allow {
+ protocol = "all"
+ }
+ target_tags = ["${var.network_tag}"]
+ destination_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf
new file mode 100644
index 0000000..b4180ad
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf
@@ -0,0 +1,39 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "region" {}
+
+variable "product" {}
+
+variable "vpc_name" {}
+
+variable "fw_ingress" {}
+
+variable "fw_egress_public" {}
+
+variable "fw_egress_private" {}
+
+variable "network_tag" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ps_roles" {
+ type = "list"
+ default = [
+ "roles/dataproc.worker"
+ ]
+}
+
+variable "ps_policy" {
+ type = "list"
+ default = [
+
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf
new file mode 100644
index 0000000..fdb6250
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf
@@ -0,0 +1,101 @@
+locals {
+ notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+ cluster_name = "${var.project_tag}-de-${var.notebook_name}-${var.cluster_name}"
+}
+
+resource "google_compute_instance" "master" {
+ name = "${local.cluster_name}-m"
+ machine_type = "${var.master_shape}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 30
+ }
+ }
+
+ labels = {
+ name = "${local.cluster_name}-m"
+ notebook_name = "${local.notebook_name}"
+ project = "${var.project_tag}"
+ product = "${var.product}"
+ type = "master"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
+
+
+resource "google_compute_instance" "slave" {
+ count = "${var.total_count - 1}"
+ name = "${local.cluster_name}-s${count.index + 1}"
+ machine_type = "${var.slave_shape}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 30
+ }
+ }
+
+ labels = {
+ name = "${local.cluster_name}-s${count.index + 1}"
+ notebook_name = "${local.notebook_name}"
+ project = "${var.project_tag}"
+ product = "${var.product}"
+ sbn = "${var.project_tag}"
+ type = "slave"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf
new file mode 100644
index 0000000..12c4f92
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf
@@ -0,0 +1,35 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf
new file mode 100644
index 0000000..415479c
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf
@@ -0,0 +1,42 @@
+locals {
+ dataproc_name = "${var.project_tag}-des-${var.notebook_name}-${var.cluster_name}"
+}
+
+resource "google_dataproc_cluster" "dataproc" {
+ name = "${local.dataproc_name}"
+ region = "${var.region}"
+ labels = {
+ computational_name = "${var.cluster_name}"
+ name = "${local.dataproc_name}"
+ sbn = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+
+ cluster_config {
+
+ master_config {
+ num_instances = 1
+ machine_type = "${var.master_shape}"
+ disk_config {
+ boot_disk_size_gb = 30
+ }
+ }
+
+ worker_config {
+ num_instances = "${var.total_count - 1}"
+ machine_type = "${var.slave_shape}"
+ disk_config {
+ boot_disk_size_gb = 30
+ }
+ }
+
+ gce_cluster_config {
+ subnetwork = "${var.subnet_name}"
+ tags = ["${var.network_tag}"]
+ }
+
+ preemptible_worker_config {
+ num_instances = "${var.preemptible_count}"
+ }
+ }
+ }
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf
new file mode 100644
index 0000000..d3dcf15
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf
@@ -0,0 +1,39 @@
+variable "region" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "preemptible_count" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf
new file mode 100644
index 0000000..0943e51
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf
@@ -0,0 +1,65 @@
+locals {
+ name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "google_compute_disk" "secondary" {
+ name = "${local.name}-secondary"
+ zone = "${var.zone_var}"
+ labels = {
+ name = "${local.name}"
+ product = "${var.product}"
+ project = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+ physical_block_size_bytes = 4096
+ size = 30
+}
+
+resource "google_compute_instance" "notebook" {
+ name = "${local.name}"
+ machine_type = "${var.machine_type}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 12
+ }
+ }
+
+ attached_disk {
+ source = "${google_compute_disk.secondary.self_link}"
+ }
+
+ labels = {
+ name = "${local.name}"
+ product = "${var.product}"
+ project = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf
new file mode 100644
index 0000000..6217480
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf
@@ -0,0 +1,29 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "machine_type" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org