You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/07/02 08:19:20 UTC
[incubator-dlab] branch DLAB-668 created (now e31002d)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a change to branch DLAB-668
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at e31002d [DLAB-668] Added terraform scripts for notebooks and computational resources
This branch includes the following new commits:
new e31002d [DLAB-668] Added terraform scripts for notebooks and computational resources
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-668] Added terraform scripts for
notebooks and computational resources
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-668
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e31002d1d3b9208a17be85deab706fbdc5c15e0e
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Tue Jul 2 11:19:07 2019 +0300
[DLAB-668] Added terraform scripts for notebooks and computational resources
---
.../terraform/gcp/main/main.tf | 88 ++++++++++++++++++
.../terraform/gcp/main/variables.tf | 55 +++++++++++
.../terraform/gcp/modules/common/iam.tf | 36 ++++++++
.../terraform/gcp/modules/common/network.tf | 39 ++++++++
.../terraform/gcp/modules/common/variables.tf | 39 ++++++++
.../terraform/gcp/modules/data_engine/instance.tf | 101 +++++++++++++++++++++
.../terraform/gcp/modules/data_engine/variables.tf | 35 +++++++
.../terraform/gcp/modules/dataproc/instance.tf | 42 +++++++++
.../terraform/gcp/modules/dataproc/variables.tf | 39 ++++++++
.../terraform/gcp/modules/notebook/instance.tf | 65 +++++++++++++
.../terraform/gcp/modules/notebook/variables.tf | 29 ++++++
11 files changed, 568 insertions(+)
diff --git a/infrastructure-provisioning/terraform/gcp/main/main.tf b/infrastructure-provisioning/terraform/gcp/main/main.tf
new file mode 100644
index 0000000..3cbca78
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/main/main.tf
@@ -0,0 +1,88 @@
+provider "google" {
+ credentials = "${var.credentials}"
+ project = "${var.project_name}"
+ region = "${var.region_var}"
+ zone = "${var.zone_var}"
+}
+
+module "common" {
+ source = "../modules/common"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ region = "${var.region_var}"
+ vpc_name = "${var.vpc_name}"
+ fw_ingress = "${var.fw_ingress}"
+ fw_egress_public = "${var.fw_egress_public}"
+ fw_egress_private = "${var.fw_egress_private}"
+ network_tag = "${var.network_tag}"
+ cidr_range = "${var.cidr_range}"
+ traefik_cidr = "${var.traefik_cidr}"
+}
+
+module "notebook" {
+ source = "../modules/notebook"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ machine_type = "${var.machine_type}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+}
+
+module "data_engine" {
+ source = "../modules/data_engine"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+ cluster_name = "${var.cluster_name}"
+ total_count = "${var.total_count}"
+ master_shape = "${var.master_shape}"
+ slave_shape = "${var.slave_shape}"
+}
+
+module "dataproc" {
+ source = "../modules/dataproc"
+ region = "${var.region_var}"
+ project_tag = "${var.project_tag}"
+ endpoint_tag = "${var.endpoint_tag}"
+ user_tag = "${var.user_tag}"
+ custom_tag = "${var.custom_tag}"
+ product = "${var.product_name}"
+ notebook_name = "${var.notebook_name}"
+ zone_var = "${var.zone_var}"
+ vpc_name = "${var.vpc_name}"
+ subnet_name = "${var.subnet_name}"
+ network_tag = "${var.network_tag}"
+ sa_email = "${var.sa_email}"
+ ami = "${var.ami}"
+ ssh_key = "${var.ssh_key}"
+ gpu_accelerator = "${var.gpu_accelerator}"
+ cluster_name = "${var.cluster_name}"
+ total_count = "${var.total_count}"
+ master_shape = "${var.master_shape}"
+ slave_shape = "${var.slave_shape}"
+ preemptible_count = "${var.preemptible_count}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/main/variables.tf b/infrastructure-provisioning/terraform/gcp/main/variables.tf
new file mode 100644
index 0000000..2b328ee
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/main/variables.tf
@@ -0,0 +1,55 @@
+variable "credentials" {}
+
+variable "project_name" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "notebook_name" {}
+
+variable "region_var" {}
+
+variable "zone_var" {}
+
+variable "product_name" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "fw_ingress" {}
+
+variable "fw_egress_public" {}
+
+variable "fw_egress_private" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ami" {}
+
+variable "machine_type" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "preemptible_count" {}
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf
new file mode 100644
index 0000000..6931f60
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf
@@ -0,0 +1,36 @@
+locals {
+ service_name = "${var.project_tag}-ps-sa"
+ role_name = "${var.project_tag}-ps-role"
+}
+
+resource "google_service_account" "ps_sa" {
+ #Create service account for notebooks and computational resources
+ account_id = "${var.project_tag}-ps-sa"
+ display_name = "${var.project_tag}-ps-sa"
+}
+
+resource "google_service_account_key" "ps_sa_key" {
+ #Create service account key
+ depends_on = ["google_project_iam_member.iam"]
+ service_account_id = google_service_account.ps_sa.name
+}
+
+resource "google_project_iam_custom_role" "ps-custom-role" {
+ #Create custom role for ps_sa
+ role_id = "${replace("${var.project_tag}-ps-role", "-", "_")}"
+ title = "${var.project_tag}-ps-role"
+ permissions = "${var.ps_policy}"
+}
+
+resource "google_project_iam_member" "role_for_member" {
+ #Grant the custom role for the ps_sa
+ member = "serviceAccount:${google_service_account.ps_sa.email}"
+ role = "${google_project_iam_custom_role.ps-custom-role.id}"
+}
+
+resource "google_project_iam_member" "iam" {
+ #Grant other roles for the ps_sa
+ count = "${length(var.ps_roles)}"
+ member = "serviceAccount:${google_service_account.ps_sa.email}"
+ role = "${element(var.ps_roles, count.index)}"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/network.tf b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf
new file mode 100644
index 0000000..8d34249
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf
@@ -0,0 +1,39 @@
+resource "google_compute_subnetwork" "subnet" {
+ name = "${var.project_tag}-subnet"
+ ip_cidr_range = "${var.cidr_range}"
+ region = "${var.region}"
+ network = "${var.vpc_name}"
+}
+
+resource "google_compute_firewall" "fw_ingress" {
+ name = "${var.fw_ingress}"
+ network = "${var.vpc_name}"
+ allow {
+ protocol = "all"
+ }
+ target_tags = ["${var.network_tag}"]
+ source_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"]
+}
+
+resource "google_compute_firewall" "fw_egress_public" {
+ name = "${var.fw_egress_public}"
+ network = "${var.vpc_name}"
+ direction = "EGRESS"
+ allow {
+ protocol = "tcp"
+ ports = ["443"]
+ }
+ target_tags = ["${var.network_tag}"]
+ destination_ranges = ["0.0.0.0/0"]
+}
+
+resource "google_compute_firewall" "fw_egress_private" {
+ name = "${var.fw_egress_private}"
+ network = "${var.vpc_name}"
+ direction = "EGRESS"
+ allow {
+ protocol = "all"
+ }
+ target_tags = ["${var.network_tag}"]
+ destination_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf
new file mode 100644
index 0000000..b4180ad
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf
@@ -0,0 +1,39 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "region" {}
+
+variable "product" {}
+
+variable "vpc_name" {}
+
+variable "fw_ingress" {}
+
+variable "fw_egress_public" {}
+
+variable "fw_egress_private" {}
+
+variable "network_tag" {}
+
+variable "cidr_range" {}
+
+variable "traefik_cidr" {}
+
+variable "ps_roles" {
+ type = "list"
+ default = [
+ "roles/dataproc.worker"
+ ]
+}
+
+variable "ps_policy" {
+ type = "list"
+ default = [
+
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf
new file mode 100644
index 0000000..fdb6250
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf
@@ -0,0 +1,101 @@
+locals {
+ notebook_name = "${var.project_tag}-nb-${var.notebook_name}"
+ cluster_name = "${var.project_tag}-de-${var.notebook_name}-${var.cluster_name}"
+}
+
+resource "google_compute_instance" "master" {
+ name = "${local.cluster_name}-m"
+ machine_type = "${var.master_shape}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 30
+ }
+ }
+
+ labels = {
+ name = "${local.cluster_name}-m"
+ notebook_name = "${local.notebook_name}"
+ project = "${var.project_tag}"
+ product = "${var.product}"
+ type = "master"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
+
+
+resource "google_compute_instance" "slave" {
+ count = "${var.total_count - 1}"
+ name = "${local.cluster_name}-s${count.index + 1}"
+ machine_type = "${var.slave_shape}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 30
+ }
+ }
+
+ labels = {
+ name = "${local.cluster_name}-s${count.index + 1}"
+ notebook_name = "${local.notebook_name}"
+ project = "${var.project_tag}"
+ product = "${var.product}"
+ sbn = "${var.project_tag}"
+ type = "slave"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf
new file mode 100644
index 0000000..12c4f92
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf
@@ -0,0 +1,35 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf
new file mode 100644
index 0000000..415479c
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf
@@ -0,0 +1,42 @@
+locals {
+ dataproc_name = "${var.project_tag}-des-${var.notebook_name}-${var.cluster_name}"
+}
+
+resource "google_dataproc_cluster" "dataproc" {
+ name = "${local.dataproc_name}"
+ region = "${var.region}"
+ labels = {
+ computational_name = "${var.cluster_name}"
+ name = "${local.dataproc_name}"
+ sbn = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+
+ cluster_config {
+
+ master_config {
+ num_instances = 1
+ machine_type = "${var.master_shape}"
+ disk_config {
+ boot_disk_size_gb = 30
+ }
+ }
+
+ worker_config {
+ num_instances = "${var.total_count - 1}"
+ machine_type = "${var.slave_shape}"
+ disk_config {
+ boot_disk_size_gb = 30
+ }
+ }
+
+ gce_cluster_config {
+ subnetwork = "${var.subnet_name}"
+ tags = ["${var.network_tag}"]
+ }
+
+ preemptible_worker_config {
+ num_instances = "${var.preemptible_count}"
+ }
+ }
+ }
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf
new file mode 100644
index 0000000..d3dcf15
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf
@@ -0,0 +1,39 @@
+variable "region" {}
+
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
+
+variable "cluster_name" {}
+
+variable "total_count" {}
+
+variable "master_shape" {}
+
+variable "slave_shape" {}
+
+variable "preemptible_count" {}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf
new file mode 100644
index 0000000..0943e51
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf
@@ -0,0 +1,65 @@
+locals {
+ name = "${var.project_tag}-nb-${var.notebook_name}"
+}
+
+resource "google_compute_disk" "secondary" {
+ name = "${local.name}-secondary"
+ zone = "${var.zone_var}"
+ labels = {
+ name = "${local.name}"
+ product = "${var.product}"
+ project = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+ physical_block_size_bytes = 4096
+ size = 30
+}
+
+resource "google_compute_instance" "notebook" {
+ name = "${local.name}"
+ machine_type = "${var.machine_type}"
+ tags = ["${var.network_tag}"]
+ zone = "${var.zone_var}"
+
+ boot_disk {
+ initialize_params {
+ image = "${var.ami}"
+ size = 12
+ }
+ }
+
+ attached_disk {
+ source = "${google_compute_disk.secondary.self_link}"
+ }
+
+ labels = {
+ name = "${local.name}"
+ product = "${var.product}"
+ project = "${var.project_tag}"
+ user = "${var.user_tag}"
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file("${var.ssh_key}")}"
+ }
+
+ service_account {
+ email = "${var.sa_email}"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = "${var.vpc_name}"
+ subnetwork = "${var.subnet_name}"
+ }
+
+ guest_accelerator {
+ count = "${var.gpu_accelerator != "false" ? 1 : 0}"
+ type = "nvidia-tesla-k80"
+ }
+
+ scheduling {
+ on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}"
+ }
+
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf
new file mode 100644
index 0000000..6217480
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf
@@ -0,0 +1,29 @@
+variable "project_tag" {}
+
+variable "endpoint_tag" {}
+
+variable "user_tag" {}
+
+variable "custom_tag" {}
+
+variable "product" {}
+
+variable "notebook_name" {}
+
+variable "zone_var" {}
+
+variable "vpc_name" {}
+
+variable "subnet_name" {}
+
+variable "network_tag" {}
+
+variable "sa_email" {}
+
+variable "ami" {}
+
+variable "machine_type" {}
+
+variable "ssh_key" {}
+
+variable "gpu_accelerator" {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org