You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by ad...@apache.org on 2019/09/13 07:50:26 UTC
[incubator-dlab] branch merge_gcp_endpoint_terraform created (now
292443f)
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a change to branch merge_gcp_endpoint_terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 292443f Added Endpoint for GCP by infrastructure-provisioning/terraform/gcp/endpoint
This branch includes the following new commits:
new 292443f Added Endpoint for GCP by infrastructure-provisioning/terraform/gcp/endpoint
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: Added Endpoint for GCP by
infrastructure-provisioning/terraform/gcp/endpoint
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a commit to branch merge_gcp_endpoint_terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 292443f4084263860a1511a66e6cd1eaaf9d19b7
Author: AdamsDisturber <ad...@gmail.com>
AuthorDate: Fri Sep 13 10:50:13 2019 +0300
Added Endpoint for GCP by
infrastructure-provisioning/terraform/gcp/endpoint
---
.../terraform/gcp/endpoint/daemon.json | 5 +
.../terraform/gcp/endpoint/main/iam.tf | 49 ++
.../terraform/gcp/endpoint/main/instance.tf | 60 +++
.../terraform/gcp/endpoint/main/main.tf | 27 ++
.../terraform/gcp/endpoint/main/network.tf | 74 +++
.../terraform/gcp/endpoint/main/variables.tf | 144 ++++++
.../terraform/gcp/endpoint/provisioning.py | 495 +++++++++++++++++++++
.../terraform/gcp/endpoint/provisioning.yml | 151 +++++++
.../terraform/gcp/endpoint/supervisor_svc.conf | 35 ++
9 files changed, 1040 insertions(+)
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/daemon.json b/infrastructure-provisioning/terraform/gcp/endpoint/daemon.json
new file mode 100644
index 0000000..94cf208
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/daemon.json
@@ -0,0 +1,5 @@
+{
+ "insecure-registries": ["REPOSITORY"],
+ "disable-legacy-registry": true,
+ "dns": ["8.8.8.8", "8.8.4.4"]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/iam.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/iam.tf
new file mode 100644
index 0000000..41587fa
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/iam.tf
@@ -0,0 +1,49 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+locals {
+ endpoint_policy_name = "${var.service_base_name}-${var.endpoint_id}-endpoint-policy"
+ endpoint_role_name = "${var.service_base_name}-${var.endpoint_id}-endpoint_role"
+ service_account_name = "${var.service_base_name}-${var.endpoint_id}-endpoint-sa"
+}
+
+resource "google_service_account" "endpoint_sa" {
+ account_id = local.service_account_name
+ display_name = local.service_account_name
+}
+
+resource "google_project_iam_custom_role" "endpoint_role" {
+ permissions = var.endpoint_policies
+ role_id = "${replace("${local.endpoint_role_name}", "-", "_")}"
+ title = local.endpoint_role_name
+}
+
+resource "google_project_iam_member" "iam" {
+ # try to set perms as file
+ count = length(var.endpoint_roles)
+ member = "serviceAccount:${google_service_account.endpoint_sa.email}"
+ role = element(var.endpoint_roles, count.index)
+}
+
+resource "google_project_iam_member" "role_for_member" {
+ member = "serviceAccount:${google_service_account.endpoint_sa.email}"
+ role = google_project_iam_custom_role.endpoint_role.id
+}
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf
new file mode 100644
index 0000000..e2c8eef
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf
@@ -0,0 +1,60 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+locals {
+ endpoint_instance_name = "${var.service_base_name}-${var.endpoint_id}-endpoint"
+}
+
+resource "google_compute_instance" "endpoint" {
+ name = local.endpoint_instance_name
+ machine_type = var.endpoint_shape
+ tags = ["${replace("${local.endpoint_instance_name}", "_", "-")}"]
+ labels = {
+ name = "${local.endpoint_instance_name}"
+ sbn = "${var.service_base_name}"
+ product = "${var.product}"
+ }
+ zone = var.zone
+
+ boot_disk {
+ initialize_params {
+ image = var.ami
+ size = var.endpoint_volume_size
+ }
+ }
+
+ metadata = {
+ ssh-keys = "ubuntu:${file(var.path_to_pub_key)}"
+ }
+
+ service_account {
+ email = google_service_account.endpoint_sa.email #"${var.project_name_var}-ssn-sa@${var.project_var}.iam.gserviceaccount.com"
+ scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
+ }
+
+ network_interface {
+ network = data.google_compute_network.endpoint_vpc_data.name
+ subnetwork = data.google_compute_subnetwork.endpoint_subnet_data.name
+ access_config {
+ nat_ip = var.endpoint_eip
+ }
+ }
+}
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf
new file mode 100644
index 0000000..76b4ada
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf
@@ -0,0 +1,27 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+provider "google" {
+ credentials = file(var.creds_file)
+ project = var.project_name
+ region = var.region
+ zone = var.zone
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/network.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/network.tf
new file mode 100644
index 0000000..417760e
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/network.tf
@@ -0,0 +1,74 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+locals {
+ vpc_name = "${var.service_base_name}-${var.endpoint_id}-endpoint-vpc"
+ subnet_name = "${var.service_base_name}-${var.endpoint_id}-endpoint-subnet"
+ firewall_ingress_name = "${var.service_base_name}-${var.endpoint_id}-ing-rule"
+ firewall_egress_name = "${var.service_base_name}-${var.endpoint_id}-eg-rule"
+}
+
+resource "google_compute_network" "endpoint_vpc" {
+ count = var.vpc_name == "" ? 1 : 0
+ name = local.vpc_name
+ auto_create_subnetworks = false
+}
+
+data "google_compute_network" "endpoint_vpc_data" {
+ name = var.vpc_name == "" ? google_compute_network.endpoint_vpc.0.name : var.vpc_name
+}
+
+resource "google_compute_subnetwork" "endpoint_subnet" {
+ count = var.subnet_name == "" ? 1 : 0
+ name = local.subnet_name
+ ip_cidr_range = var.subnet_cidr
+ region = var.region
+ network = data.google_compute_network.endpoint_vpc_data.id
+}
+
+data "google_compute_subnetwork" "endpoint_subnet_data" {
+ name = var.subnet_name == "" ? google_compute_subnetwork.endpoint_subnet.0.name : var.subnet_name
+}
+
+resource "google_compute_firewall" "firewall-ingress" {
+ count = var.vpc_name == "" ? 1 : 0
+ name = local.firewall_ingress_name
+ network = data.google_compute_network.endpoint_vpc_data.name
+ allow {
+ protocol = "all"
+ ports = ["22", "8084", "8085"]
+ }
+ target_tags = ["${var.service_base_name}-${var.endpoint_id}-endpoint"]
+ source_ranges = ["${var.firewall_ing_cidr_range}"]
+
+}
+
+resource "google_compute_firewall" "firewall-egress" {
+ count = var.vpc_name == "" ? 1 : 0
+ name = local.firewall_egress_name
+ network = data.google_compute_network.endpoint_vpc_data.name
+ direction = "EGRESS"
+ allow {
+ protocol = "all"
+ }
+ target_tags = ["${var.service_base_name}-${var.endpoint_id}-endpoint"]
+ destination_ranges = ["${var.firewall_eg_cidr_range}"]
+}
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf
new file mode 100644
index 0000000..41f6964
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf
@@ -0,0 +1,144 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+# id of gcp project
+variable "project_name" {
+ default = ""
+}
+# path to .json file with creds
+variable "creds_file" {
+ default = ""
+}
+
+variable "endpoint_shape" {
+ default = "n1-standard-2"
+}
+# for example <us-west1>
+variable "region" {
+ default = ""
+}
+# for example <us-west1-a>
+variable "zone" {
+ default = ""
+}
+
+variable "service_base_name" {
+ default = ""
+}
+
+variable "endpoint_id" {
+ default = ""
+}
+
+variable "vpc_name" {
+ default = ""
+}
+
+variable "ami" {
+ default = "/projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20190628"
+}
+
+variable "subnet_name" {
+ default = ""
+}
+
+variable "endpoint_volume_size" {
+ default = "20"
+}
+
+variable "subnet_cidr" {
+ default = "172.31.0.0/24"
+}
+# TEMPORARY
+variable "firewall_ing_cidr_range" {
+ default = "0.0.0.0/0"
+}
+# created by ssn (bcs of certs)
+variable "endpoint_eip" {
+ default = ""
+}
+
+variable "firewall_eg_cidr_range" {
+ default = "0.0.0.0/0"
+}
+
+variable "endpoint_policies" {
+ type = "list"
+ default = [
+ "storage.buckets.create",
+ "storage.buckets.delete",
+ "storage.buckets.get",
+ "storage.buckets.getIamPolicy",
+ "storage.buckets.list",
+ "storage.buckets.setIamPolicy",
+ "storage.buckets.update",
+ "storage.objects.create",
+ "storage.objects.delete",
+ "storage.objects.get",
+ "storage.objects.getIamPolicy",
+ "storage.objects.list",
+ "storage.objects.setIamPolicy",
+ "storage.objects.update",
+ "compute.autoscalers.get",
+ "compute.instances.get",
+ "compute.healthChecks.get",
+ "compute.addresses.create",
+ "compute.addresses.delete",
+ "compute.firewalls.create",
+ "compute.firewalls.delete",
+ "compute.firewalls.get",
+ "compute.firewalls.list",
+ "compute.images.create",
+ "compute.images.delete",
+ "compute.images.get",
+ "compute.images.list",
+ "compute.images.setLabels",
+ "compute.networks.create",
+ "compute.networks.delete",
+ "compute.networks.updatePolicy",
+ "compute.projects.setCommonInstanceMetadata",
+ "compute.projects.setDefaultServiceAccount",
+ "compute.subnetworks.create",
+ "compute.subnetworks.delete"
+ ]
+}
+
+variable "endpoint_roles" {
+ type = "list"
+ default = [
+ "roles/iam.serviceAccountUser",
+ "roles/iam.serviceAccountAdmin",
+ "roles/storage.admin",
+ "roles/dataproc.editor",
+ "roles/resourcemanager.projectIamAdmin",
+ "roles/iam.roleAdmin",
+ "roles/compute.instanceAdmin",
+ "roles/bigquery.dataViewer",
+ "roles/bigquery.jobUser"
+ ]
+}
+# path for public key to connect to instance
+variable "path_to_pub_key" {
+ default = ""
+}
+
+variable "product" {
+ default = "dlab"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
new file mode 100644
index 0000000..09e61d7
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
@@ -0,0 +1,495 @@
+from fabric import Connection
+from patchwork.files import exists
+import logging
+import argparse
+import sys
+import traceback
+import time
+
+conn = None
+args = None
+java_home = None
+
+
+def create_user():
+ initial_user = 'ubuntu'
+ sudo_group = 'sudo'
+ with Connection(host=args.hostname, user=initial_user,
+ connect_kwargs={'key_filename': args.pkey}) as conn:
+ try:
+ if not exists(conn,
+ '/home/{}/.ssh_user_ensured'.format(initial_user)):
+ conn.sudo('useradd -m -G {1} -s /bin/bash {0}'
+ .format(args.os_user, sudo_group))
+ conn.sudo(
+ 'bash -c \'echo "{} ALL = NOPASSWD:ALL" >> /etc/sudoers\''
+ .format(args.os_user, initial_user))
+ conn.sudo('mkdir /home/{}/.ssh'.format(args.os_user))
+ conn.sudo('chown -R {0}:{0} /home/{1}/.ssh/'
+ .format(initial_user, args.os_user))
+ conn.sudo('cat /home/{0}/.ssh/authorized_keys > '
+ '/home/{1}/.ssh/authorized_keys'
+ .format(initial_user, args.os_user))
+ conn.sudo(
+ 'chown -R {0}:{0} /home/{0}/.ssh/'.format(args.os_user))
+ conn.sudo('chmod 700 /home/{0}/.ssh'.format(args.os_user))
+ conn.sudo('chmod 600 /home/{0}/.ssh/authorized_keys'
+ .format(args.os_user))
+ conn.sudo(
+ 'touch /home/{}/.ssh_user_ensured'.format(initial_user))
+ except Exception as err:
+ logging.error('Failed to create new os_user: ', str(err))
+ sys.exit(1)
+
+
+def copy_keys():
+ try:
+ conn.put(args.pkey, '/tmp/')
+ conn.sudo('mv /tmp/{0} /home/{1}/keys/'
+ .format(args.key_name, args.os_user))
+ conn.sudo('chown -R {0}:{0} /home/{0}/keys'.format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to copy keys ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_dir_endpoint():
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir'.format(args.os_user)):
+ conn.sudo('mkdir /home/{}/.ensure_dir'.format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to create ~/.ensure_dir/: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_logs_endpoint():
+ log_root_dir = "/var/opt/dlab/log"
+ supervisor_log_file = "/var/log/application/provision-service.log"
+ try:
+ if not exists(conn, '/home/' + args.os_user + '/.ensure_dir/logs_ensured'):
+ if not exists(conn, args.dlab_path):
+ conn.sudo("mkdir -p " + args.dlab_path)
+ conn.sudo("chown -R " + args.os_user + ' ' + args.dlab_path)
+ if not exists(conn, log_root_dir):
+ conn.sudo('mkdir -p ' + log_root_dir + '/provisioning')
+ conn.sudo('touch ' + log_root_dir + '/provisioning/provisioning.log')
+ if not exists(conn, supervisor_log_file):
+ conn.sudo("mkdir -p /var/log/application")
+ conn.sudo("touch " + supervisor_log_file)
+ conn.sudo("chown -R {0} {1}".format(args.os_user, log_root_dir))
+ conn.sudo('touch /home/' + args.os_user + '/.ensure_dir/logs_ensured')
+ except Exception as err:
+ print('Failed to configure logs and dlab directory: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_jre_jdk_endpoint():
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir/jre_jdk_ensured'
+ .format(args.os_user)):
+ conn.sudo('apt-get install -y openjdk-8-jre-headless')
+ conn.sudo('apt-get install -y openjdk-8-jdk-headless')
+ conn.sudo('touch /home/{}/.ensure_dir/jre_jdk_ensured'
+ .format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to install Java JDK: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_supervisor_endpoint():
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir/superv_ensured'
+ .format(args.os_user)):
+ conn.sudo('apt-get -y install supervisor')
+ conn.sudo('update-rc.d supervisor defaults')
+ conn.sudo('update-rc.d supervisor enable')
+ conn.sudo('touch /home/{}/.ensure_dir/superv_ensured'
+ .format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to install Supervisor: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_docker_endpoint():
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir/docker_ensured'
+ .format(args.os_user)):
+ conn.sudo("bash -c "
+ "'curl -fsSL https://download.docker.com/linux/ubuntu/gpg"
+ " | apt-key add -'")
+ conn.sudo('add-apt-repository "deb [arch=amd64] '
+ 'https://download.docker.com/linux/ubuntu '
+ '$(lsb_release -cs) stable"')
+ conn.sudo('apt-get update')
+ conn.sudo('apt-cache policy docker-ce')
+ conn.sudo('apt-get install -y docker-ce={}'
+ .format(args.docker_version))
+ # dns_ip_resolve = (conn.run("systemd-resolve --status "
+ # "| grep -A 5 'Current Scopes: DNS' "
+ # "| grep 'DNS Servers:' "
+ # "| awk '{print $3}'")
+ # .stdout.rstrip("\n\r"))
+ if not exists(conn, '{}/tmp'.format(args.dlab_path)):
+ conn.run('mkdir -p {}/tmp'.format(args.dlab_path))
+ conn.put('./daemon.json',
+ '{}/tmp/daemon.json'.format(args.dlab_path))
+ conn.sudo('sed -i "s|REPOSITORY|{}:{}|g" {}/tmp/daemon.json'
+ .format(args.repository_address,
+ args.repository_port,
+ args.dlab_path))
+ # conn.sudo('sed -i "s|DNS_IP_RESOLVE|{}|g" {}/tmp/daemon.json'
+ # .format(dns_ip_resolve, args.dlab_path))
+ conn.sudo('mv {}/tmp/daemon.json /etc/docker'
+ .format(args.dlab_path))
+ conn.sudo('usermod -a -G docker ' + args.os_user)
+ conn.sudo('update-rc.d docker defaults')
+ conn.sudo('update-rc.d docker enable')
+ conn.sudo('service docker restart')
+ conn.sudo('touch /home/{}/.ensure_dir/docker_ensured'
+ .format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to install Docker: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def create_key_dir_endpoint():
+ try:
+ if not exists(conn, '/home/{}/keys'.format(args.os_user)):
+ conn.run('mkdir /home/{}/keys'.format(args.os_user))
+ except Exception as err:
+ logging.error('Failed create keys directory as ~/keys: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_keystore_endpoint(os_user):
+ try:
+ # conn.sudo('apt install -y gsutil')
+ # conn.sudo('bash -c \'echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] '
+ # 'https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a '
+ # '/etc/apt/sources.list.d/google-cloud-sdk.list \' ')
+ # conn.sudo('apt install -y apt-transport-https ca-certificates')
+ # conn.sudo('bash -c \'curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key '
+ # '--keyring /usr/share/keyrings/cloud.google.gpg add - \' ')
+ # conn.sudo('bash -c \'apt-get update && sudo apt-get install google-cloud-sdk \' ')
+ if not exists(conn, '/home/' + args.os_user + '/keys/endpoint.keystore.jks'):
+ conn.sudo('gsutil -m cp -r gs://{0}/dlab/certs/endpoint/endpoint.keystore.jks '
+ '/home/{1}/keys/'
+ .format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/dlab.crt'):
+ conn.sudo('gsutil -m cp -r gs://{0}/dlab/certs/endpoint/endpoint.crt'
+ ' /home/{1}/keys/'.format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/ssn.crt'):
+ conn.sudo('gsutil -m cp -r '
+ 'gs://{0}/dlab/certs/ssn/ssn.crt /home/{1}/keys/'
+ .format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/.ensure_dir/cert_imported'):
+ conn.sudo('keytool -importcert -trustcacerts -alias dlab -file /home/{0}/keys/endpoint.crt -noprompt \
+ -storepass changeit -keystore {1}/lib/security/cacerts'.format(os_user, java_home))
+ conn.sudo('keytool -importcert -trustcacerts -file /home/{0}/keys/ssn.crt -noprompt \
+ -storepass changeit -keystore {1}/lib/security/cacerts'.format(os_user, java_home))
+ conn.sudo('touch /home/' + args.os_user + '/.ensure_dir/cert_imported')
+ except Exception as err:
+ print('Failed to configure Keystore certificates: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_supervisor_endpoint():
+ try:
+ if not exists(conn,
+ '/home/{}/.ensure_dir/configure_supervisor_ensured'
+ .format(args.os_user)):
+ supervisor_conf = '/etc/supervisor/conf.d/supervisor_svc.conf'
+ if not exists(conn, '{}/tmp'.format(args.dlab_path)):
+ conn.run('mkdir -p {}/tmp'.format(args.dlab_path))
+ conn.put('./supervisor_svc.conf',
+ '{}/tmp/supervisor_svc.conf'.format(args.dlab_path))
+ dlab_conf_dir = '{}/conf/'.format(args.dlab_path)
+ if not exists(conn, dlab_conf_dir):
+ conn.run('mkdir -p {}'.format(dlab_conf_dir))
+ web_path = '{}/webapp'.format(args.dlab_path)
+ if not exists(conn, web_path):
+ conn.run('mkdir -p {}'.format(web_path))
+ conn.sudo('sed -i "s|OS_USR|{}|g" {}/tmp/supervisor_svc.conf'
+ .format(args.os_user, args.dlab_path))
+ conn.sudo('sed -i "s|WEB_CONF|{}|g" {}/tmp/supervisor_svc.conf'
+ .format(dlab_conf_dir, args.dlab_path))
+ conn.sudo('sed -i \'s=WEB_APP_DIR={}=\' {}/tmp/supervisor_svc.conf'
+ .format(web_path, args.dlab_path))
+ conn.sudo('cp {}/tmp/supervisor_svc.conf {}'
+ .format(args.dlab_path, supervisor_conf))
+ conn.put('./provisioning.yml', '{}provisioning.yml'
+ .format(dlab_conf_dir))
+ conn.sudo('sed -i "s|KEYNAME|{}|g" {}provisioning.yml'
+ .format(args.key_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|KEYSTORE_PASSWORD|{}|g" {}provisioning.yml'
+ .format(args.endpoint_keystore_password, dlab_conf_dir))
+ conn.sudo('sed -i "s|JRE_HOME|{}|g" {}provisioning.yml'
+ .format(java_home, dlab_conf_dir))
+ conn.sudo('sed -i "s|CLOUD_PROVIDER|{}|g" {}provisioning.yml'
+ .format(args.cloud_provider, dlab_conf_dir))
+ conn.sudo('sed -i "s|SSN_NLB|{}|g" {}provisioning.yml'
+ .format(args.ssn_k8s_nlb_dns_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|SSN_ALB|{}|g" {}provisioning.yml'
+ .format(args.ssn_k8s_alb_dns_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|CLIENT_SECRET|{}|g" {}provisioning.yml'
+ .format(args.keycloak_client_secret, dlab_conf_dir))
+ # conn.sudo('sed -i "s|MONGO_PASSWORD|{}|g" {}provisioning.yml'
+ # .format(args.mongo_password, dlab_conf_dir))
+ conn.sudo('touch /home/{}/.ensure_dir/configure_supervisor_ensured'
+ .format(args.os_user))
+ except Exception as err:
+ logging.error('Failed to configure Supervisor: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_jar_endpoint():
+ try:
+ ensure_file = ('/home/{}/.ensure_dir/backend_jar_ensured'
+ .format(args.os_user))
+ if not exists(conn, ensure_file):
+ web_path = '{}/webapp'.format(args.dlab_path)
+ if not exists(conn, web_path):
+ conn.run('mkdir -p {}'.format(web_path))
+
+ conn.run('wget -P {} --user={} --password={} '
+ 'https://{}/repository/packages/provisioning-service-'
+ '2.1.jar --no-check-certificate'
+ .format(web_path, args.repository_user,
+ args.repository_pass, args.repository_address))
+ conn.run('mv {0}/*.jar {0}/provisioning-service.jar'
+ .format(web_path))
+ conn.sudo('touch {}'.format(ensure_file))
+ except Exception as err:
+ logging.error('Failed to download jar-provisioner: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def start_supervisor_endpoint():
+ try:
+ conn.sudo("service supervisor restart")
+ except Exception as err:
+ logging.error('Unable to start Supervisor: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def pull_docker_images():
+ try:
+ ensure_file = ('/home/{}/.ensure_dir/docker_images_pulled'
+ .format(args.os_user))
+ if not exists(conn, ensure_file):
+ conn.sudo('docker login -u {} -p {} {}:{}'
+ .format(args.repository_user,
+ args.repository_pass,
+ args.repository_address,
+ args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-base'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-edge'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-project'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-jupyter'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-zeppelin'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-tensor'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-tensor-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-deeplearning'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-dataengine-service'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker pull {}:{}/docker.dlab-dataengine'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-base docker.dlab-base'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-edge docker.dlab-edge'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-project docker.dlab-project'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-jupyter docker.dlab-jupyter'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-rstudio docker.dlab-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-zeppelin '
+ 'docker.dlab-zeppelin'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-tensor docker.dlab-tensor'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-tensor-rstudio '
+ 'docker.dlab-tensor-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-deeplearning '
+ 'docker.dlab-deeplearning'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-dataengine-service '
+ 'docker.dlab-dataengine-service'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker tag {}:{}/docker.dlab-dataengine '
+ 'docker.dlab-dataengine'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-base'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-edge'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-project'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-jupyter'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-zeppelin'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-tensor'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-tensor-rstudio'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-deeplearning'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-dataengine-service'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('docker rmi {}:{}/docker.dlab-dataengine'
+ .format(args.repository_address, args.repository_port))
+ conn.sudo('chown -R {0}:docker /home/{0}/.docker/'
+ .format(args.os_user))
+ conn.sudo('touch {}'.format(ensure_file))
+ except Exception as err:
+ logging.error('Failed to pull Docker images: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def init_args():
+ global args
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--dlab_path', type=str, default='')
+ parser.add_argument('--key_name', type=str, default='')
+ parser.add_argument('--endpoint_eip_address', type=str)
+ parser.add_argument('--pkey', type=str, default='')
+ parser.add_argument('--hostname', type=str, default='')
+ parser.add_argument('--os_user', type=str, default='dlab-user')
+ parser.add_argument('--cloud_provider', type=str, default='')
+ parser.add_argument('--ssn_k8s_nlb_dns_name', type=str, default='')
+ parser.add_argument('--ssn_k8s_alb_dns_name', type=str, default='')
+ # parser.add_argument('--mongo_password', type=str, default='')
+ parser.add_argument('--repository_address', type=str, default='')
+ parser.add_argument('--repository_port', type=str, default='')
+ parser.add_argument('--repository_user', type=str, default='')
+ parser.add_argument('--repository_pass', type=str, default='')
+ parser.add_argument('--docker_version', type=str,
+ default='18.06.3~ce~3-0~ubuntu')
+ parser.add_argument('--ssn_bucket_name', type=str, default='')
+ parser.add_argument('--endpoint_keystore_password', type=str, default='')
+ parser.add_argument('--keycloak_client_secret', type=str, default='')
+ print(parser.parse_known_args())
+ args = parser.parse_known_args()[0]
+
+
+def update_system():
+ conn.sudo('apt-get update')
+
+
+def init_dlab_connection(ip=None, user=None,
+ pkey=None):
+ global conn
+ if not ip:
+ ip = args.hostname
+ if not user:
+ user = args.os_user
+ if not pkey:
+ pkey = args.pkey
+ try:
+ conn = Connection(ip, user, connect_kwargs={'key_filename': pkey})
+ except Exception as err:
+ logging.error('Failed connect as dlab-user: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def set_java_home():
+ global java_home
+ command = ('bash -c "update-alternatives --query java | grep \'Value: \' '
+ '| grep -o \'/.*/jre\'" ')
+ java_home = (conn.sudo(command).stdout.rstrip("\n\r"))
+
+
+def close_connection():
+ global conn
+ conn.close()
+
+
+def start_deploy():
+ global args
+ init_args()
+ print(args)
+ if args.hostname == "":
+ args.hostname = args.endpoint_eip_address
+
+ print("Start provisioning of Endpoint.")
+ time.sleep(40)
+
+ print(args)
+ logging.info("Creating dlab-user")
+ create_user()
+
+ init_dlab_connection()
+ update_system()
+
+ logging.info("Configuring ensure dir")
+ ensure_dir_endpoint()
+
+ logging.info("Configuring Logs")
+ ensure_logs_endpoint()
+
+ logging.info("Installing Java")
+ ensure_jre_jdk_endpoint()
+
+ set_java_home()
+
+ logging.info("Installing Supervisor")
+ ensure_supervisor_endpoint()
+
+ logging.info("Installing Docker")
+ ensure_docker_endpoint()
+
+ logging.info("Configuring Supervisor")
+ configure_supervisor_endpoint()
+
+ logging.info("Creating key directory")
+ create_key_dir_endpoint()
+
+ logging.info("Copying admin key")
+ copy_keys()
+
+ logging.info("Configuring certificates")
+ configure_keystore_endpoint(args.os_user)
+
+ logging.info("Ensure jar")
+ ensure_jar_endpoint()
+
+ logging.info("Pulling docker images")
+ pull_docker_images()
+
+ logging.info("Starting supervisor")
+ start_supervisor_endpoint()
+
+ close_connection()
+ print("Done provisioning of Endpoint.")
+
+
+if __name__ == "__main__":
+ start_deploy()
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
new file mode 100644
index 0000000..0eb5dad
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
@@ -0,0 +1,151 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+<#assign LOG_ROOT_DIR="/var/opt/dlab/log">
+<#assign KEYS_DIR="/home/${sys['user.name']}/keys">
+<#assign KEY_STORE_PATH="${KEYS_DIR}/endpoint.keystore.jks">
+<#assign KEY_STORE_PASSWORD="KEYSTORE_PASSWORD">
+<#assign TRUST_STORE_PATH="JRE_HOME/lib/security/cacerts">
+<#assign TRUST_STORE_PASSWORD="changeit">
+
+# Available options are aws, azure, gcp
+<#assign CLOUD_TYPE="CLOUD_PROVIDER">
+cloudProvider: ${CLOUD_TYPE}
+
+#Switch on/off developer mode here
+<#assign DEV_MODE="false">
+devMode: ${DEV_MODE}
+
+
+mongo:
+ host: SSN_HOST
+ port: 27017
+ username: admin
+ password: MONGO_PASSWORD
+ database: dlabdb
+
+selfService:
+ protocol: https
+ host: SSN_HOST
+ port: 8443
+ jerseyClient:
+ timeout: 3s
+ connectionTimeout: 3s
+
+securityService:
+ protocol: https
+ host: SSN_HOST
+ port: 8090
+ jerseyClient:
+ timeout: 20s
+ connectionTimeout: 20s
+
+
+provisioningService:
+ protocol: https
+ host: localhost
+ port: 8084
+ jerseyClient:
+ timeout: 3s
+ connectionTimeout: 3s
+
+# Log out user on inactivity
+inactiveUserTimeoutMillSec: 7200000
+
+backupScriptPath: /opt/dlab/tmp/backup.py
+backupDirectory: /opt/dlab/tmp/result
+keyDirectory: ${KEYS_DIR}
+responseDirectory: /opt/dlab/tmp
+handlerDirectory: /opt/dlab/handlers
+dockerLogDirectory: ${LOG_ROOT_DIR}
+warmupPollTimeout: 25s
+resourceStatusPollTimeout: 300m
+keyLoaderPollTimeout: 30m
+requestEnvStatusTimeout: 50s
+adminKey: KEYNAME
+edgeImage: docker.dlab-edge
+fileLengthCheckDelay: 500ms
+
+<#if CLOUD_TYPE == "aws">
+emrEC2RoleDefault: EMR_EC2_DefaultRole
+emrServiceRoleDefault: EMR_DefaultRole
+</#if>
+
+processMaxThreadsPerJvm: 50
+processMaxThreadsPerUser: 5
+processTimeout: 180m
+
+handlersPersistenceEnabled: true
+
+server:
+ requestLog:
+ appenders:
+ - type: file
+ currentLogFilename: ${LOG_ROOT_DIR}/provisioning/request-provisioning.log
+ archive: true
+ archivedLogFilenamePattern: ${LOG_ROOT_DIR}/provisioning/request-provisioning-%d{yyyy-MM-dd}.log.gz
+ archivedFileCount: 10
+ applicationConnectors:
+# - type: http
+ - type: https
+ port: 8084
+ certAlias: dlab
+ validateCerts: true
+ keyStorePath: ${KEY_STORE_PATH}
+ keyStorePassword: ${KEY_STORE_PASSWORD}
+ trustStorePath: ${TRUST_STORE_PATH}
+ trustStorePassword: ${TRUST_STORE_PASSWORD}
+ adminConnectors:
+# - type: http
+ - type: https
+ port: 8085
+ certAlias: dlab
+ validateCerts: true
+ keyStorePath: ${KEY_STORE_PATH}
+ keyStorePassword: ${KEY_STORE_PASSWORD}
+ trustStorePath: ${TRUST_STORE_PATH}
+ trustStorePassword: ${TRUST_STORE_PASSWORD}
+
+logging:
+ level: INFO
+ loggers:
+ com.epam: TRACE
+ com.aegisql: INFO
+ appenders:
+<#if DEV_MODE == "true">
+ - type: console
+</#if>
+ - type: file
+ currentLogFilename: ${LOG_ROOT_DIR}/provisioning/provisioning.log
+ archive: true
+ archivedLogFilenamePattern: ${LOG_ROOT_DIR}/provisioning/provisioning-%d{yyyy-MM-dd}.log.gz
+ archivedFileCount: 10
+
+keycloakConfiguration:
+ realm: dlab
+ bearer-only: true
+ auth-server-url: http://new-k8s-ssn-alb-1114650944.us-west-2.elb.amazonaws.com/auth
+ ssl-required: none
+ register-node-at-startup: true
+ register-node-period: 600
+ resource: dlab-ui
+ credentials:
+ secret: CLIENT_SECRET
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/supervisor_svc.conf b/infrastructure-provisioning/terraform/gcp/endpoint/supervisor_svc.conf
new file mode 100644
index 0000000..7bb19b2
--- /dev/null
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/supervisor_svc.conf
@@ -0,0 +1,35 @@
+; *****************************************************************************
+;
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+;
+; ******************************************************************************
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[program:provserv]
+command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 WEB_APP_DIR/provisioning-service.jar server WEB_CONFprovisioning.yml
+directory=WEB_APP_DIR
+autorestart=true
+priority=20
+user=OS_USR
+stdout_logfile=/var/log/application/provision-service.log
+redirect_stderr=true
+environment=DLAB_CONF_DIR="WEB_CONF"
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org