You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/12/09 16:33:01 UTC
[incubator-dlab] branch DLAB-1326 updated: [DLAB-1326] Refactored
Superset deployment scripts
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1326
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1326 by this push:
new 835a667 [DLAB-1326] Refactored Superset deployment scripts
835a667 is described below
commit 835a667d45dfd0c032df13b79339fa79795a3765
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Mon Dec 9 18:32:37 2019 +0200
[DLAB-1326] Refactored Superset deployment scripts
---
.../src/general/lib/os/fab.py | 8 ---
.../src/general/scripts/gcp/superset_configure.py | 52 ++++++++++++++++--
.../src/general/scripts/os/superset_start.py | 61 ++++++++++++++++++++++
.../superset/scripts/configure_superset_node.py | 2 +-
4 files changed, 109 insertions(+), 14 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 751ce5c..96fd01f 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -860,14 +860,6 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
sudo('cp -f /opt/dlab/templates/requirements-extra.txt /home/{}/incubator-superset/contrib/docker/'.format(os_user))
sudo('cp -f /opt/dlab/templates/superset_config.py /home/{}/incubator-superset/contrib/docker/'.format(os_user))
sudo('cp -f /opt/dlab/templates/docker-init.sh /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- with cd('/home/{}/incubator-superset/contrib/docker'.format(os_user)):
- sudo('docker-compose run --rm superset ./docker-init.sh')
- sudo('cp /opt/dlab/templates/superset-notebook.service /tmp/')
- sudo('sed -i \'s/OS_USER/{}/g\' /tmp/superset-notebook.service'.format(os_user))
- sudo('cp /tmp/superset-notebook.service /etc/systemd/system/')
- sudo('systemctl daemon-reload')
- sudo('systemctl enable superset-notebook')
- sudo('systemctl start superset-notebook')
sudo('touch /tmp/superset-notebook_installed')
except Exception as err:
print("Failed configure superset: " + str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
index c5b291c..cdcafea 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
@@ -54,10 +54,27 @@ if __name__ == "__main__":
notebook_config['instance_name'] = '{0}-{1}-nb-{2}'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
notebook_config['exploratory_name'])
- notebook_config['expected_primary_image_name'] = '{}-{}-notebook-primary-image'.format(
- notebook_config['service_base_name'], os.environ['application'])
- notebook_config['expected_secondary_image_name'] = '{}-{}-notebook-secondary-image'.format(
- notebook_config['service_base_name'], os.environ['application'])
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if notebook_config['shared_image_enabled'] == 'false':
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'],
+ os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'],
+ os.environ['application'])
+ notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
+ "endpoint_tag": notebook_config['endpoint_tag'],
+ "project_tag": notebook_config['project_tag'],
+ "product": "dlab"}
+ else:
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application'])
+ notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
+ "endpoint_tag": notebook_config['endpoint_tag'],
+ "product": "dlab"}
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-{2}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_tag'])
@@ -280,7 +297,12 @@ if __name__ == "__main__":
try:
print('[CONFIGURING PROXY FOR DOCKER]')
logging.info('[CONFIGURING PROXY FOR DOCKER]')
- params = "--os_user {} ".format(notebook_config['dlab_ssh_user'])
+ params = "--hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} ". \
+ format(instance_hostname,
+ notebook_config['ssh_key_path'],
+ notebook_config['dlab_ssh_user'])
try:
local("~/scripts/configure_proxy_for_docker.py {}".format(params))
except:
@@ -292,6 +314,26 @@ if __name__ == "__main__":
GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+ try:
+ print('[STARTING SUPERSET]')
+ logging.info('[STARTING SUPERSET]')
+ params = "--hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} ". \
+ format(instance_hostname,
+ notebook_config['ssh_key_path'],
+ notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/superset_start.py {}".format(params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to start Superset.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
# generating output information
ip_address = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
superset_ip_url = "http://" + ip_address + ":8088/{}/".format(notebook_config['exploratory_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/os/superset_start.py b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
new file mode 100644
index 0000000..33bbc9c
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import sys
+import os
+from dlab.notebook_lib import *
+from dlab.fab import *
+from fabric.api import *
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--hostname', type=str, default='')
+parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--os_user', type=str, default='')
+args = parser.parse_args()
+
+superset_dir = '/home/' + args.os_user + '/incubator-superset/contrib/docker'
+
+def start_superset(superset_dir):
+ try:
+ with cd('{}'.format(superset_dir)):
+ sudo('docker-compose run --rm superset ./docker-init.sh')
+ sudo('cp /opt/dlab/templates/superset-notebook.service /tmp/')
+ sudo('sed -i \'s/OS_USER/{}/g\' /tmp/superset-notebook.service'.format(args.os_user))
+ sudo('cp /tmp/superset-notebook.service /etc/systemd/system/')
+ sudo('systemctl daemon-reload')
+ sudo('systemctl enable superset-notebook')
+ sudo('systemctl start superset-notebook')
+ except: sys.exit(1)
+
+if __name__ == "__main__":
+ print("Configure connections")
+ env['connection_attempts'] = 100
+ env.key_filename = [args.keyfile]
+ env.host_string = args.os_user + '@' + args.hostname
+ print("Starting Superset")
+ try:
+ start_superset(superset_dir)
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
+
diff --git a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
index 03d8e99..a156f2f 100644
--- a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
+++ b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
@@ -68,7 +68,7 @@ if __name__ == "__main__":
# INSTALL DOCKER COMPOSE
print("Installing docker compose")
- if not ensure_docker_compose(args.dlab_path, args.os_user):
+ if not ensure_docker_compose(args.os_user):
sys.exit(1)
# INSTALL UNGIT
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org