You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/04/26 11:59:27 UTC
[incubator-dlab] 01/01: [DLAB-433] Created Containerized Jupyter
templete(aws/debian)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-433
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 978007c75e746db8cd3532c89163328a5017f8f9
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Fri Apr 26 14:58:53 2019 +0300
[DLAB-433] Created Containerized Jupyter templete(aws/debian)
---
.../general/files/aws/jupyter-docker_Dockerfile | 50 +++
.../files/aws/jupyter-docker_description.json | 29 ++
.../src/general/lib/os/fab.py | 59 +++-
.../scripts/aws/jupyter-docker_configure.py | 335 +++++++++++++++++++++
.../general/scripts/os/jupyter_container_start.py | 46 +++
.../src/jupyter-docker/Dockerfile_jupyter | 105 +++++++
.../src/jupyter-docker/fabfile.py | 226 ++++++++++++++
.../src/jupyter-docker/jupyter_run.sh | 11 +
.../scripts/configure_jupyter-docker_node.py | 131 ++++++++
9 files changed, 990 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyter-docker_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyter-docker_Dockerfile
new file mode 100644
index 0000000..bf51e2d
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/aws/jupyter-docker_Dockerfile
@@ -0,0 +1,50 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+
+FROM docker.dlab-base:latest
+
+ARG OS
+
+COPY jupyter/ /root/
+COPY general/scripts/os/* /root/scripts/
+COPY general/scripts/aws/jupyter_* /root/scripts/
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/dlab/notebook_lib.py
+COPY general/jupyter-docker/Dockerfile_jupyter /root/templates/
+COPY general/jupyter-docker/jupyter_run.sh /root/templates/
+COPY general/templates/os/${OS}/ungit.service /root/templates/
+COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
+COPY general/templates/os/pyspark_local_template.json /root/templates/
+COPY general/templates/os/py3spark_local_template.json /root/templates/
+COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/
+COPY general/templates/os/r_dataengine-service_template.json /root/templates/
+COPY general/templates/os/r_template.json /root/templates/
+COPY general/templates/os/run_template.sh /root/templates/
+COPY general/templates/os/toree_dataengine-service_* /root/templates/
+COPY general/files/os/toree-assembly-0.2.0.jar /root/files/
+COPY general/files/os/toree_kernel.tar.gz /root/files/
+COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
+COPY general/templates/os/r_dataengine_template.json /root/templates/
+COPY general/templates/os/toree_dataengine_template.json /root/templates/
+
+RUN chmod a+x /root/fabfile.py; \
+ chmod a+x /root/scripts/*
+
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyter-docker_description.json b/infrastructure-provisioning/src/general/files/aws/jupyter-docker_description.json
new file mode 100644
index 0000000..053e71d
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/aws/jupyter-docker_description.json
@@ -0,0 +1,29 @@
+{
+ "exploratory_environment_shapes" :
+ {
+ "For testing" : [
+ {"Size": "XS", "Description": "t2.medium", "Type": "t2.medium","Ram": "4 GB","Cpu": "2"}
+ ],
+ "Memory optimized" : [
+ {"Size": "S", "Description": "r3.xlarge", "Type": "r3.xlarge","Ram": "30.5 GB","Cpu": "4"},
+ {"Size": "M", "Description": "r4.2xlarge", "Type": "r4.2xlarge","Ram": "61 GB","Cpu": "8"},
+ {"Size": "M", "Description": "r3.4xlarge", "Type": "r3.4xlarge","Ram": "122 GB","Cpu": "16"},
+ {"Size": "L", "Description": "r3.8xlarge", "Type": "r3.8xlarge","Ram": "244 GB","Cpu": "32"}
+ ],
+ "Compute optimized": [
+ {"Size": "S", "Description": "c4.large", "Type": "c4.large","Ram": "3.75 GB","Cpu": "2"},
+ {"Size": "M", "Description": "c4.2xlarge", "Type": "c4.2xlarge","Ram": "15.0 GB","Cpu": "8"},
+ {"Size": "L", "Description": "c4.8xlarge", "Type": "c4.8xlarge","Ram": "60.0 GB","Cpu": "36"}
+ ]
+ },
+ "exploratory_environment_versions" :
+ [
+ {
+ "template_name": "Containerized Jupyter notebook 5.7.4",
+ "description": "Base image with Jupyter node creation routines",
+ "environment_type": "exploratory",
+ "version": "jupyter_notebook-5.7.4",
+ "vendor": "AWS"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index fc287c9..c551ab2 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -193,6 +193,55 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
print('Error:', str(err))
sys.exit(1)
+def configure_docker(os_user, docker_version):
+ try:
+ if not exists('/home/' + os_user + '/.ensure_dir/docker_ensured'):
+ sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
+ sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
+ stable"')
+ sudo('apt-get update')
+ sudo('apt-cache policy docker-ce')
+ sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(docker_version))
+ sudo('mkdir -p /etc/systemd/system/docker.service.d')
+ sudo('touch {}'.format(http_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"\'$http_proxy\'\"\' > {}'.format(http_file))
+ sudo('touch {}'.format(https_file))
+ sudo('echo -e \'[Service] \nEnvironment=\"\'$http_proxy\'\"\' > {}'.format(https_file))
+ sudo('mkdir /home/{}/.docker'.format(os_user))
+ sudo('touch {}'.format(docker_conf_file))
+ sudo('echo -e \'{\n "proxies":\n {\n "default":\n {\n "httpProxy":"\'$http_proxy\'",\n "httpsProxy":"\'$http_proxy\'"\n }\n }\n}\' > {}'.format(docker_conf_file))
+ sudo('usermod -a -G docker ' + os_user)
+ sudo('update-rc.d docker defaults')
+ sudo('update-rc.d docker enable')
+ sudo('wget https://raw.githubusercontent.com/CWSpear/local-persist/master/scripts/install.sh && chmod +x install.sh')
+ sudo('sed -i "66s/curl/sudo curl/g" install.sh')
+ sudo('sed -i "s/sudo curl/sudo -E curl/g" install.sh')
+ run('sudo -E ./install.sh')
+ sudo('rm install.sh')
+ sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user))
+ except Exception as err:
+ print('Failed to configure Docker:', str(err))
+ sys.exit(1)
+
+def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
+ try:
+ if not exists(jupyter_dir):
+ try:
+ sudo('mkdir {}'.format(jupyter_dir))
+ put('{}Dockerfile_jupyter {}/Dockerfile_jupyter'.format(templates_dir, jupyter_dir))
+ put('{}jupyter_run.sh {}/jupyter_run.sh'.format(templates_dir, jupyter_dir))
+ sudo('sed -i \'s/jup_version/{}/\' Dockerfile_jupyter'.format(jupyter_version)
+ sudo('sed -i \'s/CONF_PATH/{}/\' {}jupyter'.format(docker_jupyter_conf, jupyter_dir))
+ sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
+ sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
+ sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
+ sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
+ sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
+ sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+ except Exception as err:
+ print('Failed to configure Jupyter files:', str(err))
+ sys.exit(1)
+
def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir, spark_version):
if not exists('/home/' + os_user + '/.ensure_dir/pyspark_local_kernel_ensured'):
@@ -204,7 +253,10 @@ def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir,
"PYJ=`find /opt/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g' /tmp/pyspark_local_template.json")
sudo('sed -i "s|SP_VER|' + spark_version + '|g" /tmp/pyspark_local_template.json')
sudo('sed -i \'/PYTHONPATH\"\:/s|\(.*\)"|\\1/home/{0}/caffe/python:/home/{0}/pytorch/build:"|\' /tmp/pyspark_local_template.json'.format(os_user))
- sudo('\cp /tmp/pyspark_local_template.json ' + pyspark_local_path_dir + 'kernel.json')
+ if os.environ['application'] == 'jupyter-docker':
+ sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_kernel.json')
+ else:
+ sudo('\cp /tmp/pyspark_local_template.json ' + pyspark_local_path_dir + 'kernel.json')
sudo('touch /home/' + os_user + '/.ensure_dir/pyspark_local_kernel_ensured')
except:
sys.exit(1)
@@ -220,7 +272,10 @@ def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir
"PYJ=`find /opt/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g' /tmp/py3spark_local_template.json")
sudo('sed -i "s|SP_VER|' + spark_version + '|g" /tmp/py3spark_local_template.json')
sudo('sed -i \'/PYTHONPATH\"\:/s|\(.*\)"|\\1/home/{0}/caffe/python:/home/{0}/pytorch/build:"|\' /tmp/py3spark_local_template.json'.format(os_user))
- sudo('\cp /tmp/py3spark_local_template.json ' + py3spark_local_path_dir + 'kernel.json')
+ if os.environ['application'] == 'jupyter-docker':
+ sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_kernel.json')
+ else:
+ sudo('\cp /tmp/py3spark_local_template.json ' + py3spark_local_path_dir + 'kernel.json')
sudo('touch /home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py
new file mode 100644
index 0000000..33387dd
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter-docker_configure.py
@@ -0,0 +1,335 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--uuid', type=str, default='')
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ notebook_config = dict()
+ try:
+ notebook_config['exploratory_name'] = os.environ['exploratory_name']
+ except:
+ notebook_config['exploratory_name'] = ''
+ notebook_config['service_base_name'] = os.environ['conf_service_base_name']
+ notebook_config['instance_type'] = os.environ['aws_notebook_instance_type']
+ notebook_config['key_name'] = os.environ['conf_key_name']
+ notebook_config['user_keyname'] = os.environ['edge_user_name']
+ notebook_config['network_type'] = os.environ['conf_network_type']
+ notebook_config['instance_name'] = '{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
+ os.environ['edge_user_name'],
+ notebook_config['exploratory_name'], args.uuid)
+ notebook_config['expected_image_name'] = '{}-{}-notebook-image'.format(notebook_config['service_base_name'],
+ os.environ['application'])
+ notebook_config['notebook_image_name'] = str(os.environ.get('notebook_image_name'))
+ notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
+ .format(notebook_config['service_base_name'].lower().replace('-', '_'), os.environ['edge_user_name'])
+ notebook_config['security_group_name'] = '{}-{}-nb-SG'.format(notebook_config['service_base_name'],
+ os.environ['edge_user_name'])
+ notebook_config['tag_name'] = '{}-Tag'.format(notebook_config['service_base_name'])
+ notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+
+ # generating variables regarding EDGE proxy on Notebook instance
+ instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
+ edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ if notebook_config['network_type'] == 'private':
+ edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
+ else:
+ edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
+ keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+
+ try:
+ if os.environ['conf_os_family'] == 'debian':
+ initial_user = 'ubuntu'
+ sudo_group = 'sudo'
+ if os.environ['conf_os_family'] == 'redhat':
+ initial_user = 'ec2-user'
+ sudo_group = 'wheel'
+
+ logging.info('[CREATING DLAB SSH USER]')
+ print('[CREATING DLAB SSH USER]')
+ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
+ (instance_hostname, os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem", initial_user,
+ notebook_config['dlab_ssh_user'], sudo_group)
+
+ try:
+ local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed creating ssh user 'dlab'.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ # configuring proxy on Notebook instance
+ try:
+ logging.info('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]')
+ print('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]')
+ additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
+ params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
+ .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ # updating repositories & installing python packages
+ try:
+ logging.info('[INSTALLING PREREQUISITES TO JUPYTER-DOCKER NOTEBOOK INSTANCE]')
+ print('[INSTALLING PREREQUISITES TO JUPYTER-DOCKER NOTEBOOK INSTANCE]')
+ params = "--hostname {} --keyfile {} --user {} --region {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ try:
+ local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed installing apps: apt & pip.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ # installing and configuring jupiter and all dependencies
+ try:
+ logging.info('[CONFIGURE JUPYTER-DOCKER NOTEBOOK INSTANCE]')
+ print('[CONFIGURE JUPYTER-DOCKER NOTEBOOK INSTANCE]')
+ params = "--hostname {} " \
+ "--keyfile {} " \
+ "--region {} " \
+ "--spark_version {} " \
+ "--hadoop_version {} " \
+ "--os_user {} " \
+ "--scala_version {} " \
+ "--r_mirror {} " \
+ "--exploratory_name {}".\
+ format(instance_hostname,
+ keyfile_name,
+ os.environ['aws_region'],
+ os.environ['notebook_spark_version'],
+ os.environ['notebook_hadoop_version'],
+ notebook_config['dlab_ssh_user'],
+ os.environ['notebook_scala_version'],
+ os.environ['notebook_r_mirror'],
+ notebook_config['exploratory_name'])
+ try:
+ local("~/scripts/{}.py {}".format('configure_jupyter-docker_node', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure jupyter-docker.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
+ print('[INSTALLING USERs KEY]')
+ logging.info('[INSTALLING USERs KEY]')
+ additional_config = {"user_keyname": notebook_config['user_keyname'],
+ "user_keydir": os.environ['conf_key_dir']}
+ params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
+ instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('install_user_key', params))
+ except:
+ append_result("Failed installing users key")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed installing users key.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
+ print('[SETUP USER GIT CREDENTIALS]')
+ logging.info('[SETUP USER GIT CREDENTIALS]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name)
+ try:
+ local("~/scripts/{}.py {}".format('common_download_git_certfile', params))
+ local("~/scripts/{}.py {}".format('manage_git_creds', params))
+ except:
+ append_result("Failed setup git credentials")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to setup git credentials.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
+ logging.info('[POST CONFIGURING PROCESS]')
+ print('[POST CONFIGURING PROCESS')
+ if notebook_config['notebook_image_name'] not in [notebook_config['expected_image_name'], 'None']:
+ params = "--hostname {} --keyfile {} --os_user {} --nb_tag_name {} --nb_tag_value {}" \
+ .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
+ notebook_config['tag_name'], notebook_config['instance_name'])
+ try:
+ local("~/scripts/{}.py {}".format('common_remove_remote_kernels', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to post configuring instance.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'jupyter',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ if notebook_config['shared_image_enabled'] == 'true':
+ try:
+ print('[CREATING AMI]')
+ ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
+ if ami_id == '':
+ print("Looks like it's first time we configure notebook server. Creating image.")
+ image_id = create_image_from_instance(tag_name=notebook_config['tag_name'],
+ instance_name=notebook_config['instance_name'],
+ image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is {}".format(image_id))
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed creating image.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ try:
+ print('[STARTING JUPYTER CONTAINER]')
+ logging.info('[STARTING JUPYTER CONTAINER]')
+ try:
+ local("~/scripts/jupyter_container_start.py")
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to start Jupyter container.", str(err))
+ remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
+ # generating output information
+ ip_address = get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
+ dns_name = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
+ jupyter_dns_url = "http://" + dns_name + ":8888/{}/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_acces_url = "http://" + edge_instance_ip + "/{}/".format(notebook_config['exploratory_name'])
+ jupyter_ungit_acces_url = "http://" + edge_instance_ip + "/{}-ungit/".format(notebook_config['exploratory_name'])
+ ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ print('[SUMMARY]')
+ logging.info('[SUMMARY]')
+ print("Instance name: {}".format(notebook_config['instance_name']))
+ print("Private DNS: {}".format(dns_name))
+ print("Private IP: {}".format(ip_address))
+ print("Instance ID: {}".format(get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name'])))
+ print("Instance type: {}".format(notebook_config['instance_type']))
+ print("Key name: {}".format(notebook_config['key_name']))
+ print("User key name: {}".format(notebook_config['user_keyname']))
+ print("Image name: {}".format(notebook_config['notebook_image_name']))
+ print("Profile name: {}".format(notebook_config['role_profile_name']))
+ print("SG name: {}".format(notebook_config['security_group_name']))
+ print("Jupyter URL: {}".format(jupyter_ip_url))
+ print("Jupyter URL: {}".format(jupyter_dns_url))
+ print("Ungit URL: {}".format(ungit_ip_url))
+ print("ReverseProxyNotebook".format(jupyter_notebook_acces_url))
+ print("ReverseProxyUngit".format(jupyter_ungit_acces_url))
+ print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.
+ format(notebook_config['key_name'], notebook_config['dlab_ssh_user'], ip_address))
+ print('SSH access (from Edge node, via FQDN): ssh -i {0}.pem {1}@{2}'.
+ format(notebook_config['key_name'], notebook_config['dlab_ssh_user'], dns_name))
+
+ with open("/root/result.json", 'w') as result:
+ res = {"hostname": dns_name,
+ "ip": ip_address,
+ "instance_id": get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name']),
+ "master_keyname": os.environ['conf_key_name'],
+ "notebook_name": notebook_config['instance_name'],
+ "notebook_image_name": notebook_config['notebook_image_name'],
+ "Action": "Create new notebook server",
+ "exploratory_url": [
+ {"description": "Jupyter",
+ "url": jupyter_notebook_acces_url},
+ {"description": "Ungit",
+ "url": jupyter_ungit_acces_url},
+ {"description": "Jupyter (via tunnel)",
+ "url": jupyter_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
+ result.write(json.dumps(res))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
new file mode 100644
index 0000000..0eed1a7
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import sys
+import os
+from dlab.notebook_lib import *
+from dlab.fab import *
+from fabric.api import *
+
+jupyter_dir = '/home/' + args.os_user + '/.jupyter/'
+
+def start_jupyter_container():
+ try:
+ with cd('{}'.format(jupyter_dir))
+ run('docker volume create -d local-persist -o mountpoint=/opt --name=jup_volume')
+ run('docker build --file /home/dlab-user/jupyter/Dockerfile.jupyter -t jupyter-notebook .')
+ run('docker run -d -p 8888:8888 -v jup_volume:/opt/ jupyter-notebook:latest')
+
+if __name__ == "__main__":
+ print("Starting Jupyter container")
+ try:
+ start_jupyter_container()
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
+
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
new file mode 100644
index 0000000..102d8f4
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -0,0 +1,105 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM ubuntu:18.04
+
+ARG NB_USER="dlab-user"
+ENV CONF_FILE="/home/dlab-user/.local/share/jupyter/jupyter_notebook_config.py"
+ENV PATH /usr/local/bin:$PATH
+ENV LANG C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ tk-dev wget gnupg dirmngr build-essential openssl \
+ libreadline-gplv2-dev libncursesw5-dev libssl-dev \
+ libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev \
+ python python-dev python-pip python-virtualenv \
+ python3 python3-dev python3-pip default-jre default-jdk \
+ && rm -rf /var/lib/apt/lists/*
+
+
+RUN useradd -u 1001 -ms /bin/bash $NB_USER
+
+
+#Installing Python libraries
+RUN pip install --upgrade setuptools \
+ && pip install tornado==4.5.3 ipython ipython matplotlib==2.0.2 \
+ boto3 fabvenv fabric-virtualenv future NumPy==1.14.3 \
+ SciPy pandas Sympy Pillow sklearn pyopenssl --no-cache-dir \
+ && pip3 install --upgrade setuptools \
+ && pip3 install tornado==4.5.3 ipython ipython matplotlib==2.0.2 \
+ boto3 fabvenv fabric-virtualenv future NumPy==1.14.3 \
+ SciPy pandas Sympy Pillow sklearn --no-cache-dir
+
+
+#Installing SBT
+#RUN apt-get install -y apt-transport-https \
+# && echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list \
+# && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 642AC823 \
+# && apt-get install -y sbt
+
+
+
+#Installing Scala
+RUN wget http://www.scala-lang.org/files/archive/scala-2.12.8.deb -O /tmp/scala.deb \
+ && dpkg -i /tmp/scala.deb
+
+
+
+#Installing Jupyter
+RUN pip install notebook==jup_version --no-cache-dir \
+ && pip install jupyter --no-cache-dir \
+ && pip3 install notebook==jup_version --no-cache-dir \
+ && pip3 install jupyter --no-cache-dir \
+ && rm -rf $CONF_FILE \
+ && jupyter notebook --generate-config --config $CONF_FILE \
+ && mkdir -p /home/$NB_USER/.jupyter/custom/ \
+ && echo "#notebook-container { width: auto; }" > /home/$NB_USER/.jupyter/custom/custom.css \
+ && echo "c.NotebookApp.ip = \'0.0.0.0\'" >> $CONF_FILE \
+ && echo "c.NotebookApp.base_url = \'/jup/\'" >> $CONF_FILE \
+ && echo c.NotebookApp.open_browser = False >> $CONF_FILE \
+ && echo "c.NotebookApp.cookie_secret = NzMwZTI5MW" >> $CONF_FILE \
+ && ''echo "c.NotebookApp.token = u''" >> $CONF_FILE'' \
+ && echo 'c.KernelSpecManager.ensure_native_kernel = False' >> $CONF_FILE \
+ && chown -R $NB_USER:$NB_USER /home/$NB_USER/.local \
+ && mkdir -p /mnt/var \
+ && chown $NB_USER:$NB_USER /mnt/var \
+ && jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists" \
+ && jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"
+
+#Installing local kernel
+RUN mkdir -p /home/$NB_USER/.local/share/jupyter/kernels/py3spark_local/ \
+ /home/$NB_USER/.local/share/jupyter/kernels/pyspark_local/
+COPY py3spark_local_kernel.json /home/$NB_USER/.local/share/jupyter/kernels/py3spark_local/kernel.json
+COPY pyspark_local_kernel.json /home/$NB_USER/.local/share/jupyter/kernels/pyspark_local/kernel.json
+RUN chown -R 1001:1001 /home/$NB_USER/.local/share/jupyter/*
+
+EXPOSE 8888
+
+COPY jupyter_run.sh /jupyter_run.sh
+RUN sed -i 's|CONF_PATH|/home/dlab-user/.local/share/jupyter/jupyter_notebook_config.py|' /jupyter_run.sh \
+ && chmod +x /jupyter_run.sh
+RUN cat /jupyter_run.sh
+
+USER $NB_USER
+
+ENTRYPOINT ["/jupyter_run.sh", "-d"]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter-docker/fabfile.py b/infrastructure-provisioning/src/jupyter-docker/fabfile.py
new file mode 100644
index 0000000..ddc0d60
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-docker/fabfile.py
@@ -0,0 +1,226 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import uuid
+
+
+# Main function for provisioning notebook server
+def run():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ notebook_config = dict()
+ notebook_config['uuid'] = str(uuid.uuid4())[:5]
+
+ try:
+ params = "--uuid {}".format(notebook_config['uuid'])
+ local("~/scripts/{}.py {}".format('common_prepare_notebook', params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed preparing Notebook node.", str(err))
+ sys.exit(1)
+
+ try:
+ params = "--uuid {}".format(notebook_config['uuid'])
+ local("~/scripts/{}.py {}".format('jupyter-docker_configure', params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed configuring Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for terminating exploratory environment
+def terminate():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ local("~/scripts/{}.py".format('common_terminate_notebook'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed terminating Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for stopping notebook server
+def stop():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ local("~/scripts/{}.py".format('common_stop_notebook'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed stopping Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for starting notebook server
+def start():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('common_start_notebook'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed starting Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for configuring notebook server after deploying DataEngine service
+def configure():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ if os.environ['conf_resource'] == 'dataengine-service':
+ local("~/scripts/{}.py".format('common_notebook_configure_dataengine-service'))
+ elif os.environ['conf_resource'] == 'dataengine':
+ local("~/scripts/{}.py".format('common_notebook_configure_dataengine'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed configuring analytical tool on Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for installing additional libraries for notebook
+def install_libs():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('notebook_install_libs'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed installing additional libs for Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for get available libraries for notebook
+def list_libs():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('notebook_list_libs'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed get available libraries for notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for manage git credentials on notebook
+def git_creds():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('notebook_git_creds'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to manage git credentials for notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for creating image from notebook
+def create_image():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('common_create_notebook_image'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to create image from notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for deleting existing notebook image
+def terminate_image():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('common_terminate_notebook_image'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to create image from notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for reconfiguring Spark for notebook
+def reconfigure_spark():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('notebook_reconfigure_spark'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to reconfigure Spark for Notebook node.", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
new file mode 100644
index 0000000..f8b6fff
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+jupyter notebook --config CONF_PATH
+
+if [[$1 == '-d']];then
+ while true; do sleep 1000; done
+fi
+
+if [[$1 == "-bash"]];then
+ /bin/bash
+fi
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
new file mode 100644
index 0000000..384fe86
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import argparse
+import json
+import sys
+from dlab.notebook_lib import *
+from dlab.actions_lib import *
+from dlab.fab import *
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--hostname', type=str, default='')
+parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--region', type=str, default='')
+parser.add_argument('--spark_version', type=str, default='')
+parser.add_argument('--hadoop_version', type=str, default='')
+parser.add_argument('--os_user', type=str, default='')
+parser.add_argument('--scala_version', type=str, default='')
+parser.add_argument('--r_mirror', type=str, default='')
+parser.add_argument('--exploratory_name', type=str, default='')
+args = parser.parse_args()
+
+spark_version = args.spark_version
+hadoop_version = args.hadoop_version
+jupyter_version = os.environ['notebook_jupyter_version']
+scala_link = "http://www.scala-lang.org/files/archive/"
+if args.region == 'cn-north-1':
+ spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
+ "-bin-hadoop" + hadoop_version + ".tgz"
+else:
+ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
+ "-bin-hadoop" + hadoop_version + ".tgz"
+
+docker_version = '18.09.4'
+http_file = '/etc/systemd/system/docker.service.d/http-proxy.conf'
+https_file = '/etc/systemd/system/docker.service.d/https-proxy.conf'
+docker_conf_file = '/home/' + os_user + '/.docker/config.json'
+jupyter_dir = '/home/' + args.os_user + '/.jupyter/'
+pyspark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/pyspark_local/'
+py3spark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/py3spark_local/'
+jupyter_conf_file = '/home/' + args.os_user + '/.jupyter/jupyter_notebook_config.py'
+docker_jupyter_conf = '/home/' + args.os_user + '/.local/share/jupyter/jupyter_notebook_config.py'
+scala_kernel_path = '/usr/local/share/jupyter/kernels/apache_toree_scala/'
+r_kernels_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/'
+jars_dir = '/opt/jars/'
+templates_dir = '/root/templates/'
+files_dir = '/root/files/'
+local_spark_path = '/opt/spark/'
+toree_link = 'http://archive.apache.org/dist/incubator/toree/0.2.0-incubating/toree-pip/toree-0.2.0.tar.gz'
+r_libs = ['R6', 'pbdZMQ', 'RCurl', 'devtools', 'reshape2', 'caTools', 'rJava', 'ggplot2']
+gitlab_certfile = os.environ['conf_gitlab_certfile']
+
+
+##############
+# Run script #
+##############
+if __name__ == "__main__":
+ print("Configure connections")
+ env['connection_attempts'] = 100
+ env.key_filename = [args.keyfile]
+ env.host_string = args.os_user + '@' + args.hostname
+
+ # PREPARE DISK
+ print("Prepare .ensure directory")
+ try:
+ if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
+ except:
+ sys.exit(1)
+ print("Mount additional volume")
+ prepare_disk(args.os_user)
+
+ # INSTALL LANGUAGES
+ print("Install Java")
+ ensure_jre_jdk(args.os_user)
+
+ # INSTALL DOCKER
+ print ("Install Docker")
+ configure_docker(args.os_user, docker_version)
+
+ # CONFIGURE JUPYTER FILES
+ print("Configure jupyter files")
+ ensure_jupyter_docker_files(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name)
+
+ # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
+ print("Install local Spark")
+ ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path)
+ print("Install storage jars")
+ ensure_local_jars(args.os_user, jars_dir)
+ print("Configure local Spark")
+ configure_local_spark(jars_dir, templates_dir)
+
+ # INSTALL JUPYTER KERNELS
+ print("Install pyspark local kernel for Jupyter")
+ ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
+ print("Install py3spark local kernel for Jupyter")
+ ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version)
+ print("Install Toree-Scala kernel for Jupyter")
+ ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, args.scala_version, spark_version)
+ if os.environ['notebook_r_enabled'] == 'true':
+ print("Install R kernel for Jupyter")
+ ensure_r_local_kernel(spark_version, args.os_user, templates_dir, r_kernels_dir)
+
+ # INSTALL UNGIT
+ print("Install nodejs")
+ install_nodejs(args.os_user)
+ print("Install ungit")
+ install_ungit(args.os_user, args.exploratory_name)
+ if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
+ install_gitlab_cert(args.os_user, gitlab_certfile)
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org