You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/07/16 08:25:49 UTC
[incubator-dlab] branch DLAB-627 created (now 22202c4)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a change to branch DLAB-627
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 22202c4 [DLAB-627] Created jupyter-docker templete form GCP
This branch includes the following new commits:
new 81dda0e [DLAB-433] Updated image ID
new 22202c4 [DLAB-627] Created jupyter-docker templete form GCP
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/02: [DLAB-627] Created jupyter-docker templete
form GCP
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-627
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 22202c45a3d6ef1712ee6b77d322bf94bcd1bba9
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Tue Jul 16 11:24:57 2019 +0300
[DLAB-627] Created jupyter-docker templete form GCP
---
.../general/files/gcp/jupyter-docker_Dockerfile | 48 ++++
.../files/gcp/jupyter-docker_description.json | 33 +++
.../scripts/gcp/jupyter-docker_configure.py | 300 +++++++++++++++++++++
3 files changed, 381 insertions(+)
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile
new file mode 100644
index 0000000..a44f03a
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile
@@ -0,0 +1,48 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+
+FROM docker.dlab-base:latest
+
+ARG OS
+
+COPY jupyter-docker/ /root/
+COPY general/scripts/os/* /root/scripts/
+COPY general/scripts/gcp/jupyter* /root/scripts/
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/dlab/notebook_lib.py
+COPY general/templates/os/${OS}/ungit.service /root/templates/
+COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
+COPY general/templates/os/pyspark_local_template.json /root/templates/
+COPY general/templates/os/py3spark_local_template.json /root/templates/
+COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/
+COPY general/templates/os/r_dataengine-service_template.json /root/templates/
+COPY general/templates/os/r_template.json /root/templates/
+COPY general/templates/os/run_template.sh /root/templates/
+COPY general/templates/os/toree_dataengine-service_* /root/templates/
+COPY general/files/os/toree-assembly-0.2.0.jar /root/files/
+COPY general/files/os/toree_kernel.tar.gz /root/files/
+COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
+COPY general/templates/os/r_dataengine_template.json /root/templates/
+COPY general/templates/os/toree_dataengine_template.json /root/templates/
+
+RUN chmod a+x /root/fabfile.py; \
+ chmod a+x /root/scripts/*
+
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json
new file mode 100644
index 0000000..3f202f7
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json
@@ -0,0 +1,33 @@
+{
+ "exploratory_environment_shapes" :
+ {
+ "For testing" : [
+ {"Size": "S", "Description": "n1-standard-2", "Type": "n1-standard-2","Ram": "7.50 GB","Cpu": "2"}
+ ],
+ "Memory optimized" : [
+ {"Size": "S", "Description": "n1-highmem-4", "Type": "n1-highmem-4","Ram": "26 GB","Cpu": "4"},
+ {"Size": "M", "Description": "n1-highmem-16", "Type": "n1-highmem-16","Ram": "104 GB","Cpu": "16"},
+ {"Size": "L", "Description": "n1-highmem-32", "Type": "n1-highmem-32","Ram": "208 GB","Cpu": "32"}
+ ],
+ "GPU optimized": [
+ {"Size": "S", "Description": "n1-standard-2", "Type": "n1-standard-2","Ram": "7.50 GB","Cpu": "2"},
+ {"Size": "M", "Description": "n1-highcpu-8", "Type": "n1-highcpu-8","Ram": "7.20 GB","Cpu": "8"},
+ {"Size": "L", "Description": "n1-highmem-32", "Type": "n1-highmem-32","Ram": "208 GB","Cpu": "32"}
+ ],
+ "Compute optimized": [
+ {"Size": "S", "Description": "n1-highcpu-2", "Type": "n1-highcpu-2","Ram": "1.80 GB","Cpu": "2"},
+ {"Size": "M", "Description": "n1-highcpu-8", "Type": "n1-highcpu-8","Ram": "7.20 GB","Cpu": "8"},
+ {"Size": "L", "Description": "n1-highcpu-32", "Type": "n1-highcpu-32","Ram": "28.8 GB","Cpu": "32"}
+ ]
+ },
+ "exploratory_environment_versions" :
+ [
+ {
+ "template_name": "JupyterLab 0.35.6",
+ "description": "Base image with JupyterLab node creation routines",
+ "environment_type": "exploratory",
+ "version": "jupyter_lab-0.35.6",
+ "vendor": "GCP"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py
new file mode 100644
index 0000000..a344df9
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py
@@ -0,0 +1,300 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+# #
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# #
+# http://www.apache.org/licenses/LICENSE-2.0
+# #
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# #
+# ******************************************************************************
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ notebook_config = dict()
+ try:
+ notebook_config['exploratory_name'] = (os.environ['exploratory_name']).lower().replace('_', '-')
+ except:
+ notebook_config['exploratory_name'] = ''
+ notebook_config['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ notebook_config['instance_type'] = os.environ['gcp_notebook_instance_size']
+ notebook_config['key_name'] = os.environ['conf_key_name']
+ notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower().replace('_', '-')
+ notebook_config['instance_name'] = '{0}-{1}-nb-{2}'.format(notebook_config['service_base_name'],
+ notebook_config['edge_user_name'],
+ notebook_config['exploratory_name'])
+ notebook_config['expected_primary_image_name'] = '{}-{}-notebook-primary-image'.format(
+ notebook_config['service_base_name'], os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-notebook-secondary-image'.format(
+ notebook_config['service_base_name'], os.environ['application'])
+ instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
+ edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
+ edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
+ notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['zone'] = os.environ['gcp_zone']
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ try:
+ if os.environ['conf_os_family'] == 'debian':
+ initial_user = 'ubuntu'
+ sudo_group = 'sudo'
+ if os.environ['conf_os_family'] == 'redhat':
+ initial_user = 'ec2-user'
+ sudo_group = 'wheel'
+
+ logging.info('[CREATING DLAB SSH USER]')
+ print('[CREATING DLAB SSH USER]')
+ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
+ (instance_hostname, notebook_config['ssh_key_path'], initial_user,
+ notebook_config['dlab_ssh_user'], sudo_group)
+
+ try:
+ local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed creating ssh user 'dlab'.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # configuring proxy on Notebook instance
+ try:
+ logging.info('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]')
+ print('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]')
+ additional_config = {"proxy_host": edge_instance_name, "proxy_port": "3128"}
+ params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
+ .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
+ json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure proxy.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # updating repositories & installing python packages
+ try:
+ logging.info('[INSTALLING PREREQUISITES TO JUPYTER-DOCKER NOTEBOOK INSTANCE]')
+ print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
+ try:
+ local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed installing apps: apt & pip.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # installing and configuring jupiter and all dependencies
+ try:
+ logging.info('[CONFIGURE JUPYTER NOTEBOOK INSTANCE]')
+ print('[CONFIGURE JUPYTER NOTEBOOK INSTANCE]')
+ params = "--hostname {} --keyfile {} " \
+ "--region {} --spark_version {} " \
+ "--hadoop_version {} --os_user {} " \
+ "--scala_version {} --r_mirror {} " \
+ "--exploratory_name {}" "--edge_ip {} ".\
+ format(instance_hostname, notebook_config['ssh_key_path'],
+ os.environ['gcp_region'], os.environ['notebook_spark_version'],
+ os.environ['notebook_hadoop_version'], notebook_config['dlab_ssh_user'],
+ os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'],
+ notebook_config['exploratory_name'], edge_instance_private_ip)
+ try:
+ local("~/scripts/{}.py {}".format('configure_jupyter_node', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure jupyter.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ print('[INSTALLING USERs KEY]')
+ logging.info('[INSTALLING USERs KEY]')
+ additional_config = {"user_keyname": os.environ['edge_user_name'],
+ "user_keydir": os.environ['conf_key_dir']}
+ params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
+ instance_hostname, notebook_config['ssh_key_path'], json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('install_user_key', params))
+ except:
+ append_result("Failed installing users key")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed installing users key.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ print('[SETUP USER GIT CREDENTIALS]')
+ logging.info('[SETUP USER GIT CREDENTIALS]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(notebook_config['dlab_ssh_user'], instance_hostname, notebook_config['ssh_key_path'])
+ try:
+ local("~/scripts/{}.py {}".format('common_download_git_certfile', params))
+ local("~/scripts/{}.py {}".format('manage_git_creds', params))
+ except:
+ append_result("Failed setup git credentials")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to setup git credentials.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ if notebook_config['shared_image_enabled'] == 'true':
+ try:
+ print('[CREATING IMAGE]')
+ primary_image_id = GCPMeta().get_image_by_name(notebook_config['expected_primary_image_name'])
+ if primary_image_id == '':
+ print("Looks like it's first time we configure notebook server. Creating images.")
+ image_id_list = GCPActions().create_image_from_instance_disks(
+ notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
+ notebook_config['instance_name'], notebook_config['zone'])
+ if image_id_list and image_id_list[0] != '':
+ print("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
+ else:
+ print("Looks like another image creating operation for your template have been started a moment ago.")
+ if image_id_list and image_id_list[1] != '':
+ print("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1]))
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed creating image.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ GCPActions().remove_image(notebook_config['expected_primary_image_name'])
+ GCPActions().remove_image(notebook_config['expected_secondary_image_name'])
+ sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ notebook_config['ssh_key_path'],
+ notebook_config['dlab_ssh_user'],
+ 'jupyter',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # generating output information
+ ip_address = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
+ jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
+ ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(notebook_config['exploratory_name'])
+ jupyter_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
+ print('[SUMMARY]')
+ logging.info('[SUMMARY]')
+ print("Instance name: {}".format(notebook_config['instance_name']))
+ print("Private IP: {}".format(ip_address))
+ print("Instance type: {}".format(notebook_config['instance_type']))
+ print("Key name: {}".format(notebook_config['key_name']))
+ print("User key name: {}".format(os.environ['edge_user_name']))
+ print("JupyterLab URL: {}".format(jupyter_ip_url))
+ print("Ungit URL: {}".format(ungit_ip_url))
+ print("ReverseProxyNotebook".format(jupyter_notebook_acces_url))
+ print("ReverseProxyUngit".format(jupyter_ungit_acces_url))
+ print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(notebook_config['key_name'],
+ notebook_config['dlab_ssh_user'],
+ ip_address))
+
+ with open("/root/result.json", 'w') as result:
+ res = {"hostname": ip_address,
+ "ip": ip_address,
+ "instance_id": notebook_config['instance_name'],
+ "master_keyname": os.environ['conf_key_name'],
+ "notebook_name": notebook_config['instance_name'],
+ "Action": "Create new notebook server",
+ "exploratory_url": [
+ {"description": "JupyterLab",
+ "url": jupyter_notebook_acces_url},
+ {"description": "Ungit",
+ "url": jupyter_ungit_acces_url},
+ {"description": "JupyterLab (via tunnel)",
+ "url": jupyter_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
+ result.write(json.dumps(res))
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/02: [DLAB-433] Updated image ID
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-627
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 81dda0e3f96706b06c1c5a271c3cfa49c564eda1
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Mon Jul 15 17:26:55 2019 +0300
[DLAB-433] Updated image ID
---
infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
index 68ac706..1651623 100644
--- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -19,7 +19,7 @@
#
# ******************************************************************************
-FROM nexus.cc.epm.kharlamov.biz:443/legion/python-toolchain:1.0.0-20190709105425.523.e22cd7d
+FROM nexus.cc.epm.kharlamov.biz:443/legion/python-toolchain:1.0.0-20190715020857.542.3935d27
ARG NB_USER="dlab-user"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org