You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by mh...@apache.org on 2022/03/07 13:10:43 UTC
[incubator-datalab] branch gcp-jupyter-highgpu-template-1 updated: Added files for jupyter-gpu-conda
This is an automated email from the ASF dual-hosted git repository.
mhladun pushed a commit to branch gcp-jupyter-highgpu-template-1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/gcp-jupyter-highgpu-template-1 by this push:
new bc92559 Added files for jupyter-gpu-conda
bc92559 is described below
commit bc92559d4616a0cb1b9c32dceea6932d8911cfc5
Author: Marian_Hladun <ma...@gmail.com>
AuthorDate: Mon Mar 7 15:10:23 2022 +0200
Added files for jupyter-gpu-conda
---
.../general/files/gcp/jupyter-gpu-conda_Dockerfile | 54 ++++
.../files/gcp/jupyter-gpu-conda_description.json | 18 ++
.../scripts/gcp/jupyter-gpu-conda_configure.py | 322 +++++++++++++++++++++
.../src/jupyter-gpu-conda/fabfile.py | 255 ++++++++++++++++
.../scripts/configure_jupyter-gpu-conda_node.py | 159 ++++++++++
5 files changed, 808 insertions(+)
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile
new file mode 100644
index 0000000..170cda9
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile
@@ -0,0 +1,54 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+
+FROM docker.datalab-base:latest
+
+ARG OS
+
+COPY jupyter-gpu-conda/ /root/
+COPY general/scripts/os/* /root/scripts/
+COPY general/scripts/gcp/jupyter-gpu-conda_* /root/scripts/
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
+COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
+COPY general/templates/os/${OS}/ungit.service /root/templates/
+COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
+COPY general/templates/os/pyspark_local_template.json /root/templates/
+COPY general/templates/os/py3spark_local_template.json /root/templates/
+COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/
+COPY general/templates/os/sparkmagic_config_template.json /root/templates/
+COPY general/templates/os/r_dataengine-service_template.json /root/templates/
+COPY general/templates/os/r_template.json /root/templates/
+COPY general/templates/os/run_template.sh /root/templates/
+COPY general/templates/os/toree_dataengine-service_* /root/templates/
+COPY general/templates/os/inactive.sh /root/templates/
+COPY general/templates/os/inactive.service /root/templates/
+COPY general/templates/os/inactive.timer /root/templates/
+COPY general/files/os/toree-assembly-0.5.0.jar /root/files/
+COPY general/files/os/toree_kernel.tar.gz /root/files/
+COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
+COPY general/templates/os/r_dataengine_template.json /root/templates/
+COPY general/templates/os/toree_dataengine_template.json /root/templates/
+COPY general/templates/gcp/core-site.xml /root/templates/
+
+RUN chmod a+x /root/fabfile.py; \
+ chmod a+x /root/scripts/*
+
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json
new file mode 100644
index 0000000..01c6cc2
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json
@@ -0,0 +1,18 @@
+{
+ "exploratory_environment_shapes" :
+ {
+ "GPU optimized" : [
+ {"Size": "S", "Description": "a2-highgpu-1g", "Type": "a2-highgpu-1g","Ram": "85 GB","Cpu": "12"}
+ ]
+ },
+ "exploratory_environment_versions" :
+ [
+ {
+ "template_name": "Jupyter notebook 6.1.6 with GPU and Anaconda",
+ "description": "Base image with jupyter node creation routines",
+ "environment_type": "exploratory",
+ "version": "jupyter_notebook_gpu_conda",
+ "vendor": "GCP"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py
new file mode 100644
index 0000000..2f1c6f1
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py
@@ -0,0 +1,322 @@
+#!/usr/bin/python3
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import datalab.fab
+import datalab.actions_lib
+import datalab.meta_lib
+import json
+from datalab.logger import logging
+import os
+import sys
+import traceback
+import subprocess
+from fabric import *
+
+if __name__ == "__main__":
+ try:
+ GCPMeta = datalab.meta_lib.GCPMeta()
+ GCPActions = datalab.actions_lib.GCPActions()
+ notebook_config = dict()
+ try:
+ notebook_config['exploratory_name'] = (os.environ['exploratory_name']).replace('_', '-').lower()
+ except:
+ notebook_config['exploratory_name'] = ''
+ notebook_config['service_base_name'] = (os.environ['conf_service_base_name'])
+ notebook_config['instance_type'] = os.environ['gcp_notebook_instance_size']
+ notebook_config['key_name'] = os.environ['conf_key_name']
+ notebook_config['edge_user_name'] = (os.environ['edge_user_name'])
+ notebook_config['project_name'] = (os.environ['project_name']).replace('_', '-').lower()
+ notebook_config['project_tag'] = notebook_config['project_name']
+ notebook_config['endpoint_name'] = (os.environ['endpoint_name']).replace('_', '-').lower()
+ notebook_config['endpoint_tag'] = notebook_config['endpoint_name']
+ notebook_config['instance_name'] = '{0}-{1}-{2}-nb-{3}'.format(notebook_config['service_base_name'],
+ notebook_config['project_name'],
+ notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ notebook_config['image_enabled'] = os.environ['conf_image_enabled']
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if notebook_config['shared_image_enabled'] == 'false':
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['project_name'], notebook_config['endpoint_name'],
+ os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'], notebook_config['project_name'], notebook_config['endpoint_name'],
+ os.environ['application'])
+ notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
+ "endpoint_tag": notebook_config['endpoint_tag'],
+ "project_tag": notebook_config['project_tag'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
+ else:
+ notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_name'], os.environ['application'])
+ notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'], notebook_config['endpoint_name'], os.environ['application'])
+ notebook_config['image_labels'] = {"sbn": notebook_config['service_base_name'],
+ "endpoint_tag": notebook_config['endpoint_tag'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']}
+ # generating variables regarding EDGE proxy on Notebook instance
+ instance_hostname = GCPMeta.get_private_ip_address(notebook_config['instance_name'])
+ edge_instance_name = '{0}-{1}-{2}-edge'.format(notebook_config['service_base_name'],
+ notebook_config['project_name'],
+ notebook_config['endpoint_name'])
+ edge_instance_hostname = GCPMeta.get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta.get_private_ip_address(edge_instance_name)
+ notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['zone'] = os.environ['gcp_zone']
+ notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
+ except Exception as err:
+ datalab.fab.append_result("Failed to generate variables dictionary", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ if os.environ['conf_os_family'] == 'debian':
+ notebook_config['initial_user'] = 'ubuntu'
+ notebook_config['sudo_group'] = 'sudo'
+ if os.environ['conf_os_family'] == 'redhat':
+ notebook_config['initial_user'] = 'ec2-user'
+ notebook_config['sudo_group'] = 'wheel'
+
+ logging.info('[CREATING DATALAB SSH USER]')
+ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format(
+ instance_hostname, notebook_config['ssh_key_path'], notebook_config['initial_user'],
+ notebook_config['datalab_ssh_user'], notebook_config['sudo_group'])
+
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed creating ssh user 'datalab'.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # configuring proxy on Notebook instance
+ try:
+ logging.info('[CONFIGURE PROXY ON JUPYTER INSTANCE]')
+ additional_config = {"proxy_host": edge_instance_name, "proxy_port": "3128"}
+ params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
+ .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
+ json.dumps(additional_config), notebook_config['datalab_ssh_user'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_proxy', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed to configure proxy.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # updating repositories & installing python packages
+ try:
+ logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['datalab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed installing apps: apt & pip.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # installing and configuring jupiter and all dependencies
+ try:
+ logging.info('[CONFIGURE JUPYTER NOTEBOOK INSTANCE]')
+ params = "--hostname {} --keyfile {} " \
+ "--region {} --spark_version {} " \
+ "--hadoop_version {} --os_user {} " \
+ "--scala_version {} " \
+ "--exploratory_name {} "\
+ "--edge_ip {}".\
+ format(instance_hostname, notebook_config['ssh_key_path'],
+ os.environ['gcp_region'], os.environ['notebook_spark_version'],
+ os.environ['notebook_hadoop_version'], notebook_config['datalab_ssh_user'],
+ os.environ['notebook_scala_version'],
+ notebook_config['exploratory_name'], edge_instance_private_ip)
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_jupyter-gpu_node', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed to configure jupyter.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ logging.info('[INSTALLING USERs KEY]')
+ additional_config = {"user_keyname": os.environ['project_name'],
+ "user_keydir": os.environ['conf_key_dir']}
+ params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
+ instance_hostname, notebook_config['ssh_key_path'], json.dumps(additional_config),
+ notebook_config['datalab_ssh_user'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed installing users key")
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed installing users key.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ logging.info('[SETUP USER GIT CREDENTIALS]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(notebook_config['datalab_ssh_user'], instance_hostname, notebook_config['ssh_key_path'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('common_download_git_certfile', params), shell=True, check=True)
+ subprocess.run("~/scripts/{}.py {}".format('manage_git_creds', params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup git credentials")
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed to setup git credentials.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ # installing gpu
+ try:
+ logging.info('[INSTALLING GPU DRIVERS]')
+ params = "--hostname {} --keyfile {} --os_user {}".format(
+ instance_hostname, notebook_config['ssh_key_path'], notebook_config['datalab_ssh_user'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('common_install_gpu', params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed installing gpu drivers")
+ raise Exception
+
+ except Exception as err:
+ datalab.fab.append_result("Failed to install GPU drivers.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+
+ if notebook_config['image_enabled'] == 'true':
+ try:
+ logging.info('[CREATING IMAGE]')
+ primary_image_id = GCPMeta.get_image_by_name(notebook_config['expected_primary_image_name'])
+ if primary_image_id == '':
+ logging.info("Looks like it's first time we configure notebook server. Creating images.")
+ image_id_list = GCPActions.create_image_from_instance_disks(
+ notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
+ notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
+ notebook_config['gcp_wrapped_csek'])
+ if image_id_list and image_id_list[0] != '':
+ logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
+ else:
+ logging.info("Looks like another image creating operation for your template have been started a "
+ "moment ago.")
+ if image_id_list and image_id_list[1] != '':
+ logging.info("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1]))
+ except Exception as err:
+ datalab.fab.append_result("Failed creating image.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ GCPActions.remove_image(notebook_config['expected_primary_image_name'])
+ GCPActions.remove_image(notebook_config['expected_secondary_image_name'])
+ sys.exit(1)
+
+
+ try:
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ notebook_config['ssh_key_path'],
+ notebook_config['datalab_ssh_user'],
+ 'jupyter-gpu',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ datalab.fab.append_result("Failed to set edge reverse proxy template.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
+ try:
+ # generating output information
+ ip_address = GCPMeta.get_private_ip_address(notebook_config['instance_name'])
+ jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
+ ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_access_url = "https://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ jupyter_ungit_access_url = "https://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
+ logging.info('[SUMMARY]')
+ logging.info("Instance name: {}".format(notebook_config['instance_name']))
+ logging.info("Private IP: {}".format(ip_address))
+ logging.info("Instance type: {}".format(notebook_config['instance_type']))
+ logging.info("Key name: {}".format(notebook_config['key_name']))
+ logging.info("User key name: {}".format(os.environ['project_name']))
+ logging.info("Jupyter URL: {}".format(jupyter_ip_url))
+ logging.info("Ungit URL: {}".format(ungit_ip_url))
+ logging.info("ReverseProxyNotebook".format(jupyter_notebook_access_url))
+ logging.info("ReverseProxyUngit".format(jupyter_ungit_access_url))
+ logging.info('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(
+ notebook_config['key_name'], notebook_config['datalab_ssh_user'], ip_address))
+
+ with open("/root/result.json", 'w') as result:
+ res = {"hostname": ip_address,
+ "ip": ip_address,
+ "instance_id": notebook_config['instance_name'],
+ "master_keyname": os.environ['conf_key_name'],
+ "notebook_name": notebook_config['instance_name'],
+ "Action": "Create new notebook server",
+ "exploratory_url": [
+ {"description": "Jupyter",
+ "url": jupyter_notebook_access_url},
+ {"description": "Ungit",
+ "url": jupyter_ungit_access_url}#,
+ #{"description": "Jupyter (via tunnel)",
+ # "url": jupyter_ip_url},
+ #{"description": "Ungit (via tunnel)",
+ # "url": ungit_ip_url}
+ ]}
+ result.write(json.dumps(res))
+ except Exception as err:
+ datalab.fab.append_result("Failed to generate output information", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py b/infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py
new file mode 100644
index 0000000..cee6a85
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py
@@ -0,0 +1,255 @@
+#!/usr/bin/python3
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import os
+import sys
+import uuid
+import subprocess
+from datalab.actions_lib import *
+from datalab.fab import *
+from datalab.meta_lib import *
+
+
+# Main function for provisioning notebook server
+@task
+def run(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ notebook_config = dict()
+ notebook_config['uuid'] = str(uuid.uuid4())[:5]
+
+ try:
+ params = "--uuid {}".format(notebook_config['uuid'])
+ subprocess.run("~/scripts/{}.py {}".format('common_prepare_notebook', params), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed preparing Notebook node.", str(err))
+ sys.exit(1)
+
+ try:
+ params = "--uuid {}".format(notebook_config['uuid'])
+ subprocess.run("~/scripts/{}.py {}".format('jupyter-gpu_configure', params), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed configuring Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for terminating exploratory environment
+@task
+def terminate(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ subprocess.run("~/scripts/{}.py".format('common_terminate_notebook'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed terminating Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for stopping notebook server
+@task
+def stop(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ subprocess.run("~/scripts/{}.py".format('common_stop_notebook'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed stopping Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for starting notebook server
+@task
+def start(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('common_start_notebook'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed starting Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for configuring notebook server after deploying DataEngine service
+@task
+def configure(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ if os.environ['conf_resource'] == 'dataengine-service':
+ subprocess.run("~/scripts/{}.py".format('common_notebook_configure_dataengine-service'), shell=True, check=True)
+ elif os.environ['conf_resource'] == 'dataengine':
+ subprocess.run("~/scripts/{}.py".format('common_notebook_configure_dataengine'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed configuring analytical tool on Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for installing additional libraries for notebook
+@task
+def install_libs(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('notebook_install_libs'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed installing additional libs for Notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for get available libraries for notebook
+@task
+def list_libs(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('notebook_list_libs'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed get available libraries for notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for manage git credentials on notebook
+@task
+def git_creds(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('notebook_git_creds'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to manage git credentials for notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for creating image from notebook
+@task
+def create_image(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('common_create_notebook_image'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to create image from notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for deleting existing notebook image
+@task
+def terminate_image(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('common_terminate_notebook_image'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to create image from notebook node.", str(err))
+ sys.exit(1)
+
+
+# Main function for reconfiguring Spark for notebook
+@task
+def reconfigure_spark(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('notebook_reconfigure_spark'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to reconfigure Spark for Notebook node.", str(err))
+ sys.exit(1)
+
+# Main function for checking inactivity status
+@task
+def check_inactivity(ctx):
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ subprocess.run("~/scripts/{}.py".format('notebook_inactivity_check'), shell=True, check=True)
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to check inactivity status.", str(err))
+ sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py b/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py
new file mode 100644
index 0000000..a40fc81
--- /dev/null
+++ b/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py
@@ -0,0 +1,159 @@
+#!/usr/bin/python3
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import argparse
+import os
+import sys
+from datalab.actions_lib import *
+from datalab.fab import *
+from datalab.notebook_lib import *
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--hostname', type=str, default='')
+parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--region', type=str, default='')
+parser.add_argument('--spark_version', type=str, default='')
+parser.add_argument('--hadoop_version', type=str, default='')
+parser.add_argument('--os_user', type=str, default='')
+parser.add_argument('--scala_version', type=str, default='')
+parser.add_argument('--ip_address', type=str, default='')
+parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
+args = parser.parse_args()
+
+spark_version = args.spark_version
+hadoop_version = args.hadoop_version
+jupyter_version = os.environ['notebook_jupyter_version']
+python_venv_version = os.environ['notebook_python_venv_version']
+scala_link = "https://www.scala-lang.org/files/archive/"
+if args.region == 'cn-north-1':
+ spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
+ "-bin-hadoop" + hadoop_version + ".tgz"
+else:
+ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
+ "-bin-hadoop" + hadoop_version + ".tgz"
+python_venv_path = '/opt/python/python{0}/bin/python{1}'.format(python_venv_version, python_venv_version[:3])
+pyspark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/pyspark_local/'
+py3spark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/py3spark_local/'
+jupyter_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyter_notebook_config.py'
+scala_kernel_path = '/usr/local/share/jupyter/kernels/apache_toree_scala/'
+r_kernels_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/'
+jars_dir = '/opt/jars/'
+templates_dir = '/root/templates/'
+files_dir = '/root/files/'
+local_spark_path = '/opt/spark/'
+toree_link = 'https://dist.apache.org/repos/dist/dev/incubator/toree/0.5.0-incubating-rc1/toree-pip/toree-0.5.0.tar.gz'
+r_libs = ['R6', 'pbdZMQ={}'.format(os.environ['notebook_pbdzmq_version']), 'RCurl', 'reshape2', 'caTools={}'.format(os.environ['notebook_catools_version']), 'rJava', 'ggplot2']
+gitlab_certfile = os.environ['conf_gitlab_certfile']
+venv_libs = 'numpy scipy pandas scikit-learn python-git transformers==4.4.2 gensim==4.0.1 tokenizers==0.10.1 python-levenshtein==0.12.2'
+
+
+##############
+# Run script #
+##############
+if __name__ == "__main__":
+ print("Configure connections")
+ global conn
+ conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile)
+
+ # PREPARE DISK
+ print("Prepare .ensure directory")
+ try:
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
+ conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
+ except:
+ sys.exit(1)
+ print("Mount additional volume")
+ prepare_disk(args.os_user)
+
+ # INSTALL LANGUAGES
+ print("Install Java")
+ ensure_jre_jdk(args.os_user)
+ print("Install Scala")
+ ensure_scala(scala_link, args.scala_version, args.os_user)
+ print("Install Python 3 modules")
+ ensure_python3_libraries(args.os_user)
+
+ # INSTALL PYTHON IN VIRTUALENV
+ print("Configure Python Virtualenv")
+ ensure_python_venv(python_venv_version)
+
+ # INSTALL JUPYTER NOTEBOOK
+ print("Install Jupyter")
+ configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name)
+
+ # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
+ print("Install local Spark")
+ ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path)
+ local_spark_scala_version = conn.run(
+ 'export PATH=$PATH:' + local_spark_path + 'bin/; spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"').stdout.replace(
+ '\n', '')
+ print("Install storage jars")
+ ensure_local_jars(args.os_user, jars_dir)
+ print("Configure local Spark")
+ configure_local_spark(jars_dir, templates_dir)
+
+ # INSTALL JUPYTER KERNELS
+ #print("Install pyspark local kernel for Jupyter")
+ #ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
+ print("Install py3spark local kernel for Jupyter")
+ ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version)
+ print("Install Toree-Scala kernel for Jupyter")
+ ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, local_spark_scala_version, spark_version)
+
+ # INSTALL UNGIT
+ print("Install nodejs")
+ install_nodejs(args.os_user)
+ print("Install ungit")
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
+ if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
+ install_gitlab_cert(args.os_user, gitlab_certfile)
+
+ # INSTALL INACTIVITY CHECKER
+ print("Install inactivity checker")
+ install_inactivity_checker(args.os_user, args.ip_address)
+
+ # INSTALL OPTIONAL PACKAGES
+ print("Installing additional Python packages")
+ ensure_additional_python_libs(args.os_user)
+ print("Install Matplotlib")
+ ensure_matplot(args.os_user)
+ print("Install SBT")
+ ensure_sbt(args.os_user)
+ print("Install Breeze")
+ add_breeze_library_local(args.os_user)
+ if os.environ['conf_cloud_provider'] == 'gcp':
+ print('Installing Pytorch')
+ ensure_pytorch(args.os_user)
+
+ # INSTALL PIP PACKAGES
+ print("Install python venv required libs")
+ ensure_venv_libs(args.os_user, venv_libs)
+
+ #POST INSTALLATION PROCESS
+ print("Updating pyOpenSSL library")
+ update_pyopenssl_lib(args.os_user)
+ print("Removing unexisting kernels")
+ remove_unexisting_kernel(args.os_user)
+
+ conn.close()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org