You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by ol...@apache.org on 2022/02/15 11:13:55 UTC
[incubator-datalab] branch 2.5.1-tensor-jupyterlab updated: added jupyterlab config function
This is an automated email from the ASF dual-hosted git repository.
olehmykolaishyn pushed a commit to branch 2.5.1-tensor-jupyterlab
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/2.5.1-tensor-jupyterlab by this push:
new 683ae33 added jupyterlab config function
new 06d8d2f Merge branch '2.5.1-tensor-jupyterlab' of https://github.com/apache/incubator-datalab into 2.5.1-tensor-jupyterlab
683ae33 is described below
commit 683ae33d0ea7c08060e83800c40ffae6220b0552
Author: oleh_mykolaishyn <ow...@gmail.com>
AuthorDate: Tue Feb 15 13:13:10 2022 +0200
added jupyterlab config function
---
.../src/general/lib/os/fab.py | 74 ++++++++++++++++++++++
.../src/ssn/scripts/docker_build.py | 15 +++--
.../scripts/configure_tensor-jupyterlab_node.py | 12 +++-
3 files changed, 91 insertions(+), 10 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 22215e6..6444a7b 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -1043,6 +1043,80 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
sys.exit(1)
+def configure_jupyterlab(os_user, jupyterlab_conf_file, templates_dir, jupyterlab_version, exploratory_name):
+ if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyterlab_ensured'):
+ try:
+ # conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ conn.sudo('source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}').format(python_version, pip_version)
+ #conn.sudo('pip3 install jupyterlab --no-cache-dir') # create external var with version
+ conn.sudo('rm -rf {}'.format(jupyterlab_conf_file))
+ #elif os.environ['application'] != 'tensor':
+ # conn.sudo('pip3 install environment_kernels')
+ #if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
+ # conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+ conn.run('jupyter lab --generate-config --config {}'.format(jupyter_conf_file))
+ conn.run('mkdir -p ~/.jupyter/custom/')
+ conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
+ conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
+ conn.sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
+ conn.sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
+ conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
+ conn.sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
+ conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+ if os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
+ conn.sudo(
+ '''echo "c.NotebookApp.kernel_spec_manager_class = 'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
+ jupyter_conf_file))
+ conn.sudo(
+ '''echo "c.EnvironmentKernelSpecManager.conda_env_dirs=['/home/ubuntu/anaconda3/envs']" >> {}'''.format(
+ jupyter_conf_file))
+ conn.put(templates_dir + 'jupyter-notebook.service', '/tmp/jupyter-notebook.service')
+ conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
+ if os.environ['application'] == 'tensor':
+ conn.sudo(
+ "sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; |g' /tmp/jupyter-notebook.service")
+ elif os.environ['application'] == 'deeplearning' and os.environ['conf_deeplearning_cloud_ami'] == 'false':
+ conn.sudo(
+ "sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib: ; export PYTHONPATH=/home/" + os_user +
+ "/caffe/python:/home/" + os_user + "/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
+ conn.sudo("sed -i 's|CONF_PATH|{}|' /tmp/jupyter-notebook.service".format(jupyter_conf_file))
+ conn.sudo("sed -i 's|OS_USR|{}|' /tmp/jupyter-notebook.service".format(os_user))
+ if os.environ['application'] == 'deeplearning' and os.environ['conf_cloud_provider'] == 'azure':
+ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-11.*/bin/java\'").stdout.splitlines()[0]
+ else:
+ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
+ conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyter-notebook.service'.format(
+ java_home))
+ conn.sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
+ conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+ conn.sudo('mkdir -p /mnt/var')
+ conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
+ if os.environ['application'] == 'jupyter' or os.environ['application'] == 'deeplearning':
+ try:
+ conn.sudo('jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists"')
+ conn.sudo('jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"')
+ except Exception as err:
+ logging.error('Error:', str(err))
+ conn.sudo("systemctl daemon-reload")
+ conn.sudo("systemctl enable jupyter-notebook")
+ conn.sudo("systemctl start jupyter-notebook")
+ conn.sudo('touch /home/{}/.ensure_dir/jupyter_ensured'.format(os_user))
+ except Exception as err:
+ logging.error('Function configure_jupyter error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+ else:
+ try:
+ conn.sudo(
+ 'sed -i "s/c.NotebookApp.base_url =.*/c.NotebookApp.base_url = \'\/{0}\/\'/" {1}'.format(
+ exploratory_name, jupyter_conf_file))
+ conn.sudo("systemctl restart jupyter-notebook")
+ except Exception as err:
+ logging.error('Function configure_jupyter error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path,
python_venv_version):
if not exists(conn, '/home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured'):
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index f0ddafe..57daf72 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -38,13 +38,14 @@ if sys.argv[1] == 'all':
'jupyterlab',
'rstudio',
'zeppelin',
- 'tensor',
- 'tensor-rstudio',
- 'deeplearning',
- 'dataengine',
- 'dataengine-service',
- 'superset'
- ]
+ 'tensor',
+ 'tensor-rstudio',
+ 'deeplearning',
+ 'dataengine',
+ 'dataengine-service',
+ 'tensor-jupyterlab',
+ 'superset'
+ ]
else:
node = sys.argv[1:]
diff --git a/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py b/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
index 3686231..3258a89 100644
--- a/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
+++ b/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
@@ -45,7 +45,7 @@ args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
hadoop_version = os.environ['notebook_hadoop_version']
tensorflow_version = os.environ['notebook_tensorflow_version']
-jupyter_version = os.environ['notebook_jupyter_version']
+jupyterlab_version = os.environ['notebook_jupyterlab_version']
nvidia_version = os.environ['notebook_nvidia_version']
theano_version = os.environ['notebook_theano_version']
keras_version = os.environ['notebook_keras_version']
@@ -63,13 +63,16 @@ local_spark_path = '/opt/spark/'
jars_dir = '/opt/jars/'
templates_dir = '/root/templates/'
files_dir = '/root/files/'
-jupyter_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyter_notebook_config.py'
+#jupyterlab_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyterlab_notebook_config.py'
+
+#
gitlab_certfile = os.environ['conf_gitlab_certfile']
cuda_version = os.environ['notebook_cuda_version']
cuda_file_name = os.environ['notebook_cuda_file_name']
cudnn_version = os.environ['notebook_cudnn_version']
cudnn_file_name = os.environ['notebook_cudnn_file_name']
venv_libs = 'numpy scipy matplotlib pandas scikit-learn opencv-python'
+jupyterlab_pip = 'jupyterlab'
##############
# Run script #
@@ -111,7 +114,7 @@ if __name__ == "__main__":
# INSTALL JUPYTER NOTEBOOK
print("Install Jupyter")
- configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name)
+ configure_jupyterlab(args.os_user, jupyterlab_conf_file, templates_dir, jupyterlab_version, args.exploratory_name)
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
print("Install local Spark")
@@ -151,6 +154,9 @@ if __name__ == "__main__":
print("Install python venv required libs")
ensure_venv_libs(args.os_user, venv_libs)
+ print("Install python venv required libs")
+ ensure_venv_libs(args.os_user, 'jupyterlab')
+
#POST INSTALLATION PROCESS
print("Updating pyOpenSSL library")
update_pyopenssl_lib(args.os_user)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org