You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/04/27 08:52:32 UTC
[incubator-datalab] branch DATALAB-2372 updated: [DataLab-2372] -
[AWS] Deeplearning notebook updated for using aws predefined image
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2372
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2372 by this push:
new dc9d8c2 [DataLab-2372] - [AWS] Deeplearning notebook updated for using aws predefined image
dc9d8c2 is described below
commit dc9d8c201611e9a136c4a2e0f791a64731db41cc
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Tue Apr 27 11:52:01 2021 +0300
[DataLab-2372] - [AWS] Deeplearning notebook updated for using aws predefined image
---
.../scripts/configure_deep_learning_node.py | 103 +++++++++++----------
.../src/general/lib/os/debian/notebook_lib.py | 2 +-
.../src/general/lib/os/fab.py | 20 ++--
3 files changed, 69 insertions(+), 56 deletions(-)
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index 3ac031d..39642f0 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -106,49 +106,60 @@ if __name__ == "__main__":
print("Mount additional volume")
prepare_disk(args.os_user)
- # INSTALL LANGUAGES
- print("Install Java")
- ensure_jre_jdk(args.os_user)
- print("Install Python 3 modules")
- ensure_python3_libraries(args.os_user)
-
- # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES AND FRAMEWORKS
- print("Install TensorFlow")
- install_tensor(args.os_user, cuda_version, cuda_file_name,
- cudnn_version, cudnn_file_name, tensorflow_version,
- templates_dir, nvidia_version)
- print("Install Theano")
- install_theano(args.os_user, theano_version)
- print("Installing Keras")
- install_keras(args.os_user, keras_version)
- print("Installing Caffe2")
- install_caffe2(args.os_user, caffe2_version, cmake_version)
- #print("Installing Torch")
- #install_torch(args.os_user)
- print("Install CNTK Python library")
- install_cntk(args.os_user, cntk_version)
- print("Installing MXNET")
- install_mxnet(args.os_user, mxnet_version)
-
- # INSTALL JUPYTER NOTEBOOK
- print("Install Jupyter")
- configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, args.jupyter_version, args.exploratory_name)
-
- # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
- print("Install local Spark")
- ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path)
- print("Install storage jars")
- ensure_local_jars(args.os_user, jars_dir)
- print("Configure local Spark")
- configure_local_spark(jars_dir, templates_dir)
-
- # INSTALL JUPYTER KERNELS
- print("Install pyspark local kernel for Jupyter")
- ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
- print("Install py3spark local kernel for Jupyter")
- ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version)
- #print("Installing ITorch kernel for Jupyter")
- #install_itorch(args.os_user)
+ if not os.environ['aws_deeplearning_image_name']:
+ # INSTALL LANGUAGES
+ print("Install Java")
+ ensure_jre_jdk(args.os_user)
+ print("Install Python 3 modules")
+ ensure_python3_libraries(args.os_user)
+
+ # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES AND FRAMEWORKS
+ print("Install TensorFlow")
+ install_tensor(args.os_user, cuda_version, cuda_file_name,
+ cudnn_version, cudnn_file_name, tensorflow_version,
+ templates_dir, nvidia_version)
+ print("Install Theano")
+ install_theano(args.os_user, theano_version)
+ print("Installing Keras")
+ install_keras(args.os_user, keras_version)
+ print("Installing Caffe2")
+ install_caffe2(args.os_user, caffe2_version, cmake_version)
+ #print("Installing Torch")
+ #install_torch(args.os_user)
+ print("Install CNTK Python library")
+ install_cntk(args.os_user, cntk_version)
+ print("Installing MXNET")
+ install_mxnet(args.os_user, mxnet_version)
+
+ # INSTALL JUPYTER NOTEBOOK
+ print("Install Jupyter")
+ configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, args.jupyter_version, args.exploratory_name)
+
+ # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
+ print("Install local Spark")
+ ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path)
+ print("Install storage jars")
+ ensure_local_jars(args.os_user, jars_dir)
+ print("Configure local Spark")
+ configure_local_spark(jars_dir, templates_dir)
+
+ # INSTALL JUPYTER KERNELS
+ print("Install pyspark local kernel for Jupyter")
+ ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
+ print("Install py3spark local kernel for Jupyter")
+ ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version)
+ #print("Installing ITorch kernel for Jupyter")
+ #install_itorch(args.os_user)
+
+ # INSTALL OPTIONAL PACKAGES
+ print("Installing additional Python packages")
+ ensure_additional_python_libs(args.os_user)
+ print("Install Matplotlib")
+ ensure_matplot(args.os_user)
+ else:
+ # CONFIGURE JUPYTER NOTEBOOK
+ print("Configure Jupyter")
+ configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, args.jupyter_version, args.exploratory_name)
# INSTALL UNGIT
print("Install nodejs")
@@ -162,12 +173,6 @@ if __name__ == "__main__":
print("Install inactivity checker")
install_inactivity_checker(args.os_user, args.ip_address)
- # INSTALL OPTIONAL PACKAGES
- print("Installing additional Python packages")
- ensure_additional_python_libs(args.os_user)
- print("Install Matplotlib")
- ensure_matplot(args.os_user)
-
#POST INSTALLATION PROCESS
print("Updating pyOpenSSL library")
update_pyopenssl_lib(args.os_user)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 62d1730..0a199f6 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -268,7 +268,7 @@ def ensure_python3_libraries(os_user):
except:
sys.exit(1)
-def install_nvidia_drivers(os_user)
+def install_nvidia_drivers(os_user):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/nvidia_ensured'.format(os_user)):
try:
# install nvidia drivers
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 2f9a3f0..18d3818 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -191,9 +191,10 @@ def put_resource_status(resource, status, datalab_path, os_user, hostname):
def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
- conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
- conn.sudo('pip3 install jupyter --no-cache-dir')
- conn.sudo('rm -rf {}'.format(jupyter_conf_file))
+ if not os.environ['aws_deeplearning_image_name']:
+ conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ conn.sudo('pip3 install jupyter --no-cache-dir')
+ conn.sudo('rm -rf {}'.format(jupyter_conf_file))
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
@@ -203,11 +204,18 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
conn.sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+ if os.environ['aws_deeplearning_image_name']:
+ conn.sudo(
+ '''echo "c.NotebookApp.kernel_spec_manager_class = 'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
+ jupyter_conf_file))
+ conn.sudo(
+ '''echo "c.EnvironmentKernelSpecManager.conda_env_dirs=['/home/ubuntu/anaconda3/envs']" >> {}'''.format(
+ jupyter_conf_file))
conn.put(templates_dir + 'jupyter-notebook.service', '/tmp/jupyter-notebook.service')
conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
if os.environ['application'] == 'tensor':
conn.sudo("sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; |g' /tmp/jupyter-notebook.service")
- elif os.environ['application'] == 'deeplearning':
+ elif os.environ['application'] == 'deeplearning' and not os.environ['aws_plearning_image_name']:
conn.sudo("sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib: ; export PYTHONPATH=/home/" + os_user +
"/caffe/python:/home/" + os_user + "/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
conn.sudo("sed -i 's|CONF_PATH|{}|' /tmp/jupyter-notebook.service".format(jupyter_conf_file))
@@ -225,9 +233,10 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
conn.sudo('mkdir -p /mnt/var')
conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
- if os.environ['application'] == 'jupyter':
+ if os.environ['application'] == 'jupyter' or os.environ['application'] == 'deeplearning':
try:
conn.sudo('jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists"')
+ conn.sudo('jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"')
except Exception as err:
print('Error:', str(err))
conn.sudo("systemctl daemon-reload")
@@ -245,7 +254,6 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
print('Error:', str(err))
sys.exit(1)
-
def remove_unexisting_kernel(os_user):
if not exists(conn,'/home/{}/.ensure_dir/unexisting_kernel_removed'.format(os_user)):
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org