You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/22 13:46:01 UTC

[incubator-datalab] 01/01: [DATALAB-2572]: fixed libs install for gcp deeplearning

This is an automated email from the ASF dual-hosted git repository.

lfrolov pushed a commit to branch DATALAB-2572
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git

commit fca8c124a55079d4802132397772b477e953a40e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 22 15:45:42 2022 +0200

    [DATALAB-2572]: fixed libs install for gcp deeplearning
---
 .../src/general/lib/os/debian/notebook_lib.py      | 26 +++++++++++-----------
 .../src/general/lib/os/fab.py                      | 10 ++++-----
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index cb468f7..66f347c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -273,24 +273,24 @@ def ensure_python3_libraries(os_user):
             manage_pkg('-y install', 'remote', 'libkrb5-dev')
             manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev '
                                                'liblzma-dev uuid-dev lzma-dev libgdbm-dev')  #necessary for python build
-            datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
+            datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff')
             if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true': 
-                datalab.fab.conn.sudo('pip3 install --upgrade --user pyqt5==5.12')
-                datalab.fab.conn.sudo('pip3 install --upgrade --user pyqtwebengine==5.12')
-                datalab.fab.conn.sudo('pip3 install setuptools')
+                datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12')
+                datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12')
+                datalab.fab.conn.sudo('-i pip3 install setuptools')
             else:
-                datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
+                datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
             try:
-                datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+                datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
                      .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
                              os.environ['notebook_nbconvert_version']))
             except:
-                datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+                datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
                      .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
                              os.environ['notebook_nbconvert_version']))
-            datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
-            datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir')
-            datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
+            datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+            datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir')
+            datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
             datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured')
         except:
             sys.exit(1)
@@ -561,11 +561,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
                    'libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev '
                    'libopencv-dev libopenmpi-dev libsnappy-dev openmpi-bin openmpi-doc python-pydot')
         datalab.fab.conn.sudo(
-            'pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
+            '-i pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
             'scipy tornado --no-cache-dir'.format(os.environ['notebook_matplotlib_version']))
         if os.environ['application'] == 'deeplearning':
             manage_pkg('-y install', 'remote', 'cmake')
-            datalab.fab.conn.sudo('pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+            datalab.fab.conn.sudo('-i pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
         else:
             # datalab.fab.conn.sudo('mkdir /opt/cuda-{}'.format(os.environ['notebook_cuda_version']))
             # datalab.fab.conn.sudo('mkdir /opt/cuda-{}/include/'.format(os.environ['notebook_cuda_version']))
@@ -605,7 +605,7 @@ def install_theano(os_user, theano_version):
 
 def install_mxnet(os_user, mxnet_version):
     if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
-        datalab.fab.conn.sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
+        datalab.fab.conn.sudo('-i pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
         datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
 
 
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index df8b3c4..7617b93 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -969,13 +969,13 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
     if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
         try:
             if os.environ['conf_deeplearning_cloud_ami'] == 'false' or os.environ['application'] != 'deeplearning':
-                conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
-                conn.sudo('pip3 install jupyter --no-cache-dir')
+                conn.sudo('-i pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+                conn.sudo('-i pip3 install jupyter --no-cache-dir')
                 conn.sudo('rm -rf {}'.format(jupyter_conf_file))
             elif os.environ['application'] != 'tensor':
-                conn.sudo('pip3 install environment_kernels')
-            if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
-                conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+                conn.sudo('-i pip3 install environment_kernels')
+            #if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
+            #    conn.sudo( pip3 install flask'chown -R {0}:{0} /home/{0}/.local'.format(os_user))
             conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
             conn.run('mkdir -p ~/.jupyter/custom/')
             conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org