You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/22 13:46:01 UTC
[incubator-datalab] 01/01: [DATALAB-2572]: fixed libs install for gcp deeplearning
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2572
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit fca8c124a55079d4802132397772b477e953a40e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 22 15:45:42 2022 +0200
[DATALAB-2572]: fixed libs install for gcp deeplearning
---
.../src/general/lib/os/debian/notebook_lib.py | 26 +++++++++++-----------
.../src/general/lib/os/fab.py | 10 ++++-----
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index cb468f7..66f347c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -273,24 +273,24 @@ def ensure_python3_libraries(os_user):
manage_pkg('-y install', 'remote', 'libkrb5-dev')
manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev '
'liblzma-dev uuid-dev lzma-dev libgdbm-dev') #necessary for python build
- datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
+ datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff')
if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true':
- datalab.fab.conn.sudo('pip3 install --upgrade --user pyqt5==5.12')
- datalab.fab.conn.sudo('pip3 install --upgrade --user pyqtwebengine==5.12')
- datalab.fab.conn.sudo('pip3 install setuptools')
+ datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12')
+ datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12')
+ datalab.fab.conn.sudo('-i pip3 install setuptools')
else:
- datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
+ datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
try:
- datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
os.environ['notebook_nbconvert_version']))
except:
- datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
os.environ['notebook_nbconvert_version']))
- datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
- datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir')
- datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
+ datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir')
+ datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured')
except:
sys.exit(1)
@@ -561,11 +561,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
'libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev '
'libopencv-dev libopenmpi-dev libsnappy-dev openmpi-bin openmpi-doc python-pydot')
datalab.fab.conn.sudo(
- 'pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
+ '-i pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
'scipy tornado --no-cache-dir'.format(os.environ['notebook_matplotlib_version']))
if os.environ['application'] == 'deeplearning':
manage_pkg('-y install', 'remote', 'cmake')
- datalab.fab.conn.sudo('pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+ datalab.fab.conn.sudo('-i pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
else:
# datalab.fab.conn.sudo('mkdir /opt/cuda-{}'.format(os.environ['notebook_cuda_version']))
# datalab.fab.conn.sudo('mkdir /opt/cuda-{}/include/'.format(os.environ['notebook_cuda_version']))
@@ -605,7 +605,7 @@ def install_theano(os_user, theano_version):
def install_mxnet(os_user, mxnet_version):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
- datalab.fab.conn.sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
+ datalab.fab.conn.sudo('-i pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index df8b3c4..7617b93 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -969,13 +969,13 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
if os.environ['conf_deeplearning_cloud_ami'] == 'false' or os.environ['application'] != 'deeplearning':
- conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
- conn.sudo('pip3 install jupyter --no-cache-dir')
+ conn.sudo('-i pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ conn.sudo('-i pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
elif os.environ['application'] != 'tensor':
- conn.sudo('pip3 install environment_kernels')
- if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
- conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+ conn.sudo('-i pip3 install environment_kernels')
+ #if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
+ # conn.sudo( pip3 install flask'chown -R {0}:{0} /home/{0}/.local'.format(os_user))
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org