You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/22 13:46:00 UTC
[incubator-datalab] branch DATALAB-2572 created (now fca8c12)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2572
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at fca8c12 [DATALAB-2572]: fixed libs install for gcp deeplearning
This branch includes the following new commits:
new fca8c12 [DATALAB-2572]: fixed libs install for gcp deeplearning
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2572]: fixed libs install for gcp deeplearning
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2572
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit fca8c124a55079d4802132397772b477e953a40e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 22 15:45:42 2022 +0200
[DATALAB-2572]: fixed libs install for gcp deeplearning
---
.../src/general/lib/os/debian/notebook_lib.py | 26 +++++++++++-----------
.../src/general/lib/os/fab.py | 10 ++++-----
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index cb468f7..66f347c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -273,24 +273,24 @@ def ensure_python3_libraries(os_user):
manage_pkg('-y install', 'remote', 'libkrb5-dev')
manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev '
'liblzma-dev uuid-dev lzma-dev libgdbm-dev') #necessary for python build
- datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
+ datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff')
if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true':
- datalab.fab.conn.sudo('pip3 install --upgrade --user pyqt5==5.12')
- datalab.fab.conn.sudo('pip3 install --upgrade --user pyqtwebengine==5.12')
- datalab.fab.conn.sudo('pip3 install setuptools')
+ datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12')
+ datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12')
+ datalab.fab.conn.sudo('-i pip3 install setuptools')
else:
- datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
+ datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
try:
- datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
os.environ['notebook_nbconvert_version']))
except:
- datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
os.environ['notebook_nbconvert_version']))
- datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
- datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir')
- datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
+ datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir')
+ datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured')
except:
sys.exit(1)
@@ -561,11 +561,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
'libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev '
'libopencv-dev libopenmpi-dev libsnappy-dev openmpi-bin openmpi-doc python-pydot')
datalab.fab.conn.sudo(
- 'pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
+ '-i pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image '
'scipy tornado --no-cache-dir'.format(os.environ['notebook_matplotlib_version']))
if os.environ['application'] == 'deeplearning':
manage_pkg('-y install', 'remote', 'cmake')
- datalab.fab.conn.sudo('pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+ datalab.fab.conn.sudo('-i pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
else:
# datalab.fab.conn.sudo('mkdir /opt/cuda-{}'.format(os.environ['notebook_cuda_version']))
# datalab.fab.conn.sudo('mkdir /opt/cuda-{}/include/'.format(os.environ['notebook_cuda_version']))
@@ -605,7 +605,7 @@ def install_theano(os_user, theano_version):
def install_mxnet(os_user, mxnet_version):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
- datalab.fab.conn.sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
+ datalab.fab.conn.sudo('-i pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index df8b3c4..7617b93 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -969,13 +969,13 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
if os.environ['conf_deeplearning_cloud_ami'] == 'false' or os.environ['application'] != 'deeplearning':
- conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
- conn.sudo('pip3 install jupyter --no-cache-dir')
+ conn.sudo('-i pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ conn.sudo('-i pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
elif os.environ['application'] != 'tensor':
- conn.sudo('pip3 install environment_kernels')
- if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
- conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+ conn.sudo('-i pip3 install environment_kernels')
+ #if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
+ # conn.sudo( pip3 install flask'chown -R {0}:{0} /home/{0}/.local'.format(os_user))
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org