You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/15 17:27:07 UTC
[incubator-datalab] branch DATALAB-2691 created (now 00a0e58)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 00a0e58 [DATALAB-2691]: fixed python libs install for gcp dataproc on zeppelin
This branch includes the following new commits:
new 00a0e58 [DATALAB-2691]: fixed python libs install for gcp dataproc on zeppelin
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2691]: fixed python libs install for gcp dataproc on zeppelin
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 00a0e58cb4975fbffd9dc6185a857edc96836a4f
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 15 19:22:52 2022 +0200
[DATALAB-2691]: fixed python libs install for gcp dataproc on zeppelin
---
.../src/general/lib/gcp/actions_lib.py | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 8d3543c..6bfc177 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1355,15 +1355,16 @@ class GCPActions:
subprocess.run('sudo -i virtualenv /opt/python/python{}'.format(python_version), shell=True, check=True)
venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
- subprocess.run('{0} && sudo -i {1} install -U pip==9.0.3'.format(venv_command, pip_command), shell=True, check=True)
- subprocess.run('{0} && sudo -i {1} install pyzmq==17.0.0'.format(venv_command, pip_command), shell=True, check=True)
- subprocess.run('{0} && sudo -i {1} install ipython ipykernel --no-cache-dir'.format(venv_command, pip_command), shell=True, check=True)
- subprocess.run('{0} && sudo -i {1} install boto boto3 NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir'
- .format(venv_command, pip_command, numpy_version), shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install -U pip==9.0.3"'.format(venv_command, pip_command), shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install pyzmq==17.0.0"'.format(venv_command, pip_command), shell=True, check=True)
+ for lib in ['ipython', 'ipykernel', 'NumPy=={}'.format(numpy_version), 'boto', 'boto3', 'pybind11',
+ 'pythran', 'cython', 'SciPy', 'Matplotlib', 'pandas', 'Sympy', 'Pillow', 'sklearn']:
+ subprocess.run('bash -c "{0} && sudo -i {1} install {2} --no-cache-dir"'
+ .format(venv_command, pip_command, lib), shell=True, check=True)
if application == 'deeplearning':
- subprocess.run('{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano --no-cache-dir'.format(venv_command, pip_command), shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano --no-cache-dir"'.format(venv_command, pip_command), shell=True, check=True)
python_without_dots = python_version.replace('.', '')
- subprocess.run('{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{2}-cp{2}m-linux_x86_64.whl --no-cache-dir'
+ subprocess.run('bash -c "{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{2}-cp{2}m-linux_x86_64.whl --no-cache-dir"'
.format(venv_command, pip_command, python_without_dots[:2]), shell=True, check=True)
subprocess.run('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]), shell=True, check=True)
subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version, python_version[0:3]), shell=True, check=True)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org