You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/16 14:24:35 UTC
[incubator-datalab] 01/01: [DATALAB-2691]: fixed python libs install for aws emr on zeppelin
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c5a62ff056c64c96b328fafa9922efb4b8ad56e3
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Mar 16 16:18:29 2022 +0200
[DATALAB-2691]: fixed python libs install for aws emr on zeppelin
---
.../src/general/lib/aws/actions_lib.py | 34 +++++++++++-----------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index d74b6dc..dfddc10 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1591,7 +1591,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('wget https://www.python.org/ftp/python/' + python_version +
'/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz', shell=True, check=True)
subprocess.run('tar zxvf /tmp/Python-' + python_version + '.tgz -C /tmp/', shell=True, check=True)
- subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version), shell=True, check=True)
+ subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ '
+ '--with-ensurepip=install'.format(python_version), shell=True, check=True)
subprocess.run('cd /tmp/Python-{0}; sudo make altinstall'.format(python_version), shell=True, check=True)
subprocess.run('cd /tmp/; sudo rm -rf Python-' + python_version + '/', shell=True, check=True)
if region == 'cn-north-1':
@@ -1602,8 +1603,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('sudo echo "[global]" >> /etc/pip.conf', shell=True, check=True)
subprocess.run('sudo echo "timeout = 600" >> /etc/pip.conf', shell=True, check=True)
subprocess.run('sudo -i virtualenv /opt/python/python' + python_version, shell=True, check=True)
- venv_command = '/bin/bash source /opt/python/python' + python_version + '/bin/activate'
- pip_command = '/opt/python/python' + python_version + '/bin/pip' + python_version[:3]
+ venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
+ pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
if region == 'cn-north-1':
try:
subprocess.run(venv_command + ' && sudo -i ' + pip_command +
@@ -1634,24 +1635,23 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('sudo rm -rf /opt/python/python{}/'.format(python_version), shell=True, check=True)
sys.exit(1)
else:
- print(subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install -U pip==9.0.3', shell=True, check=True))
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0', shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install ipython ipykernel --no-cache-dir', shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install NumPy=={}'.format(numpy_version), shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install boto boto3 SciPy Matplotlib=={} pandas Sympy Pillow sklearn '
- '--no-cache-dir'.format(os.environ['notebook_matplotlib_version']), shell=True, check=True)
+ for lib in ['-U pip==9.0.3', 'pyzmq==17.0.0',
+ 'ipython ipykernel boto boto3 pybind11 pythran cython NumPy=={} Matplotlib=={} --no-cache-dir'
+ .format(numpy_version, os.environ['notebook_matplotlib_version']),
+ 'SciPy pandas Sympy Pillow', 'sklearn --no-cache-dir']:
+ subprocess.run('bash -c "{0} && sudo -i {1} install {2}"'
+ .format(venv_command, pip_command, lib), shell=True, check=True)
# Need to refactor when we add GPU cluster
if application == 'deeplearning':
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install mxnet-cu80 opencv-python keras Theano --no-cache-dir', shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano '
+ '--no-cache-dir"'.format(venv_command, pip_command), shell=True, check=True)
python_without_dots = python_version.replace('.', '')
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl '
- '--no-cache-dir'.format(python_without_dots[:2]), shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}'
+ '-cp{0}m-linux_x86_64.whl --no-cache-dir"'
+ .format(venv_command, pip_command, python_without_dots[:2]), shell=True, check=True)
subprocess.run('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]), shell=True, check=True)
- subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version,
- python_version[0:3]), shell=True, check=True)
+ subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'
+ .format(python_version, python_version[0:3]), shell=True, check=True)
def spark_defaults(args):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org