You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/16 14:24:34 UTC
[incubator-datalab] branch DATALAB-2691 created (now c5a62ff)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at c5a62ff [DATALAB-2691]: fixed python libs install for aws emr on zeppelin
This branch includes the following new commits:
new c5a62ff [DATALAB-2691]: fixed python libs install for aws emr on zeppelin
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2691]: fixed python libs install for aws emr on zeppelin
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c5a62ff056c64c96b328fafa9922efb4b8ad56e3
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Mar 16 16:18:29 2022 +0200
[DATALAB-2691]: fixed python libs install for aws emr on zeppelin
---
.../src/general/lib/aws/actions_lib.py | 34 +++++++++++-----------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index d74b6dc..dfddc10 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1591,7 +1591,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('wget https://www.python.org/ftp/python/' + python_version +
'/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz', shell=True, check=True)
subprocess.run('tar zxvf /tmp/Python-' + python_version + '.tgz -C /tmp/', shell=True, check=True)
- subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version), shell=True, check=True)
+ subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ '
+ '--with-ensurepip=install'.format(python_version), shell=True, check=True)
subprocess.run('cd /tmp/Python-{0}; sudo make altinstall'.format(python_version), shell=True, check=True)
subprocess.run('cd /tmp/; sudo rm -rf Python-' + python_version + '/', shell=True, check=True)
if region == 'cn-north-1':
@@ -1602,8 +1603,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('sudo echo "[global]" >> /etc/pip.conf', shell=True, check=True)
subprocess.run('sudo echo "timeout = 600" >> /etc/pip.conf', shell=True, check=True)
subprocess.run('sudo -i virtualenv /opt/python/python' + python_version, shell=True, check=True)
- venv_command = '/bin/bash source /opt/python/python' + python_version + '/bin/activate'
- pip_command = '/opt/python/python' + python_version + '/bin/pip' + python_version[:3]
+ venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
+ pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
if region == 'cn-north-1':
try:
subprocess.run(venv_command + ' && sudo -i ' + pip_command +
@@ -1634,24 +1635,23 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('sudo rm -rf /opt/python/python{}/'.format(python_version), shell=True, check=True)
sys.exit(1)
else:
- print(subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install -U pip==9.0.3', shell=True, check=True))
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0', shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install ipython ipykernel --no-cache-dir', shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install NumPy=={}'.format(numpy_version), shell=True, check=True)
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install boto boto3 SciPy Matplotlib=={} pandas Sympy Pillow sklearn '
- '--no-cache-dir'.format(os.environ['notebook_matplotlib_version']), shell=True, check=True)
+ for lib in ['-U pip==9.0.3', 'pyzmq==17.0.0',
+ 'ipython ipykernel boto boto3 pybind11 pythran cython NumPy=={} Matplotlib=={} --no-cache-dir'
+ .format(numpy_version, os.environ['notebook_matplotlib_version']),
+ 'SciPy pandas Sympy Pillow', 'sklearn --no-cache-dir']:
+ subprocess.run('bash -c "{0} && sudo -i {1} install {2}"'
+ .format(venv_command, pip_command, lib), shell=True, check=True)
# Need to refactor when we add GPU cluster
if application == 'deeplearning':
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install mxnet-cu80 opencv-python keras Theano --no-cache-dir', shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano '
+ '--no-cache-dir"'.format(venv_command, pip_command), shell=True, check=True)
python_without_dots = python_version.replace('.', '')
- subprocess.run(venv_command + ' && sudo -i ' + pip_command +
- ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl '
- '--no-cache-dir'.format(python_without_dots[:2]), shell=True, check=True)
+ subprocess.run('bash -c "{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}'
+ '-cp{0}m-linux_x86_64.whl --no-cache-dir"'
+ .format(venv_command, pip_command, python_without_dots[:2]), shell=True, check=True)
subprocess.run('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]), shell=True, check=True)
- subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version,
- python_version[0:3]), shell=True, check=True)
+ subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'
+ .format(python_version, python_version[0:3]), shell=True, check=True)
def spark_defaults(args):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org