You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/05/28 09:20:53 UTC
[incubator-datalab] 03/03: [DATALAB-2398] - Installing libs for
python venv fixed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 294316085f3257cdc4929f88c1b4187b59326cd6
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Fri May 28 12:19:29 2021 +0300
[DATALAB-2398] - Installing libs for python venv fixed
---
.../src/general/lib/os/fab.py | 54 ++++++++++++++--------
1 file changed, 34 insertions(+), 20 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 8b54a2e..9447242 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -79,13 +79,14 @@ def install_pip_pkg(requisites, pip_version, lib_group):
status = list()
error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:"
try:
- if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
- for v in range(4, 8):
- if exists(conn, '/bin/pip3.{}'.format(v)):
- conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v))
- conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version']))
- conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version']))
- conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version']))
+ venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
+ #if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
+ # for v in range(4, 8):
+ # if exists(conn, '/bin/pip3.{}'.format(v)):
+ # conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v))
+ #conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version']))
+ #conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version']))
+ #conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version']))
for pip_pkg in requisites:
name, vers = pip_pkg
if pip_pkg[1] == '' or pip_pkg[1] == 'N/A':
@@ -94,20 +95,30 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
version = pip_pkg[1]
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
- conn.sudo('{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > '
- ' /tmp/{0}install_{3}.log; then echo "" > /tmp/{0}install_{3}.log;fi'.format(pip_version, pip_pkg, error_parser, name))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace('"', "'").replace('\n', ' ')
- conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "not_found" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
- conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- installed_out = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
+ conn.sudo(
+ '''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
+ venv_install_command, pip_pkg, error_parser, name, pip_version))
+ err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
+ '"', "'").replace('\n', ' ')
+ conn.sudo(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
+ venv_install_command, name, pip_version))
+ res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
+ '\n', '')
+ conn.sudo(
+ '''bash -l -c 'cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi' '''.format(
+ pip_version, name))
+ installed_out = conn.sudo(
+ '''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
- conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > '
- '/tmp/{0}install_{1}.list;fi'.format(pip_version, changed_pip_pkg))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace('\n', '')
+ conn.sudo(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
+ venv_install_command, changed_pip_pkg, pip_version))
+ res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
+ '\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
if 'ERROR: No matching distribution found for {}'.format(name) in err:
@@ -120,10 +131,11 @@ def install_pip_pkg(requisites, pip_version, lib_group):
version = [i for i in ver if changed_pip_pkg.lower() in i][0].split('==')[1]
else:
version = \
- [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1]
+ [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1]
status_msg = "installed"
versions = []
- if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format(name) in err:
+ if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format(
+ name) in err:
versions = err[err.find("(from versions: ") + 16: err.find(") ")]
if versions != '' and versions != 'none':
versions = versions.split(', ')
@@ -132,7 +144,9 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
versions = []
- conn.sudo('cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, name))
+ conn.sudo(
+ '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi' '''.format(
+ pip_version, name))
dep = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
if dep == '':
dep = []
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org