You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by lf...@apache.org on 2020/07/08 16:22:01 UTC
[incubator-dlab] branch DLAB-1748 created (now 03f6e24)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DLAB-1748
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 03f6e24 [DLAB-1748]: added installation of specific pip lib and list of versions and dependencies to response file
This branch includes the following new commits:
new 011f939 [DLAB-1748]: added installation of specific debian os pkg and dependencies to response file
new 03f6e24 [DLAB-1748]: added installation of specific pip lib and list of versions and dependencies to response file
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/02: [DLAB-1748]: added installation of specific
pip lib and list of versions and dependencies to response file
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1748
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 03f6e240bef1e958b6f286b0103b8ca0aec2b5d9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Jul 8 19:19:21 2020 +0300
[DLAB-1748]: added installation of specific pip lib and list of versions and dependencies to response file
---
.../src/general/lib/os/debian/notebook_lib.py | 2 +-
infrastructure-provisioning/src/general/lib/os/fab.py | 19 +++++++++++++++----
.../src/general/scripts/os/install_additional_libs.py | 3 +++
3 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 985b311..9558c7c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -384,7 +384,7 @@ def install_os_pkg(requisites):
dep = sudo('cat /tmp/os_install_{}_dep.log'.format(os_pkg))
dep = dep[len(add_pkgs_parser):dep.find("Suggested packages:")].replace('\r', '').replace('\n', '').replace(' ', ' ').strip()
if dep == '':
- dep = "N/A"
+ dep = "none"
sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{1}.list; then echo "" > /tmp/os_install_{1}.list;fi'.format(os_pkg.split("=")[0], os_pkg))
res = sudo('cat /tmp/os_install_{}.list'.format(os_pkg))
if res:
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index cd15d42..b800448 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -62,13 +62,22 @@ def install_pip_pkg(requisites, pip_version, lib_group):
sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version']))
sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version']))
for pip_pkg in requisites:
+ if pip_pkg[1] == '' or pip_pkg[1] == 'N/A':
+ pip_pkg = pip_pkg[0]
+ else:
+ pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
sudo('{0} install {1} --no-cache-dir 2>&1 | if ! grep -w -i -E "({2})" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, pip_pkg, error_parser))
err = sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg)).replace('"', "'")
sudo('{0} freeze | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > /tmp/{0}install_{1}.list;fi'.format(pip_version, pip_pkg))
res = sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, pip_pkg))
changed_pip_pkg = False
+ if "Could not find a version that satisfies the requirement" in err:
+ versions = [err[err.find("(from versions: ") + 16:err.find(")\r\n")]]
+ status.append({"group": "{}".format(lib_group), "name": pip_pkg.split("=")[0], "status": "failed",
+ "error_message": err, "available_versions": versions})
+ return status
if res == '':
- changed_pip_pkg = pip_pkg.replace("_", "-").split('-')
+ changed_pip_pkg = pip_pkg.split("=")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
sudo(
'{0} freeze | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > /tmp/{0}install_{1}.list;fi'.format(
@@ -83,11 +92,13 @@ def install_pip_pkg(requisites, pip_version, lib_group):
version = [i for i in ver if changed_pip_pkg.lower() in i][0].split('==')[1]
else:
version = \
- [i for i in ver if pip_pkg.lower() in i][0].split(
+ [i for i in ver if pip_pkg.split("=")[0].lower() in i][0].split(
'==')[1]
- status.append({"group": "{}".format(lib_group), "name": pip_pkg, "version": version, "status": "installed"})
+ dep = sudo('{0} show {1} 2>&1 | grep "Requires: "'.format(pip_version, pip_pkg.split("=")[0])).replace(
+ '\r', '').replace('\n', '').replace('Requires: ', '').strip()
+ status.append({"group": "{}".format(lib_group), "name": pip_pkg.split("=")[0], "version": version, "status": "installed", "add_pkgs": dep})
else:
- status.append({"group": "{}".format(lib_group), "name": pip_pkg, "status": "failed", "error_message": err})
+ status.append({"group": "{}".format(lib_group), "name": pip_pkg.split("=")[0], "status": "failed", "error_message": err})
return status
except Exception as err:
append_result("Failed to install {} packages".format(pip_version), str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
index b3a787f..84db583 100644
--- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
@@ -58,6 +58,9 @@ if __name__ == "__main__":
pkgs['libraries'][data[row]['group']].append(
[data[row]['name'].split(':')[0], data[row]['name'].split(':')[1],
data[row]['version'], data[row]['override']])
+ elif data[row]['group'] == "pip2" or data[row]['group'] == "pip3" or data[row]['group'] == "os_pkg":
+ pkgs['libraries'][data[row]['group']].append(
+ [data[row]['name'], data[row]['version']])
else:
pkgs['libraries'][data[row]['group']].append(data[row]['name'])
except Exception as err:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/02: [DLAB-1748]: added installation of specific
debian os pkg and dependencies to response file
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1748
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 011f93948bea9434c44ebd08bf78100492dba673
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Jul 8 17:28:46 2020 +0300
[DLAB-1748]: added installation of specific debian os pkg and dependencies to response file
---
.../src/general/lib/os/debian/notebook_lib.py | 29 ++++++++++++++++------
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index d1fa44a..985b311 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -365,22 +365,35 @@ def install_nodejs(os_user):
def install_os_pkg(requisites):
status = list()
- error_parser = "Could not|No matching|Error:|failed|Requires:"
+ error_parser = "Could not|No matching|Error:|E:|failed|Requires:"
+ add_pkgs_parser = "The following additional packages will be installed:"
try:
print("Updating repositories and installing requested tools: {}".format(requisites))
manage_pkg('update', 'remote', '')
for os_pkg in requisites:
- sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install {0} 2>&1 | if ! grep -w -E "({1})" > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, error_parser))
- err = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)).replace('"', "'")
- sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{0}.list; then echo "" > /tmp/os_install_{0}.list;fi'.format(os_pkg))
+ if os_pkg[1] != '':
+ os_pkg = "{}={}".format(os_pkg[0], os_pkg[1])
+ else:
+ os_pkg = os_pkg[0]
+ sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install {0} 2>&1 > /tmp/os_install_{0}.log'.format(os_pkg))
+ sudo('cat /tmp/os_install_{0}.log | if ! grep -w -E "({1})" /tmp/os_install_{0}.log > '
+ '/tmp/os_install_{0}_err.log; then echo "" > /tmp/os_install_{0}_err.log;fi'.format(os_pkg, error_parser))
+ sudo('cat /tmp/os_install_{0}.log | if ! grep -w -E -A 10 "({1})" /tmp/os_install_{0}.log > '
+ '/tmp/os_install_{0}_dep.log; then echo "" > /tmp/os_install_{0}_dep.log;fi'.format(os_pkg, add_pkgs_parser))
+ err = sudo('cat /tmp/os_install_{}_err.log'.format(os_pkg)).replace('"', "'")
+ dep = sudo('cat /tmp/os_install_{}_dep.log'.format(os_pkg))
+ dep = dep[len(add_pkgs_parser):dep.find("Suggested packages:")].replace('\r', '').replace('\n', '').replace(' ', ' ').strip()
+ if dep == '':
+ dep = "N/A"
+ sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{1}.list; then echo "" > /tmp/os_install_{1}.list;fi'.format(os_pkg.split("=")[0], os_pkg))
res = sudo('cat /tmp/os_install_{}.list'.format(os_pkg))
if res:
ansi_escape = re.compile(r'\x1b[^m]*m')
ver = ansi_escape.sub('', res).split("\r\n")
- version = [i for i in ver if os_pkg in i][0].split(' ')[1]
- status.append({"group": "os_pkg", "name": os_pkg, "version": version, "status": "installed"})
+ version = [i for i in ver if os_pkg.split("=")[0] in i][0].split(' ')[1]
+ status.append({"group": "os_pkg", "name": os_pkg.split("=")[0], "version": version, "status": "installed", "add_pkgs": dep})
else:
- status.append({"group": "os_pkg", "name": os_pkg, "status": "failed", "error_message": err})
+ status.append({"group": "os_pkg", "name": os_pkg.split("=")[0], "status": "failed", "error_message": err})
sudo('unattended-upgrades -v')
sudo('export LC_ALL=C')
return status
@@ -401,7 +414,7 @@ def get_available_os_pkgs():
os_pkgs = dict()
ansi_escape = re.compile(r'\x1b[^m]*m')
manage_pkg('update', 'remote', '')
- apt_raw = sudo("apt list")
+ apt_raw = sudo("apt list -a")
apt_list = ansi_escape.sub('', apt_raw).split("\r\n")
for pkg in apt_list:
if "/" in pkg:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org