You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/04/01 13:09:58 UTC
[incubator-datalab] branch DATALAB-2091 updated (4e42505 -> d176547)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 4e42505 Merge pull request #1120 from apache/DATALAB-2339
new ce28c05 [DATALAB-2091]: fixed apt list
new e5b78e8 [DATALAB-2091]: fixed apt lib install
new 4740572 [DATALAB-2091]: fixed R and pip lib install
new d176547 Merge branch 'DATALAB-2091' of https://github.com/apache/incubator-dlab into DATALAB-2091
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/general/lib/os/debian/notebook_lib.py | 22 ++++++++--------
.../src/general/lib/os/fab.py | 29 +++++++++++-----------
2 files changed, 25 insertions(+), 26 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/04: [DATALAB-2091]: fixed R and pip lib
install
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 4740572ddd327fadce057470c7943f2436de938d
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Apr 1 16:08:01 2021 +0300
[DATALAB-2091]: fixed R and pip lib install
---
.../src/general/lib/os/fab.py | 29 +++++++++++-----------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 35c3c07..dcbb92e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -76,18 +76,18 @@ def install_pip_pkg(requisites, pip_version, lib_group):
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
conn.sudo('{0} install -U {1} --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > '
' /tmp/{0}install_{3}.log; then echo "" > /tmp/{0}install_{3}.log;fi'.format(pip_version, pip_pkg, error_parser, name))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace('"', "'")
- conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout
- conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- installed_out = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout
+ err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace('"', "'").replace('\n', '')
+ conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "not_found" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
+ res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
+ conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
+ installed_out = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
- if res == '':
+ if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > '
'/tmp/{0}install_{1}.list;fi'.format(pip_version, changed_pip_pkg))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout
+ res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace('\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
if 'ERROR: No matching distribution found for {}'.format(name) in err:
@@ -111,9 +111,8 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
versions = []
- conn.sudo('if ! grep -w -i -E "Installing collected packages:" /tmp/tee.tmp > /tmp/{0}install_{1}.log; '
- 'then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, name))
- dep = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, name)).replace('\r\n', '').strip()[31:]
+ conn.sudo('cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, name))
+ dep = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
if dep == '':
dep = []
else:
@@ -454,8 +453,8 @@ def install_r_pkg(requisites):
else:
conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | '
'tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
- dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $1}\'').stdout.replace('\r\n', ' ')
- dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $4}\'').stdout.replace('\r\n', ' ').replace(')', '').split(' ')
+ dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
+ dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
if dep == '':
dep = []
else:
@@ -466,9 +465,9 @@ def install_r_pkg(requisites):
else:
dep[n] = '{} v.{}'.format(dep[n], dep_ver[n])
dep = [i for i in dep if i]
- err = conn.sudo('cat /tmp/install_{0}.log'.format(name)).stdout.replace('"', "'")
+ err = conn.sudo('cat /tmp/install_{0}.log'.format(name)).stdout.replace('"', "'").replace('\n', '')
conn.sudo('R -e \'installed.packages()[,c(3:4)]\' | if ! grep -w {0} > /tmp/install_{0}.list; then echo "" > /tmp/install_{0}.list;fi'.format(name))
- res = conn.sudo('cat /tmp/install_{0}.list'.format(name)).stdout
+ res = conn.sudo('cat /tmp/install_{0}.list'.format(name)).stdout.replace('\n', '')
if err:
status_msg = 'installation_error'
if 'couldn\'t find package \'{}\''.format(name) in err:
@@ -549,7 +548,7 @@ def install_java_pkg(requisites):
err = conn.sudo('cat /tmp/install_{0}.log'.format(artifact)).stdout.replace('"', "'").strip()
conn.sudo('find {0} -name "{1}*.jar" | head -n 1 | rev | cut -f1 -d "/" | rev | \
if ! grep -w -i {1} > /tmp/install_{1}.list; then echo "" > /tmp/install_{1}.list;fi'.format(ivy_cache_dir, artifact))
- res = conn.sudo('cat /tmp/install_{0}.list'.format(artifact)).stdout
+ res = conn.sudo('cat /tmp/install_{0}.list'.format(artifact)).stdout.replace('\n','')
if res:
conn.sudo('cp -f $(find {0} -name "*.jar" | xargs) {1}'.format(ivy_cache_dir, dest_dir))
status.append({"group": "java", "name": "{0}:{1}".format(group, artifact), "version": version, "status": "installed"})
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/04: [DATALAB-2091]: fixed apt list
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit ce28c0581b87b93ae68b1c027c96523732ae7282
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Mar 31 11:53:28 2021 +0300
[DATALAB-2091]: fixed apt list
---
infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index c87a4f0..0305f88 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -459,7 +459,7 @@ def get_available_os_pkgs():
ansi_escape = re.compile(r'\x1b[^m]*m')
manage_pkg('update', 'remote', '')
apt_raw = datalab.fab.conn.sudo("apt list").stdout
- apt_list = ansi_escape.sub('', apt_raw).split("\r\n")
+ apt_list = ansi_escape.sub('', apt_raw).split("\n")
for pkg in apt_list:
if "/" in pkg:
os_pkgs[pkg.split('/')[0]] = pkg.split(' ')[1]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/04: [DATALAB-2091]: fixed apt lib install
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit e5b78e80196c52f1a7185d7a233785126d4dc433
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Apr 1 14:35:07 2021 +0300
[DATALAB-2091]: fixed apt lib install
---
.../src/general/lib/os/debian/notebook_lib.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 0305f88..81976cf 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -396,12 +396,12 @@ def install_os_pkg(requisites):
version = 'N/A'
os_pkg = os_pkg[0]
datalab.fab.conn.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{2}.log; then echo "" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
+ '/tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
datalab.fab.conn.sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
+ '/tmp/os_install_{0}.log; then echo "no_new_pkgs" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
- if dep == '':
+ if 'no_new_pkgs' in dep:
dep = []
else:
dep = dep[len(new_pkgs_parser): dep.find(" upgraded, ") - 1].replace('\r', '') \
@@ -411,17 +411,17 @@ def install_os_pkg(requisites):
dep[n] = ''
else:
datalab.fab.conn.sudo('apt show {0} 2>&1 | if ! grep Version: > '
- '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(i))
- dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version: ', '{} v.'.format(i))
+ '/tmp/os_install_{0}.log; then echo "no_version" > /tmp/os_install_{0}.log;fi'.format(i))
+ dep[n] = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(i)).stdout.replace('Version: ', '{} v.'.format(i)).replace('\n', '')
dep = [i for i in dep if i]
versions = []
- datalab.fab.conn.sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{0}.list; then echo "" > /tmp/os_install_{0}.list;fi'.format(name))
- res = datalab.fab.conn.sudo('cat /tmp/os_install_{}.list'.format(name))
- if err:
+ datalab.fab.conn.sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{0}.list; then echo "not_installed" > /tmp/os_install_{0}.list;fi'.format(name))
+ res = datalab.fab.conn.sudo('cat /tmp/os_install_{}.list'.format(name)).stdout.replace('\n', '')
+ if "no_error" not in err:
status_msg = 'installation_error'
if 'E: Unable to locate package {}'.format(name) in err:
status_msg = 'invalid_name'
- elif res:
+ elif "not_installed" not in res:
ansi_escape = re.compile(r'\x1b[^m]*m')
ver = ansi_escape.sub('', res).split("\r\n")
version = [i for i in ver if os_pkg.split("=")[0] in i][0].split(' ')[1]
@@ -434,7 +434,7 @@ def install_os_pkg(requisites):
status.append({"group": "os_pkg", "name": name, "version": version, "status": status_msg,
"error_message": err, "add_pkgs": dep, "available_versions": versions})
datalab.fab.conn.sudo('unattended-upgrades -v')
- datalab.fab.conn.sudo('export LC_ALL=C')
+ #datalab.fab.conn.sudo('export LC_ALL=C')
return status
except Exception as err:
for os_pkg in requisites:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 04/04: Merge branch 'DATALAB-2091' of
https://github.com/apache/incubator-dlab into DATALAB-2091
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d1765470e79c33fe60e6732a90ff7e67a00279b0
Merge: 4740572 4e42505
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Apr 1 16:09:44 2021 +0300
Merge branch 'DATALAB-2091' of https://github.com/apache/incubator-dlab into DATALAB-2091
.../exploratory/install-libraries/install-libraries.component.scss | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org