You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/07/09 08:44:27 UTC
[incubator-datalab] branch DATALAB-2515 created (now c1b6dec)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2515
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at c1b6dec [DATALAB-2515]: fixed emr yum packages install errors
This branch includes the following new commits:
new c1b6dec [DATALAB-2515]: fixed emr yum packages install errors
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2515]: fixed emr yum packages
install errors
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2515
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c1b6dec923322e06e9ed599dd5904049f8ff5474
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Jul 9 11:44:04 2021 +0300
[DATALAB-2515]: fixed emr yum packages install errors
---
.../src/general/lib/os/redhat/notebook_lib.py | 44 ++++++++++++----------
.../scripts/aws/dataengine-service_install_libs.py | 2 +-
2 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index c621760..f90df73 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -320,7 +320,7 @@ def install_os_pkg(requisites):
try:
print("Updating repositories and installing requested tools: {}".format(requisites))
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
- datalab.fab.conn.sudo('export LC_ALL=C')
+ #datalab.fab.conn.sudo('export LC_ALL=C')
for os_pkg in requisites:
name, vers = os_pkg
if vers != '' and vers !='N/A':
@@ -329,14 +329,14 @@ def install_os_pkg(requisites):
else:
version = 'N/A'
os_pkg = name
- manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" '
- '/tmp/tee.tmp > /tmp/os_install_{2}.log; then echo "" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
- install_output = datalab.fab.conn.sudo('cat /tmp/tee.tmp').stdout
+ manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee /tmp/os_install_{2}.tmp; if ! grep -w -E "({1})" '
+ '/tmp/os_install_{2}.tmp > /tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
+ install_output = datalab.fab.conn.sudo('cat /tmp/os_install_{}.tmp'.format(name)).stdout
err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
- datalab.fab.conn.sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
+ datalab.fab.conn.sudo('cat /tmp/os_install_{0}.tmp | if ! grep -w -E -A 30 "({1})" /tmp/os_install_{0}.tmp > '
+ '/tmp/os_install_{0}.log; then echo "no_dependencies" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
- if dep == '':
+ if 'no_dependencies' in dep:
dep = []
else:
dep = dep[len(new_pkgs_parser): dep.find("Complete!") - 1].replace(' ', '').strip().split('\r\n')
@@ -346,13 +346,15 @@ def install_os_pkg(requisites):
dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version : ', '{} v.'.format(i))
dep = [i for i in dep if i]
versions = []
- res = datalab.fab.conn.sudo(
- 'python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(
- name)).stdout.split('\r\n')[1]
- if err:
+ datalab.fab.conn.sudo(
+ 'yum list installed | if ! grep "{0}\." > /tmp/os_install_{0}.list; then echo "not_installed" > /tmp/os_install_{0}.list;fi'.format(
+ name))
+ res = datalab.fab.conn.sudo('cat /tmp/os_install_{}.list '.format(name) + '| awk \'{print $1":"$2}\'').stdout.replace('\n', '')
+ #res = datalab.fab.conn.sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(name)).stdout.split('\r\n')[1]
+ if "no_error" not in err:
status_msg = 'installation_error'
- elif res != []:
- version = res.split("'")[1].split("-")[0]
+ elif "not_installed" not in res:
+ version = res.split(":")[1]
status_msg = "installed"
if 'No package {} available'.format(os_pkg) in install_output:
versions = datalab.fab.conn.sudo('yum --showduplicates list ' + name + ' | expand | grep ' + name + ' | awk \'{print $2}\'').stdout.replace('\r\n', '')
@@ -369,6 +371,7 @@ def install_os_pkg(requisites):
status_msg = 'invalid_name'
status.append({"group": "os_pkg", "name": name, "version": version, "status": status_msg,
"error_message": err, "add_pkgs": dep, "available_versions": versions})
+ datalab.fab.conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for os_pkg in requisites:
@@ -387,14 +390,15 @@ def remove_os_pkg(pkgs):
def get_available_os_pkgs():
try:
+ os_pkgs = dict()
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
- downgrade_python_version()
- yum_raw = datalab.fab.conn.sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); '
- 'print {pkg.name:pkg.vr for pkg in pl.available}"').stdout
- yum_re = re.sub\
- (r'\w*\s\w*\D\s\w*.\w*.\s\w*.\w*.\w.\w*.\w*.\w*', '', yum_raw)
- yum_list = yum_re.replace("'", "\"")
- os_pkgs = json.loads(yum_list)
+ #downgrade_python_version()
+ yum_names = datalab.fab.conn.sudo("yum list available | grep -v \"Loaded plugins:\|Available Packages\" | xargs -n3 | column -t | awk '{print $1}'").stdout.split('\n')
+ for pkg in yum_names:
+ if "." in pkg:
+ os_pkgs[pkg.split('.')[0]] = 'N/A'
+ elif pkg != '':
+ os_pkgs[pkg] = 'N/A'
return os_pkgs
except Exception as err:
append_result("Failed to get available os packages.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
index f47e231..d687c8a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
@@ -35,7 +35,7 @@ from fabric import *
def install_libs(instance, data_engine):
data_engine['instance_ip'] = instance.get('PrivateIpAddress')
- params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"' \
+ params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}" --dataengine_service True' \
.format(data_engine['os_user'], data_engine['instance_ip'],
data_engine['keyfile'], data_engine['libs'])
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org