You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/03/02 09:42:43 UTC
[incubator-datalab] branch DATALAB-2091 updated (029a5f7 -> 4d7b577)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 029a5f7 [DATALAB-2091]: corrected roles` names
new eb507fe [DATALAB-2091]: fixed connection usage
new 4d7b577 [DATALAB-2091]: added traceback
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/base/scripts/install_prerequisites.py | 7 +-
.../src/general/lib/os/redhat/common_lib.py | 81 +++++++++++-----------
2 files changed, 47 insertions(+), 41 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2091]: fixed connection usage
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit eb507fe6af6e95d828b75273f5a694a545828ad9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 2 11:42:08 2021 +0200
[DATALAB-2091]: fixed connection usage
---
.../src/general/lib/os/redhat/common_lib.py | 81 +++++++++++-----------
1 file changed, 41 insertions(+), 40 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index 835c681..bd1669d 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -26,6 +26,7 @@ from patchwork.files import exists
import sys
import os
import subprocess
+import datalab.fab
def manage_pkg(command, environment, requisites):
try:
@@ -38,12 +39,12 @@ def manage_pkg(command, environment, requisites):
else:
print('Package manager is:')
if environment == 'remote':
- if conn.sudo('pgrep yum -a && echo "busy" || echo "ready"') == 'busy':
+ if datalab.fab.conn.sudo('pgrep yum -a && echo "busy" || echo "ready"') == 'busy':
counter += 1
time.sleep(10)
else:
allow = True
- conn.sudo('yum {0} {1}'.format(command, requisites))
+ datalab.fab.conn.sudo('yum {0} {1}'.format(command, requisites))
elif environment == 'local':
if subprocess.run('sudo pgrep yum -a && echo "busy" || echo "ready"', capture_output=True, shell=True, check=True) == 'busy':
counter += 1
@@ -58,44 +59,44 @@ def manage_pkg(command, environment, requisites):
def ensure_pkg(user, requisites='git vim gcc python-devel openssl-devel nmap libffi libffi-devel unzip libxml2-devel'):
try:
- if not exists(conn,'/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
+ if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
print("Updating repositories and installing requested tools: {}".format(requisites))
- if conn.sudo("systemctl list-units --all | grep firewalld | awk '{print $1}'") != '':
- conn.sudo('systemctl disable firewalld.service')
- conn.sudo('systemctl stop firewalld.service')
- conn.sudo('setenforce 0')
- conn.sudo("sed -i '/^SELINUX=/s/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config")
+ if datalab.fab.conn.sudo("systemctl list-units --all | grep firewalld | awk '{print $1}'") != '':
+ datalab.fab.conn.sudo('systemctl disable firewalld.service')
+ datalab.fab.conn.sudo('systemctl stop firewalld.service')
+ datalab.fab.conn.sudo('setenforce 0')
+ datalab.fab.conn.sudo("sed -i '/^SELINUX=/s/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config")
mirror = 'mirror.centos.org'
- conn.sudo('echo "[Centos-repo]" > /etc/yum.repos.d/centOS-base.repo')
- conn.sudo('echo "name=Centos 7 Repository" >> /etc/yum.repos.d/centOS-base.repo')
- conn.sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> /etc/yum.repos.d/centOS-base.repo'.format(mirror))
- conn.sudo('echo "enabled=1" >> /etc/yum.repos.d/centOS-base.repo')
- conn.sudo('echo "gpgcheck=1" >> /etc/yum.repos.d/centOS-base.repo')
- conn.sudo('echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> /etc/yum.repos.d/centOS-base.repo'.format(mirror))
- conn.sudo('yum-config-manager --enable rhui-REGION-rhel-server-optional')
+ datalab.fab.conn.sudo('echo "[Centos-repo]" > /etc/yum.repos.d/centOS-base.repo')
+ datalab.fab.conn.sudo('echo "name=Centos 7 Repository" >> /etc/yum.repos.d/centOS-base.repo')
+ datalab.fab.conn.sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> /etc/yum.repos.d/centOS-base.repo'.format(mirror))
+ datalab.fab.conn.sudo('echo "enabled=1" >> /etc/yum.repos.d/centOS-base.repo')
+ datalab.fab.conn.sudo('echo "gpgcheck=1" >> /etc/yum.repos.d/centOS-base.repo')
+ datalab.fab.conn.sudo('echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> /etc/yum.repos.d/centOS-base.repo'.format(mirror))
+ datalab.fab.conn.sudo('yum-config-manager --enable rhui-REGION-rhel-server-optional')
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install', 'remote', 'wget')
- conn.sudo('wget --no-check-certificate https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm')
- conn.sudo('rpm -ivh epel-release-latest-7.noarch.rpm')
+ datalab.fab.conn.sudo('wget --no-check-certificate https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm')
+ datalab.fab.conn.sudo('rpm -ivh epel-release-latest-7.noarch.rpm')
manage_pkg('repolist', 'remote', '')
manage_pkg('-y install', 'remote', 'python3-pip gcc')
- conn.sudo('rm -f epel-release-latest-7.noarch.rpm')
- conn.sudo('export LC_ALL=C')
+ datalab.fab.conn.sudo('rm -f epel-release-latest-7.noarch.rpm')
+ datalab.fab.conn.sudo('export LC_ALL=C')
manage_pkg('-y install', 'remote', requisites)
- conn.sudo('touch /home/{}/.ensure_dir/pkg_upgraded'.format(user))
+ datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/pkg_upgraded'.format(user))
except:
sys.exit(1)
def change_pkg_repos():
- if not exists(conn,'/tmp/pkg_china_ensured'):
- conn.put('/root/files/sources.list', '/tmp/sources.list')
- conn.sudo('mv /tmp/sources.list /etc/yum.repos.d/CentOS-Base-aliyun.repo')
- conn.sudo('touch /tmp/pkg_china_ensured')
+ if not exists(datalab.fab.conn,'/tmp/pkg_china_ensured'):
+ datalab.fab.conn.put('/root/files/sources.list', '/tmp/sources.list')
+ datalab.fab.conn.sudo('mv /tmp/sources.list /etc/yum.repos.d/CentOS-Base-aliyun.repo')
+ datalab.fab.conn.sudo('touch /tmp/pkg_china_ensured')
def find_java_path_remote():
- java_path = conn.sudo("alternatives --display java | grep 'slave jre: ' | awk '{print $3}'").stdout.replace('\n','')
+ java_path = datalab.fab.conn.sudo("alternatives --display java | grep 'slave jre: ' | awk '{print $3}'").stdout.replace('\n','')
return java_path
@@ -106,38 +107,38 @@ def find_java_path_local():
def ensure_ntpd(user, edge_private_ip=''):
try:
- if not exists(conn,'/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
- conn.sudo('systemctl disable chronyd')
+ if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
+ datalab.fab.conn.sudo('systemctl disable chronyd')
manage_pkg('-y install', 'remote', 'ntp')
- conn.sudo('echo "tinker panic 0" >> /etc/ntp.conf')
- conn.sudo('systemctl start ntpd')
+ datalab.fab.conn.sudo('echo "tinker panic 0" >> /etc/ntp.conf')
+ datalab.fab.conn.sudo('systemctl start ntpd')
if os.environ['conf_resource'] != 'ssn' and os.environ['conf_resource'] != 'edge':
- conn.sudo('echo "server {} prefer iburst" >> /etc/ntp.conf'.format(edge_private_ip))
- conn.sudo('systemctl restart ntpd')
- conn.sudo('systemctl enable ntpd')
- conn.sudo('touch /home/{}/.ensure_dir/ntpd_ensured'.format(user))
+ datalab.fab.conn.sudo('echo "server {} prefer iburst" >> /etc/ntp.conf'.format(edge_private_ip))
+ datalab.fab.conn.sudo('systemctl restart ntpd')
+ datalab.fab.conn.sudo('systemctl enable ntpd')
+ datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/ntpd_ensured'.format(user))
except:
sys.exit(1)
def ensure_java(user):
try:
- if not exists(conn,'/home/{}/.ensure_dir/java_ensured'.format(user)):
+ if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/java_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel')
- conn.sudo('touch /home/{}/.ensure_dir/java_ensured'.format(user))
+ datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/java_ensured'.format(user))
except:
sys.exit(1)
def ensure_step(user):
try:
- if not exists(conn,'/home/{}/.ensure_dir/step_ensured'.format(user)):
+ if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/step_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'wget')
- conn.sudo('wget https://github.com/smallstep/cli/releases/download/v0.13.3/step_0.13.3_linux_amd64.tar.gz '
+ datalab.fab.conn.sudo('wget https://github.com/smallstep/cli/releases/download/v0.13.3/step_0.13.3_linux_amd64.tar.gz '
'-O /tmp/step_0.13.3_linux_amd64.tar.gz')
- conn.sudo('tar zxvf /tmp/step_0.13.3_linux_amd64.tar.gz -C /tmp/')
- conn.sudo('mv /tmp/step_0.13.3/bin/step /usr/bin/')
- conn.sudo('touch /home/{}/.ensure_dir/step_ensured'.format(user))
+ datalab.fab.conn.sudo('tar zxvf /tmp/step_0.13.3_linux_amd64.tar.gz -C /tmp/')
+ datalab.fab.conn.sudo('mv /tmp/step_0.13.3/bin/step /usr/bin/')
+ datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/step_ensured'.format(user))
except:
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2091]: added traceback
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 4d7b5778153f95704ed1e0923abba520550e34c1
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 2 11:42:27 2021 +0200
[DATALAB-2091]: added traceback
---
.../src/base/scripts/install_prerequisites.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index c3ccb8e..a0b7f90 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -28,6 +28,7 @@ from datalab.common_lib import *
from datalab.fab import *
from fabric import *
from patchwork.files import exists
+import traceback
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
@@ -64,7 +65,11 @@ if __name__ == "__main__":
update_hosts_file(args.user)
print("Updating repositories and installing requested tools.")
- ensure_pkg(args.user)
+ try:
+ ensure_pkg(args.user)
+ except:
+ traceback.print_exc()
+ sys.exit(1)
print("Installing python packages: {}".format(args.pip_packages))
ensure_pip(args.pip_packages)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org