You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/06/17 13:37:13 UTC
[incubator-datalab] 01/01: [DATALAB-2429]: fixed additional lib
installation for gcp dataengine-service
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2429
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c720897f1b77a9901fccf96c8269310ad8261397
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Jun 17 16:36:30 2021 +0300
[DATALAB-2429]: fixed additional lib installation for gcp dataengine-service
---
infrastructure-provisioning/src/general/lib/os/fab.py | 15 +++++++++------
.../scripts/gcp/dataengine-service_install_libs.py | 6 +++---
.../src/general/scripts/os/install_additional_libs.py | 7 ++++---
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 10420b8..b69c515 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -99,12 +99,15 @@ def dataengine_dir_prepare(cluster_dir):
subprocess.run('mkdir -p ' + cluster_dir, shell=True, check=True)
-def install_pip_pkg(requisites, pip_version, lib_group):
+def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = False):
status = list()
error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:"
try:
- venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(
- os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
+ if dataengine_service:
+ install_command = pip_version
+ else:
+ install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(
+ os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
#if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
# for v in range(4, 8):
# if exists(conn, '/bin/pip3.{}'.format(v)):
@@ -124,13 +127,13 @@ def install_pip_pkg(requisites, pip_version, lib_group):
'''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | '''
'''tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; '''
'''then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
- venv_install_command, pip_pkg, error_parser, name, pip_version))
+ install_command, pip_pkg, error_parser, name, pip_version))
err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
'"', "'").replace('\n', ' ')
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; '''
'''then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
- venv_install_command, name, pip_version))
+ install_command, name, pip_version))
res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
'\n', '')
conn.sudo(
@@ -145,7 +148,7 @@ def install_pip_pkg(requisites, pip_version, lib_group):
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
- venv_install_command, changed_pip_pkg, pip_version))
+ install_command, changed_pip_pkg, pip_version))
res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
'\n', '')
if err and name not in installed_out:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
index bcc2eb7..32475e1 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
@@ -34,8 +34,8 @@ from fabric import *
def install_libs(instance, data_engine):
- data_engine['instance_ip'] = meta_lib.GCPMeta().get_private_ip_address(instance)
- params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"' \
+ data_engine['instance_ip'] = GCPMeta().get_private_ip_address(instance)
+ params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}" --dataengine_service True' \
.format(data_engine['os_user'], data_engine['instance_ip'],
data_engine['keyfile'], data_engine['libs'])
try:
@@ -65,7 +65,7 @@ if __name__ == "__main__":
data_engine['gcp_project_id'] = os.environ['gcp_project_id']
data_engine['gcp_region'] = os.environ['gcp_region']
data_engine['gcp_zone'] = os.environ['gcp_zone']
- res = meta_lib.GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name'])
+ res = GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name'])
data_engine['cluster_instances'] = [i.get('name') for i in res['items']]
data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
data_engine['libs'] = os.environ['libs']
diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
index 764f86d..cf443c5 100644
--- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
@@ -34,6 +34,7 @@ parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--instance_ip', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--libs', type=str, default='')
+parser.add_argument('--dataengine_service', type=bool, default=False)
args = parser.parse_args()
@@ -78,14 +79,14 @@ if __name__ == "__main__":
try:
print('Installing pip2 packages: {}'.format(pkgs['libraries']['pip2']))
- status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2')
+ status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2', args.dataengine_service)
general_status = general_status + status
except KeyError:
pass
try:
print('Installing pip3 packages: {}'.format(pkgs['libraries']['pip3']))
- status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3')
+ status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3', args.dataengine_service)
general_status = general_status + status
except KeyError:
pass
@@ -93,7 +94,7 @@ if __name__ == "__main__":
try:
print('Installing other packages (only tries pip3): {}'.format(pkgs['libraries']['others']))
for pkg in pkgs['libraries']['others']:
- status_pip3 = install_pip_pkg([pkg], 'pip3', 'others')
+ status_pip3 = install_pip_pkg([pkg], 'pip3', 'others', args.dataengine_service)
general_status = general_status + status_pip3
except KeyError:
pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org