You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/06/17 13:37:12 UTC
[incubator-datalab] branch DATALAB-2429 created (now c720897)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2429
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at c720897 [DATALAB-2429]: fixed additional lib installation for gcp dataengine-service
This branch includes the following new commits:
new c720897 [DATALAB-2429]: fixed additional lib installation for gcp dataengine-service
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2429]: fixed additional lib
installation for gcp dataengine-service
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2429
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c720897f1b77a9901fccf96c8269310ad8261397
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Jun 17 16:36:30 2021 +0300
[DATALAB-2429]: fixed additional lib installation for gcp dataengine-service
---
infrastructure-provisioning/src/general/lib/os/fab.py | 15 +++++++++------
.../scripts/gcp/dataengine-service_install_libs.py | 6 +++---
.../src/general/scripts/os/install_additional_libs.py | 7 ++++---
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 10420b8..b69c515 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -99,12 +99,15 @@ def dataengine_dir_prepare(cluster_dir):
subprocess.run('mkdir -p ' + cluster_dir, shell=True, check=True)
-def install_pip_pkg(requisites, pip_version, lib_group):
+def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = False):
status = list()
error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:"
try:
- venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(
- os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
+ if dataengine_service:
+ install_command = pip_version
+ else:
+ install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(
+ os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
#if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
# for v in range(4, 8):
# if exists(conn, '/bin/pip3.{}'.format(v)):
@@ -124,13 +127,13 @@ def install_pip_pkg(requisites, pip_version, lib_group):
'''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | '''
'''tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; '''
'''then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
- venv_install_command, pip_pkg, error_parser, name, pip_version))
+ install_command, pip_pkg, error_parser, name, pip_version))
err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
'"', "'").replace('\n', ' ')
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; '''
'''then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
- venv_install_command, name, pip_version))
+ install_command, name, pip_version))
res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
'\n', '')
conn.sudo(
@@ -145,7 +148,7 @@ def install_pip_pkg(requisites, pip_version, lib_group):
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
- venv_install_command, changed_pip_pkg, pip_version))
+ install_command, changed_pip_pkg, pip_version))
res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
'\n', '')
if err and name not in installed_out:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
index bcc2eb7..32475e1 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
@@ -34,8 +34,8 @@ from fabric import *
def install_libs(instance, data_engine):
- data_engine['instance_ip'] = meta_lib.GCPMeta().get_private_ip_address(instance)
- params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"' \
+ data_engine['instance_ip'] = GCPMeta().get_private_ip_address(instance)
+ params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}" --dataengine_service True' \
.format(data_engine['os_user'], data_engine['instance_ip'],
data_engine['keyfile'], data_engine['libs'])
try:
@@ -65,7 +65,7 @@ if __name__ == "__main__":
data_engine['gcp_project_id'] = os.environ['gcp_project_id']
data_engine['gcp_region'] = os.environ['gcp_region']
data_engine['gcp_zone'] = os.environ['gcp_zone']
- res = meta_lib.GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name'])
+ res = GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name'])
data_engine['cluster_instances'] = [i.get('name') for i in res['items']]
data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
data_engine['libs'] = os.environ['libs']
diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
index 764f86d..cf443c5 100644
--- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
@@ -34,6 +34,7 @@ parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--instance_ip', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--libs', type=str, default='')
+parser.add_argument('--dataengine_service', type=bool, default=False)
args = parser.parse_args()
@@ -78,14 +79,14 @@ if __name__ == "__main__":
try:
print('Installing pip2 packages: {}'.format(pkgs['libraries']['pip2']))
- status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2')
+ status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2', args.dataengine_service)
general_status = general_status + status
except KeyError:
pass
try:
print('Installing pip3 packages: {}'.format(pkgs['libraries']['pip3']))
- status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3')
+ status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3', args.dataengine_service)
general_status = general_status + status
except KeyError:
pass
@@ -93,7 +94,7 @@ if __name__ == "__main__":
try:
print('Installing other packages (only tries pip3): {}'.format(pkgs['libraries']['others']))
for pkg in pkgs['libraries']['others']:
- status_pip3 = install_pip_pkg([pkg], 'pip3', 'others')
+ status_pip3 = install_pip_pkg([pkg], 'pip3', 'others', args.dataengine_service)
general_status = general_status + status_pip3
except KeyError:
pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org