You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/23 14:47:34 UTC
[incubator-datalab] branch DATALAB-2091 updated: [DATALAB-2091]:
removed use_sudo from part of fabric put() where content of directory has
to be put on remote host
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new 30c56cb [DATALAB-2091]: removed use_sudo from part of fabric put() where content of directory has to be put on remote host
30c56cb is described below
commit 30c56cb9d6b7649b2e9fcaf214d4c66aba2f0e3e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 23 16:47:22 2021 +0200
[DATALAB-2091]: removed use_sudo from part of fabric put() where content of directory has to be put on remote host
---
.../scripts/deploy_repository/deploy_repository.py | 3 ++-
infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py | 8 +++++---
infrastructure-provisioning/src/general/lib/os/fab.py | 7 +++++--
infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py | 8 +++++---
.../src/general/scripts/os/common_configure_reverse_proxy.py | 3 ++-
.../general/scripts/os/deeplearning_install_dataengine_kernels.py | 4 +++-
.../src/general/scripts/os/jupyter_install_dataengine_kernels.py | 8 +++++---
.../src/general/scripts/os/rstudio_install_dataengine_kernels.py | 8 +++++---
.../scripts/os/tensor-rstudio_install_dataengine_kernels.py | 4 +++-
.../src/general/scripts/os/tensor_install_dataengine_kernels.py | 8 +++++---
.../src/general/scripts/os/zeppelin_install_dataengine_kernels.py | 4 +++-
11 files changed, 43 insertions(+), 22 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index 40601ac..a34e43a 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -1291,7 +1291,8 @@ def install_squid():
try:
if not exists(conn,'/home/{}/.ensure_dir/squid_installed'.format(configuration['conf_os_user'])):
conn.sudo('apt-get -y install squid')
- conn.put('templates/squid.conf', '/etc/squid/', use_sudo=True)
+ conn.put('templates/squid.conf', '/tmp/')
+ conn.sudo('cp -f /tmp/squid.conf /etc/squid/')
replace_string = ''
for cidr in get_vpc_cidr_by_id(args.vpc_id):
replace_string += 'acl AWS_VPC_CIDR src {}\\n'.format(cidr)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 17a2996..d0634ec 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -108,8 +108,8 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
'/usr/local/bin/manage_step_certs.sh'.format(user))
datalab.fab.conn.sudo('bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> '
'/var/log/renew_certificates.log 2>&1" >> /etc/crontab \'')
- datalab.fab.conn.put('/root/templates/step-cert-manager.service', '/etc/systemd/system/step-cert-manager.service',
- use_sudo=True)
+ datalab.fab.conn.put('/root/templates/step-cert-manager.service', '/tmp/step-cert-manager.service')
+ datalab.fab.conn.sudo('cp -f /tmp/step-cert-manager.service /etc/systemd/system/step-cert-manager.service')
datalab.fab.conn.sudo('systemctl daemon-reload')
datalab.fab.conn.sudo('systemctl enable step-cert-manager.service')
else:
@@ -138,7 +138,9 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
datalab.fab.conn.sudo('useradd -r nginx')
datalab.fab.conn.sudo('mkdir -p /opt/datalab/templates')
- datalab.fab.conn.put('/root/templates', '/opt/datalab', use_sudo=True)
+ datalab.fab.conn.local('cd /root/templates; tar -zcvf /tmp/templates.tar.gz *')
+ datalab.fab.conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
+ datalab.fab.conn.sudo('tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates/')
datalab.fab.conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(edge_ip))
datalab.fab.conn.sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_auth_server_url))
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 7544502..101f3a4 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -530,7 +530,8 @@ def install_java_pkg(requisites):
try:
ivy_jar = conn.sudo('find /opt /usr -name "*ivy-{0}.jar" | head -n 1'.format(os.environ['notebook_ivy_version'])).stdout.replace('\n','')
conn.sudo('mkdir -p {0} {1}'.format(ivy_dir, dest_dir))
- conn.put('{0}{1}'.format(templates_dir, ivy_settings), '{0}/{1}'.format(ivy_dir, ivy_settings), use_sudo=True)
+ conn.put('{0}{1}'.format(templates_dir, ivy_settings), '/tmp/{}'.format(ivy_settings))
+ conn.sudo('cp -f /tmp/{1} {0}/{1}'.format(ivy_dir, ivy_settings))
proxy_string = conn.sudo('cat /etc/profile | grep http_proxy | cut -f2 -d"="').stdout.replace('\n','')
proxy_re = '(?P<proto>http.*)://(?P<host>[^:/ ]+):(?P<port>[0-9]*)'
proxy_find = re.search(proxy_re, proxy_string)
@@ -951,7 +952,9 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
conn.sudo('ln -sf incubator-superset-{} incubator-superset'.format(os.environ['notebook_superset_version']))
if not exists(conn,'/tmp/superset-notebook_installed'):
conn.sudo('mkdir -p /opt/datalab/templates')
- conn.put('/root/templates', '/opt/datalab', use_sudo=True)
+ conn.local('cd /root/templates; tar -zcvf /tmp/templates.tar.gz *')
+ conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
+ conn.sudo('tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates')
conn.sudo('sed -i \'s/OS_USER/{}/g\' /opt/datalab/templates/.env'.format(os_user))
proxy_string = '{}:3128'.format(edge_instance_private_ip)
conn.sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/datalab/templates/id_provider.json'.format(
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 182146b..b9159f3 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -107,8 +107,8 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
'/usr/local/bin/manage_step_certs.sh'.format(user))
conn.sudo('bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> '
'/var/log/renew_certificates.log 2>&1" >> /etc/crontab \'')
- conn.put('/root/templates/step-cert-manager.service', '/etc/systemd/system/step-cert-manager.service',
- use_sudo=True)
+ conn.put('/root/templates/step-cert-manager.service', '/tmp/step-cert-manager.service')
+ conn.sudo('cd -f /tmp/step-cert-manager.service /etc/systemd/system/step-cert-manager.service')
conn.sudo('systemctl daemon-reload')
conn.sudo('systemctl enable step-cert-manager.service')
else:
@@ -169,7 +169,9 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('useradd -r nginx')
conn.sudo('rm -f /etc/nginx/nginx.conf')
conn.sudo('mkdir -p /opt/datalab/templates')
- conn.put('/root/templates', '/opt/datalab', use_sudo=True)
+ conn.local('cd /root/templates; tar -zcvf /tmp/templates.tar.gz *')
+ conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
+ conn.sudo('tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates/')
conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(edge_ip))
conn.sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_auth_server_url))
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
index 92c0e0e..a7fa7a6 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
@@ -105,7 +105,8 @@ if __name__ == "__main__":
print("Configure connections")
global conn
conn = datalab.fab.init_datalab_connection(args.edge_hostname, args.os_user, args.keyfile)
- conn.put('/tmp/{}.conf'.format(conf_file_name), '/usr/local/openresty/nginx/conf/locations', use_sudo=True)
+ conn.put('/tmp/{}.conf'.format(conf_file_name), '/tmp/{}.conf'.format(conf_file_name))
+ conn.sudo('cp -f /tmp/{}.conf /usr/local/openresty/nginx/conf/locations'.format(conf_file_name))
conn.sudo('service openresty reload')
conn.close()
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
index f1ae2f4..401ccb7 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
@@ -60,7 +60,9 @@ def configure_notebook(keyfile, hoststring):
conn.sudo('chmod 755 /usr/local/bin/deeplearning_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index ecfe354..a9b0f4b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -59,12 +59,14 @@ def configure_notebook(keyfile, hoststring):
# spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
# conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
if not exists(conn,'/usr/local/bin/jupyter_dataengine_create_configs.py'):
- conn.put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py',
- use_sudo=True)
+ conn.put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/tmp/jupyter_dataengine_create_configs.py')
+ conn.sudo('cp -f /tmp/jupyter_dataengine_create_configs.py /usr/local/bin/jupyter_dataengine_create_configs.py')
conn.sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
index e64abaa..e2f9d82 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
@@ -52,12 +52,14 @@ def configure_notebook(keyfile, hoststring):
conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory,
args.cluster_name))
if not exists(conn,'/usr/local/bin/rstudio_dataengine_create_configs.py'):
- conn.put(scripts_dir + 'rstudio_dataengine_create_configs.py', '/usr/local/bin/rstudio_dataengine_create_configs.py',
- use_sudo=True)
+ conn.put(scripts_dir + 'rstudio_dataengine_create_configs.py', '/tmp/rstudio_dataengine_create_configs.py')
+ conn.sudo('cp -f /tmp/rstudio_dataengine_create_configs.py /usr/local/bin/rstudio_dataengine_create_configs.py')
conn.sudo('chmod 755 /usr/local/bin/rstudio_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
index 5e6be28..496fd8a 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
@@ -59,7 +59,9 @@ def configure_notebook(keyfile, hoststring):
conn.sudo('chmod 755 /usr/local/bin/tensor-rstudio_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
index e9c2782..8a9f9d0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
@@ -52,12 +52,14 @@ def configure_notebook(keyfile, hoststring):
# spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
# conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
if not exists(conn,'/usr/local/bin/tensor_dataengine_create_configs.py'):
- conn.put(scripts_dir + 'tensor_dataengine_create_configs.py', '/usr/local/bin/tensor_dataengine_create_configs.py',
- use_sudo=True)
+ conn.put(scripts_dir + 'tensor_dataengine_create_configs.py', '/tmp/tensor_dataengine_create_configs.py')
+ conn.sudo('cp -f /tmp/tensor_dataengine_create_configs.py /usr/local/bin/tensor_dataengine_create_configs.py')
conn.sudo('chmod 755 /usr/local/bin/tensor_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
index 16712c8..bca7566 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
@@ -64,7 +64,9 @@ def configure_notebook(keyfile, hoststring):
conn.sudo('chmod 755 /usr/local/bin/zeppelin_dataengine_create_configs.py')
if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ conn.local('cd /usr/lib/python3.8/datalab/; tar -zcvf /tmp/datalab.tar.gz *')
+ conn.put('/tmp/datalab.tar.gz', '/tmp/datalab.tar.gz')
+ conn.sudo('tar -zxvf /tmp/datalab.tar.gz -C /usr/lib/python3.8/datalab/')
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org