You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/12/07 09:33:33 UTC
[incubator-datalab] 01/01: [DATALAB-2567]: replaced scp with rsync and disabled scp
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2567
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 574662f5270c30b0277fb5c4c8b5897558bedffd
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Dec 7 11:33:18 2021 +0200
[DATALAB-2567]: replaced scp with rsync and disabled scp
---
.../src/base/scripts/install_prerequisites.py | 4 ++--
.../src/general/lib/os/debian/ssn_lib.py | 2 +-
.../src/general/lib/os/redhat/ssn_lib.py | 2 +-
.../aws/jupyter_install_dataengine-service_kernels.py | 2 +-
.../aws/rstudio_install_dataengine-service_kernels.py | 2 +-
.../aws/zeppelin_install_dataengine-service_kernels.py | 2 +-
.../gcp/jupyter_install_dataengine-service_kernels.py | 2 +-
.../gcp/rstudio_install_dataengine-service_kernels.py | 2 +-
.../gcp/zeppelin_install_dataengine-service_kernels.py | 2 +-
.../src/ssn/scripts/configure_docker.py | 8 ++++----
.../src/ssn/scripts/configure_ssn_node.py | 4 ++--
infrastructure-provisioning/src/ssn/scripts/configure_ui.py | 12 ++++++------
12 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index 430805f..007f0e1 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -60,8 +60,8 @@ if __name__ == "__main__":
traceback.print_exc()
sys.exit(1)
- #logging.info("Disable scp on nodes")
- #disable_edge_scp_binary(args.user)
+ logging.info("Disable scp on nodes")
+ disable_edge_scp_binary(args.user)
#logging.info("Updating openssh to version")
#ensure_openssh_version(args.user)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 5439abc..ae4f6bb 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -270,7 +270,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
append_result("Unable to upload webapp jars")
sys.exit(1)
if billing_enabled:
- datalab.fab.conn.local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
+ datalab.fab.conn.local('rsync -e "ssh -i {}" /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
host_string))
params = '--cloud_provider {} ' \
'--infrastructure_tag {} ' \
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index 39ad178..4ebed0f 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -289,7 +289,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
sys.exit(1)
if billing_enabled:
- conn.local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
+ conn.local('rsync -e "ssh -i {}" /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
host_string))
params = '--cloud_provider {} ' \
'--infrastructure_tag {} ' \
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index 0efb00f..6e76b31 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@ -67,7 +67,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/jupyter_dataengine-service_create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- subprocess.run('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs'.format(args.keyfile, args.os_user, args.notebook_ip), shell=True, check=True)
+ subprocess.run('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs'.format(args.keyfile, args.os_user, args.notebook_ip), shell=True, check=True)
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists(conn, '/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
index 66eccb5..75d4db6 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
@@ -53,7 +53,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
+ conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists(conn, '/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index ae2616c..311cb04 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@ -58,7 +58,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
+ conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists(conn, '/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 7fc3ae9..a209d90 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -63,7 +63,7 @@ def configure_notebook(args):
datalab.fab.conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
datalab.fab.conn.run('mkdir -p /tmp/datalab_libs/')
host_string = args.os_user + "@" + args.notebook_ip
- datalab.fab.conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string))
+ datalab.fab.conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string))
datalab.fab.conn.run('chmod a+x /tmp/datalab_libs/*')
datalab.fab.conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists(datalab.fab.conn, '/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index be225cd..f098a95 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -55,7 +55,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /home/{}/datalab_libs/'.format(args.os_user))
- conn.local('scp -i {0} /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
+ conn.local('rsync -e "ssh -i {0}" /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
conn.run('chmod a+x /home/{}/datalab_libs/*'.format(args.os_user))
conn.sudo('mv /home/{}/datalab_libs/* /usr/lib/python3.8/datalab/'.format(args.os_user))
conn.sudo('rm -rf /home/{}/datalab_libs/'.format(args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index 8269b93..e17520a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -58,7 +58,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /home/{}/datalab_libs/'.format(args.os_user))
- conn.local('scp -i {0} /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
+ conn.local('rsync -e "ssh -i {0}" /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip))
conn.run('chmod a+x /home/{}/datalab_libs/*'.format(args.os_user))
conn.sudo('mv /home/{}/datalab_libs/* /usr/lib/python3.8/datalab/'.format(args.os_user))
conn.sudo('rm -rf /home/{}/datalab_libs/'.format(args.os_user))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 1e6e31d..e379440 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -52,8 +52,8 @@ def modify_conf_file(args):
for os_var in os.environ:
if "'" not in os.environ[os_var] and os_var != 'aws_access_key' and os_var != 'aws_secret_access_key':
variables_list[os_var] = os.environ[os_var]
- conn.local('scp -r -i {} /project_tree/* {}:{}sources/'.format(args.keyfile, host_string, args.datalab_path))
- conn.local('scp -i {} /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile,
+ conn.local('rsync -r -e "ssh -i {}" /project_tree/* {}:{}sources/'.format(args.keyfile, host_string, args.datalab_path))
+ conn.local('rsync -e "ssh -i {}" /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile,
host_string))
conn.sudo("python3 /tmp/configure_conf_file.py --datalab_dir {} --variables_list '{}'".format(
args.datalab_path, json.dumps(variables_list)))
@@ -92,7 +92,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider):
host_string = '{}@{}'.format(args.os_user, args.hostname)
with open('/tmp/config', 'w') as f:
f.write(base64.b64decode(gcr_creds))
- conn.local('scp -i {} /tmp/config {}:/tmp/config'.format(args.keyfile, host_string, os_user))
+ conn.local('rsync -e "ssh -i {}" /tmp/config {}:/tmp/config'.format(args.keyfile, host_string, os_user))
conn.sudo('mkdir /home/{}/.docker'.format(os_user))
conn.sudo('cp /tmp/config /home/{}/.docker/config.json'.format(os_user))
conn.sudo('sed -i "s|ODAHU_IMAGE|{}|" {}sources/infrastructure-provisioning/src/general/files/{}/odahu_Dockerfile'
@@ -110,7 +110,7 @@ def build_docker_images(image_list):
try:
host_string = '{}@{}'.format(args.os_user, args.hostname)
if os.environ['conf_cloud_provider'] == 'azure':
- conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
+ conn.local('rsync -e "ssh -i {}" /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
'/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 4b0cde6..f411fbb 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -63,7 +63,7 @@ def cp_key(keyfile, host_string, os_user):
key_name=keyfile.split("/")
conn.sudo('mkdir -p /home/' + os_user + '/keys')
conn.sudo('chown -R ' + os_user + ':' + os_user + ' /home/' + os_user + '/keys')
- conn.local('scp -r -q -i {0} {0} {1}:/home/{3}/keys/{2}'.format(keyfile, host_string, key_name[-1], os_user))
+ conn.local('rsync -r -q -e "ssh -i {0}" {0} {1}:/home/{3}/keys/{2}'.format(keyfile, host_string, key_name[-1], os_user))
conn.sudo('chmod 600 /home/' + os_user + '/keys/*.pem')
except Exception as err:
traceback.print_exc()
@@ -210,7 +210,7 @@ def copy_ssn_libraries():
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
subprocess.run(
- 'scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string),
+ 'rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string),
shell=True, check=True)
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 4a4476e..0c3a871 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -95,7 +95,7 @@ def copy_ssn_libraries():
try:
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- subprocess.run('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), shell=True, check=True)
+ subprocess.run('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), shell=True, check=True)
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists(conn, '/usr/lib64'):
@@ -113,23 +113,23 @@ def configure_mongo(mongo_passwd, default_endpoint_name):
subprocess.run('sed -i "s/MONGO_USR/mongodb/g" /root/templates/mongod.service_template', shell=True, check=True)
elif os.environ['conf_os_family'] == 'redhat':
subprocess.run('sed -i "s/MONGO_USR/mongod/g" /root/templates/mongod.service_template', shell=True, check=True)
- subprocess.run('scp -i {} /root/templates/mongod.service_template {}:/tmp/mongod.service'.format(args.keyfile,
+ subprocess.run('rsync -e "ssh -i {}" /root/templates/mongod.service_template {}:/tmp/mongod.service'.format(args.keyfile,
host_string), shell=True, check=True)
conn.sudo('mv /tmp/mongod.service /lib/systemd/system/mongod.service')
conn.sudo('systemctl daemon-reload')
conn.sudo('systemctl enable mongod.service')
subprocess.run('sed -i "s|PASSWORD|{}|g" /root/scripts/resource_status.py'.format(mongo_passwd), shell=True, check=True)
- subprocess.run('scp -i {} /root/scripts/resource_status.py {}:/tmp/resource_status.py'.format(args.keyfile,
+ subprocess.run('rsync -e "ssh -i {}" /root/scripts/resource_status.py {}:/tmp/resource_status.py'.format(args.keyfile,
host_string), shell=True, check=True)
conn.sudo('mv /tmp/resource_status.py ' + os.environ['ssn_datalab_path'] + 'tmp/')
subprocess.run('sed -i "s|PASSWORD|{}|g" /root/scripts/configure_mongo.py'.format(mongo_passwd), shell=True, check=True)
- subprocess.run('scp -i {} /root/scripts/configure_mongo.py {}:/tmp/configure_mongo.py'.format(args.keyfile,
+ subprocess.run('rsync -e "ssh -i {}" /root/scripts/configure_mongo.py {}:/tmp/configure_mongo.py'.format(args.keyfile,
host_string), shell=True, check=True)
conn.sudo('mv /tmp/configure_mongo.py ' + args.datalab_path + 'tmp/')
- subprocess.run('scp -i {} /root/files/{}/mongo_roles.json {}:/tmp/mongo_roles.json'.format(args.keyfile,
+ subprocess.run('rsync -e "ssh -i {}" /root/files/{}/mongo_roles.json {}:/tmp/mongo_roles.json'.format(args.keyfile,
args.cloud_provider,
host_string), shell=True, check=True)
- subprocess.run('scp -i {} /root/files/local_endpoint.json {}:/tmp/local_endpoint.json'.format(args.keyfile,
+ subprocess.run('rsync -e "ssh -i {}" /root/files/local_endpoint.json {}:/tmp/local_endpoint.json'.format(args.keyfile,
host_string), shell=True, check=True)
conn.sudo('mv /tmp/mongo_roles.json ' + args.datalab_path + 'tmp/')
conn.sudo('sed -i "s|DEF_ENDPOINT_NAME|{0}|g" /tmp/local_endpoint.json'.format(default_endpoint_name))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org