You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/05 15:43:41 UTC
[incubator-datalab] 08/08: [DATALAB-2091]: partially replaced local
with subprocess.run(
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091-fab2
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 0f42c4cfeb99dce6a3ace3f3bc05ab1de540235f
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 5 17:43:16 2021 +0200
[DATALAB-2091]: partially replaced local with subprocess.run(
---
.../gcp/common_notebook_configure_dataengine.py | 4 +-
.../general/scripts/gcp/common_prepare_notebook.py | 2 +-
.../src/general/scripts/gcp/common_reupload_key.py | 3 +-
.../general/scripts/gcp/common_start_notebook.py | 5 +-
.../scripts/gcp/dataengine-service_install_libs.py | 3 +-
.../scripts/gcp/dataengine-service_list_libs.py | 3 +-
.../scripts/gcp/dataengine-service_prepare.py | 11 +++--
.../src/general/scripts/gcp/dataengine_prepare.py | 5 +-
.../src/general/scripts/gcp/edge_configure.py | 15 +++---
.../src/general/scripts/gcp/ssn_prepare.py | 13 ++---
.../src/general/scripts/gcp/ssn_terminate.py | 3 +-
.../zeppelin_install_dataengine-service_kernels.py | 2 +-
.../os/tensor-rstudio_dataengine_create_configs.py | 38 +++++++--------
.../scripts/os/tensor_dataengine_create_configs.py | 56 +++++++++++-----------
14 files changed, 86 insertions(+), 77 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
index 6ad175f..291d282 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
@@ -100,7 +100,7 @@ if __name__ == "__main__":
notebook_config['spark_master_url'], notebook_config['key_path'],
notebook_config['notebook_ip'], notebook_config['spark_master_ip'])
try:
- local("~/scripts/{}_{}.py {}".format(os.environ['application'], 'install_dataengine_kernels', params))
+ subprocess.run("~/scripts/{}_{}.py {}".format(os.environ['application'], 'install_dataengine_kernels', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -121,7 +121,7 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'],
notebook_config['cluster_name'])
try:
- local("~/scripts/{0}.py {1}".format('common_configure_spark', params))
+ subprocess.run("~/scripts/{0}.py {1}".format('common_configure_spark', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index 3407eb7..87b0277 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -192,7 +192,7 @@ if __name__ == "__main__":
notebook_config['network_tag'], json.dumps(notebook_config['labels']),
notebook_config['service_base_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py b/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
index a5e6653..6721c0b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
@@ -23,6 +23,7 @@
import argparse
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -45,7 +46,7 @@ if __name__ == "__main__":
params = "--user {} --hostname {} --keyfile '{}' --additional_config '{}'".format(
args.os_user, ip, args.keyfile, args.additional_config)
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
index aebbc9e..734cd44 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
@@ -29,6 +29,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -68,7 +69,7 @@ if __name__ == "__main__":
params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
.format(os.environ['conf_os_user'], notebook_config['notebook_ip'], notebook_config['keyfile'])
try:
- local("~/scripts/{}.py {}".format('manage_git_creds', params))
+ subprocess.run("~/scripts/{}.py {}".format('manage_git_creds', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to setup git credentials.", str(err))
@@ -82,7 +83,7 @@ if __name__ == "__main__":
params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
.format(os.environ['conf_os_user'], notebook_config['notebook_ip'], notebook_config['keyfile'])
try:
- local("~/scripts/{}.py {}".format('update_inactivity_on_start', params))
+ subprocess.run("~/scripts/{}.py {}".format('update_inactivity_on_start', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to update last activity time.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
index 477cbba..9027269 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
@@ -26,6 +26,7 @@ import multiprocessing
import os
import sys
import traceback
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -39,7 +40,7 @@ def install_libs(instance, data_engine):
data_engine['keyfile'], data_engine['libs'])
try:
# Run script to install additional libs
- local("~/scripts/{}.py {}".format('install_additional_libs', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_additional_libs', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
index fe7f3ff..d84b3e0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
@@ -25,6 +25,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -60,7 +61,7 @@ if __name__ == "__main__":
.format(data_engine['os_user'], data_engine['master_ip'], data_engine['keyfile'], data_engine['group_name'])
try:
# Run script to get available libs
- local("~/scripts/{}.py {}".format('get_list_available_pkgs', params))
+ subprocess.run("~/scripts/{}.py {}".format('get_list_available_pkgs', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
index 09123ff..5eed753 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
@@ -30,6 +30,7 @@ import os
import sys
import time
import traceback
+import subprocess
from Crypto.PublicKey import RSA
from fabric import *
@@ -131,13 +132,13 @@ if __name__ == "__main__":
try:
GCPMeta.dataproc_waiter(dataproc_conf['cluster_labels'])
- local('touch /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ subprocess.run('touch /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Dataproc waiter fail.", str(err))
sys.exit(1)
- local("echo Waiting for changes to propagate; sleep 10")
+ subprocess.run("echo Waiting for changes to propagate; sleep 10", shell=True)
dataproc_cluster = json.loads(open('/root/templates/dataengine-service_cluster.json').read())
print(dataproc_cluster)
@@ -178,14 +179,14 @@ if __name__ == "__main__":
json.dumps(dataproc_cluster))
try:
- local("~/scripts/{}.py {}".format('dataengine-service_create', params))
+ subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True)
except:
traceback.print_exc()
raise Exception
keyfile_name = "/root/keys/{}.pem".format(dataproc_conf['key_name'])
- local('rm /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ subprocess.run('rm /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']), shell=True)
except Exception as err:
datalab.fab.append_result("Failed to create Dataproc Cluster.", str(err))
- local('rm /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ subprocess.run('rm /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']), shell=True)
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index 0070324..7c6b55d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -29,6 +29,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -194,7 +195,7 @@ if __name__ == "__main__":
data_engine['network_tag'], data_engine['cluster_name'],
json.dumps(data_engine['master_labels']), data_engine['service_base_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -222,7 +223,7 @@ if __name__ == "__main__":
data_engine['network_tag'], data_engine['cluster_name'],
json.dumps(data_engine['slave_labels']), data_engine['service_base_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
index 08ca1df..d05d351 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
@@ -30,6 +30,7 @@ import os
import sys
import traceback
import uuid
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -161,7 +162,7 @@ if __name__ == "__main__":
edge_conf['initial_user'], edge_conf['datalab_ssh_user'], edge_conf['sudo_group'])
try:
- local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -177,7 +178,7 @@ if __name__ == "__main__":
edge_conf['instance_hostname'], edge_conf['ssh_key_path'], edge_conf['datalab_ssh_user'],
os.environ['gcp_region'])
try:
- local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -202,7 +203,7 @@ if __name__ == "__main__":
.format(edge_conf['instance_hostname'], edge_conf['ssh_key_path'], json.dumps(additional_config),
edge_conf['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('configure_http_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_http_proxy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -221,7 +222,7 @@ if __name__ == "__main__":
edge_conf['instance_hostname'], edge_conf['ssh_key_path'], json.dumps(additional_config),
edge_conf['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -243,7 +244,7 @@ if __name__ == "__main__":
edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
try:
- local("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -261,7 +262,7 @@ if __name__ == "__main__":
edge_conf['keycloak_client_secret'], edge_conf['instance_hostname'], edge_conf['project_name'],
edge_conf['endpoint_name'], edge_conf['edge_hostname'])
try:
- local("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -280,7 +281,7 @@ if __name__ == "__main__":
edge_conf['instance_hostname'], edge_conf['ssh_key_path'], json.dumps(additional_config),
edge_conf['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('configure_nftables', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_nftables', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
index d355250..9898020 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
@@ -30,6 +30,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
parser = argparse.ArgumentParser()
@@ -97,7 +98,7 @@ if __name__ == "__main__":
print('[CREATE VPC]')
params = "--vpc_name {}".format(ssn_conf['vpc_name'])
try:
- local("~/scripts/{}.py {}".format('ssn_create_vpc', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_vpc', params), shell=True)
os.environ['gcp_vpc_name'] = ssn_conf['vpc_name']
except:
traceback.print_exc()
@@ -126,7 +127,7 @@ if __name__ == "__main__":
format(ssn_conf['subnet_name'], ssn_conf['region'], ssn_conf['vpc_selflink'], ssn_conf['subnet_prefix'],
ssn_conf['vpc_cidr'], True)
try:
- local("~/scripts/{}.py {}".format('common_create_subnet', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_subnet', params), shell=True)
os.environ['gcp_subnet_name'] = ssn_conf['subnet_name']
except:
traceback.print_exc()
@@ -192,7 +193,7 @@ if __name__ == "__main__":
params = "--firewall '{}'".format(json.dumps(firewall_rules))
try:
- local("~/scripts/{}.py {}".format('common_create_firewall', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_firewall', params), shell=True)
os.environ['gcp_firewall_name'] = ssn_conf['firewall_name']
except:
traceback.print_exc()
@@ -213,7 +214,7 @@ if __name__ == "__main__":
ssn_conf['ssn_policy_path'], ssn_conf['ssn_roles_path'],
ssn_conf['ssn_unique_index'], ssn_conf['service_base_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_service_account', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_service_account', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -238,7 +239,7 @@ if __name__ == "__main__":
print('[CREATING STATIC IP ADDRESS]')
params = "--address_name {} --region {}".format(ssn_conf['static_address_name'], ssn_conf['region'])
try:
- local("~/scripts/{}.py {}".format('ssn_create_static_ip', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_static_ip', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -283,7 +284,7 @@ if __name__ == "__main__":
ssn_conf['static_ip'], ssn_conf['network_tag'], json.dumps(ssn_conf['instance_labels']), '20',
ssn_conf['service_base_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
index 0c04bb0..0c5d235 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
@@ -27,6 +27,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -58,7 +59,7 @@ if __name__ == "__main__":
params = "--service_base_name {} --region {} --zone {} --pre_defined_vpc {} --vpc_name {}".format(
ssn_conf['service_base_name'], ssn_conf['region'], ssn_conf['zone'], pre_defined_vpc, ssn_conf['vpc_name'])
try:
- local("~/scripts/{}.py {}".format('ssn_terminate_gcp_resources', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_terminate_gcp_resources', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index f022b1a..82df27b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -58,7 +58,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
index 802e651..b7561c1 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
@@ -52,30 +52,30 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
def configure_rstudio():
if not os.path.exists('/home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured'):
try:
- local('echo "export R_LIBS_USER=' + cluster_dir + 'spark/R/lib:" >> /home/' + args.os_user + '/.bashrc')
- local("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron')
- local("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile")
- local('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
- args.os_user + '/.Rprofile')
- local('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user))
- local('touch /home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured')
+ subprocess.run('echo "export R_LIBS_USER=' + cluster_dir + 'spark/R/lib:" >> /home/' + args.os_user + '/.bashrc', shell=True)
+ subprocess.run("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron', shell=True)
+ subprocess.run("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile", shell=True)
+ subprocess.run('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
+ args.os_user + '/.Rprofile', shell=True)
+ subprocess.run('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user), shell=True)
+ subprocess.run('touch /home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured', shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
else:
try:
- local("sed -i '/R_LIBS_USER/ { s|=\(.*\)|=\\1" + cluster_dir + "spark/R/lib:| }' /home/" + args.os_user + "/.bashrc")
- local("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron')
- local("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile")
- local('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
- args.os_user + '/.Rprofile')
- local('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user))
+ subprocess.run("sed -i '/R_LIBS_USER/ { s|=\(.*\)|=\\1" + cluster_dir + "spark/R/lib:| }' /home/" + args.os_user + "/.bashrc", shell=True)
+ subprocess.run("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron', shell=True)
+ subprocess.run("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile", shell=True)
+ subprocess.run('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
+ args.os_user + '/.Rprofile', shell=True)
+ subprocess.run('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user), shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
index a0f0f4f..a6df4ee 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
@@ -54,7 +54,7 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
def pyspark_kernel(args):
spark_path = '/opt/' + args.cluster_name + '/spark/'
- local('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "pyspark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -68,13 +68,13 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
- local('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "py3spark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -88,40 +88,40 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
def install_sparkamagic_kernels(args):
try:
- local('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
- sparkmagic_dir = local("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True)
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
+ subprocess.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension', shell=True)
+ sparkmagic_dir = subprocess.run("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True, shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir), shell=True)
pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
- local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
- pyspark_kernel_name, args.os_user))
- scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True)
+ subprocess.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True)
+ scala_version = subprocess.run('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True, shell=True)
spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(scala_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
- spark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True)
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name))
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
- local('mkdir -p /home/' + args.os_user + '/.sparkmagic')
- local('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json')
+ subprocess.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True)
+ subprocess.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True)
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
- local('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
- spark_master_ip, args.os_user))
- local('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user))
+ subprocess.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ spark_master_ip, args.os_user), shell=True)
+ subprocess.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True)
except:
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org