You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/05 15:43:38 UTC
[incubator-datalab] 05/08: [DATALAB-2091]: partially replaced local
with subprocess.run(
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091-fab2
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 3d93bf7b30379e328cb3183e5de1f74f58b9ee0e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 5 16:54:14 2021 +0200
[DATALAB-2091]: partially replaced local with subprocess.run(
---
.../src/deeplearning/fabfile.py | 27 +++----
.../general/scripts/aws/common_prepare_notebook.py | 3 +-
.../scripts/aws/dataengine-service_list_libs.py | 3 +-
.../rstudio_install_dataengine-service_kernels.py | 2 +-
.../zeppelin_install_dataengine-service_kernels.py | 2 +-
.../scripts/azure/common_create_notebook_image.py | 5 +-
.../src/general/scripts/azure/ssn_prepare.py | 17 ++--
.../src/general/scripts/gcp/dataengine_start.py | 3 +-
.../src/general/scripts/gcp/ssn_configure.py | 12 +--
.../general/scripts/os/dataengine_install_libs.py | 5 +-
.../os/jupyter_dataengine_create_configs.py | 91 +++++++++++-----------
11 files changed, 89 insertions(+), 81 deletions(-)
diff --git a/infrastructure-provisioning/src/deeplearning/fabfile.py b/infrastructure-provisioning/src/deeplearning/fabfile.py
index deaa516..3e9f2df 100644
--- a/infrastructure-provisioning/src/deeplearning/fabfile.py
+++ b/infrastructure-provisioning/src/deeplearning/fabfile.py
@@ -25,6 +25,7 @@ import logging
import os
import sys
import uuid
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -44,7 +45,7 @@ def run():
try:
params = "--uuid {}".format(notebook_config['uuid'])
- local("~/scripts/{}.py {}".format('common_prepare_notebook', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_prepare_notebook', params), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed preparing Notebook node.", str(err))
@@ -52,7 +53,7 @@ def run():
try:
params = "--uuid {}".format(notebook_config['uuid'])
- local("~/scripts/{}.py {}".format('deeplearning_configure', params))
+ subprocess.run("~/scripts/{}.py {}".format('deeplearning_configure', params), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed configuring Notebook node.", str(err))
@@ -67,7 +68,7 @@ def terminate():
level=logging.DEBUG,
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('common_terminate_notebook'))
+ subprocess.run("~/scripts/{}.py".format('common_terminate_notebook'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed terminating Notebook node.", str(err))
@@ -82,7 +83,7 @@ def stop():
level=logging.DEBUG,
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('common_stop_notebook'))
+ subprocess.run("~/scripts/{}.py".format('common_stop_notebook'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed stopping Notebook node.", str(err))
@@ -98,7 +99,7 @@ def start():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('common_start_notebook'))
+ subprocess.run("~/scripts/{}.py".format('common_start_notebook'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed starting Notebook node.", str(err))
@@ -116,7 +117,7 @@ def configure():
try:
if os.environ['conf_resource'] == 'dataengine':
- local("~/scripts/{}.py".format('common_notebook_configure_dataengine'))
+ subprocess.run("~/scripts/{}.py".format('common_notebook_configure_dataengine'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed configuring dataengine on Notebook node.", str(err))
@@ -133,7 +134,7 @@ def install_libs():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('notebook_install_libs'))
+ subprocess.run("~/scripts/{}.py".format('notebook_install_libs'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed installing additional libs for Notebook node.", str(err))
@@ -150,7 +151,7 @@ def list_libs():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('notebook_list_libs'))
+ subprocess.run("~/scripts/{}.py".format('notebook_list_libs'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed get available libraries for notebook node.", str(err))
@@ -167,7 +168,7 @@ def git_creds():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('notebook_git_creds'))
+ subprocess.run("~/scripts/{}.py".format('notebook_git_creds'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed to manage git credentials for notebook node.", str(err))
@@ -184,7 +185,7 @@ def create_image():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('common_create_notebook_image'))
+ subprocess.run("~/scripts/{}.py".format('common_create_notebook_image'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed to create image from notebook node.", str(err))
@@ -201,7 +202,7 @@ def terminate_image():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('common_terminate_notebook_image'))
+ subprocess.run("~/scripts/{}.py".format('common_terminate_notebook_image'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed to create image from notebook node.", str(err))
@@ -218,7 +219,7 @@ def reconfigure_spark():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('notebook_reconfigure_spark'))
+ subprocess.run("~/scripts/{}.py".format('notebook_reconfigure_spark'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed to reconfigure Spark for Notebook node.", str(err))
@@ -234,7 +235,7 @@ def check_inactivity():
filename=local_log_filepath)
try:
- local("~/scripts/{}.py".format('notebook_inactivity_check'))
+ subprocess.run("~/scripts/{}.py".format('notebook_inactivity_check'), shell=True)
except Exception as err:
traceback.print_exc()
append_result("Failed to check inactivity status.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index e50c268..edc4a89 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -30,6 +30,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
parser = argparse.ArgumentParser()
@@ -147,7 +148,7 @@ if __name__ == "__main__":
notebook_config['tag_name'], notebook_config['instance_name'], instance_class,
os.environ['notebook_disk_size'], notebook_config['primary_disk_size'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except:
traceback.print_exc()
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
index eeec353..3302458 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
@@ -25,6 +25,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -59,7 +60,7 @@ if __name__ == "__main__":
.format(data_engine['os_user'], data_engine['master_ip'], data_engine['keyfile'], data_engine['group_name'])
try:
# Run script to get available libs
- local("~/scripts/{}.py {}".format('get_list_available_pkgs', params))
+ subprocess.run("~/scripts/{}.py {}".format('get_list_available_pkgs', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
index 05f6c49..a2d7e7f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
@@ -53,7 +53,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index 6cc8c9c..fe89e33 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@ -59,7 +59,7 @@ def configure_notebook(args):
conn.sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index 562b98c..c23bd08 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -27,6 +27,7 @@ import datalab.meta_lib
import json
import os
import sys
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -85,7 +86,7 @@ if __name__ == "__main__":
json.dumps(image_conf['tags']))
print("Image was successfully created.")
try:
- local("~/scripts/{}.py".format('common_prepare_notebook'))
+ subprocess.run("~/scripts/{}.py".format('common_prepare_notebook'), shell=True)
instance_running = False
while not instance_running:
if AzureMeta.get_instance_status(image_conf['resource_group_name'],
@@ -101,7 +102,7 @@ if __name__ == "__main__":
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, image_conf['instance_name'], keyfile_name,
json.dumps(additional_config), image_conf['datalab_ssh_user'])
- local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_proxy', params), shell=True)
print("Image was successfully created. It's name is {}".format(image_conf['full_image_name']))
except Exception as err:
AzureActions.remove_instance(image_conf['resource_group_name'], image_conf['instance_name'])
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
index 63865ff..d78fb48 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
@@ -29,6 +29,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from Crypto.PublicKey import RSA
from fabric import *
@@ -110,7 +111,7 @@ if __name__ == "__main__":
logging.info('[CREATING RESOURCE GROUP]')
print("[CREATING RESOURCE GROUP]")
params = "--resource_group_name {} --region {}".format(ssn_conf['resource_group_name'], ssn_conf['region'])
- local("~/scripts/{}.py {}".format('ssn_create_resource_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_resource_group', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to create Resource Group.", str(err))
@@ -125,7 +126,7 @@ if __name__ == "__main__":
print("[CREATING VIRTUAL NETWORK]")
params = "--resource_group_name {} --vpc_name {} --region {} --vpc_cidr {}".format(
ssn_conf['resource_group_name'], ssn_conf['vpc_name'], ssn_conf['region'], ssn_conf['vpc_cidr'])
- local("~/scripts/{}.py {}".format('ssn_create_vpc', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_vpc', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to create VPC.", str(err))
@@ -146,7 +147,7 @@ if __name__ == "__main__":
params = "--resource_group_name {} --vpc_name {} --region {} --vpc_cidr {} --subnet_name {} --prefix {}".\
format(ssn_conf['resource_group_name'], ssn_conf['vpc_name'], ssn_conf['region'],
ssn_conf['vpc_cidr'], ssn_conf['subnet_name'], ssn_conf['subnet_prefix'])
- local("~/scripts/{}.py {}".format('common_create_subnet', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_subnet', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to create Subnet.", str(err))
@@ -168,7 +169,7 @@ if __name__ == "__main__":
"--source_virtual_network_name {} --destination_virtual_network_name {}".format(
ssn_conf['source_resource_group_name'], ssn_conf['resource_group_name'],
os.environ['azure_source_vpc_name'], ssn_conf['vpc_name'])
- local("~/scripts/{}.py {}".format('ssn_create_peering', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_peering', params), shell=True)
except Exception as err:
traceback.print_exc()
try:
@@ -238,7 +239,7 @@ if __name__ == "__main__":
params = "--resource_group_name {} --security_group_name {} --region {} --tags '{}' --list_rules '{}'".\
format(ssn_conf['resource_group_name'], ssn_conf['security_group_name'], ssn_conf['region'],
json.dumps(ssn_conf['instance_tags']), json.dumps(list_rules))
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_security_group', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Error creating Security group", str(err))
@@ -263,7 +264,7 @@ if __name__ == "__main__":
format(ssn_conf['datalake_store_name'], json.dumps(ssn_conf['datalake_store_tags']),
ssn_conf['resource_group_name'], ssn_conf['region'])
try:
- local("~/scripts/{}.py {}".format('ssn_create_datalake', params))
+ subprocess.run("~/scripts/{}.py {}".format('ssn_create_datalake', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -275,7 +276,7 @@ if __name__ == "__main__":
ssn_conf['datalake_shared_directory_name'], ssn_conf['service_base_name'],
os.environ['azure_ad_group_id'])
try:
- local("~/scripts/{}.py {}".format('common_create_datalake_directory', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_datalake_directory', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -316,7 +317,7 @@ if __name__ == "__main__":
initial_user, ssn_conf['static_public_ip_name'], ssn_conf['public_ssh_key'],
ssn_conf['primary_disk_size'], 'ssn', ssn_conf['instance_storage_account_type'],
ssn_conf['ssn_image_name'], json.dumps(ssn_conf['instance_tags']))
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to create instance.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
index 095ba98..27e69dc 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
@@ -29,6 +29,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
@@ -97,7 +98,7 @@ if __name__ == "__main__":
.format(os.environ['conf_os_user'], data_engine['notebook_ip'], data_engine['keyfile'],
data_engine['computational_ip'])
try:
- local("~/scripts/{}.py {}".format('update_inactivity_on_start', params))
+ subprocess.run("~/scripts/{}.py {}".format('update_inactivity_on_start', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to update last activity time.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 9da2932..21a7d2f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -148,7 +148,7 @@ if __name__ == "__main__":
ssn_conf['datalab_ssh_user'], ssn_conf['sudo_group'])
try:
- local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -167,7 +167,7 @@ if __name__ == "__main__":
ssn_conf['datalab_ssh_user'], ssn_conf['region'])
try:
- local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -190,7 +190,7 @@ if __name__ == "__main__":
ssn_conf['step_cert_sans'])
try:
- local("~/scripts/{}.py {}".format('configure_ssn_node', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_ssn_node', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -222,7 +222,7 @@ if __name__ == "__main__":
os.environ['conf_cloud_provider'], ssn_conf['region'])
try:
- local("~/scripts/{}.py {}".format('configure_docker', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_docker', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -515,7 +515,7 @@ if __name__ == "__main__":
json.dumps(cloud_params), os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
os.environ['keycloak_auth_server_url'])
try:
- local("~/scripts/{}.py {}".format('configure_ui', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_ui', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -568,7 +568,7 @@ if __name__ == "__main__":
params = "--instance_name {} --local_log_filepath {} --os_user {} --instance_hostname {}". \
format(ssn_conf['instance_name'], local_log_filepath, ssn_conf['datalab_ssh_user'],
ssn_conf['instance_hostname'])
- local("~/scripts/{}.py {}".format('upload_response_file', params))
+ subprocess.run("~/scripts/{}.py {}".format('upload_response_file', params), shell=True)
except Exception as err:
datalab.fab.append_result("Error with writing results.", str(err))
clear_resources()
diff --git a/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py b/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
index bd90687..74ceebe 100644
--- a/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
@@ -26,6 +26,7 @@ import multiprocessing
import os
import sys
import traceback
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -41,7 +42,7 @@ def install_libs_on_slaves(slave, data_engine):
data_engine['keyfile'], data_engine['libs'])
try:
# Run script to install additional libs
- local("~/scripts/{}.py {}".format('install_additional_libs', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_additional_libs', params), shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
@@ -80,7 +81,7 @@ if __name__ == "__main__":
data_engine['keyfile'], data_engine['libs'])
try:
# Run script to install additional libs
- local("~/scripts/{}.py {}".format('install_additional_libs', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_additional_libs', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
index 0bea54e..a5b3c11 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
@@ -23,6 +23,7 @@
import argparse
import sys
+import subprocess
from datalab.actions_lib import *
from datalab.common_lib import *
from datalab.fab import *
@@ -55,10 +56,10 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
def r_kernel(args):
spark_path = '/opt/{}/spark/'.format(args.cluster_name)
- local('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name))
+ subprocess.run('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = "{}/r_{}/kernel.json".format(kernels_dir, args.cluster_name)
template_file = "/tmp/{}/r_dataengine_template.json".format(args.cluster_name)
- r_version = local("R --version | awk '/version / {print $3}'", capture = True)
+ r_version = subprocess.run("R --version | awk '/version / {print $3}'", capture_output = True, shell=True)
with open(template_file, 'r') as f:
text = f.read()
@@ -74,11 +75,11 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/' + args.cluster_name + '/spark/'
- scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True)
- local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/')
- local('tar zxvf /tmp/{}/toree_kernel.tar.gz -C '.format(args.cluster_name) + kernels_dir + 'toree_' + args.cluster_name + '/')
- local('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
- local('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name))
+ scala_version = subprocess.run('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True, shell=True)
+ subprocess.run('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/', shell=True)
+ subprocess.run('tar zxvf /tmp/{}/toree_kernel.tar.gz -C '.format(args.cluster_name) + kernels_dir + 'toree_' + args.cluster_name + '/', shell=True)
+ subprocess.run('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name), shell=True)
+ subprocess.run('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = kernels_dir + "toree_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/toree_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -91,12 +92,12 @@ def toree_kernel(args):
text = text.replace('SCALA_VERSION', scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/" + args.cluster_name +
"/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat " + kernel_path +
- " | sed 's|PY4J|'$PYJ'|g' > /tmp/{}/kernel_var.json".format(args.cluster_name))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ " | sed 's|PY4J|'$PYJ'|g' > /tmp/{}/kernel_var.json".format(args.cluster_name), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
run_sh_path = kernels_dir + "toree_" + args.cluster_name + "/bin/run.sh"
template_sh_file = '/tmp/{}/run_template.sh'.format(args.cluster_name)
with open(template_sh_file, 'r') as f:
@@ -109,7 +110,7 @@ def toree_kernel(args):
def pyspark_kernel(args):
spark_path = '/opt/' + args.cluster_name + '/spark/'
- local('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "pyspark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -123,13 +124,13 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
- local('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "py3spark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -143,45 +144,45 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
def install_sparkamagic_kernels(args):
try:
- local('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
- sparkmagic_dir = local("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True)
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir))
+ subprocess.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension', shell=True)
+ sparkmagic_dir = subprocess.run("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True, shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir), shell=True)
pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
- local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
- pyspark_kernel_name, args.os_user))
- scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True)
+ subprocess.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True)
+ scala_version = subprocess.run('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True, shell=True)
spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(scala_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
- spark_kernel_name, args.os_user))
- r_version = local("R --version | awk '/version / {print $3}'", capture_output=True)
+ subprocess.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True)
+ r_version = subprocess.run("R --version | awk '/version / {print $3}'", capture_output=True, shell=True)
sparkr_kernel_name = 'SparkR (R-{0} / Spark-{1} ) [{2}]'.format(str(r_version), args.spark_version,
args.cluster_name)
- local('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
- sparkr_kernel_name, args.os_user))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkrkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkrkernel_{1}'.format(args.os_user, args.cluster_name))
- local('mkdir -p /home/' + args.os_user + '/.sparkmagic')
- local('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json')
+ subprocess.run('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
+ sparkr_kernel_name, args.os_user), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkrkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkrkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True)
+ subprocess.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True)
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
- local('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
- spark_master_ip, args.os_user))
- local('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user))
+ subprocess.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ spark_master_ip, args.os_user), shell=True)
+ subprocess.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True)
except:
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org