You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/05 15:43:39 UTC
[incubator-datalab] 06/08: [DATALAB-2091]: partially replaced local
with subprocess.run(
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091-fab2
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 876d6bdbc3ba8296cd784a09af4ef637e7b65e9f
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 5 17:09:34 2021 +0200
[DATALAB-2091]: partially replaced local with subprocess.run(
---
.../src/general/lib/aws/meta_lib.py | 3 +-
.../src/general/scripts/aws/dataengine_prepare.py | 5 +-
.../jupyter_dataengine-service_create_configs.py | 117 +++++++++++----------
.../general/scripts/aws/jupyterlab_configure.py | 23 ++--
.../src/general/scripts/aws/project_prepare.py | 31 +++---
.../general/scripts/azure/common_reupload_key.py | 3 +-
.../general/scripts/azure/dataengine_configure.py | 26 ++---
.../jupyter_dataengine-service_create_configs.py | 51 ++++-----
.../os/deeplearning_dataengine_create_configs.py | 57 +++++-----
.../os/rstudio_dataengine_create_configs.py | 39 +++----
.../src/ssn/scripts/configure_docker.py | 8 +-
11 files changed, 186 insertions(+), 177 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index 4f35072..27ec19f 100644
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@ -27,6 +27,7 @@ import logging
import sys
import time
import traceback
+import subprocess
from botocore.client import Config
from datalab.fab import *
@@ -678,7 +679,7 @@ def check_security_group(security_group_name, count=0):
def emr_waiter(tag_name, tag_value):
if len(get_emr_list(tag_value, 'Value', False, True)) > 0 or os.path.exists('/response/.emr_creating_' + os.environ['exploratory_name']) or get_not_configured_emr(tag_name, tag_value):
with hide('stderr', 'running', 'warnings'):
- local("echo 'Some EMR cluster is still being created/terminated, waiting..'")
+ subprocess.run("echo 'Some EMR cluster is still being created/terminated, waiting..'", shell=True)
time.sleep(60)
emr_waiter(tag_name, tag_value)
else:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
index df48a08..313cf6f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
@@ -29,6 +29,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -156,7 +157,7 @@ if __name__ == "__main__":
data_engine['notebook_dataengine_role_profile_name'], data_engine['tag_name'],
data_engine['master_node_name'], data_engine['primary_disk_size'], data_engine['instance_class'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
data_engine['master_id'] = datalab.meta_lib.get_instance_by_name(data_engine['tag_name'],
data_engine['master_node_name'])
datalab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_tag'], False)
@@ -188,7 +189,7 @@ if __name__ == "__main__":
data_engine['notebook_dataengine_role_profile_name'], data_engine['tag_name'],
slave_name, data_engine['primary_disk_size'], data_engine['instance_class'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
data_engine['slave_id'] = datalab.meta_lib.get_instance_by_name(data_engine['tag_name'], slave_name)
datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_tag'], False)
datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_resource_tag'],
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 7a1a1a2..273de3a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@ -23,6 +23,7 @@
import argparse
import sys
+import subprocess
from datalab.actions_lib import *
from datalab.common_lib import *
from datalab.fab import *
@@ -57,7 +58,7 @@ yarn_dir = '/opt/' + args.emr_version + '/' + args.cluster_name + '/conf/'
def r_kernel(args):
spark_path = '/opt/{}/{}/spark/'.format(args.emr_version, args.cluster_name)
- local('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name))
+ subprocess.run('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = "{}/r_{}/kernel.json".format(kernels_dir, args.cluster_name)
template_file = "/tmp/r_dataengine-service_template.json"
@@ -80,9 +81,9 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/' + args.emr_version + '/' + args.cluster_name + '/spark/'
- scala_version = local("Spark-submit --version 2>&1 | awk '/Scala version / {gsub(/,/, \"\"); print $4}'")
+ scala_version = subprocess.run("Spark-submit --version 2>&1 | awk '/Scala version / {gsub(/,/, \"\"); print $4}'", shell=True)
if args.emr_version == 'emr-4.3.0' or args.emr_version == 'emr-4.6.0' or args.emr_version == 'emr-4.8.0':
- local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "toree_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/toree_dataengine-service_template.json"
with open(template_file, 'r') as f:
@@ -94,15 +95,15 @@ def toree_kernel(args):
text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/kernel_var.json')
- local(
- "PYJ=`find /opt/" + args.emr_version + "/" + args.cluster_name + "/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat " + kernel_path + " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json")
- local('sudo mv /tmp/kernel_var.json ' + kernel_path)
+ subprocess.run('touch /tmp/kernel_var.json', shell=True)
+ subprocess.run(
+ "PYJ=`find /opt/" + args.emr_version + "/" + args.cluster_name + "/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat " + kernel_path + " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json", shell=True)
+ subprocess.run('sudo mv /tmp/kernel_var.json ' + kernel_path, shell=True)
else:
- local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/')
- local('tar zxvf /tmp/toree_kernel.tar.gz -C ' + kernels_dir + 'toree_' + args.cluster_name + '/')
- local('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
- local('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name))
+ subprocess.run('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/', shell=True)
+ subprocess.run('tar zxvf /tmp/toree_kernel.tar.gz -C ' + kernels_dir + 'toree_' + args.cluster_name + '/', shell=True)
+ subprocess.run('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name), shell=True)
+ subprocess.run('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = kernels_dir + "toree_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/toree_dataengine-service_templatev2.json"
with open(template_file, 'r') as f:
@@ -115,12 +116,12 @@ def toree_kernel(args):
text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/kernel_var.json')
- local(
+ subprocess.run('touch /tmp/kernel_var.json', shell=True)
+ subprocess.run(
"PYJ=`find /opt/" + args.emr_version + "/" + args.cluster_name +
"/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat " + kernel_path +
- " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json")
- local('sudo mv /tmp/kernel_var.json ' + kernel_path)
+ " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json", shell=True)
+ subprocess.run('sudo mv /tmp/kernel_var.json ' + kernel_path, shell=True)
run_sh_path = kernels_dir + "toree_" + args.cluster_name + "/bin/run.sh"
template_sh_file = '/tmp/run_template.sh'
with open(template_sh_file, 'r') as f:
@@ -135,58 +136,58 @@ def add_breeze_library_emr(args):
spark_defaults_path = '/opt/' + args.emr_version + '/' + args.cluster_name + '/spark/conf/spark-defaults.conf'
new_jars_directory_path = '/opt/' + args.emr_version + '/jars/usr/other/'
breeze_tmp_dir = '/tmp/breeze_tmp_emr/'
- local('sudo mkdir -p ' + new_jars_directory_path)
- local('mkdir -p ' + breeze_tmp_dir)
- local('wget https://repo1.maven.org/maven2/org/scalanlp/breeze_2.11/0.12/breeze_2.11-0.12.jar -O ' +
- breeze_tmp_dir + 'breeze_2.11-0.12.jar')
- local('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-natives_2.11/0.12/breeze-natives_2.11-0.12.jar -O '
- + breeze_tmp_dir + 'breeze-natives_2.11-0.12.jar')
- local('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-viz_2.11/0.12/breeze-viz_2.11-0.12.jar -O ' +
- breeze_tmp_dir + 'breeze-viz_2.11-0.12.jar')
- local('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-macros_2.11/0.12/breeze-macros_2.11-0.12.jar -O ' +
- breeze_tmp_dir + 'breeze-macros_2.11-0.12.jar')
- local('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-parent_2.11/0.12/breeze-parent_2.11-0.12.jar -O ' +
- breeze_tmp_dir + 'breeze-parent_2.11-0.12.jar')
- local('wget https://repo1.maven.org/maven2/org/jfree/jfreechart/1.0.19/jfreechart-1.0.19.jar -O ' +
- breeze_tmp_dir + 'jfreechart-1.0.19.jar')
- local('wget https://repo1.maven.org/maven2/org/jfree/jcommon/1.0.24/jcommon-1.0.24.jar -O ' +
- breeze_tmp_dir + 'jcommon-1.0.24.jar')
- local('wget --no-check-certificate https://brunelvis.org/jar/spark-kernel-brunel-all-2.3.jar -O ' +
- breeze_tmp_dir + 'spark-kernel-brunel-all-2.3.jar')
- local('sudo mv ' + breeze_tmp_dir + '* ' + new_jars_directory_path)
- local(""" sudo bash -c "sed -i '/spark.driver.extraClassPath/s/$/:\/opt\/""" + args.emr_version +
- """\/jars\/usr\/other\/*/' """ + spark_defaults_path + """" """)
+ subprocess.run('sudo mkdir -p ' + new_jars_directory_path, shell=True)
+ subprocess.run('mkdir -p ' + breeze_tmp_dir, shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/scalanlp/breeze_2.11/0.12/breeze_2.11-0.12.jar -O ' +
+ breeze_tmp_dir + 'breeze_2.11-0.12.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-natives_2.11/0.12/breeze-natives_2.11-0.12.jar -O '
+ + breeze_tmp_dir + 'breeze-natives_2.11-0.12.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-viz_2.11/0.12/breeze-viz_2.11-0.12.jar -O ' +
+ breeze_tmp_dir + 'breeze-viz_2.11-0.12.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-macros_2.11/0.12/breeze-macros_2.11-0.12.jar -O ' +
+ breeze_tmp_dir + 'breeze-macros_2.11-0.12.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/scalanlp/breeze-parent_2.11/0.12/breeze-parent_2.11-0.12.jar -O ' +
+ breeze_tmp_dir + 'breeze-parent_2.11-0.12.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/jfree/jfreechart/1.0.19/jfreechart-1.0.19.jar -O ' +
+ breeze_tmp_dir + 'jfreechart-1.0.19.jar', shell=True)
+ subprocess.run('wget https://repo1.maven.org/maven2/org/jfree/jcommon/1.0.24/jcommon-1.0.24.jar -O ' +
+ breeze_tmp_dir + 'jcommon-1.0.24.jar', shell=True)
+ subprocess.run('wget --no-check-certificate https://brunelvis.org/jar/spark-kernel-brunel-all-2.3.jar -O ' +
+ breeze_tmp_dir + 'spark-kernel-brunel-all-2.3.jar', shell=True)
+ subprocess.run('sudo mv ' + breeze_tmp_dir + '* ' + new_jars_directory_path, shell=True)
+ subprocess.run(""" sudo bash -c "sed -i '/spark.driver.extraClassPath/s/$/:\/opt\/""" + args.emr_version +
+ """\/jars\/usr\/other\/*/' """ + spark_defaults_path + """" """, shell=True)
def install_sparkamagic_kernels(args):
try:
- local('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
- sparkmagic_dir = local("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True)
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir))
+ subprocess.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension', shell=True)
+ sparkmagic_dir = subprocess.run("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True, shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir), shell=True)
pyspark_kernel_name = 'PySpark (Python-{0} / Spark-{1} ) [{2}]'.format(args.python_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
- pyspark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True)
spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(args.scala_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
- spark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True)
sparkr_kernel_name = 'SparkR (R-{0} / Spark-{1} ) [{2}]'.format(args.r_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
- sparkr_kernel_name, args.os_user))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkrkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkrkernel_{1}'.format(args.os_user, args.cluster_name))
- local('mkdir -p /home/' + args.os_user + '/.sparkmagic')
- local('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json')
- local('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
- args.master_ip, args.os_user))
- local('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user))
+ subprocess.run('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
+ sparkr_kernel_name, args.os_user), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkrkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkrkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True)
+ subprocess.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True)
+ subprocess.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ args.master_ip, args.os_user), shell=True)
+ subprocess.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True)
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
index 267eec7..2214975 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
@@ -30,6 +30,7 @@ import logging
import os
import sys
import traceback
+import subprocess
from fabric import *
parser = argparse.ArgumentParser()
@@ -118,7 +119,7 @@ if __name__ == "__main__":
notebook_config['initial_user'], notebook_config['datalab_ssh_user'], notebook_config['sudo_group'])
try:
- local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -136,7 +137,7 @@ if __name__ == "__main__":
.format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
notebook_config['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_proxy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -152,7 +153,7 @@ if __name__ == "__main__":
params = "--hostname {} --keyfile {} --user {} --region {}". \
format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['aws_region'])
try:
- local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -188,7 +189,7 @@ if __name__ == "__main__":
notebook_config['ip_address'],
notebook_config['exploratory_name'])
try:
- local("~/scripts/{}.py {}".format('configure_jupyterlab_node', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_jupyterlab_node', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -205,7 +206,7 @@ if __name__ == "__main__":
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except:
datalab.fab.append_result("Failed installing users key")
raise Exception
@@ -220,8 +221,8 @@ if __name__ == "__main__":
params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
.format(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name)
try:
- local("~/scripts/{}.py {}".format('common_download_git_certfile', params))
- local("~/scripts/{}.py {}".format('manage_git_creds', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_download_git_certfile', params), shell=True)
+ subprocess.run("~/scripts/{}.py {}".format('manage_git_creds', params), shell=True)
except:
datalab.fab.append_result("Failed setup git credentials")
raise Exception
@@ -238,7 +239,7 @@ if __name__ == "__main__":
.format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'],
notebook_config['tag_name'], notebook_config['instance_name'])
try:
- local("~/scripts/{}.py {}".format('common_remove_remote_kernels', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_remove_remote_kernels', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -267,7 +268,7 @@ if __name__ == "__main__":
notebook_config['exploratory_name'],
json.dumps(additional_info))
try:
- local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params), shell=True)
except:
datalab.fab.append_result("Failed edge reverse proxy template")
raise Exception
@@ -302,7 +303,7 @@ if __name__ == "__main__":
keyfile_name,
notebook_config['datalab_ssh_user'])
try:
- local("~/scripts/configure_proxy_for_docker.py {}".format(params))
+ subprocess.run("~/scripts/configure_proxy_for_docker.py {}".format(params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -321,7 +322,7 @@ if __name__ == "__main__":
keyfile_name,
notebook_config['datalab_ssh_user'])
try:
- local("~/scripts/jupyterlab_container_start.py {}".format(params))
+ subprocess.run("~/scripts/jupyterlab_container_start.py {}".format(params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 3f40045..1724199 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -31,6 +31,7 @@ import os
import sys
import time
import traceback
+import subprocess
from fabric import *
if __name__ == "__main__":
@@ -137,8 +138,8 @@ if __name__ == "__main__":
try:
project_conf['user_key'] = os.environ['key']
try:
- local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
- project_conf['project_name']))
+ subprocess.run('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'],
+ project_conf['project_name']), shell=True)
except:
print("ADMINSs PUBLIC KEY DOES NOT INSTALLED")
except KeyError:
@@ -196,7 +197,7 @@ if __name__ == "__main__":
project_conf['private_subnet_name'],
project_conf['zone'])
try:
- local("~/scripts/{}.py {}".format('common_create_subnet', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_subnet', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -223,7 +224,7 @@ if __name__ == "__main__":
project_conf['edge_policy_name'], os.environ['aws_region'], project_conf['tag_name'],
project_conf['service_base_name'], user_tag)
try:
- local("~/scripts/{}.py {}".format('common_create_role_policy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_role_policy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -243,7 +244,7 @@ if __name__ == "__main__":
project_conf['notebook_dataengine_policy_name'], os.environ['aws_region'],
project_conf['tag_name'], project_conf['service_base_name'], user_tag)
try:
- local("~/scripts/{}.py {}".format('common_create_role_policy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_role_policy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -418,7 +419,7 @@ if __name__ == "__main__":
project_conf['service_base_name'], project_conf['edge_instance_name'], json.dumps(edge_sg_egress),
True, project_conf['notebook_instance_name'], 'edge')
try:
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_security_group', params), shell=True)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed creating security group for edge node.", str(err))
@@ -500,7 +501,7 @@ if __name__ == "__main__":
project_conf['service_base_name'],
project_conf['notebook_instance_name'], True)
try:
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_security_group', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -526,7 +527,7 @@ if __name__ == "__main__":
project_conf['service_base_name'],
project_conf['dataengine_instances_name'], True)
try:
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_security_group', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -548,7 +549,7 @@ if __name__ == "__main__":
project_conf['service_base_name'],
project_conf['dataengine_instances_name'], True)
try:
- local("~/scripts/{}.py {}".format('common_create_security_group', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_security_group', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -572,7 +573,7 @@ if __name__ == "__main__":
format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'],
project_conf['shared_bucket_name_tag'])
try:
- local("~/scripts/{}.py {}".format('common_create_bucket', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -584,7 +585,7 @@ if __name__ == "__main__":
.format(project_conf['bucket_name'], project_conf['bucket_tags'], project_conf['region'],
project_conf['bucket_name_tag'])
try:
- local("~/scripts/{}.py {}".format('common_create_bucket', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -608,7 +609,7 @@ if __name__ == "__main__":
project_conf['service_base_name'], project_conf['region'],
os.environ['aws_user_predefined_s3_policies'], project_conf['endpoint_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_policy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_policy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -636,7 +637,7 @@ if __name__ == "__main__":
project_conf['edge_role_profile_name'], project_conf['tag_name'],
project_conf['edge_instance_name'])
try:
- local("~/scripts/{}.py {}".format('common_create_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True)
edge_instance = datalab.meta_lib.get_instance_by_name(project_conf['tag_name'],
project_conf['edge_instance_name'])
if os.environ['edge_is_nat']:
@@ -672,7 +673,7 @@ if __name__ == "__main__":
project_conf['elastic_ip'], project_conf['edge_id'], project_conf['tag_name'],
project_conf['elastic_ip_name'])
try:
- local("~/scripts/{}.py {}".format('edge_associate_elastic_ip', params))
+ subprocess.run("~/scripts/{}.py {}".format('edge_associate_elastic_ip', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -703,7 +704,7 @@ if __name__ == "__main__":
params = "--vpc_id {} --infra_tag_value {} --edge_instance_id {} --private_subnet_id {}".format(
project_conf['vpc2_id'], project_conf['nat_rt_name'], edge_instance, subnet_id)
try:
- local("~/scripts/{}.py {}".format('edge_configure_route_table', params))
+ subprocess.run("~/scripts/{}.py {}".format('edge_configure_route_table', params), shell=True)
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py b/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
index 494b5ab..3141cb3 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
@@ -23,6 +23,7 @@
import argparse
+import subprocess
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
@@ -44,7 +45,7 @@ if __name__ == "__main__":
params = "--user {} --hostname {} --keyfile '{}' --additional_config '{}'".format(
args.os_user, ip, args.keyfile, args.additional_config)
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index b2625e0..444b206 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -45,7 +45,7 @@ def configure_slave(slave_number, data_engine):
data_engine['datalab_ssh_user'], sudo_group)
try:
- local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -63,7 +63,7 @@ def configure_slave(slave_number, data_engine):
slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem",
json.dumps(additional_config), data_engine['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -78,7 +78,7 @@ def configure_slave(slave_number, data_engine):
params = '--hostname {} --keyfile {} --os_user {} --application {}' \
.format(slave_hostname, keyfile_name, data_engine['datalab_ssh_user'], os.environ['application'])
try:
- local("~/scripts/{}.py {}".format('common_clean_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_clean_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -95,7 +95,7 @@ def configure_slave(slave_number, data_engine):
.format(slave_hostname, slave_name, keyfile_name, json.dumps(additional_config),
data_engine['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_proxy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -111,7 +111,7 @@ def configure_slave(slave_number, data_engine):
format(slave_hostname, keyfile_name, data_engine['datalab_ssh_user'], data_engine['region'],
edge_instance_private_hostname)
try:
- local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -130,7 +130,7 @@ def configure_slave(slave_number, data_engine):
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
'slave')
try:
- local("~/scripts/{}.py {}".format('configure_dataengine', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_dataengine', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -236,7 +236,7 @@ if __name__ == "__main__":
data_engine['datalab_ssh_user'], sudo_group)
try:
- local("~/scripts/{}.py {}".format('create_ssh_user', params))
+ subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -254,7 +254,7 @@ if __name__ == "__main__":
master_node_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(
additional_config), data_engine['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('install_user_key', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -270,7 +270,7 @@ if __name__ == "__main__":
params = '--hostname {} --keyfile {} --os_user {} --application {}' \
.format(master_node_hostname, keyfile_name, data_engine['datalab_ssh_user'], os.environ['application'])
try:
- local("~/scripts/{}.py {}".format('common_clean_instance', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_clean_instance', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -287,7 +287,7 @@ if __name__ == "__main__":
.format(master_node_hostname, data_engine['master_node_name'], keyfile_name, json.dumps(additional_config),
data_engine['datalab_ssh_user'])
try:
- local("~/scripts/{}.py {}".format('common_configure_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_proxy', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -303,7 +303,7 @@ if __name__ == "__main__":
format(master_node_hostname, keyfile_name, data_engine['datalab_ssh_user'], data_engine['region'],
edge_instance_private_hostname)
try:
- local("~/scripts/{}.py {}".format('install_prerequisites', params))
+ subprocess.run("~/scripts/{}.py {}".format('install_prerequisites', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -322,7 +322,7 @@ if __name__ == "__main__":
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
'master')
try:
- local("~/scripts/{}.py {}".format('configure_dataengine', params))
+ subprocess.run("~/scripts/{}.py {}".format('configure_dataengine', params), shell=True)
except:
traceback.print_exc()
raise Exception
@@ -374,7 +374,7 @@ if __name__ == "__main__":
data_engine['exploratory_name'],
json.dumps(additional_info))
try:
- local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ subprocess.run("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params), shell=True)
except:
datalab.fab.append_result("Failed edge reverse proxy template")
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
index f915706..313d4ef 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
@@ -23,6 +23,7 @@
import argparse
import sys
+import subprocess
from datalab.actions_lib import *
from datalab.common_lib import *
from datalab.fab import *
@@ -56,7 +57,7 @@ yarn_dir = '/opt/{}/{}/conf/'.format(args.dataproc_version, args.cluster_name)
def r_kernel(args):
spark_path = '/opt/{}/{}/spark/'.format(args.dataproc_version, args.cluster_name)
- local('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name))
+ subprocess.run('mkdir -p {}/r_{}/'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = "{}/r_{}/kernel.json".format(kernels_dir, args.cluster_name)
template_file = "/tmp/r_dataengine-service_template.json"
@@ -75,10 +76,10 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/{0}/{1}/spark/'.format(args.dataproc_version, args.cluster_name)
- local('mkdir -p {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
- local('tar zxvf /tmp/toree_kernel.tar.gz -C {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
- local('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
- local('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name))
+ subprocess.run('mkdir -p {0}toree_{1}/'.format(kernels_dir, args.cluster_name), shell=True)
+ subprocess.run('tar zxvf /tmp/toree_kernel.tar.gz -C {0}toree_{1}/'.format(kernels_dir, args.cluster_name), shell=True)
+ subprocess.run('sudo mv {0}toree_{1}/toree-0.3.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name), shell=True)
+ subprocess.run('sudo rm -r {0}toree_{1}/toree-0.3.0-incubating'.format(kernels_dir, args.cluster_name), shell=True)
kernel_path = '{0}toree_{1}/kernel.json'.format(kernels_dir, args.cluster_name)
template_file = "/tmp/toree_dataengine-service_templatev2.json"
with open(template_file, 'r') as f:
@@ -91,12 +92,12 @@ def toree_kernel(args):
text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/kernel_var.json')
- local(
+ subprocess.run('touch /tmp/kernel_var.json', shell=True)
+ subprocess.run(
"PYJ=`find /opt/" + args.dataproc_version + "/" + args.cluster_name +
"/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat " + kernel_path +
- " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json")
- local('sudo mv /tmp/kernel_var.json ' + kernel_path)
+ " | sed 's|PY4J|'$PYJ'|g' > /tmp/kernel_var.json", shell=True)
+ subprocess.run('sudo mv /tmp/kernel_var.json ' + kernel_path, shell=True)
run_sh_path = kernels_dir + "toree_" + args.cluster_name + "/bin/run.sh"
template_sh_file = '/tmp/run_template.sh'
with open(template_sh_file, 'r') as f:
@@ -108,28 +109,28 @@ def toree_kernel(args):
def install_sparkamagic_kernels(args):
try:
- local('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
- sparkmagic_dir = local("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True)
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir))
+ subprocess.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
+ sparkmagic_dir = subprocess.run("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True, shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir), shell=True)
pyspark_kernel_name = 'PySpark (Python-{0} / Spark-{1} ) [{2}]'.format(args.python_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
- pyspark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True)
spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(args.scala_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
- spark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True)
sparkr_kernel_name = 'SparkR (R-{0} / Spark-{1} ) [{2}]'.format(args.r_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
- sparkr_kernel_name, args.os_user))
- local('mkdir -p /home/' + args.os_user + '/.sparkmagic')
- local('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json')
- local('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
- args.master_ip, args.os_user))
- local('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user))
+ subprocess.run('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
+ sparkr_kernel_name, args.os_user), shell=True)
+ subprocess.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True)
+ subprocess.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True)
+ subprocess.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ args.master_ip, args.os_user), shell=True)
+ subprocess.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True)
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
index bdbf328..25bdd6d 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
@@ -23,6 +23,7 @@
import argparse
import sys
+import subprocess
from datalab.actions_lib import *
from datalab.common_lib import *
from datalab.fab import *
@@ -54,7 +55,7 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
def pyspark_kernel(args):
spark_path = '/opt/' + args.cluster_name + '/spark/'
- local('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'pyspark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "pyspark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -68,13 +69,13 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
- local('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/')
+ subprocess.run('mkdir -p ' + kernels_dir + 'py3spark_' + args.cluster_name + '/', shell=True)
kernel_path = kernels_dir + "py3spark_" + args.cluster_name + "/kernel.json"
template_file = "/tmp/{}/pyspark_dataengine_template.json".format(args.cluster_name)
with open(template_file, 'r') as f:
@@ -88,40 +89,40 @@ def pyspark_kernel(args):
text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
- local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
- local(
+ subprocess.run('touch /tmp/{}/kernel_var.json'.format(args.cluster_name), shell=True)
+ subprocess.run(
"PYJ=`find /opt/{0}/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; cat {1} | sed 's|PY4J|'$PYJ'|g' | sed \'/PYTHONPATH\"\:/s|\(.*\)\"|\\1/home/{2}/caffe/python:/home/{2}/pytorch/build:\"|\' > /tmp/{0}/kernel_var.json".
- format(args.cluster_name, kernel_path, args.os_user))
- local('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path)
+ format(args.cluster_name, kernel_path, args.os_user), shell=True)
+ subprocess.run('sudo mv /tmp/{}/kernel_var.json '.format(args.cluster_name) + kernel_path, shell=True)
def install_sparkamagic_kernels(args):
try:
- local('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
- sparkmagic_dir = local("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True)
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
- local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
+ subprocess.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension', shell=True)
+ sparkmagic_dir = subprocess.run("sudo pip3 show sparkmagic | grep 'Location: ' | awk '{print $2}'", capture_output=True, shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir), shell=True)
+ subprocess.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir), shell=True)
pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
- local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
- pyspark_kernel_name, args.os_user))
- scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True)
+ subprocess.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True)
+ scala_version = subprocess.run('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture_output=True, shell=True)
spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(scala_version, args.spark_version,
args.cluster_name)
- local('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
- spark_kernel_name, args.os_user))
+ subprocess.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True)
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name))
- local('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
- '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name))
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/pysparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/pysparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
+ subprocess.run('sudo mv -f /home/{0}/.local/share/jupyter/kernels/sparkkernel '
+ '/home/{0}/.local/share/jupyter/kernels/sparkkernel_{1}'.format(args.os_user, args.cluster_name), shell=True)
- local('mkdir -p /home/' + args.os_user + '/.sparkmagic')
- local('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json')
+ subprocess.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True)
+ subprocess.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True)
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
- local('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
- spark_master_ip, args.os_user))
- local('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user))
+ subprocess.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ spark_master_ip, args.os_user), shell=True)
+ subprocess.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True)
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
index 7fe1050..f94c9f8 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
@@ -24,6 +24,7 @@
import argparse
import os
import sys
+import subprocess
from datalab.actions_lib import *
from datalab.common_lib import *
from datalab.fab import *
@@ -52,30 +53,30 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
def configure_rstudio():
if not os.path.exists('/home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured'):
try:
- local('echo "export R_LIBS_USER=' + cluster_dir + 'spark/R/lib:" >> /home/' + args.os_user + '/.bashrc')
- local("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron')
- local("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile")
- local('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
- args.os_user + '/.Rprofile')
- local('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user))
- local('touch /home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured')
+ subprocess.run('echo "export R_LIBS_USER=' + cluster_dir + 'spark/R/lib:" >> /home/' + args.os_user + '/.bashrc', shell=True)
+ subprocess.run("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron', shell=True)
+ subprocess.run("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile", shell=True)
+ subprocess.run('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
+ args.os_user + '/.Rprofile', shell=True)
+ subprocess.run('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user), shell=True)
+ subprocess.run('touch /home/' + args.os_user + '/.ensure_dir/rstudio_dataengine_ensured', shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
else:
try:
- local("sed -i '/R_LIBS_USER/ { s|=\(.*\)|=\\1" + cluster_dir + "spark/R/lib:| }' /home/" + args.os_user + "/.bashrc")
- local("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron")
- local('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron')
- local("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile")
- local('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
- args.os_user + '/.Rprofile')
- local('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user))
+ subprocess.run("sed -i '/R_LIBS_USER/ { s|=\(.*\)|=\\1" + cluster_dir + "spark/R/lib:| }' /home/" + args.os_user + "/.bashrc", shell=True)
+ subprocess.run("sed -i 's/^SPARK_HOME/#SPARK_HOME/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^YARN_CONF_DIR/#YARN_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run("sed -i 's/^HADOOP_CONF_DIR/#HADOOP_CONF_DIR/' /home/" + args.os_user + "/.Renviron", shell=True)
+ subprocess.run('echo \'SPARK_HOME="' + cluster_dir + 'spark/"\' >> /home/' + args.os_user + '/.Renviron', shell=True)
+ subprocess.run("sed -i 's/^master/#master/' /home/" + args.os_user + "/.Rprofile", shell=True)
+ subprocess.run('echo \'master="' + args.spark_master + '" # Cluster - "' + args.cluster_name + '" \' >> /home/' +
+ args.os_user + '/.Rprofile', shell=True)
+ subprocess.run('''R -e "source('/home/{}/.Rprofile')"'''.format(args.os_user), shell=True)
except Exception as err:
print('Error: {0}'.format(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 419b521..3ced3ef 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -51,8 +51,8 @@ def modify_conf_file(args):
for os_var in os.environ:
if "'" not in os.environ[os_var] and os_var != 'aws_access_key' and os_var != 'aws_secret_access_key':
variables_list[os_var] = os.environ[os_var]
- local('scp -r -i {} /project_tree/* {}:{}sources/'.format(args.keyfile, env.host_string, args.datalab_path))
- local('scp -i {} /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile,
+ conn.local('scp -r -i {} /project_tree/* {}:{}sources/'.format(args.keyfile, env.host_string, args.datalab_path))
+ conn.local('scp -i {} /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile,
env.host_string))
conn.sudo("python3 /tmp/configure_conf_file.py --datalab_dir {} --variables_list '{}'".format(
args.datalab_path, json.dumps(variables_list)))
@@ -98,7 +98,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider):
try:
with open('/tmp/config', 'w') as f:
f.write(base64.b64decode(gcr_creds))
- local('scp -i {} /tmp/config {}:/tmp/config'.format(args.keyfile, env.host_string, os_user))
+ conn.local('scp -i {} /tmp/config {}:/tmp/config'.format(args.keyfile, env.host_string, os_user))
conn.sudo('mkdir /home/{}/.docker'.format(os_user))
conn.sudo('cp /tmp/config /home/{}/.docker/config.json'.format(os_user))
conn.sudo('sed -i "s|ODAHU_IMAGE|{}|" {}sources/infrastructure-provisioning/src/general/files/{}/odahu_Dockerfile'
@@ -115,7 +115,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider):
def build_docker_images(image_list, region, datalab_path):
try:
if os.environ['conf_cloud_provider'] == 'azure':
- local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
+ conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
'azure_auth.json'.format(args.keyfile, env.host_string, args.datalab_path))
conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
'/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org