You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/17 09:42:46 UTC
[incubator-datalab] branch DATALAB-2091 updated: [DATALAB-2091]:
fixed exists() usage for fabric2
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new 38ef258 [DATALAB-2091]: fixed exists() usage for fabric2
38ef258 is described below
commit 38ef258eb5743258ab27fa033751ba8db8f2eddc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Feb 17 11:42:31 2021 +0200
[DATALAB-2091]: fixed exists() usage for fabric2
---
README.md | 2 +-
.../scripts/deploy_keycloak/deploy_keycloak.py | 2 +-
.../scripts/deploy_repository/deploy_repository.py | 28 +++++++-------
.../src/base/scripts/install_prerequisites.py | 2 +-
.../src/dataengine/scripts/configure_dataengine.py | 8 ++--
.../scripts/configure_deep_learning_node.py | 4 +-
.../src/general/lib/aws/actions_lib.py | 8 ++--
.../src/general/lib/azure/actions_lib.py | 8 ++--
.../src/general/lib/gcp/actions_lib.py | 8 ++--
.../src/general/lib/os/debian/common_lib.py | 10 ++---
.../src/general/lib/os/debian/edge_lib.py | 4 +-
.../src/general/lib/os/debian/notebook_lib.py | 42 ++++++++++-----------
.../src/general/lib/os/debian/ssn_lib.py | 20 +++++-----
.../src/general/lib/os/fab.py | 44 +++++++++++-----------
.../src/general/lib/os/redhat/common_lib.py | 10 ++---
.../src/general/lib/os/redhat/edge_lib.py | 2 +-
.../src/general/lib/os/redhat/notebook_lib.py | 42 ++++++++++-----------
.../src/general/lib/os/redhat/ssn_lib.py | 20 +++++-----
.../os/deeplearning_install_dataengine_kernels.py | 6 +--
.../os/jupyter_install_dataengine_kernels.py | 4 +-
.../src/general/scripts/os/reconfigure_spark.py | 2 +-
.../os/rstudio_install_dataengine_kernels.py | 4 +-
.../tensor-rstudio_install_dataengine_kernels.py | 4 +-
.../os/tensor_install_dataengine_kernels.py | 4 +-
.../os/zeppelin_install_dataengine_kernels.py | 4 +-
.../src/jupyter/scripts/configure_jupyter_node.py | 2 +-
.../scripts/configure_jupyterlab_node.py | 2 +-
.../src/rstudio/scripts/configure_rstudio_node.py | 2 +-
.../src/ssn/scripts/configure_ssn_node.py | 4 +-
.../src/ssn/scripts/configure_ui.py | 2 +-
.../src/ssn/scripts/docker_build.py | 2 +-
.../superset/scripts/configure_superset_node.py | 2 +-
.../scripts/configure_tensor-rstudio_node.py | 2 +-
.../src/tensor/scripts/configure_tensor_node.py | 2 +-
.../zeppelin/scripts/configure_zeppelin_node.py | 10 ++---
35 files changed, 161 insertions(+), 161 deletions(-)
diff --git a/README.md b/README.md
index 9981194..3b8291f 100644
--- a/README.md
+++ b/README.md
@@ -2584,7 +2584,7 @@ if __name__ == "__main__":
print("Configuring notebook server.")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py b/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
index 74ab0a0..48ade87 100644
--- a/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
+++ b/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
@@ -45,7 +45,7 @@ internal_port = "8080"
private_ip_address = "127.0.0.1"
def ensure_jre_jdk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
try:
conn.sudo('mkdir -p /home/' + os_user + '/.ensure_dir')
conn.sudo('apt-get update')
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index d91669d..d32c28b 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -780,7 +780,7 @@ def remove_efs():
def ensure_ssh_user(initial_user):
try:
- if not exists('/home/{}/.ssh_user_ensured'.format(initial_user)):
+ if not exists(conn,'/home/{}/.ssh_user_ensured'.format(initial_user)):
conn.sudo('useradd -m -G sudo -s /bin/bash {0}'.format(configuration['conf_os_user']))
conn.sudo('echo "{} ALL = NOPASSWD:ALL" >> /etc/sudoers'.format(configuration['conf_os_user']))
conn.sudo('mkdir /home/{}/.ssh'.format(configuration['conf_os_user']))
@@ -800,7 +800,7 @@ def ensure_ssh_user(initial_user):
def install_java():
try:
- if not exists('/home/{}/.ensure_dir/java_ensured'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/java_ensured'.format(configuration['conf_os_user'])):
conn.sudo('apt-get update')
conn.sudo('apt-get install -y default-jdk ')
conn.sudo('touch /home/{}/.ensure_dir/java_ensured'.format(configuration['conf_os_user']))
@@ -812,7 +812,7 @@ def install_java():
def install_groovy():
try:
- if not exists('/home/{}/.ensure_dir/groovy_ensured'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/groovy_ensured'.format(configuration['conf_os_user'])):
conn.sudo('apt-get install -y unzip')
conn.sudo('mkdir /usr/local/groovy')
conn.sudo('wget https://bintray.com/artifact/download/groovy/maven/apache-groovy-binary-{0}.zip -O \
@@ -848,7 +848,7 @@ def nexus_service_waiter():
def install_nexus():
try:
- if not exists('/home/{}/.ensure_dir/nexus_ensured'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/nexus_ensured'.format(configuration['conf_os_user'])):
if args.efs_enabled == 'False':
mounting_disks()
else:
@@ -1033,7 +1033,7 @@ def install_nexus():
def install_nginx():
try:
- if not exists('/home/{}/.ensure_dir/nginx_ensured'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/nginx_ensured'.format(configuration['conf_os_user'])):
hostname = conn.sudo('hostname')
conn.sudo('apt-get install -y nginx')
conn.sudo('rm -f /etc/nginx/conf.d/* /etc/nginx/sites-enabled/default')
@@ -1056,7 +1056,7 @@ def install_nginx():
def mounting_disks():
try:
- if not exists('/home/{}/.ensure_dir/additional_disk_mounted'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/additional_disk_mounted'.format(configuration['conf_os_user'])):
conn.sudo('mkdir -p /opt/sonatype-work')
disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort | tail -n 1 | tr '\\n' ',' | sed 's|.$||g'")
conn.sudo('bash -c \'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}\' '.format(disk_name))
@@ -1076,7 +1076,7 @@ def mounting_disks():
def mount_efs():
try:
- if not exists('/home/{}/.ensure_dir/efs_mounted'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/efs_mounted'.format(configuration['conf_os_user'])):
conn.sudo('mkdir -p /opt/sonatype-work')
conn.sudo('apt-get -y install binutils')
with conn.cd('/tmp/'):
@@ -1105,7 +1105,7 @@ def mount_efs():
def configure_ssl():
try:
- if not exists('/home/{}/.ensure_dir/ssl_ensured'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/ssl_ensured'.format(configuration['conf_os_user'])):
hostname = conn.sudo('hostname')
private_ip = conn.sudo('curl http://169.254.169.254/latest/meta-data/local-ipv4')
subject_alt_name = 'subjectAltName = IP:{}'.format(private_ip)
@@ -1128,7 +1128,7 @@ def configure_ssl():
def set_hostname():
try:
- if not exists('/home/{}/.ensure_dir/hostname_set'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/hostname_set'.format(configuration['conf_os_user'])):
if args.hosted_zone_id and args.hosted_zone_name and args.subdomain:
hostname = '{0}.{1}'.format(args.subdomain, args.hosted_zone_name)
else:
@@ -1146,7 +1146,7 @@ def set_hostname():
def create_keystore():
try:
- if not exists('/home/{}/.ensure_dir/keystore_created'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/keystore_created'.format(configuration['conf_os_user'])):
conn.sudo('openssl pkcs12 -export -in /etc/ssl/certs/repository.crt -inkey /etc/ssl/certs/repository.key '
'-out wildcard.p12 -passout pass:{}'.format(keystore_pass))
conn.sudo('keytool -importkeystore -deststorepass {0} -destkeypass {0} -srckeystore wildcard.p12 -srcstoretype '
@@ -1160,7 +1160,7 @@ def create_keystore():
def download_packages():
try:
- if not exists('/home/{}/.ensure_dir/packages_downloaded'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/packages_downloaded'.format(configuration['conf_os_user'])):
packages_urls = [
'https://pkg.jenkins.io/debian/jenkins-ci.org.key',
'http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
@@ -1251,7 +1251,7 @@ def download_packages():
def install_docker():
try:
- if not exists('/home/{}/.ensure_dir/docker_installed'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/docker_installed'.format(configuration['conf_os_user'])):
conn.sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
conn.sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) '
'stable"')
@@ -1270,7 +1270,7 @@ def install_docker():
def prepare_images():
try:
- if not exists('/home/{}/.ensure_dir/images_prepared'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/images_prepared'.format(configuration['conf_os_user'])):
conn.put('files/Dockerfile', '/tmp/Dockerfile')
with conn.cd('/tmp/'):
conn.sudo('docker build --file Dockerfile -t pre-base .')
@@ -1287,7 +1287,7 @@ def prepare_images():
def install_squid():
try:
- if not exists('/home/{}/.ensure_dir/squid_installed'.format(configuration['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/squid_installed'.format(configuration['conf_os_user'])):
conn.sudo('apt-get -y install squid')
conn.put('templates/squid.conf', '/etc/squid/', use_sudo=True)
replace_string = ''
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index b2adfba..7268ef7 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -42,7 +42,7 @@ args = parser.parse_args()
def create_china_pip_conf_file(conn):
- if not exists('/home/{}/pip_china_ensured'.format(args.user)):
+ if not exists(conn,'/home/{}/pip_china_ensured'.format(args.user)):
conn.sudo('touch /etc/pip.conf')
conn.sudo('echo "[global]" >> /etc/pip.conf')
conn.sudo('echo "timeout = 600" >> /etc/pip.conf')
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 4f5cdcc..2329386 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -80,8 +80,8 @@ if os.environ['application'] == 'deeplearning':
def start_spark(os_user, master_ip, node):
- if not exists('/home/{0}/.ensure_dir/start_spark-{1}_ensured'.format(os_user, node)):
- if not exists('/opt/spark/conf/spark-env.sh'):
+ if not exists(conn,'/home/{0}/.ensure_dir/start_spark-{1}_ensured'.format(os_user, node)):
+ if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
conn.sudo('''echo "SPARK_MASTER_HOST='{}'" >> /opt/spark/conf/spark-env.sh'''.format(master_ip))
if os.environ['application'] in ('tensor', 'tensor-rstudio'):
@@ -122,7 +122,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
@@ -198,7 +198,7 @@ if __name__ == "__main__":
install_r_packages(args.os_user)
# INSTALL LIVY
- if not exists('/home/{0}/.ensure_dir/livy_ensured'.format(args.os_user)):
+ if not exists(conn,'/home/{0}/.ensure_dir/livy_ensured'.format(args.os_user)):
livy_version = '0.7.0'
conn.sudo(
'wget -nv --timeout=30 --tries=5 --retry-connrefused https://archive.apache.org/dist/incubator/livy/{0}-incubating/apache-livy-{0}-incubating-bin.zip -P /tmp/'.format(
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index b4cd5c6..2d4dacf 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -79,7 +79,7 @@ gitlab_certfile = os.environ['conf_gitlab_certfile']
def install_itorch(os_user):
- if not exists('/home/{}/.ensure_dir/itorch_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/itorch_ensured'.format(os_user)):
conn.run('git clone https://github.com/facebook/iTorch.git')
with conn.cd('/home/{}/iTorch/'.format(os_user)):
conn.run('luarocks install luacrypto')
@@ -98,7 +98,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
conn.sudo('touch /home/' + args.os_user + '/.ensure_dir/deep_learning')
except:
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 2132c74..00debfb 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1623,7 +1623,7 @@ def spark_defaults(args):
def ensure_local_jars(os_user, jars_dir):
- if not exists('/home/{}/.ensure_dir/local_jars_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/local_jars_ensured'.format(os_user)):
try:
conn.sudo('mkdir -p {0}'.format(jars_dir))
conn.sudo('wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/{0}/hadoop-aws-{0}.jar -O \
@@ -1658,7 +1658,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
endpoint_url))
conn.sudo('echo "spark.hadoop.fs.s3a.server-side-encryption-algorithm AES256" >> '
'/tmp/notebook_spark-defaults_local.conf')
- if not exists('/opt/spark/conf/spark-env.sh'):
+ if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
@@ -1928,7 +1928,7 @@ def remove_dataengine_kernels(tag_name, notebook_name, os_user, key_path, cluste
def prepare_disk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/disk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/disk_ensured'):
try:
disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort | tail -n 1")
conn.sudo('''bash -c 'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}' '''.format(disk_name))
@@ -1941,7 +1941,7 @@ def prepare_disk(os_user):
def ensure_local_spark(os_user, spark_link, spark_version, hadoop_version, local_spark_path):
- if not exists('/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
try:
conn.sudo('wget ' + spark_link + ' -O /tmp/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz')
conn.sudo('tar -zxvf /tmp/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz -C /opt/')
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index ae81399..16b3031 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1062,7 +1062,7 @@ class AzureActions:
def ensure_local_jars(os_user, jars_dir):
- if not exists('/home/{}/.ensure_dir/local_jars_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/local_jars_ensured'.format(os_user)):
try:
hadoop_version = conn.sudo("ls /opt/spark/jars/hadoop-common* | sed -n 's/.*\([0-9]\.[0-9]\.[0-9]\).*/\\1/p'")
print("Downloading local jars for Azure")
@@ -1145,7 +1145,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf')
conn.sudo('echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
- if not exists('/opt/spark/conf/spark-env.sh'):
+ if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
@@ -1239,7 +1239,7 @@ def prepare_vm_for_image(creds=False, os_user='', hostname='', keyfile=''):
def prepare_disk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/disk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/disk_ensured'):
try:
allow = False
counter = 0
@@ -1278,7 +1278,7 @@ def prepare_disk(os_user):
def ensure_local_spark(os_user, spark_link, spark_version, hadoop_version, local_spark_path):
- if not exists('/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
try:
if os.environ['azure_datalake_enable'] == 'false':
conn.sudo('wget ' + spark_link + ' -O /tmp/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz')
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 8fb5fd0..2f5b025 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1333,7 +1333,7 @@ class GCPActions:
def ensure_local_jars(os_user, jars_dir):
- if not exists('/home/{}/.ensure_dir/gs_kernel_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/gs_kernel_ensured'.format(os_user)):
try:
templates_dir = '/root/templates/'
conn.sudo('mkdir -p {}'.format(jars_dir))
@@ -1369,7 +1369,7 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
def prepare_disk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/disk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/disk_ensured'):
try:
disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort | tail -n 1")
conn.sudo('''bash -c 'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}' '''.format(disk_name))
@@ -1382,7 +1382,7 @@ def prepare_disk(os_user):
def ensure_local_spark(os_user, spark_link, spark_version, hadoop_version, local_spark_path):
- if not exists('/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/local_spark_ensured'):
try:
conn.sudo('wget ' + spark_link + ' -O /tmp/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz')
conn.sudo('tar -zxvf /tmp/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz -C /opt/')
@@ -1412,7 +1412,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf')
conn.sudo('echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
- if not exists('/opt/spark/conf/spark-env.sh'):
+ if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 7a37f93..534c01a 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -101,7 +101,7 @@ def ensure_pkg(user, conn, requisites='linux-headers-generic python3-pip python3
'libssl-dev unattended-upgrades nmap '
'libffi-dev unzip libxml2-dev haveged'):
try:
- if not exists('/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
count = 0
check = False
while not check:
@@ -142,7 +142,7 @@ def renew_gpg_key():
def change_pkg_repos(conn):
- if not exists('/tmp/pkg_china_ensured'):
+ if not exists(conn,'/tmp/pkg_china_ensured'):
conn.put('/root/files/sources.list', '/tmp/sources.list')
conn.sudo('mv /tmp/sources.list /etc/apt/sources.list')
manage_pkg('update', 'remote', '')
@@ -161,7 +161,7 @@ def find_java_path_local():
def ensure_ntpd(user, conn, edge_private_ip=''):
try:
- if not exists('/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
conn.sudo('timedatectl set-ntp no')
manage_pkg('-y install', 'remote', 'ntp ntpdate')
conn.sudo('echo "tinker panic 0" >> /etc/ntp.conf')
@@ -176,7 +176,7 @@ def ensure_ntpd(user, conn, edge_private_ip=''):
def ensure_java(user):
try:
- if not exists('/home/{}/.ensure_dir/java_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/java_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'openjdk-8-jdk')
conn.sudo('touch /home/{}/.ensure_dir/java_ensured'.format(user))
except:
@@ -185,7 +185,7 @@ def ensure_java(user):
def ensure_step(user):
try:
- if not exists('/home/{}/.ensure_dir/step_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/step_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'wget')
conn.sudo('wget https://github.com/smallstep/cli/releases/download/v0.13.3/step-cli_0.13.3_amd64.deb '
'-O /tmp/step-cli_0.13.3_amd64.deb')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 917d6e1..ce3a376 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -33,7 +33,7 @@ from patchwork.files import exists
def configure_http_proxy_server(config):
try:
- if not exists('/tmp/http_proxy_ensured'):
+ if not exists(conn,'/tmp/http_proxy_ensured'):
manage_pkg('-y install', 'remote', 'squid')
template_file = config['template_file']
proxy_subnet = config['exploratory_subnet']
@@ -168,7 +168,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
def configure_nftables(config):
try:
- if not exists('/tmp/nftables_ensured'):
+ if not exists(conn,'/tmp/nftables_ensured'):
manage_pkg('-y install', 'remote', 'nftables')
conn.sudo('systemctl enable nftables.service')
conn.sudo('systemctl start nftables')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index f0a2301..90ed3ff 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -53,7 +53,7 @@ def enable_proxy(proxy_host, proxy_port):
def ensure_r_local_kernel(spark_version, os_user, templates_dir, kernels_dir):
- if not exists('/home/' + os_user + '/.ensure_dir/r_local_kernel_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/r_local_kernel_ensured'):
try:
conn.sudo('R -e "IRkernel::installspec()"')
r_version = conn.sudo("R --version | awk '/version / {print $3}'")
@@ -80,7 +80,7 @@ def add_marruter_key():
sys.exit(1)
def ensure_r(os_user, r_libs, region, r_mirror):
- if not exists('/home/' + os_user + '/.ensure_dir/r_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/r_ensured'):
try:
if region == 'cn-north-1':
r_repository = r_mirror
@@ -120,7 +120,7 @@ def ensure_r(os_user, r_libs, region, r_mirror):
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
- if not exists('/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
try:
manage_pkg('-y install', 'remote', 'r-base')
manage_pkg('-y install', 'remote', 'gdebi-core')
@@ -169,7 +169,7 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
def ensure_matplot(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/matplot_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/matplot_ensured'):
try:
conn.sudo("sudo sed -i~orig -e 's/# deb-src/deb-src/' /etc/apt/sources.list")
manage_pkg('update', 'remote', '')
@@ -187,7 +187,7 @@ def add_sbt_key():
'curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add')
def ensure_sbt(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/sbt_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/sbt_ensured'):
try:
manage_pkg('-y install', 'remote', 'apt-transport-https')
conn.sudo('echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list')
@@ -201,7 +201,7 @@ def ensure_sbt(os_user):
def ensure_scala(scala_link, scala_version, os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/scala_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/scala_ensured'):
try:
conn.sudo('wget {}scala-{}.deb -O /tmp/scala.deb'.format(scala_link, scala_version))
conn.sudo('dpkg -i /tmp/scala.deb')
@@ -211,7 +211,7 @@ def ensure_scala(scala_link, scala_version, os_user):
def ensure_jre_jdk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
try:
manage_pkg('-y install', 'remote', 'default-jre')
manage_pkg('-y install', 'remote', 'default-jdk')
@@ -223,7 +223,7 @@ def ensure_jre_jdk(os_user):
def ensure_additional_python_libs(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'):
try:
manage_pkg('-y install', 'remote', 'libjpeg8-dev zlib1g-dev')
if os.environ['application'] in ('jupyter', 'zeppelin'):
@@ -236,7 +236,7 @@ def ensure_additional_python_libs(os_user):
def ensure_python3_specific_version(python3_version, os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'):
try:
if len(python3_version) < 4:
python3_version = python3_version + ".0"
@@ -247,7 +247,7 @@ def ensure_python3_specific_version(python3_version, os_user):
sys.exit(1)
def ensure_python3_libraries(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
try:
#manage_pkg('-y install', 'remote', 'python3-setuptools')
manage_pkg('-y install', 'remote', 'python3-pip')
@@ -271,7 +271,7 @@ def ensure_python3_libraries(os_user):
def install_tensor(os_user, cuda_version, cuda_file_name,
cudnn_version, cudnn_file_name, tensorflow_version,
templates_dir, nvidia_version):
- if not exists('/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
try:
# install nvidia drivers
conn.sudo('echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf')
@@ -335,19 +335,19 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
def install_maven(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/maven_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/maven_ensured'):
manage_pkg('-y install', 'remote', 'maven')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/maven_ensured')
def install_gcloud(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/gcloud_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/gcloud_ensured'):
conn.sudo('echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list')
conn.sudo('curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -')
manage_pkg('-y install', 'remote', 'google-cloud-sdk')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/gcloud_ensured')
def install_livy_dependencies(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
manage_pkg('-y install', 'remote', 'libkrb5-dev')
conn.sudo('pip3 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -367,7 +367,7 @@ def install_livy_dependencies_emr(os_user):
def install_nodejs(os_user):
- if not exists('/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
conn.sudo('curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -')
manage_pkg('-y install', 'remote', 'nodejs')
conn.sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
@@ -462,7 +462,7 @@ def get_available_os_pkgs():
def install_caffe2(os_user, caffe2_version, cmake_version):
- if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
manage_pkg('update', 'remote', '')
manage_pkg('-y install --no-install-recommends', 'remote', 'build-essential cmake git libgoogle-glog-dev '
@@ -492,31 +492,31 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
def install_cntk(os_user, cntk_version):
- if not exists('/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
conn.sudo('pip3 install cntk-gpu=={} --no-cache-dir'.format(cntk_version))
conn.sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def install_keras(os_user, keras_version):
- if not exists('/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
conn.sudo('pip3 install keras=={} --no-cache-dir'.format(keras_version))
conn.sudo('touch /home/{}/.ensure_dir/keras_ensured'.format(os_user))
def install_theano(os_user, theano_version):
- if not exists('/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
conn.sudo('python3 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
conn.sudo('touch /home/{}/.ensure_dir/theano_ensured'.format(os_user))
def install_mxnet(os_user, mxnet_version):
- if not exists('/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
conn.sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
#def install_torch(os_user):
-# if not exists('/home/{}/.ensure_dir/torch_ensured'.format(os_user)):
+# if not exists(conn,'/home/{}/.ensure_dir/torch_ensured'.format(os_user)):
# run('git clone https://github.com/nagadomi/distro.git ~/torch --recursive')
# with cd('/home/{}/torch/'.format(os_user)):
# run('bash install-deps;')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 6dfa000..0443126 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -34,7 +34,7 @@ import subprocess
def ensure_docker_daemon(datalab_path, os_user, region):
try:
- if not exists(datalab_path + 'tmp/docker_daemon_ensured'):
+ if not exists(conn,datalab_path + 'tmp/docker_daemon_ensured'):
docker_version = os.environ['ssn_docker_version']
conn.sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
conn.sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
@@ -53,7 +53,7 @@ def ensure_docker_daemon(datalab_path, os_user, region):
def ensure_nginx(datalab_path):
try:
- if not exists(datalab_path + 'tmp/nginx_ensured'):
+ if not exists(conn,datalab_path + 'tmp/nginx_ensured'):
manage_pkg('-y install', 'remote', 'nginx')
conn.sudo('service nginx restart')
conn.sudo('update-rc.d nginx defaults')
@@ -67,7 +67,7 @@ def ensure_nginx(datalab_path):
def ensure_jenkins(datalab_path):
try:
- if not exists(datalab_path + 'tmp/jenkins_ensured'):
+ if not exists(conn,datalab_path + 'tmp/jenkins_ensured'):
conn.sudo('wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | apt-key add -')
conn.sudo('echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list')
manage_pkg('-y update', 'remote', '')
@@ -81,7 +81,7 @@ def ensure_jenkins(datalab_path):
def configure_jenkins(datalab_path, os_user, config, tag_resource_id):
try:
- if not exists(datalab_path + 'tmp/jenkins_configured'):
+ if not exists(conn,datalab_path + 'tmp/jenkins_configured'):
conn.sudo('echo \'JENKINS_ARGS="--prefix=/jenkins --httpPort=8070"\' >> /etc/default/jenkins')
conn.sudo('rm -rf /var/lib/jenkins/*')
conn.sudo('mkdir -p /var/lib/jenkins/jobs/')
@@ -106,7 +106,7 @@ def configure_jenkins(datalab_path, os_user, config, tag_resource_id):
def configure_nginx(config, datalab_path, hostname):
try:
random_file_part = id_generator(size=20)
- if not exists("/etc/nginx/conf.d/nginx_proxy.conf"):
+ if not exists(conn,"/etc/nginx/conf.d/nginx_proxy.conf"):
conn.sudo('useradd -r nginx')
conn.sudo('rm -f /etc/nginx/conf.d/*')
conn.put(config['nginx_template_dir'] + 'ssn_nginx.conf', '/tmp/nginx.conf')
@@ -124,7 +124,7 @@ def configure_nginx(config, datalab_path, hostname):
sys.exit(1)
try:
- if not exists("/etc/nginx/locations/proxy_location_jenkins.conf"):
+ if not exists(conn,"/etc/nginx/locations/proxy_location_jenkins.conf"):
nginx_password = id_generator()
template_file = config['nginx_template_dir'] + 'proxy_location_jenkins_template.conf'
with open("/tmp/%s-tmpproxy_location_jenkins_template.conf" % random_file_part, 'w') as out:
@@ -150,7 +150,7 @@ def configure_nginx(config, datalab_path, hostname):
def ensure_supervisor():
try:
- if not exists(os.environ['ssn_datalab_path'] + 'tmp/superv_ensured'):
+ if not exists(conn,os.environ['ssn_datalab_path'] + 'tmp/superv_ensured'):
manage_pkg('-y install', 'remote', 'supervisor')
conn.sudo('update-rc.d supervisor defaults')
conn.sudo('update-rc.d supervisor enable')
@@ -163,7 +163,7 @@ def ensure_supervisor():
def ensure_mongo():
try:
- if not exists(os.environ['ssn_datalab_path'] + 'tmp/mongo_ensured'):
+ if not exists(conn,os.environ['ssn_datalab_path'] + 'tmp/mongo_ensured'):
conn.sudo('wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -')
conn.sudo('ver=`lsb_release -cs`; echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu '
'$ver/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list; '
@@ -188,7 +188,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
usage_type, usage, cost, resource_id, tags, billing_dataset_name, keycloak_client_id,
keycloak_client_secret, keycloak_auth_server_url, report_path=''):
try:
- if not exists(os.environ['ssn_datalab_path'] + 'tmp/ss_started'):
+ if not exists(conn,os.environ['ssn_datalab_path'] + 'tmp/ss_started'):
java_path = conn.sudo("update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'")
supervisor_conf = '/etc/supervisor/conf.d/supervisor_svc.conf'
conn.local('sed -i "s|MONGO_PASSWORD|{}|g" /root/templates/ssn.yml'.format(mongo_passwd))
@@ -370,7 +370,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
def install_build_dep():
try:
- if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
+ if not exists(conn,'{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'openjdk-8-jdk git wget unzip')
with conn.cd('/opt/'):
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 8a8b030..48251fd 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -40,7 +40,7 @@ from patchwork.files import exists
def ensure_pip(requisites, conn):
try:
- if not exists('/home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user'])):
+ if not exists(conn,'/home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user'])):
conn.sudo('echo PATH=$PATH:/usr/local/bin/:/opt/spark/bin/ >> /etc/profile')
conn.sudo('echo export PATH >> /etc/profile')
conn.sudo('pip3 install -UI pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
@@ -190,7 +190,7 @@ def put_resource_status(resource, status, datalab_path, os_user, hostname):
def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
- if not exists('/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
conn.sudo('pip3 install jupyter --no-cache-dir')
@@ -248,7 +248,7 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
def configure_docker(os_user):
try:
- if not exists('/home/' + os_user + '/.ensure_dir/docker_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/docker_ensured'):
docker_version = os.environ['ssn_docker_version']
conn.sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
conn.sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
@@ -262,7 +262,7 @@ def configure_docker(os_user):
sys.exit(1)
def ensure_jupyterlab_files(os_user, jupyterlab_dir, jupyterlab_image, jupyter_conf_file, jupyterlab_conf_file, exploratory_name, edge_ip):
- if not exists(jupyterlab_dir):
+ if not exists(conn,jupyterlab_dir):
try:
conn.sudo('mkdir {}'.format(jupyterlab_dir))
# conn.put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
@@ -342,7 +342,7 @@ def ensure_jupyterlab_files(os_user, jupyterlab_dir, jupyterlab_image, jupyter_c
def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir, spark_version):
- if not exists('/home/' + os_user + '/.ensure_dir/pyspark_local_kernel_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/pyspark_local_kernel_ensured'):
try:
conn.sudo('mkdir -p ' + pyspark_local_path_dir)
conn.sudo('touch ' + pyspark_local_path_dir + 'kernel.json')
@@ -358,7 +358,7 @@ def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir,
def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version):
- if not exists('/home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured'):
try:
conn.sudo('mkdir -p ' + py3spark_local_path_dir)
conn.sudo('touch ' + py3spark_local_path_dir + 'kernel.json')
@@ -579,7 +579,7 @@ def get_available_r_pkgs():
def ensure_toree_local_kernel(os_user, toree_link, scala_kernel_path, files_dir, scala_version, spark_version):
- if not exists('/home/' + os_user + '/.ensure_dir/toree_local_kernel_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/toree_local_kernel_ensured'):
try:
conn.sudo('pip install ' + toree_link + ' --no-cache-dir')
conn.sudo('ln -s /opt/spark/ /usr/local/spark')
@@ -596,7 +596,7 @@ def ensure_toree_local_kernel(os_user, toree_link, scala_kernel_path, files_dir,
def install_ungit(os_user, notebook_name, edge_ip):
- if not exists('/home/{}/.ensure_dir/ungit_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/ungit_ensured'.format(os_user)):
try:
manage_npm_pkg('-g install ungit@{}'.format(os.environ['notebook_ungit_version']))
conn.put('/root/templates/ungit.service', '/tmp/ungit.service')
@@ -648,9 +648,9 @@ def install_ungit(os_user, notebook_name, edge_ip):
def install_inactivity_checker(os_user, ip_address, rstudio=False):
- if not exists('/home/{}/.ensure_dir/inactivity_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/inactivity_ensured'.format(os_user)):
try:
- if not exists('/opt/inactivity'):
+ if not exists(conn,'/opt/inactivity'):
conn.sudo('mkdir /opt/inactivity')
conn.put('/root/templates/inactive.service', '/etc/systemd/system/inactive.service', use_sudo=True)
conn.put('/root/templates/inactive.timer', '/etc/systemd/system/inactive.timer', use_sudo=True)
@@ -685,7 +685,7 @@ def set_mongo_parameters(client, mongo_parameters):
def install_r_packages(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/r_packages_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/r_packages_ensured'):
conn.sudo('R -e "install.packages(\'devtools\', repos = \'https://cloud.r-project.org\')"')
conn.sudo('R -e "install.packages(\'knitr\', repos = \'https://cloud.r-project.org\')"')
conn.sudo('R -e "install.packages(\'ggplot2\', repos = \'https://cloud.r-project.org\')"')
@@ -696,7 +696,7 @@ def install_r_packages(os_user):
def add_breeze_library_local(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/breeze_local_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/breeze_local_ensured'):
try:
breeze_tmp_dir = '/tmp/breeze_tmp_local/'
jars_dir = '/opt/jars/'
@@ -726,15 +726,15 @@ def add_breeze_library_local(os_user):
def configure_data_engine_service_pip(hostname, os_user, keyfile, emr=False):
init_datalab_connection(hostname, os_user, keyfile)
manage_pkg('-y install', 'remote', 'python3-pip')
- if not exists('/usr/bin/pip3') and conn.sudo("python3.4 -V 2>/dev/null | awk '{print $2}'"):
+ if not exists(conn,'/usr/bin/pip3') and conn.sudo("python3.4 -V 2>/dev/null | awk '{print $2}'"):
conn.sudo('ln -s /usr/bin/pip-3.4 /usr/bin/pip3')
- elif not exists('/usr/bin/pip3') and conn.sudo("python3.5 -V 2>/dev/null | awk '{print $2}'"):
+ elif not exists(conn,'/usr/bin/pip3') and conn.sudo("python3.5 -V 2>/dev/null | awk '{print $2}'"):
conn.sudo('ln -s /usr/bin/pip-3.5 /usr/bin/pip3')
- elif not exists('/usr/bin/pip3') and conn.sudo("python3.6 -V 2>/dev/null | awk '{print $2}'"):
+ elif not exists(conn,'/usr/bin/pip3') and conn.sudo("python3.6 -V 2>/dev/null | awk '{print $2}'"):
conn.sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
- elif not exists('/usr/bin/pip3') and conn.sudo("python3.7 -V 2>/dev/null | awk '{print $2}'"):
+ elif not exists(conn,'/usr/bin/pip3') and conn.sudo("python3.7 -V 2>/dev/null | awk '{print $2}'"):
conn.sudo('ln -s /usr/bin/pip-3.7 /usr/bin/pip3')
- elif not exists('/usr/bin/pip3') and conn.sudo("python3.8 -V 2>/dev/null | awk '{print $2}'"):
+ elif not exists(conn,'/usr/bin/pip3') and conn.sudo("python3.8 -V 2>/dev/null | awk '{print $2}'"):
conn.sudo('ln -s /usr/bin/pip-3.8 /usr/bin/pip3')
if emr:
conn.sudo('pip3 install -U pip=={}'.format(os.environ['conf_pip_version']))
@@ -842,7 +842,7 @@ def replace_multi_symbols(string, symbol, symbol_cut=False):
def update_pyopenssl_lib(os_user):
- if not exists('/home/{}/.ensure_dir/pyopenssl_updated'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/pyopenssl_updated'.format(os_user)):
try:
if exists('/usr/bin/pip3'):
conn.sudo('pip3 install -U pyopenssl')
@@ -914,7 +914,7 @@ def update_zeppelin_interpreters(multiple_clusters, r_enabled, interpreter_mode=
def update_hosts_file(os_user, conn):
try:
- if not exists('/home/{}/.ensure_dir/hosts_file_updated'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/hosts_file_updated'.format(os_user)):
conn.sudo('ls -lah; pwd; hostname; whoami')
conn.sudo('sed -i "s/^127.0.0.1 localhost/127.0.0.1 localhost localhost.localdomain/g" /etc/hosts')
conn.sudo('touch /home/{}/.ensure_dir/hosts_file_updated'.format(os_user))
@@ -926,7 +926,7 @@ def update_hosts_file(os_user, conn):
def ensure_docker_compose(os_user):
try:
configure_docker(os_user)
- if not exists('/home/{}/.ensure_dir/docker_compose_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/docker_compose_ensured'.format(os_user)):
docker_compose_version = "1.24.1"
conn.sudo('curl -L https://github.com/docker/compose/releases/download/{}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose'.format(docker_compose_version))
conn.sudo('chmod +x /usr/local/bin/docker-compose')
@@ -940,13 +940,13 @@ def ensure_docker_compose(os_user):
def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id, keycloak_client_secret, edge_instance_private_ip, edge_instance_public_ip, superset_name):
print('Superset configuring')
try:
- if not exists('/home/{}/incubator-superset'.format(os_user)):
+ if not exists(conn,'/home/{}/incubator-superset'.format(os_user)):
with conn.cd('/home/{}'.format(os_user)):
conn.sudo('wget https://github.com/apache/incubator-superset/archive/{}.tar.gz'.format(
os.environ['notebook_superset_version']))
conn.sudo('tar -xzf {}.tar.gz'.format(os.environ['notebook_superset_version']))
conn.sudo('ln -sf incubator-superset-{} incubator-superset'.format(os.environ['notebook_superset_version']))
- if not exists('/tmp/superset-notebook_installed'):
+ if not exists(conn,'/tmp/superset-notebook_installed'):
conn.sudo('mkdir -p /opt/datalab/templates')
conn.put('/root/templates', '/opt/datalab', use_sudo=True)
conn.sudo('sed -i \'s/OS_USER/{}/g\' /opt/datalab/templates/.env'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index 8f40045..ec33387 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -58,7 +58,7 @@ def manage_pkg(command, environment, requisites):
def ensure_pkg(user, conn, requisites='git vim gcc python-devel openssl-devel nmap libffi libffi-devel unzip libxml2-devel'):
try:
- if not exists('/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/pkg_upgraded'.format(user)):
print("Updating repositories and installing requested tools: {}".format(requisites))
if conn.sudo("systemctl list-units --all | grep firewalld | awk '{print $1}'") != '':
conn.sudo('systemctl disable firewalld.service')
@@ -89,7 +89,7 @@ def ensure_pkg(user, conn, requisites='git vim gcc python-devel openssl-devel nm
def change_pkg_repos(conn):
- if not exists('/tmp/pkg_china_ensured'):
+ if not exists(conn,'/tmp/pkg_china_ensured'):
conn.put('/root/files/sources.list', '/tmp/sources.list')
conn.sudo('mv /tmp/sources.list /etc/yum.repos.d/CentOS-Base-aliyun.repo')
conn.sudo('touch /tmp/pkg_china_ensured')
@@ -107,7 +107,7 @@ def find_java_path_local():
def ensure_ntpd(user, conn, edge_private_ip=''):
try:
- if not exists('/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
conn.sudo('systemctl disable chronyd')
manage_pkg('-y install', 'remote', 'ntp')
conn.sudo('echo "tinker panic 0" >> /etc/ntp.conf')
@@ -123,7 +123,7 @@ def ensure_ntpd(user, conn, edge_private_ip=''):
def ensure_java(user):
try:
- if not exists('/home/{}/.ensure_dir/java_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/java_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel')
conn.sudo('touch /home/{}/.ensure_dir/java_ensured'.format(user))
except:
@@ -132,7 +132,7 @@ def ensure_java(user):
def ensure_step(user):
try:
- if not exists('/home/{}/.ensure_dir/step_ensured'.format(user)):
+ if not exists(conn,'/home/{}/.ensure_dir/step_ensured'.format(user)):
manage_pkg('-y install', 'remote', 'wget')
conn.sudo('wget https://github.com/smallstep/cli/releases/download/v0.13.3/step_0.13.3_linux_amd64.tar.gz '
'-O /tmp/step_0.13.3_linux_amd64.tar.gz')
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index ae5c35e..9f84133 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -30,7 +30,7 @@ from patchwork.files import exists
def configure_http_proxy_server(config):
try:
- if not exists('/tmp/http_proxy_ensured'):
+ if not exists(conn,'/tmp/http_proxy_ensured'):
manage_pkg('-y install', 'remote', 'squid')
template_file = config['template_file']
proxy_subnet = config['exploratory_subnet']
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index b1e4a6a..4ffcd36 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -57,7 +57,7 @@ def downgrade_python_version():
def ensure_r_local_kernel(spark_version, os_user, templates_dir, kernels_dir):
- if not exists('/home/{}/.ensure_dir/r_kernel_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/r_kernel_ensured'.format(os_user)):
try:
conn.sudo('chown -R ' + os_user + ':' + os_user + ' /home/' + os_user + '/.local')
conn.run('R -e "IRkernel::installspec()"')
@@ -80,7 +80,7 @@ def ensure_r_local_kernel(spark_version, os_user, templates_dir, kernels_dir):
def ensure_r(os_user, r_libs, region, r_mirror):
- if not exists('/home/{}/.ensure_dir/r_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/r_ensured'.format(os_user)):
try:
if region == 'cn-north-1':
r_repository = r_mirror
@@ -104,7 +104,7 @@ def ensure_r(os_user, r_libs, region, r_mirror):
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
- if not exists('/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
try:
manage_pkg('-y install --nogpgcheck', 'remote', 'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm'.format(rstudio_version))
conn.sudo('mkdir -p /mnt/var')
@@ -140,7 +140,7 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
def ensure_matplot(os_user):
- if not exists('/home/{}/.ensure_dir/matplot_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/matplot_ensured'.format(os_user)):
try:
conn.sudo('python3.5 -m pip install matplotlib==2.0.2 --no-cache-dir')
if os.environ['application'] in ('tensor', 'deeplearning'):
@@ -151,7 +151,7 @@ def ensure_matplot(os_user):
def ensure_sbt(os_user):
- if not exists('/home/{}/.ensure_dir/sbt_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/sbt_ensured'.format(os_user)):
try:
conn.sudo('curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo')
manage_pkg('-y install', 'remote', 'sbt')
@@ -161,7 +161,7 @@ def ensure_sbt(os_user):
def ensure_jre_jdk(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
try:
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk')
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel')
@@ -171,7 +171,7 @@ def ensure_jre_jdk(os_user):
def ensure_scala(scala_link, scala_version, os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/scala_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/scala_ensured'):
try:
conn.sudo('wget {}scala-{}.rpm -O /tmp/scala.rpm'.format(scala_link, scala_version))
conn.sudo('rpm -i /tmp/scala.rpm')
@@ -181,7 +181,7 @@ def ensure_scala(scala_link, scala_version, os_user):
def ensure_additional_python_libs(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'):
try:
manage_pkg('clean', 'remote', 'all')
manage_pkg('-y install', 'remote', 'zlib-devel libjpeg-turbo-devel --nogpgcheck')
@@ -195,7 +195,7 @@ def ensure_additional_python_libs(os_user):
def ensure_python3_specific_version(python3_version, os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'):
try:
manage_pkg('-y install', 'remote', 'yum-utils python34 openssl-devel')
manage_pkg('-y groupinstall', 'remote', 'development --nogpgcheck')
@@ -208,7 +208,7 @@ def ensure_python3_specific_version(python3_version, os_user):
sys.exit(1)
def ensure_python3_libraries(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
try:
manage_pkg('-y install', 'remote', 'https://centos7.iuscommunity.org/ius-release.rpm')
manage_pkg('-y install', 'remote', 'python35u python35u-pip python35u-devel')
@@ -229,7 +229,7 @@ def ensure_python3_libraries(os_user):
def install_tensor(os_user, cuda_version, cuda_file_name,
cudnn_version, cudnn_file_name, tensorflow_version,
templates_dir, nvidia_version):
- if not exists('/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
try:
# install nvidia drivers
conn.sudo('echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf')
@@ -276,7 +276,7 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
def install_maven(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/maven_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/maven_ensured'):
conn.sudo('wget http://apache.volia.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -O /tmp/maven.tar.gz')
conn.sudo('tar -zxvf /tmp/maven.tar.gz -C /opt/')
conn.sudo('ln -fs /opt/apache-maven-3.3.9/bin/mvn /usr/bin/mvn')
@@ -284,7 +284,7 @@ def install_maven(os_user):
def install_livy_dependencies(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
conn.sudo('pip3.5 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -304,7 +304,7 @@ def install_livy_dependencies_emr(os_user):
def install_nodejs(os_user):
- if not exists('/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
conn.sudo('curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -')
manage_pkg('-y install', 'remote', 'nodejs')
conn.sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
@@ -398,7 +398,7 @@ def get_available_os_pkgs():
def install_opencv(os_user):
- if not exists('/home/{}/.ensure_dir/opencv_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/opencv_ensured'.format(os_user)):
manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++')
conn.sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
conn.sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
@@ -414,7 +414,7 @@ def install_opencv(os_user):
def install_caffe2(os_user, caffe2_version, cmake_version):
- if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote', 'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz')
@@ -439,7 +439,7 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
def install_cntk(os_user, cntk_version):
- if not exists('/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
conn.sudo('echo "exclude=*.i386 *.i686" >> /etc/yum.conf')
manage_pkg('clean', 'remote', 'all')
manage_pkg('update-minimal --security -y', 'remote', '')
@@ -449,25 +449,25 @@ def install_cntk(os_user, cntk_version):
def install_keras(os_user, keras_version):
- if not exists('/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
conn.sudo('pip3.5 install keras=={} --no-cache-dir'.format(keras_version))
conn.sudo('touch /home/{}/.ensure_dir/keras_ensured'.format(os_user))
def install_theano(os_user, theano_version):
- if not exists('/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
conn.sudo('python3.8 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
conn.sudo('touch /home/{}/.ensure_dir/theano_ensured'.format(os_user))
def install_mxnet(os_user, mxnet_version):
- if not exists('/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
+ if not exists(conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
conn.sudo('pip3.5 install mxnet-cu80=={} opencv-python --no-cache-dir'.format(mxnet_version))
conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
#def install_torch(os_user):
-# if not exists('/home/{}/.ensure_dir/torch_ensured'.format(os_user)):
+# if not exists(conn,'/home/{}/.ensure_dir/torch_ensured'.format(os_user)):
# run('git clone https://github.com/torch/distro.git ~/torch --recursive')
# with cd('/home/{}/torch/'.format(os_user)):
# manage_pkg('-y install --nogpgcheck', 'remote', 'cmake curl readline-devel ncurses-devel gcc-c++ gcc-gfortran git gnuplot unzip libjpeg-turbo-devel libpng-devel ImageMagick GraphicsMagick-devel fftw-devel sox-devel sox zeromq3-devel qt-devel qtwebkit-devel sox-plugins-freeworld qt-devel')
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index 8009cd4..bebeb93 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -35,7 +35,7 @@ from fabric import *
def ensure_docker_daemon(datalab_path, os_user, region):
try:
- if not exists('{}tmp/docker_daemon_ensured'.format(datalab_path)):
+ if not exists(conn,'{}tmp/docker_daemon_ensured'.format(datalab_path)):
docker_version = os.environ['ssn_docker_version']
if region == 'cn-north-1':
mirror = 'mirror.lzu.edu.cn'
@@ -63,7 +63,7 @@ def ensure_docker_daemon(datalab_path, os_user, region):
def ensure_nginx(datalab_path):
try:
- if not exists('{}tmp/nginx_ensured'.format(datalab_path)):
+ if not exists(conn,'{}tmp/nginx_ensured'.format(datalab_path)):
manage_pkg('-y install', 'remote', 'nginx')
conn.sudo('systemctl restart nginx.service')
conn.sudo('chkconfig nginx on')
@@ -76,7 +76,7 @@ def ensure_nginx(datalab_path):
def ensure_jenkins(datalab_path):
try:
- if not exists('{}tmp/jenkins_ensured'.format(datalab_path)):
+ if not exists(conn,'{}tmp/jenkins_ensured'.format(datalab_path)):
conn.sudo('wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo')
try:
conn.sudo('rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key')
@@ -93,7 +93,7 @@ def ensure_jenkins(datalab_path):
def configure_jenkins(datalab_path, os_user, config, tag_resource_id):
try:
- if not exists('{}tmp/jenkins_configured'.format(datalab_path)):
+ if not exists(conn,'{}tmp/jenkins_configured'.format(datalab_path)):
conn.sudo('rm -rf /var/lib/jenkins/*')
conn.sudo('mkdir -p /var/lib/jenkins/jobs/')
conn.sudo('chown -R {0}:{0} /var/lib/jenkins/'.format(os_user))
@@ -123,7 +123,7 @@ def configure_jenkins(datalab_path, os_user, config, tag_resource_id):
def configure_nginx(config, datalab_path, hostname):
try:
random_file_part = id_generator(size=20)
- if not exists("/etc/nginx/conf.d/nginx_proxy.conf"):
+ if not exists(conn,"/etc/nginx/conf.d/nginx_proxy.conf"):
conn.sudo('rm -f /etc/nginx/conf.d/*')
conn.put(config['nginx_template_dir'] + 'nginx_proxy.conf', '/tmp/nginx_proxy.conf')
conn.put(config['nginx_template_dir'] + 'ssn_nginx.conf', '/tmp/nginx.conf')
@@ -139,7 +139,7 @@ def configure_nginx(config, datalab_path, hostname):
sys.exit(1)
try:
- if not exists("/etc/nginx/locations/proxy_location_jenkins.conf"):
+ if not exists(conn,"/etc/nginx/locations/proxy_location_jenkins.conf"):
nginx_password = id_generator()
template_file = config['nginx_template_dir'] + 'proxy_location_jenkins_template.conf'
with open("/tmp/%s-tmpproxy_location_jenkins_template.conf" % random_file_part, 'w') as out:
@@ -164,7 +164,7 @@ def configure_nginx(config, datalab_path, hostname):
def ensure_supervisor():
try:
- if not exists('{}tmp/superv_ensured'.format(os.environ['ssn_datalab_path'])):
+ if not exists(conn,'{}tmp/superv_ensured'.format(os.environ['ssn_datalab_path'])):
manage_pkg('-y install', 'remote', 'supervisor')
# conn.sudo('pip install supervisor')
conn.sudo('chkconfig supervisord on')
@@ -178,7 +178,7 @@ def ensure_supervisor():
def ensure_mongo():
try:
- if not exists('{}tmp/mongo_ensured'.format(os.environ['ssn_datalab_path'])):
+ if not exists(conn,'{}tmp/mongo_ensured'.format(os.environ['ssn_datalab_path'])):
conn.sudo('echo -e "[mongodb-org-3.2]\nname=MongoDB Repository'
'\nbaseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/'
'\ngpgcheck=1'
@@ -209,7 +209,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
usage_type, usage, cost, resource_id, tags, billing_dataset_name, keycloak_client_id,
keycloak_client_secret, keycloak_auth_server_url, report_path=''):
try:
- if not exists('{}tmp/ss_started'.format(os.environ['ssn_datalab_path'])):
+ if not exists(conn,'{}tmp/ss_started'.format(os.environ['ssn_datalab_path'])):
java_path = conn.sudo("alternatives --display java | grep 'slave jre: ' | awk '{print $3}'")
supervisor_conf = '/etc/supervisord.d/supervisor_svc.ini'
conn.local('sed -i "s|MONGO_PASSWORD|{}|g" /root/templates/ssn.yml'.format(mongo_passwd))
@@ -387,7 +387,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
def install_build_dep():
try:
- if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
+ if not exists(conn,'{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
with conn.cd('/opt/'):
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
index 6e8b9f2..0fa513e 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
@@ -46,7 +46,7 @@ def configure_notebook(keyfile, hoststring):
scripts_dir = '/root/scripts/'
conn.run('mkdir -p /tmp/{}/'.format(args.cluster_name))
conn.put(templates_dir + 'sparkmagic_config_template.json', '/tmp/sparkmagic_config_template.json')
- if not exists('/tmp/deeplearning_dataengine_create_configs.py'):
+ if not exists(conn,'/tmp/deeplearning_dataengine_create_configs.py'):
conn.put(scripts_dir + 'deeplearning_dataengine_create_configs.py',
'/tmp/deeplearning_dataengine_create_configs.py')
# conn.put(templates_dir + 'pyspark_dataengine_template.json', '/tmp/{}/pyspark_dataengine_template.json'.format(args.cluster_name))
@@ -54,11 +54,11 @@ def configure_notebook(keyfile, hoststring):
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
# spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
# conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
- if not exists('/usr/local/bin/deeplearning_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/deeplearning_dataengine_create_configs.py'):
conn.put(scripts_dir + 'deeplearning_dataengine_create_configs.py',
'/usr/local/bin/deeplearning_dataengine_create_configs.py', use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/deeplearning_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index 7e003b1..ecfe354 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -58,11 +58,11 @@ def configure_notebook(keyfile, hoststring):
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
# spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
# conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
- if not exists('/usr/local/bin/jupyter_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/jupyter_dataengine_create_configs.py'):
conn.put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py',
use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py b/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
index 73ce8af..63079f6 100644
--- a/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
@@ -50,7 +50,7 @@ if __name__ == "__main__":
if args.spark_type == 'local':
configure_local_spark(jars_dir, templates_dir, memory_type)
elif args.spark_type == 'dataengine':
- if not exists('/usr/local/bin/notebook_reconfigure_dataengine_spark.py'):
+ if not exists(conn,'/usr/local/bin/notebook_reconfigure_dataengine_spark.py'):
conn.put('/root/scripts/notebook_reconfigure_dataengine_spark.py',
'/tmp/notebook_reconfigure_dataengine_spark.py')
conn.sudo('mv /tmp/notebook_reconfigure_dataengine_spark.py '
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
index 15d19f2..e64abaa 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
@@ -51,11 +51,11 @@ def configure_notebook(keyfile, hoststring):
spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory,
args.cluster_name))
- if not exists('/usr/local/bin/rstudio_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/rstudio_dataengine_create_configs.py'):
conn.put(scripts_dir + 'rstudio_dataengine_create_configs.py', '/usr/local/bin/rstudio_dataengine_create_configs.py',
use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
index cd7a574..fa7a27b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
@@ -51,11 +51,11 @@ def configure_notebook(keyfile, hoststring):
spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory,
args.cluster_name))
- if not exists('/usr/local/bin/tensor-rstudio_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/tensor-rstudio_dataengine_create_configs.py'):
conn.put(scripts_dir + 'tensor-rstudio_dataengine_create_configs.py',
'/usr/local/bin/tensor-rstudio_dataengine_create_configs.py', use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/tensor-rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
index d3b0a41..e9c2782 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
@@ -51,11 +51,11 @@ def configure_notebook(keyfile, hoststring):
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
# spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
# conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
- if not exists('/usr/local/bin/tensor_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/tensor_dataengine_create_configs.py'):
conn.put(scripts_dir + 'tensor_dataengine_create_configs.py', '/usr/local/bin/tensor_dataengine_create_configs.py',
use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/tensor_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
index 0ca738f..5fe1aa5 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
@@ -57,11 +57,11 @@ def configure_notebook(keyfile, hoststring):
conn.run('sed -i "s|EXECUTOR_MEMORY|{}m|g " /tmp/{}/dataengine_interpreter.json'.format(spark_memory, args.cluster_name))
conn.run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory,
args.cluster_name))
- if not exists('/usr/local/bin/zeppelin_dataengine_create_configs.py'):
+ if not exists(conn,'/usr/local/bin/zeppelin_dataengine_create_configs.py'):
conn.put(scripts_dir + 'zeppelin_dataengine_create_configs.py',
'/usr/local/bin/zeppelin_dataengine_create_configs.py', use_sudo=True)
conn.sudo('chmod 755 /usr/local/bin/zeppelin_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.8/datalab/'):
+ if not exists(conn,'/usr/lib/python3.8/datalab/'):
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
conn.put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
conn.sudo('chmod a+x /usr/lib/python3.8/datalab/*')
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index e28d6d6..18e3053 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
index 86f907f..b649181 100644
--- a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
+++ b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
@@ -85,7 +85,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
index e401eae..dd11c19 100644
--- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
+++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
@@ -71,7 +71,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 55459b4..f3ef456 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -85,7 +85,7 @@ def cp_backup_scripts(datalab_path):
def cp_gitlab_scripts(datalab_path):
try:
- if not exists('{}tmp/gitlab'.format(datalab_path)):
+ if not exists(conn,'{}tmp/gitlab'.format(datalab_path)):
conn.run('mkdir -p {}tmp/gitlab'.format(datalab_path))
with conn.cd('{}tmp/gitlab'.format(datalab_path)):
conn.put('/root/scripts/gitlab_deploy.py', 'gitlab_deploy.py')
@@ -106,7 +106,7 @@ def cp_gitlab_scripts(datalab_path):
def creating_service_directories(datalab_path, os_user):
try:
- if not exists(datalab_path):
+ if not exists(conn,datalab_path):
conn.sudo('mkdir -p ' + datalab_path)
conn.sudo('mkdir -p ' + datalab_path + 'conf')
conn.sudo('mkdir -p ' + datalab_path + 'webapp/static')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 68b3fec..cab6f99 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -107,7 +107,7 @@ def copy_ssn_libraries():
def configure_mongo(mongo_passwd, default_endpoint_name):
try:
- if not exists("/lib/systemd/system/mongod.service"):
+ if not exists(conn,"/lib/systemd/system/mongod.service"):
if os.environ['conf_os_family'] == 'debian':
subprocess.run('sed -i "s/MONGO_USR/mongodb/g" /root/templates/mongod.service_template', shell=True, check=True)
elif os.environ['conf_os_family'] == 'redhat':
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index 010f257..5942adc 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -57,7 +57,7 @@ def image_build(src_path, node):
cloud_provider = 'aws'
elif subprocess.run("uname -r | awk -F '-' '{print $3}'", capture_output=True, shell=True, check=True).stdout == 'azure':
cloud_provider = 'azure'
- if not exists('{}base/azure_auth.json'.format(src_path)):
+ if not exists(conn,'{}base/azure_auth.json'.format(src_path)):
subprocess.run('cp /home/datalab-user/keys/azure_auth.json {}base/azure_auth.json'.format(src_path), shell=True, check=True)
else:
cloud_provider = 'gcp'
diff --git a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
index 1f163c9..3f21f4f 100644
--- a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
+++ b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
@@ -56,7 +56,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
index 8f7f103..1211bb9 100644
--- a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
+++ b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
@@ -79,7 +79,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
index 137e6d4..b5d1b89 100644
--- a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
+++ b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
@@ -78,7 +78,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index 4a0df1b..1c90d14 100644
--- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -77,7 +77,7 @@ gitlab_certfile = os.environ['conf_gitlab_certfile']
def configure_zeppelin(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/zeppelin_ensured'):
+ if not exists(conn,'/home/' + os_user + '/.ensure_dir/zeppelin_ensured'):
try:
conn.sudo('wget ' + zeppelin_link + ' -O /tmp/zeppelin-' + zeppelin_version + '-bin-netinst.tgz')
conn.sudo('tar -zxvf /tmp/zeppelin-' + zeppelin_version + '-bin-netinst.tgz -C /opt/')
@@ -127,7 +127,7 @@ def configure_zeppelin(os_user):
def configure_local_livy_kernels(args):
- if not exists('/home/' + args.os_user + '/.ensure_dir/local_livy_kernel_ensured'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir/local_livy_kernel_ensured'):
port_number_found = False
default_port = 8998
livy_port = ''
@@ -159,7 +159,7 @@ def configure_local_livy_kernels(args):
def configure_local_spark_kernels(args):
- if not exists('/home/' + args.os_user + '/.ensure_dir/local_spark_kernel_ensured'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir/local_spark_kernel_ensured'):
conn.put(templates_dir + 'interpreter_spark.json', '/tmp/interpreter.json')
conn.sudo('sed -i "s|ENDPOINTURL|' + args.endpoint_url + '|g" /tmp/interpreter.json')
conn.sudo('sed -i "s|OS_USER|' + args.os_user + '|g" /tmp/interpreter.json')
@@ -174,7 +174,7 @@ def configure_local_spark_kernels(args):
def install_local_livy(args):
- if not exists('/home/' + args.os_user + '/.ensure_dir/local_livy_ensured'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir/local_livy_ensured'):
conn.sudo('wget http://archive.cloudera.com/beta/livy/livy-server-' + args.livy_version + '.zip -O /opt/livy-server-'
+ args.livy_version + '.zip')
conn.sudo('unzip /opt/livy-server-' + args.livy_version + '.zip -d /opt/')
@@ -205,7 +205,7 @@ if __name__ == "__main__":
# PREPARE DISK
print("Prepare .ensure directory")
try:
- if not exists('/home/' + args.os_user + '/.ensure_dir'):
+ if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'):
conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org