You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/08/20 13:24:10 UTC
[incubator-datalab] branch DATALAB-1864 created (now c128f89)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-1864
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at c128f89 [DATALAB-1864]: changed where partition mount point check is called
This branch includes the following new commits:
new f75d742 [DATALAB-1864]: added part mount place check for dataengine nodes
new c128f89 [DATALAB-1864]: changed where partition mount point check is called
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-1864]: added part mount place
check for dataengine nodes
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1864
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f75d74204c5daf9264b1fe0caab26120a2d62407
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Aug 20 15:10:06 2021 +0300
[DATALAB-1864]: added part mount place check for dataengine nodes
---
.../src/base/scripts/install_user_key.py | 2 +-
.../src/general/lib/azure/actions_lib.py | 2 +-
.../general/scripts/azure/dataengine_configure.py | 29 ++++++++++++++++++++--
3 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/install_user_key.py b/infrastructure-provisioning/src/base/scripts/install_user_key.py
index d7a5faf..2e68abd 100644
--- a/infrastructure-provisioning/src/base/scripts/install_user_key.py
+++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py
@@ -75,7 +75,7 @@ if __name__ == "__main__":
print("Installing users key...")
try:
copy_key(deeper_config)
- conn.close()
+ #conn.close()
except:
print("Users keyfile {0} could not be found at {1}/{0}".format(args.keyfile, deeper_config['user_keydir']))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 3a5accf..b83aba7 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1139,7 +1139,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
datalab.fab.conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
datalab.fab.conn.sudo("jar_list=`find {} -name '*.jar' | tr '\\n' ','` ; echo \"spark.jars $jar_list\" >> \
/tmp/notebook_spark-defaults_local.conf".format(jars_dir))
- datalab.fab.conn.sudo('\cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
+ datalab.fab.conn.sudo('cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
if memory_type == 'driver':
spark_memory = datalab.fab.get_spark_memory()
datalab.fab.conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf')
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index f678c9f..61d3cf5 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -74,8 +74,21 @@ def configure_slave(slave_number, data_engine):
sys.exit(1)
try:
- logging.info('[CLEANING INSTANCE FOR MASTER NODE]')
- print('[CLEANING INSTANCE FOR MASTER NODE]')
+ logging.info('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
+ print('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
+ try:
+ AzureActions.ensure_right_mount_paths()
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ clear_resources()
+ datalab.fab.append_result("Failed to ensure_right_mount_paths on slave instance..", str(err))
+ sys.exit(1)
+
+ try:
+ logging.info('[CLEANING INSTANCE FOR SLAVE NODE]')
+ print('[CLEANING INSTANCE FOR SLAVE NODE]')
params = '--hostname {} --keyfile {} --os_user {} --application {}' \
.format(slave_hostname, keyfile_name, data_engine['datalab_ssh_user'], os.environ['application'])
try:
@@ -264,6 +277,18 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to install ssh user key on master.", str(err))
sys.exit(1)
+ try:
+ logging.info('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
+ print('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
+ try:
+ AzureActions.ensure_right_mount_paths()
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ clear_resources()
+ datalab.fab.append_result("Failed to ensure right mount paths on master instance..", str(err))
+ sys.exit(1)
try:
logging.info('[CLEANING INSTANCE FOR MASTER NODE]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-1864]: changed where partition
mount point check is called
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1864
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c128f8917a3776600cfad5cc3d1cd62b2689d7cc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Aug 20 16:24:02 2021 +0300
[DATALAB-1864]: changed where partition mount point check is called
---
.../general/scripts/azure/dataengine_configure.py | 25 ----------------------
.../general/scripts/os/common_clean_instance.py | 2 ++
2 files changed, 2 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index 61d3cf5..088fbec 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -74,19 +74,6 @@ def configure_slave(slave_number, data_engine):
sys.exit(1)
try:
- logging.info('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
- print('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
- try:
- AzureActions.ensure_right_mount_paths()
- except:
- traceback.print_exc()
- raise Exception
- except Exception as err:
- clear_resources()
- datalab.fab.append_result("Failed to ensure_right_mount_paths on slave instance..", str(err))
- sys.exit(1)
-
- try:
logging.info('[CLEANING INSTANCE FOR SLAVE NODE]')
print('[CLEANING INSTANCE FOR SLAVE NODE]')
params = '--hostname {} --keyfile {} --os_user {} --application {}' \
@@ -277,18 +264,6 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to install ssh user key on master.", str(err))
sys.exit(1)
- try:
- logging.info('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
- print('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
- try:
- AzureActions.ensure_right_mount_paths()
- except:
- traceback.print_exc()
- raise Exception
- except Exception as err:
- clear_resources()
- datalab.fab.append_result("Failed to ensure right mount paths on master instance..", str(err))
- sys.exit(1)
try:
logging.info('[CLEANING INSTANCE FOR MASTER NODE]')
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py b/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
index 6b24356..b1fa4eb 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
@@ -25,6 +25,7 @@ import argparse
import os
import sys
from datalab.notebook_lib import *
+from datalab.actions_lib import ensure_right_mount_paths
from fabric import *
parser = argparse.ArgumentParser()
@@ -121,6 +122,7 @@ if __name__ == "__main__":
conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile)
if os.environ['conf_cloud_provider'] == 'azure':
+ ensure_right_mount_paths()
de_master_name = '{}-{}-{}-de-{}-m'.format(
os.environ['conf_service_base_name'],
os.environ['project_name'],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org