You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/08/20 13:24:11 UTC
[incubator-datalab] 01/02: [DATALAB-1864]: added part mount place
check for dataengine nodes
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1864
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f75d74204c5daf9264b1fe0caab26120a2d62407
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Aug 20 15:10:06 2021 +0300
[DATALAB-1864]: added part mount place check for dataengine nodes
---
.../src/base/scripts/install_user_key.py | 2 +-
.../src/general/lib/azure/actions_lib.py | 2 +-
.../general/scripts/azure/dataengine_configure.py | 29 ++++++++++++++++++++--
3 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/install_user_key.py b/infrastructure-provisioning/src/base/scripts/install_user_key.py
index d7a5faf..2e68abd 100644
--- a/infrastructure-provisioning/src/base/scripts/install_user_key.py
+++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py
@@ -75,7 +75,7 @@ if __name__ == "__main__":
print("Installing users key...")
try:
copy_key(deeper_config)
- conn.close()
+ #conn.close()
except:
print("Users keyfile {0} could not be found at {1}/{0}".format(args.keyfile, deeper_config['user_keydir']))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 3a5accf..b83aba7 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1139,7 +1139,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
datalab.fab.conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
datalab.fab.conn.sudo("jar_list=`find {} -name '*.jar' | tr '\\n' ','` ; echo \"spark.jars $jar_list\" >> \
/tmp/notebook_spark-defaults_local.conf".format(jars_dir))
- datalab.fab.conn.sudo('\cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
+ datalab.fab.conn.sudo('cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
if memory_type == 'driver':
spark_memory = datalab.fab.get_spark_memory()
datalab.fab.conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf')
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index f678c9f..61d3cf5 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -74,8 +74,21 @@ def configure_slave(slave_number, data_engine):
sys.exit(1)
try:
- logging.info('[CLEANING INSTANCE FOR MASTER NODE]')
- print('[CLEANING INSTANCE FOR MASTER NODE]')
+ logging.info('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
+ print('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]')
+ try:
+ AzureActions.ensure_right_mount_paths()
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ clear_resources()
+ datalab.fab.append_result("Failed to ensure_right_mount_paths on slave instance..", str(err))
+ sys.exit(1)
+
+ try:
+ logging.info('[CLEANING INSTANCE FOR SLAVE NODE]')
+ print('[CLEANING INSTANCE FOR SLAVE NODE]')
params = '--hostname {} --keyfile {} --os_user {} --application {}' \
.format(slave_hostname, keyfile_name, data_engine['datalab_ssh_user'], os.environ['application'])
try:
@@ -264,6 +277,18 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to install ssh user key on master.", str(err))
sys.exit(1)
+ try:
+ logging.info('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
+ print('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]')
+ try:
+ AzureActions.ensure_right_mount_paths()
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ clear_resources()
+ datalab.fab.append_result("Failed to ensure right mount paths on master instance..", str(err))
+ sys.exit(1)
try:
logging.info('[CLEANING INSTANCE FOR MASTER NODE]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org