You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/10/08 09:42:42 UTC
[incubator-datalab] 01/01: [DATALAB-1837]: added additional disk
mount checks for azure notebooks
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1837
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 22f8380b4cf38aa17dd795ba1c64c1ff91f27c52
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Oct 8 12:42:28 2021 +0300
[DATALAB-1837]: added additional disk mount checks for azure notebooks
---
.../src/general/lib/azure/actions_lib.py | 23 ++++++++++++++--------
.../scripts/azure/common_create_notebook_image.py | 2 ++
.../scripts/azure/deeplearning_configure.py | 6 +++++-
.../src/general/scripts/azure/jupyter_configure.py | 6 +++++-
.../general/scripts/azure/jupyterlab_configure.py | 6 +++++-
.../src/general/scripts/azure/rstudio_configure.py | 6 +++++-
.../src/general/scripts/azure/tensor_configure.py | 6 +++++-
.../general/scripts/azure/zeppelin_configure.py | 6 +++++-
8 files changed, 47 insertions(+), 14 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index b83aba7..09ec650 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1232,15 +1232,22 @@ def remount_azure_disk(creds=False, os_user='', hostname='', keyfile=''):
if creds:
conn.close()
-def ensure_right_mount_paths():
- opt_disk = datalab.fab.conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2]
- if opt_disk not in datalab.fab.conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
- disk_names = datalab.fab.conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n')
+def ensure_right_mount_paths(creds=False, os_user='', hostname='', keyfile=''):
+ if creds:
+ global conn
+ conn = datalab.fab.init_datalab_connection(hostname, os_user, keyfile)
+ else:
+ conn = datalab.fab.conn
+ opt_disk = conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2]
+ if opt_disk not in conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
+ disk_names = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n')
for disk in disk_names:
- if disk != '' and disk not in datalab.fab.conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
- datalab.fab.conn.sudo("umount -l /opt")
- datalab.fab.conn.sudo("mount /dev/{}1 /opt".format(disk))
- datalab.fab.conn.sudo('sed -i "/opt/ s|/dev/{}|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk))
+ if disk != '' and disk not in conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
+ conn.sudo("umount -l /opt")
+ conn.sudo("mount /dev/{}1 /opt".format(disk))
+ conn.sudo('sed -i "/opt/ s|/dev/{}|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk))
+ if creds:
+ conn.close()
def prepare_vm_for_image(creds=False, os_user='', hostname='', keyfile=''):
if creds:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index 294d7a8..4b538f9 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -96,6 +96,8 @@ if __name__ == "__main__":
image_conf['instance_name'])
datalab.actions_lib.remount_azure_disk(True, image_conf['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(image_conf['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_hostname))
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index be91e28..97cb24f 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -218,7 +218,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_deep_learning_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -283,6 +285,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 2e939b9..1388b57 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -201,7 +201,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_jupyter_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -285,6 +287,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
index 8c694e1..ca55247 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
@@ -201,7 +201,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_jupyterlab_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -266,6 +268,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 4ed3ad7..38d253a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -201,7 +201,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_rstudio_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -284,6 +286,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index 137f355..6b46434 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -199,7 +199,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_tensor_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -282,6 +284,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index b4a29b4..9ece655 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -212,7 +212,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_zeppelin_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -295,6 +297,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org