You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/10/15 11:55:29 UTC
[incubator-datalab] branch DATALAB-2409 updated (ebfbcac -> 17e5c20)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from ebfbcac [DATALAB-2409]: replaced yaml.load with yaml.safe_load
new 43353f6 [DATALAB-2409]: some tensor fixes
new 17e5c20 [DATALAB-2409]: fixed merge conflict
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/general/files/azure/tensor_Dockerfile | 1 -
.../src/general/lib/azure/actions_lib.py | 23 ++++++++++++++--------
.../src/general/lib/os/fab.py | 2 ++
.../scripts/azure/common_create_notebook_image.py | 2 ++
.../scripts/azure/deeplearning_configure.py | 6 +++++-
.../src/general/scripts/azure/jupyter_configure.py | 6 +++++-
.../general/scripts/azure/jupyterlab_configure.py | 6 +++++-
.../src/general/scripts/azure/rstudio_configure.py | 6 +++++-
.../src/general/scripts/azure/tensor_configure.py | 6 +++++-
.../general/scripts/azure/zeppelin_configure.py | 6 +++++-
10 files changed, 49 insertions(+), 15 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2409]: some tensor fixes
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 43353f6ff7240f9bc55c799959c56b5c16d61022
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Oct 15 14:53:54 2021 +0300
[DATALAB-2409]: some tensor fixes
---
infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile | 1 -
infrastructure-provisioning/src/general/lib/os/fab.py | 2 ++
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
index 4bb12ee..3ecb0ff 100644
--- a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
@@ -35,7 +35,6 @@ COPY general/templates/os/pyspark_local_template.json /root/templates/
COPY general/templates/os/py3spark_local_template.json /root/templates/
COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
COPY general/templates/os/sparkmagic_config_template.json /root/templates/
-COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/tensorboard.service /root/templates/
COPY general/templates/os/inactive.sh /root/templates/
COPY general/templates/os/inactive.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 6717ece..3d43914 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -949,6 +949,8 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
conn.sudo('pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
+ elif os.environ['application'] != 'tensor':
+ conn.sudo('pip3 install environment_kernels')
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2409]: fixed merge conflict
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 17e5c2098aaed05c16c301c602d35755791979df
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Oct 8 12:42:28 2021 +0300
[DATALAB-2409]: fixed merge conflict
---
.../src/general/lib/azure/actions_lib.py | 23 ++++++++++++++--------
.../scripts/azure/common_create_notebook_image.py | 2 ++
.../scripts/azure/deeplearning_configure.py | 6 +++++-
.../src/general/scripts/azure/jupyter_configure.py | 6 +++++-
.../general/scripts/azure/jupyterlab_configure.py | 6 +++++-
.../src/general/scripts/azure/rstudio_configure.py | 6 +++++-
.../src/general/scripts/azure/tensor_configure.py | 6 +++++-
.../general/scripts/azure/zeppelin_configure.py | 6 +++++-
8 files changed, 47 insertions(+), 14 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 756dd27..09ec650 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1232,15 +1232,22 @@ def remount_azure_disk(creds=False, os_user='', hostname='', keyfile=''):
if creds:
conn.close()
-def ensure_right_mount_paths():
- opt_disk = datalab.fab.conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2]
- if opt_disk not in datalab.fab.conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
- disk_names = datalab.fab.conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n')
+def ensure_right_mount_paths(creds=False, os_user='', hostname='', keyfile=''):
+ if creds:
+ global conn
+ conn = datalab.fab.init_datalab_connection(hostname, os_user, keyfile)
+ else:
+ conn = datalab.fab.conn
+ opt_disk = conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2]
+ if opt_disk not in conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
+ disk_names = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n')
for disk in disk_names:
- if disk != '' and disk not in datalab.fab.conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
- datalab.fab.conn.sudo("umount -l /opt")
- datalab.fab.conn.sudo("mount /dev/{}1 /opt".format(disk))
- datalab.fab.conn.sudo('sed -i "/opt/ s|/dev/{}1|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk))
+ if disk != '' and disk not in conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout:
+ conn.sudo("umount -l /opt")
+ conn.sudo("mount /dev/{}1 /opt".format(disk))
+ conn.sudo('sed -i "/opt/ s|/dev/{}|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk))
+ if creds:
+ conn.close()
def prepare_vm_for_image(creds=False, os_user='', hostname='', keyfile=''):
if creds:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index a80c43e..b7c5ee9 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -97,6 +97,8 @@ if __name__ == "__main__":
image_conf['instance_name'])
datalab.actions_lib.remount_azure_disk(True, image_conf['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(image_conf['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_hostname))
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index ae83807..1849e4b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -206,7 +206,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_deep_learning_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -269,6 +271,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index bc26e72..6f42803 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -191,7 +191,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_jupyter_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -272,6 +274,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
index d8cefb6..077f662 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
@@ -191,7 +191,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_jupyterlab_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -254,6 +256,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 9395f6d..b1e92c5 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -191,7 +191,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_rstudio_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -271,6 +273,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index e3f8fec..4ba5015 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -187,7 +187,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_tensor_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -267,6 +269,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index c2b9b01..21d3902 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -202,7 +202,9 @@ if __name__ == "__main__":
try:
subprocess.run("~/scripts/{}.py {}".format('configure_zeppelin_node', params), shell=True, check=True)
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname,
+ keyfile_name)
except:
traceback.print_exc()
raise Exception
@@ -282,6 +284,8 @@ if __name__ == "__main__":
notebook_config['instance_name'])
datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
keyfile_name)
+ datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'],
+ instance_hostname, keyfile_name)
datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org