You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/04/27 12:48:01 UTC
[incubator-dlab] branch DLAB-515 updated: [DLAB-515]: Notebooks
deploing fixed/Hrrps_proxy added
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-515
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-515 by this push:
new b5ee9e8 [DLAB-515]: Notebooks deploing fixed/Hrrps_proxy added
b5ee9e8 is described below
commit b5ee9e8153e4b06a1d080dc6b267c94fe27dcfc5
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Mon Apr 27 15:47:41 2020 +0300
[DLAB-515]: Notebooks deploing fixed/Hrrps_proxy added
---
infrastructure-provisioning/src/general/conf/dlab.ini | 10 +++++-----
.../src/general/lib/os/debian/common_lib.py | 12 +++++++++++-
.../src/general/lib/os/debian/notebook_lib.py | 10 ++++++++--
infrastructure-provisioning/src/general/lib/os/fab.py | 6 ++++++
.../src/zeppelin/scripts/configure_zeppelin_node.py | 6 ++++++
5 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index b415425..c0b8f43 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -245,7 +245,7 @@ multiple_clusters = false
### R China mirror
r_mirror = http://mirror.lzu.edu.cn/CRAN/
### NVidia driver version for Tensor/DeepLearning notebooks
-nvidia_version = 418.43
+nvidia_version = 440.64.00
### Caffe library version for DeepLearning notebook
caffe_version = 1.0
### Caffe2 library version for DeepLearning notebook
@@ -262,13 +262,13 @@ keras_version = 2.1.6
### Theano library version for Tensor/DeepLearning notebook
theano_version = 1.0.3
### Version of CUDA
-cuda_version = 9.0
+cuda_version = 10.2
### Name of CUDA file
-cuda_file_name = cuda_9.0.176_384.81_linux-run
+cuda_file_name = cuda_10.2.89_440.33.01_linux.run
### Version of CUDNN
-cudnn_version = 7.1.4
+cudnn_version = 7.6.5
### Name of CUDNN file
-cudnn_file_name = cudnn-9.0-linux-x64-v7.1.tgz
+cudnn_file_name = cudnn-10.2-linux-x64-v7.6.5.tgz
### R enabled on Jupyter/Zeppelin notebook
r_enabled = true
### Temporary fixed python libraries due to dependencies
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 870a500..d7e129b 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -44,7 +44,17 @@ def manage_pkg(command, environment, requisites):
time.sleep(10)
else:
allow = True
- sudo('apt-get {0} {1}'.format(command, requisites))
+ sudo('sudo dpkg --configure -a')
+ sudo('sudo apt update')
+ try:
+ sudo('apt-get {0} {1}'.format(command, requisites))
+ except:
+ sudo('lsof /var/lib/dpkg/lock')
+ sudo('lsof /var/lib/apt/lists/lock')
+ sudo('lsof /var/cache/apt/archives/lock')
+ sudo('rm -f /var/lib/apt/lists/lock')
+ sudo('rm -f /var/cache/apt/archives/lock')
+ sudo('rm -f /var/lib/dpkg/lock')
elif environment == 'local':
if local('sudo pgrep "^apt" -a && echo "busy" || echo "ready"', capture=True) == 'busy':
counter += 1
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index b63dd3b..f6af16d 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -118,9 +118,17 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
sudo('gdebi -n rstudio-server-{}-amd64.deb'.format(rstudio_version))
sudo('mkdir -p /mnt/var')
sudo('chown {0}:{0} /mnt/var'.format(os_user))
+ http_proxy = run('echo $http_proxy')
+ https_proxy = run('echo $https_proxy')
sudo("sed -i '/Type=forking/a \Environment=USER=dlab-user' /etc/systemd/system/rstudio-server.service")
sudo("sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
sudo("sed -i '/ExecStart/s|$|\"|g' /etc/systemd/system/rstudio-server.service")
+ sudo(
+ 'sed -i \'/\[Service\]/a Environment=\"HTTP_PROXY={}\"\' /etc/systemd/system/rstudio-server.service'.format(
+ http_proxy))
+ sudo(
+ 'sed -i \'/\[Service\]/a Environment=\"HTTPS_PROXY={}\"\' /etc/systemd/system/rstudio-server.service'.format(
+ https_proxy))
sudo("systemctl daemon-reload")
sudo('touch /home/{}/.Renviron'.format(os_user))
sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user))
@@ -128,8 +136,6 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
sudo('touch /home/{}/.Rprofile'.format(os_user))
sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user))
sudo('''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''.format(os_user))
- http_proxy = run('echo $http_proxy')
- https_proxy = run('echo $https_proxy')
sudo('''echo 'Sys.setenv(http_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(http_proxy, os_user))
sudo('''echo 'Sys.setenv(https_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(https_proxy, os_user))
sudo('rstudio-server start')
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 03b1d85..54cfe1c 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -175,6 +175,12 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
"/caffe/python:/home/" + os_user + "/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
sudo("sed -i 's|CONF_PATH|{}|' /tmp/jupyter-notebook.service".format(jupyter_conf_file))
sudo("sed -i 's|OS_USR|{}|' /tmp/jupyter-notebook.service".format(os_user))
+ http_proxy = run('echo $http_proxy')
+ https_proxy = run('echo $https_proxy')
+ sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\' /tmp/jupyter-notebook.service'.format(
+ http_proxy))
+ sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/jupyter-notebook.service'.format(
+ https_proxy))
sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
sudo('mkdir -p /mnt/var')
diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index fda8b1f..178f457 100644
--- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -106,6 +106,12 @@ def configure_zeppelin(os_user):
try:
put(templates_dir + 'zeppelin-notebook.service', '/tmp/zeppelin-notebook.service')
sudo("sed -i 's|OS_USR|" + os_user + "|' /tmp/zeppelin-notebook.service")
+ http_proxy = run('echo $http_proxy')
+ https_proxy = run('echo $https_proxy')
+ sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\' /tmp/zeppelin-notebook.service'.format(
+ http_proxy))
+ sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/zeppelin-notebook.service'.format(
+ https_proxy))
sudo("chmod 644 /tmp/zeppelin-notebook.service")
sudo('cp /tmp/zeppelin-notebook.service /etc/systemd/system/zeppelin-notebook.service')
sudo('chown ' + os_user + ':' + os_user + ' -R /opt/zeppelin/')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org