You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/05/14 07:44:27 UTC
[incubator-dlab] branch DLAB-433 updated: [DLAB-623] Removed Spark
installation and added it to Dockerfile_jupyter [bugfixing and refactoring]
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-433
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-433 by this push:
new b08f43a [DLAB-623] Removed Spark installation and added it to Dockerfile_jupyter [bugfixing and refactoring]
b08f43a is described below
commit b08f43a5a20b891ce62c3ead27563160a90af3f3
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Tue May 14 10:44:14 2019 +0300
[DLAB-623] Removed Spark installation and added it to Dockerfile_jupyter [bugfixing and refactoring]
---
infrastructure-provisioning/src/general/lib/os/fab.py | 7 +------
.../src/general/scripts/os/jupyter_container_start.py | 3 +--
infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter | 2 +-
.../src/jupyter-docker/scripts/configure_jupyter-docker_node.py | 4 ++--
4 files changed, 5 insertions(+), 11 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index fb21896..3d2894a 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -213,17 +213,12 @@ def configure_docker(os_user, http_file, https_file):
sudo('usermod -a -G docker ' + os_user)
sudo('update-rc.d docker defaults')
sudo('update-rc.d docker enable')
- sudo('wget https://raw.githubusercontent.com/CWSpear/local-persist/master/scripts/install.sh && chmod +x install.sh')
- sudo('sed -i "66s/curl/sudo curl/g" install.sh')
- sudo('sed -i "s/sudo curl/sudo -E curl/g" install.sh')
- run('sudo -E ./install.sh')
- sudo('rm install.sh')
sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user))
except Exception as err:
print('Failed to configure Docker:', str(err))
sys.exit(1)
-def ensure_jupyter_docker_files(os_user, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name):
+def ensure_jupyter_docker_files(os_user, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name):
if not exists(jupyter_dir):
try:
sudo('mkdir {}'.format(jupyter_dir))
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
index 29f146a..1412e1c 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py
@@ -38,9 +38,8 @@ jupyter_dir = '/home/' + args.os_user + '/.jupyter/'
def start_jupyter_container(jupyter_dir):
try:
with cd('{}'.format(jupyter_dir)):
- run('docker volume create -d local-persist -o mountpoint=/opt --name=jup_volume')
run('docker build --file /home/{}/.jupyter/Dockerfile_jupyter -t jupyter-notebook .'.format(args.os_user))
- run('docker run -d -p 8888:8888 -v jup_volume:/opt/ jupyter-notebook:latest')
+ run('docker run -d -p 8888:8888 jupyter-notebook:latest')
except: sys.exit(1)
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
index 77d3048..9582721 100644
--- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -29,7 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
- tk-dev wget gnupg dirmngr build-essential openssl \
+ tk-dev wget curl gnupg dirmngr build-essential openssl \
libreadline-gplv2-dev libncursesw5-dev libssl-dev \
libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev \
python python-dev python-pip python-virtualenv \
diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
index aee28e8..e0ebe7f 100644
--- a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
@@ -52,11 +52,11 @@ else:
spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
-spark_script = jupyter_dir + 'spark.sh'
docker_version = '18.09.4'
http_file = '/etc/systemd/system/docker.service.d/http-proxy.conf'
https_file = '/etc/systemd/system/docker.service.d/https-proxy.conf'
jupyter_dir = '/home/' + args.os_user + '/.jupyter/'
+spark_script = jupyter_dir + 'spark.sh'
pyspark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/pyspark_local/'
py3spark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/py3spark_local/'
jupyter_conf_file = jupyter_dir + 'jupyter_notebook_config.py'
@@ -104,7 +104,7 @@ if __name__ == "__main__":
# CONFIGURE JUPYTER FILES
print("Configure jupyter files")
- ensure_jupyter_docker_files(args.os_user, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name)
+ ensure_jupyter_docker_files(args.os_user, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name)
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
# print("Install local Spark")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org