You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/05/20 14:47:33 UTC
[incubator-dlab] branch DLAB-433 updated: [DLAB-433] Demo-version
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-433
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-433 by this push:
new 1e07736 [DLAB-433] Demo-version
1e07736 is described below
commit 1e07736a2fb8e1fe1903143a82109618c190eb1e
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Mon May 20 17:47:24 2019 +0300
[DLAB-433] Demo-version
---
.../src/general/lib/os/fab.py | 114 ++++++++++----------
.../src/jupyter-docker/Dockerfile_jupyter | 117 +--------------------
.../src/jupyter-docker/jupyter_run.sh | 2 +-
.../scripts/configure_jupyter-docker_node.py | 2 +-
4 files changed, 60 insertions(+), 175 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index a0edd3c..016c010 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -218,65 +218,65 @@ def configure_docker(os_user, http_file, https_file):
print('Failed to configure Docker:', str(err))
sys.exit(1)
-def ensure_jupyter_docker_files(os_user, legion_dir, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name):
+def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, exploratory_name):
if not exists(jupyter_dir):
try:
- sudo('mkdir {}'.format(jupyter_dir))
- put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
- put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
- put('/root/Dockerfile_jupyter', '/tmp/Dockerfile_jupyter')
- put('/root/jupyter_run.sh', '/tmp/jupyter_run.sh')
- sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_template.json')
- sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_template.json')
- sudo('sed -i \'s/3.5/3.6/g\' {}py3spark_local_template.json'.format(jupyter_dir))
- sudo('mv /tmp/jupyter_run.sh {}jupyter_run.sh'.format(jupyter_dir))
- sudo('mv /tmp/Dockerfile_jupyter {}Dockerfile_jupyter'.format(jupyter_dir))
- sudo('sed -i \'s/nb_user/{}/g\' {}Dockerfile_jupyter'.format(os_user, jupyter_dir))
- sudo('sed -i \'s/jup_version/{}/g\' {}Dockerfile_jupyter'.format(jupyter_version, jupyter_dir))
- sudo('sed -i \'s/hadoop_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_hadoop_version'], jupyter_dir))
- sudo('sed -i \'s/tornado_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_tornado_version'], jupyter_dir))
- sudo('sed -i \'s/matplotlib_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_matplotlib_version'], jupyter_dir))
- sudo('sed -i \'s/numpy_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_numpy_version'], jupyter_dir))
- sudo('sed -i \'s/spark_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_spark_version'], jupyter_dir))
- sudo('sed -i \'s/scala_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_scala_version'], jupyter_dir))
- sudo('sed -i \'s/CONF_PATH/{}/g\' {}jupyter_run.sh'.format(docker_jupyter_conf, jupyter_dir))
- sudo('touch {}'.format(jupyter_conf_file))
- sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
- sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
- sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
- sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
- sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
- sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
- sudo('touch {}'.format(spark_script))
- sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
- sudo(
- 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/pyspark_local_template.json" >> {}'.format(
- spark_script))
- sudo(
- 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
- spark_script))
- sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/pyspark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
- sudo(
- 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/py3spark_local_template.json" >> {}'.format(
- spark_script))
- sudo(
- 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
- spark_script))
- sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/py3spark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
- sudo('echo "cp /tmp/pyspark_local_template.json /home/{}/.local/share/jupyter/kernels/pyspark_local/kernel.json" >> {}'.format(os_user, spark_script))
- sudo(
- 'echo "cp /tmp/py3spark_local_template.json /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" >> {}'.format(
- os_user, spark_script))
- sudo('git clone https://github.com/legion-platform/legion.git')
- sudo('cp {}sdk/Pipfile {}sdk_Pipfile'.format(legion_dir, jupyter_dir))
- sudo('cp {}sdk/Pipfile.lock {}sdk_Pipfile.lock'.format(legion_dir, jupyter_dir))
- sudo('cp {}toolchains/python/Pipfile {}toolchains_Pipfile'.format(legion_dir, jupyter_dir))
- sudo('cp {}toolchains/python/Pipfile.lock {}toolchains_Pipfile.lock'.format(legion_dir, jupyter_dir))
- sudo('cp {}cli/Pipfile {}cli_Pipfile'.format(legion_dir, jupyter_dir))
- sudo('cp {}cli/Pipfile.lock {}cli_Pipfile.lock'.format(legion_dir, jupyter_dir))
- sudo('cp -r {}sdk {}sdk'.format(legion_dir, jupyter_dir))
- sudo('cp -r {}toolchains/python {}toolchains_python'.format(legion_dir, jupyter_dir))
- sudo('cp -r {}cli {}cli'.format(legion_dir, jupyter_dir))
+ sudo('mkdir {}'.format(jupyter_dir))
+# put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
+# put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
+ put('/root/Dockerfile_jupyter', '/tmp/Dockerfile_jupyter')
+ put('/root/jupyter_run.sh', '/tmp/jupyter_run.sh')
+# sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_template.json')
+# sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_template.json')
+# sudo('sed -i \'s/3.5/3.6/g\' {}py3spark_local_template.json'.format(jupyter_dir))
+ sudo('mv /tmp/jupyter_run.sh {}jupyter_run.sh'.format(jupyter_dir))
+ sudo('mv /tmp/Dockerfile_jupyter {}Dockerfile_jupyter'.format(jupyter_dir))
+ sudo('sed -i \'s/nb_user/{}/g\' {}Dockerfile_jupyter'.format(os_user, jupyter_dir))
+# sudo('sed -i \'s/jup_version/{}/g\' {}Dockerfile_jupyter'.format(jupyter_version, jupyter_dir))
+# sudo('sed -i \'s/hadoop_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_hadoop_version'], jupyter_dir))
+# sudo('sed -i \'s/tornado_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_tornado_version'], jupyter_dir))
+# sudo('sed -i \'s/matplotlib_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_matplotlib_version'], jupyter_dir))
+# sudo('sed -i \'s/numpy_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_numpy_version'], jupyter_dir))
+# sudo('sed -i \'s/spark_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_spark_version'], jupyter_dir))
+# sudo('sed -i \'s/scala_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_scala_version'], jupyter_dir))
+ sudo('sed -i \'s/CONF_PATH/{}/g\' {}jupyter_run.sh'.format(docker_jupyter_conf, jupyter_dir))
+ sudo('touch {}'.format(jupyter_conf_file))
+ sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
+ sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
+ sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
+ sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
+ sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
+ sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+# sudo('touch {}'.format(spark_script))
+# sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
+# sudo(
+# 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/pyspark_local_template.json" >> {}'.format(
+# spark_script))
+# sudo(
+# 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
+# spark_script))
+# sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/pyspark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
+# sudo(
+# 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/py3spark_local_template.json" >> {}'.format(
+# spark_script))
+# sudo(
+# 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
+# spark_script))
+# sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/py3spark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
+# sudo('echo "cp /tmp/pyspark_local_template.json /home/{}/.local/share/jupyter/kernels/pyspark_local/kernel.json" >> {}'.format(os_user, spark_script))
+# sudo(
+# 'echo "cp /tmp/py3spark_local_template.json /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" >> {}'.format(
+# os_user, spark_script))
+# sudo('git clone https://github.com/legion-platform/legion.git')
+# sudo('cp {}sdk/Pipfile {}sdk_Pipfile'.format(legion_dir, jupyter_dir))
+# sudo('cp {}sdk/Pipfile.lock {}sdk_Pipfile.lock'.format(legion_dir, jupyter_dir))
+# sudo('cp {}toolchains/python/Pipfile {}toolchains_Pipfile'.format(legion_dir, jupyter_dir))
+# sudo('cp {}toolchains/python/Pipfile.lock {}toolchains_Pipfile.lock'.format(legion_dir, jupyter_dir))
+# sudo('cp {}cli/Pipfile {}cli_Pipfile'.format(legion_dir, jupyter_dir))
+# sudo('cp {}cli/Pipfile.lock {}cli_Pipfile.lock'.format(legion_dir, jupyter_dir))
+# sudo('cp -r {}sdk {}sdk'.format(legion_dir, jupyter_dir))
+# sudo('cp -r {}toolchains/python {}toolchains_python'.format(legion_dir, jupyter_dir))
+# sudo('cp -r {}cli {}cli'.format(legion_dir, jupyter_dir))
except:
sys.exit(1)
else:
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
index df5203b..ee7c5ea 100644
--- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -19,127 +19,12 @@
#
# ******************************************************************************
-FROM ubuntu:18.04
+FROM legionplatformtest/python-toolchain:1.0.0-rc11
ARG NB_USER="nb_user"
-ENV LEGION_DIR="/opt/legion"
-ENV CONF_FILE="/home/nb_user/.local/share/jupyter/jupyter_notebook_config.py"
-ENV PATH /usr/local/bin:$PATH
-ENV LANG C.UTF-8
-ENV DEBIAN_FRONTEND=noninteractive
-
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
- tk-dev wget curl gnupg dirmngr build-essential openssl \
- libreadline-gplv2-dev libncursesw5-dev libssl-dev \
- libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev \
- python python-dev python-pip python-virtualenv \
- python3 python3-dev python3-pip default-jre default-jdk \
- && rm -rf /var/lib/apt/lists/*
-
RUN useradd -u 1001 -ms /bin/bash $NB_USER
-
-#Installing Python libraries
-RUN pip install --upgrade setuptools \
- && pip install tornado==tornado_version ipython ipython matplotlib==matplotlib_version \
- boto3 fabvenv fabric-virtualenv future NumPy==numpy_version \
- SciPy pandas Sympy Pillow sklearn pyopenssl --no-cache-dir \
- && pip3 install --upgrade setuptools \
- && pip3 install tornado==tornado_version ipython ipython matplotlib==matplotlib_version \
- boto3 fabvenv fabric-virtualenv future NumPy==numpy_version \
- SciPy pandas Sympy Pillow sklearn --no-cache-dir
-
-
-
-#Installing SBT
-# Env variables
-ENV SCALA_VERSION scala_version
-ENV SBT_VERSION 1.2.8
-
-# Install Scala
-## Piping curl directly in tar
-RUN \
- curl -fsL https://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz | tar xfz - -C /root/ && \
- echo >> /root/.bashrc && \
- echo "export PATH=~/scala-$SCALA_VERSION/bin:$PATH" >> /root/.bashrc
-
-# Install sbt
-RUN \
- curl -L -o sbt-$SBT_VERSION.deb https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb && \
- dpkg -i sbt-$SBT_VERSION.deb && \
- rm sbt-$SBT_VERSION.deb && \
- apt-get update && \
- apt-get install sbt && \
- sbt sbtVersion && \
- mkdir project && \
- echo "scalaVersion := \"${SCALA_VERSION}\"" > build.sbt && \
- echo "sbt.version=${SBT_VERSION}" > project/build.properties && \
- echo "case object Temp" > Temp.scala && \
- sbt compile && \
-rm -r project && rm build.sbt && rm Temp.scala && rm -r target
-
-
-
-#Installing Jupyter
-RUN pip install notebook==jup_version --no-cache-dir \
- && pip install jupyter --no-cache-dir \
- && pip3 install notebook==jup_version --no-cache-dir \
- && pip3 install jupyter --no-cache-dir \
- && rm -rf $CONF_FILE \
- && mkdir -p /home/$NB_USER/.jupyter/custom/ \
- && echo "#notebook-container { width: auto; }" > /home/$NB_USER/.jupyter/custom/custom.css \
- && mkdir -p /mnt/var \
- && chown $NB_USER:$NB_USER /mnt/var \
- && jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists" \
- && jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"
-COPY jupyter_notebook_config.py /home/$NB_USER/.local/share/jupyter/jupyter_notebook_config.py
-
-#Intsalling local Spark
-RUN wget https://archive.apache.org/dist/spark/spark-spark_version/spark-spark_version-bin-hadoophadoop_version.tgz -O \
- /tmp/spark-spark_version-bin-hadoophadoop_version.tgz \
- && tar -zxvf /tmp/spark-spark_version-bin-hadoop2.7.tgz -C /opt/ \
- && mv /opt/spark-spark_version-bin-hadoop2.7 /opt/spark \
- && mkdir -p /opt/jars/ \
- && wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.4/hadoop-aws-2.7.4.jar -O \
- /opt/jars/hadoop-aws-2.7.4.jar \
- && wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar -O \
- /opt/jars/aws-java-sdk-1.7.4.jar \
- && wget https://maven.twttr.com/com/hadoop/gplcompression/hadoop-lzo/0.4.20/hadoop-lzo-0.4.20.jar -O \
- /opt/jars/hadoop-lzo-0.4.20.jar
-
-
-#Installing local kernel
-RUN mkdir -p /home/$NB_USER/.local/share/jupyter/kernels/py3spark_local/ \
- /home/$NB_USER/.local/share/jupyter/kernels/pyspark_local/
-COPY pyspark_local_template.json /tmp/pyspark_local_template.json
-COPY py3spark_local_template.json /tmp/py3spark_local_template.json
-COPY spark.sh /tmp/spark.sh
-RUN chmod +x /tmp/spark.sh
-RUN /tmp/spark.sh #this script is needed because docker is not able to handle one of the commands
-
-#Install Legion
-RUN pip3 install --disable-pip-version-check --upgrade pip==18.1 pipenv==2018.10.13 ptvsd==4.2.2
-COPY sdk_Pipfile "${LEGION_DIR}/legion/sdk/Pipfile"
-COPY sdk_Pipfile.lock "${LEGION_DIR}/legion/sdk/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/sdk && pipenv install --system --three
-COPY toolchains_Pipfile "${LEGION_DIR}/legion/toolchains/python/Pipfile"
-COPY toolchains_Pipfile.lock "${LEGION_DIR}/legion/toolchains/python/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/toolchains/python && pipenv install --system --three
-COPY cli_Pipfile "${LEGION_DIR}/legion/cli/Pipfile"
-COPY cli_Pipfile.lock "${LEGION_DIR}/legion/cli/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/cli && pipenv install --system --three
-
-COPY sdk "${LEGION_DIR}/legion/sdk"
-COPY toolchains_python "${LEGION_DIR}/legion/toolchains/python"
-COPY cli "${LEGION_DIR}/legion/cli"
-
-RUN pip3 install --no-deps -e $LEGION_DIR/legion/sdk && \
- pip3 install --no-deps -e $LEGION_DIR/legion/toolchains/python && \
- pip3 install --no-deps -e $LEGION_DIR/legion/cli
-
-
EXPOSE 8888
#Prepearing Start script
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
index 04dab7f..55c2245 100644
--- a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
+++ b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
@@ -21,7 +21,7 @@
#
# ******************************************************************************
-jupyter notebook --config CONF_PATH
+jupyter lab --config CONF_PATH
if [[$1 == '-d']];then
while true; do sleep 1000; done
diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
index 9fdb5fd..27e1d63 100644
--- a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
@@ -105,7 +105,7 @@ if __name__ == "__main__":
# CONFIGURE JUPYTER FILES
print("Configure jupyter files")
- ensure_jupyter_docker_files(args.os_user, legion_dir, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name)
+ ensure_jupyter_docker_files(args.os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, args.exploratory_name)
# INSTALL UNGIT
print("Install nodejs")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org