You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/05/20 14:47:33 UTC

[incubator-dlab] branch DLAB-433 updated: [DLAB-433] Demo-version

This is an automated email from the ASF dual-hosted git repository.

dmysakovets pushed a commit to branch DLAB-433
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-433 by this push:
     new 1e07736  [DLAB-433] Demo-version
1e07736 is described below

commit 1e07736a2fb8e1fe1903143a82109618c190eb1e
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Mon May 20 17:47:24 2019 +0300

    [DLAB-433] Demo-version
---
 .../src/general/lib/os/fab.py                      | 114 ++++++++++----------
 .../src/jupyter-docker/Dockerfile_jupyter          | 117 +--------------------
 .../src/jupyter-docker/jupyter_run.sh              |   2 +-
 .../scripts/configure_jupyter-docker_node.py       |   2 +-
 4 files changed, 60 insertions(+), 175 deletions(-)

diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index a0edd3c..016c010 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -218,65 +218,65 @@ def configure_docker(os_user, http_file, https_file):
         print('Failed to configure Docker:', str(err))
         sys.exit(1)
 
-def ensure_jupyter_docker_files(os_user, legion_dir, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name):
+def ensure_jupyter_docker_files(os_user, jupyter_dir, jupyter_conf_file, docker_jupyter_conf, exploratory_name):
     if not exists(jupyter_dir):
         try:
-           sudo('mkdir {}'.format(jupyter_dir))
-           put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
-           put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
-           put('/root/Dockerfile_jupyter', '/tmp/Dockerfile_jupyter')
-           put('/root/jupyter_run.sh', '/tmp/jupyter_run.sh')
-           sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_template.json')
-           sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_template.json')
-           sudo('sed -i \'s/3.5/3.6/g\' {}py3spark_local_template.json'.format(jupyter_dir))
-           sudo('mv /tmp/jupyter_run.sh {}jupyter_run.sh'.format(jupyter_dir))
-           sudo('mv /tmp/Dockerfile_jupyter {}Dockerfile_jupyter'.format(jupyter_dir))
-           sudo('sed -i \'s/nb_user/{}/g\' {}Dockerfile_jupyter'.format(os_user, jupyter_dir))
-           sudo('sed -i \'s/jup_version/{}/g\' {}Dockerfile_jupyter'.format(jupyter_version, jupyter_dir))
-           sudo('sed -i \'s/hadoop_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_hadoop_version'], jupyter_dir))
-           sudo('sed -i \'s/tornado_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_tornado_version'], jupyter_dir))
-           sudo('sed -i \'s/matplotlib_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_matplotlib_version'], jupyter_dir))
-           sudo('sed -i \'s/numpy_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_numpy_version'], jupyter_dir))
-           sudo('sed -i \'s/spark_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_spark_version'], jupyter_dir))
-           sudo('sed -i \'s/scala_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_scala_version'], jupyter_dir))
-           sudo('sed -i \'s/CONF_PATH/{}/g\' {}jupyter_run.sh'.format(docker_jupyter_conf, jupyter_dir))
-           sudo('touch {}'.format(jupyter_conf_file))
-           sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
-           sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
-           sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
-           sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
-           sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
-           sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
-           sudo('touch {}'.format(spark_script))
-           sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
-           sudo(
-               'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/pyspark_local_template.json" >> {}'.format(
-               spark_script))
-           sudo(
-               'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
-                   spark_script))
-           sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/pyspark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
-           sudo(
-               'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/py3spark_local_template.json" >> {}'.format(
-               spark_script))
-           sudo(
-               'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
-                   spark_script))
-           sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/py3spark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
-           sudo('echo "cp /tmp/pyspark_local_template.json /home/{}/.local/share/jupyter/kernels/pyspark_local/kernel.json" >> {}'.format(os_user, spark_script))
-           sudo(
-               'echo "cp /tmp/py3spark_local_template.json /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" >> {}'.format(
-                   os_user, spark_script))
-           sudo('git clone https://github.com/legion-platform/legion.git')
-           sudo('cp {}sdk/Pipfile {}sdk_Pipfile'.format(legion_dir, jupyter_dir))
-           sudo('cp {}sdk/Pipfile.lock {}sdk_Pipfile.lock'.format(legion_dir, jupyter_dir))
-           sudo('cp {}toolchains/python/Pipfile {}toolchains_Pipfile'.format(legion_dir, jupyter_dir))
-           sudo('cp {}toolchains/python/Pipfile.lock {}toolchains_Pipfile.lock'.format(legion_dir, jupyter_dir))
-           sudo('cp {}cli/Pipfile {}cli_Pipfile'.format(legion_dir, jupyter_dir))
-           sudo('cp {}cli/Pipfile.lock {}cli_Pipfile.lock'.format(legion_dir, jupyter_dir))
-           sudo('cp -r {}sdk {}sdk'.format(legion_dir, jupyter_dir))
-           sudo('cp -r {}toolchains/python {}toolchains_python'.format(legion_dir, jupyter_dir))
-           sudo('cp -r {}cli {}cli'.format(legion_dir, jupyter_dir))
+            sudo('mkdir {}'.format(jupyter_dir))
+#            put(templates_dir + 'pyspark_local_template.json', '/tmp/pyspark_local_template.json')
+#            put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
+            put('/root/Dockerfile_jupyter', '/tmp/Dockerfile_jupyter')
+            put('/root/jupyter_run.sh', '/tmp/jupyter_run.sh')
+#            sudo('\cp /tmp/pyspark_local_template.json ' + jupyter_dir + 'pyspark_local_template.json')
+#            sudo('\cp /tmp/py3spark_local_template.json ' + jupyter_dir + 'py3spark_local_template.json')
+#            sudo('sed -i \'s/3.5/3.6/g\' {}py3spark_local_template.json'.format(jupyter_dir))
+            sudo('mv /tmp/jupyter_run.sh {}jupyter_run.sh'.format(jupyter_dir))
+            sudo('mv /tmp/Dockerfile_jupyter {}Dockerfile_jupyter'.format(jupyter_dir))
+            sudo('sed -i \'s/nb_user/{}/g\' {}Dockerfile_jupyter'.format(os_user, jupyter_dir))
+#            sudo('sed -i \'s/jup_version/{}/g\' {}Dockerfile_jupyter'.format(jupyter_version, jupyter_dir))
+#            sudo('sed -i \'s/hadoop_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_hadoop_version'], jupyter_dir))
+#            sudo('sed -i \'s/tornado_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_tornado_version'], jupyter_dir))
+#            sudo('sed -i \'s/matplotlib_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_matplotlib_version'], jupyter_dir))
+#            sudo('sed -i \'s/numpy_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_numpy_version'], jupyter_dir))
+#            sudo('sed -i \'s/spark_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_spark_version'], jupyter_dir))
+#            sudo('sed -i \'s/scala_version/{}/g\' {}Dockerfile_jupyter'.format(os.environ['notebook_scala_version'], jupyter_dir))
+            sudo('sed -i \'s/CONF_PATH/{}/g\' {}jupyter_run.sh'.format(docker_jupyter_conf, jupyter_dir))
+            sudo('touch {}'.format(jupyter_conf_file))
+            sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
+            sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
+            sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
+            sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
+            sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
+            sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+#            sudo('touch {}'.format(spark_script))
+#            sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
+#            sudo(
+#                'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/pyspark_local_template.json" >> {}'.format(
+#                spark_script))
+#            sudo(
+#                'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
+#                    spark_script))
+#            sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/pyspark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
+#            sudo(
+#                'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/py3spark_local_template.json" >> {}'.format(
+#                spark_script))
+#            sudo(
+#                'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
+#                    spark_script))
+#            sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/py3spark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
+#            sudo('echo "cp /tmp/pyspark_local_template.json /home/{}/.local/share/jupyter/kernels/pyspark_local/kernel.json" >> {}'.format(os_user, spark_script))
+#            sudo(
+#                'echo "cp /tmp/py3spark_local_template.json /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" >> {}'.format(
+#                    os_user, spark_script))
+#            sudo('git clone https://github.com/legion-platform/legion.git')
+#            sudo('cp {}sdk/Pipfile {}sdk_Pipfile'.format(legion_dir, jupyter_dir))
+#            sudo('cp {}sdk/Pipfile.lock {}sdk_Pipfile.lock'.format(legion_dir, jupyter_dir))
+#            sudo('cp {}toolchains/python/Pipfile {}toolchains_Pipfile'.format(legion_dir, jupyter_dir))
+#            sudo('cp {}toolchains/python/Pipfile.lock {}toolchains_Pipfile.lock'.format(legion_dir, jupyter_dir))
+#            sudo('cp {}cli/Pipfile {}cli_Pipfile'.format(legion_dir, jupyter_dir))
+#            sudo('cp {}cli/Pipfile.lock {}cli_Pipfile.lock'.format(legion_dir, jupyter_dir))
+#            sudo('cp -r {}sdk {}sdk'.format(legion_dir, jupyter_dir))
+#            sudo('cp -r {}toolchains/python {}toolchains_python'.format(legion_dir, jupyter_dir))
+#            sudo('cp -r {}cli {}cli'.format(legion_dir, jupyter_dir))
         except:
            sys.exit(1)
     else:
diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
index df5203b..ee7c5ea 100644
--- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
+++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter
@@ -19,127 +19,12 @@
 #
 # ******************************************************************************
 
-FROM ubuntu:18.04
+FROM legionplatformtest/python-toolchain:1.0.0-rc11
 
 ARG NB_USER="nb_user"
-ENV LEGION_DIR="/opt/legion"
-ENV CONF_FILE="/home/nb_user/.local/share/jupyter/jupyter_notebook_config.py"
-ENV PATH /usr/local/bin:$PATH
-ENV LANG C.UTF-8
-ENV DEBIAN_FRONTEND=noninteractive
-
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-                tk-dev wget curl gnupg dirmngr build-essential openssl \
-                libreadline-gplv2-dev libncursesw5-dev libssl-dev \
-                libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev \
-                python python-dev python-pip python-virtualenv \
-                python3 python3-dev python3-pip default-jre default-jdk \
-  && rm -rf /var/lib/apt/lists/*
-
 
 RUN useradd -u 1001 -ms /bin/bash $NB_USER
 
-
-#Installing Python libraries
-RUN pip install --upgrade setuptools \
-  && pip install tornado==tornado_version ipython ipython matplotlib==matplotlib_version \
-   boto3 fabvenv fabric-virtualenv future NumPy==numpy_version \
-   SciPy pandas Sympy Pillow sklearn pyopenssl --no-cache-dir \
-  && pip3 install --upgrade setuptools \
-  && pip3 install tornado==tornado_version ipython ipython matplotlib==matplotlib_version \
-   boto3 fabvenv fabric-virtualenv future NumPy==numpy_version \
-   SciPy pandas Sympy Pillow sklearn --no-cache-dir
-
-
-
-#Installing SBT
-# Env variables
-ENV SCALA_VERSION scala_version
-ENV SBT_VERSION 1.2.8
-
-# Install Scala
-## Piping curl directly in tar
-RUN \
-  curl -fsL https://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz | tar xfz - -C /root/ && \
-  echo >> /root/.bashrc && \
-  echo "export PATH=~/scala-$SCALA_VERSION/bin:$PATH" >> /root/.bashrc
-
-# Install sbt
-RUN \
-  curl -L -o sbt-$SBT_VERSION.deb https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb && \
-  dpkg -i sbt-$SBT_VERSION.deb && \
-  rm sbt-$SBT_VERSION.deb && \
-  apt-get update && \
-  apt-get install sbt && \
-  sbt sbtVersion && \
-  mkdir project && \
-  echo "scalaVersion := \"${SCALA_VERSION}\"" > build.sbt && \
-  echo "sbt.version=${SBT_VERSION}" > project/build.properties && \
-  echo "case object Temp" > Temp.scala && \
-  sbt compile && \
-rm -r project && rm build.sbt && rm Temp.scala && rm -r target
-
-
-
-#Installing Jupyter
-RUN pip install notebook==jup_version --no-cache-dir \
-  && pip install jupyter --no-cache-dir \
-  && pip3 install notebook==jup_version --no-cache-dir \
-  && pip3 install jupyter --no-cache-dir \
-  && rm -rf $CONF_FILE \
-  && mkdir -p /home/$NB_USER/.jupyter/custom/ \
-  && echo "#notebook-container { width: auto; }" > /home/$NB_USER/.jupyter/custom/custom.css \
-  && mkdir -p /mnt/var \
-  && chown $NB_USER:$NB_USER /mnt/var \
-  && jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists" \
-  && jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"
-COPY jupyter_notebook_config.py /home/$NB_USER/.local/share/jupyter/jupyter_notebook_config.py
-
-#Intsalling local Spark
-RUN wget https://archive.apache.org/dist/spark/spark-spark_version/spark-spark_version-bin-hadoophadoop_version.tgz -O \
-  /tmp/spark-spark_version-bin-hadoophadoop_version.tgz \
-  && tar -zxvf /tmp/spark-spark_version-bin-hadoop2.7.tgz -C /opt/ \
-  && mv /opt/spark-spark_version-bin-hadoop2.7 /opt/spark \
-  && mkdir -p /opt/jars/ \
-  && wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.4/hadoop-aws-2.7.4.jar -O \
-  /opt/jars/hadoop-aws-2.7.4.jar \
-  && wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar -O \
-  /opt/jars/aws-java-sdk-1.7.4.jar \
-  && wget https://maven.twttr.com/com/hadoop/gplcompression/hadoop-lzo/0.4.20/hadoop-lzo-0.4.20.jar -O \
-  /opt/jars/hadoop-lzo-0.4.20.jar
-
-
-#Installing local kernel
-RUN mkdir -p /home/$NB_USER/.local/share/jupyter/kernels/py3spark_local/ \
-   /home/$NB_USER/.local/share/jupyter/kernels/pyspark_local/
-COPY pyspark_local_template.json /tmp/pyspark_local_template.json
-COPY py3spark_local_template.json /tmp/py3spark_local_template.json
-COPY spark.sh /tmp/spark.sh
-RUN chmod +x /tmp/spark.sh
-RUN /tmp/spark.sh  #this script is needed because docker is not able to handle one of the commands
-
-#Install Legion
-RUN pip3 install --disable-pip-version-check --upgrade pip==18.1 pipenv==2018.10.13 ptvsd==4.2.2
-COPY sdk_Pipfile "${LEGION_DIR}/legion/sdk/Pipfile"
-COPY sdk_Pipfile.lock "${LEGION_DIR}/legion/sdk/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/sdk && pipenv install --system --three
-COPY toolchains_Pipfile "${LEGION_DIR}/legion/toolchains/python/Pipfile"
-COPY toolchains_Pipfile.lock "${LEGION_DIR}/legion/toolchains/python/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/toolchains/python && pipenv install --system --three
-COPY cli_Pipfile "${LEGION_DIR}/legion/cli/Pipfile"
-COPY cli_Pipfile.lock "${LEGION_DIR}/legion/cli/Pipfile.lock"
-RUN cd ${LEGION_DIR}/legion/cli && pipenv install --system --three
-
-COPY sdk "${LEGION_DIR}/legion/sdk"
-COPY toolchains_python "${LEGION_DIR}/legion/toolchains/python"
-COPY cli "${LEGION_DIR}/legion/cli"
-
-RUN pip3 install --no-deps -e $LEGION_DIR/legion/sdk && \
-    pip3 install --no-deps -e $LEGION_DIR/legion/toolchains/python && \
-    pip3 install --no-deps -e $LEGION_DIR/legion/cli
-
-
 EXPOSE 8888
 
 #Prepearing Start script
diff --git a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
index 04dab7f..55c2245 100644
--- a/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
+++ b/infrastructure-provisioning/src/jupyter-docker/jupyter_run.sh
@@ -21,7 +21,7 @@
 #
 # ******************************************************************************
 
-jupyter notebook --config CONF_PATH
+jupyter lab --config CONF_PATH
 
 if [[$1 == '-d']];then
         while true; do sleep 1000; done
diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
index 9fdb5fd..27e1d63 100644
--- a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
+++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py
@@ -105,7 +105,7 @@ if __name__ == "__main__":
 
     # CONFIGURE JUPYTER FILES
     print("Configure jupyter files")
-    ensure_jupyter_docker_files(args.os_user, legion_dir, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name)
+    ensure_jupyter_docker_files(args.os_user,  jupyter_dir, jupyter_conf_file, docker_jupyter_conf, args.exploratory_name)
 
     # INSTALL UNGIT
     print("Install nodejs")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org