You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2016/03/08 18:30:23 UTC
incubator-toree git commit: Adding binder support to simplify
consumption of examples and play with Toree
Repository: incubator-toree
Updated Branches:
refs/heads/master d7ddc08ea -> 463f8ac48
Adding binder support to simplify consumption of examples and play with Toree
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/463f8ac4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/463f8ac4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/463f8ac4
Branch: refs/heads/master
Commit: 463f8ac48c3ea54ded64c4c5429b0f747d40572a
Parents: d7ddc08
Author: Gino Bustelo <lb...@apache.org>
Authored: Mon Mar 7 10:36:22 2016 -0600
Committer: Gino Bustelo <lb...@apache.org>
Committed: Tue Mar 8 10:04:43 2016 -0600
----------------------------------------------------------------------
Dockerfile | 91 ++++---
Makefile | 9 +-
README.md | 1 +
.../notebooks/meetup-streaming-toree.ipynb | 273 ++++---------------
index.ipynb | 48 ++++
5 files changed, 175 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/463f8ac4/Dockerfile
----------------------------------------------------------------------
diff --git a/Dockerfile b/Dockerfile
index dcb6340..0d12f80 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,31 +1,60 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-#
-
-FROM ubuntu:14.04
-# Setup
-RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E56151BF
-RUN echo deb http://repos.mesosphere.io/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) main >> /etc/apt/sources.list.d/mesosphere.list
-RUN apt-get update
-RUN apt-get --no-install-recommends -y --force-yes install openjdk-7-jre mesos=0.20.1-1.0.ubuntu1404
-ENV MESOS_NATIVE_LIBRARY /usr/local/lib/libmesos.so
-
-# Setup the binary we will run
-ENTRYPOINT JVM_OPT="-XX:PermSize=256m -Dlog4j.logLevel=${LOG_LEVEL} ${JVM_OPT}" /app/bin/sparkkernel
-
-# Install the pack elements
-ADD kernel/target/pack /app
-RUN chmod +x /app/bin/sparkkernel
+# This Dockerfile is for MyBinder support
+
+FROM andrewosh/binder-base
+
+USER root
+
+# for declarativewidgets
+RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \
+ apt-get install -y nodejs npm && \
+ npm install -g bower
+
+# for pyspark demos
+ENV APACHE_SPARK_VERSION 1.5.1
+RUN apt-get -y update && \
+ apt-get install -y --no-install-recommends openjdk-7-jre-headless && \
+ apt-get clean
+RUN cd /tmp && \
+ wget -q http://d3kbcqa49mib13.cloudfront.net/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz && \
+ tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz -C /usr/local && \
+ rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz
+RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6 spark
+
+ENV SPARK_HOME /usr/local/spark
+ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.9-src.zip
+ENV PYSPARK_PYTHON /home/main/anaconda2/envs/python3/bin/python
+
+USER main
+
+ENV DASHBOARDS_VERSION ==0.4.1
+ENV DASHBOARDS_BUNDLERS_VERSION ==0.2.2
+
+ENV TOREE_VERSION >=0.1.0.dev0, <=0.1.0
+
+# get to the latest jupyter release and necessary libraries
+RUN conda install -y jupyter seaborn futures && \
+ bash -c "source activate python3 && \
+ conda install seaborn"
+
+ENV DECL_WIDGETS_VERSION 0.4.2
+
+# install incubator extensions
+RUN pip install jupyter_dashboards==0.4.1 \
+ jupyter_declarativewidgets==$DECL_WIDGETS_VERSION \
+ jupyter_dashboards_bundlers==0.2.2
+RUN jupyter dashboards install --user --symlink && \
+ jupyter declarativewidgets install --user --symlink && \
+ jupyter dashboards activate && \
+ jupyter declarativewidgets activate && \
+ jupyter dashboards_bundlers activate
+
+# install kernel-side incubator extensions for python3 environment too
+RUN bash -c "source activate python3 && pip install jupyter_declarativewidgets==$DECL_WIDGETS_VERSION"
+
+# install Toree
+RUN pip install 'toree>=0.1.0.dev0, <=0.1.0'
+RUN jupyter toree install --user
+
+
+# include nice intro notebook
+COPY index.ipynb $HOME/notebooks/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/463f8ac4/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 0a0d210..3330457 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
# limitations under the License
#
-.PHONY: help clean clean-dist build dev test test-travis release pip-release bin-release
+.PHONY: help clean clean-dist build dev test test-travis release pip-release bin-release dev-binder .binder-image
VERSION?=0.1.0.dev5
COMMIT=$(shell git rev-parse --short=12 --verify HEAD)
@@ -29,6 +29,7 @@ endif
APACHE_SPARK_VERSION?=1.5.1
IMAGE?=jupyter/pyspark-notebook:2988869079e6
EXAMPLE_IMAGE?=apache/toree-examples
+BINDER_IMAGE?=apache/toree-binder
DOCKER_WORKDIR?=/srv/toree
DOCKER_ARGS?=
define DOCKER
@@ -81,6 +82,12 @@ clean: clean-dist
@-docker rm -f examples_image
touch $@
+.binder-image:
+ @docker build --rm -t $(BINDER_IMAGE) .
+
+dev-binder: .binder-image
+ docker run --rm -it -p 8888:8888 -v `pwd`:/home/main/notebooks --workdir /home/main/notebooks $(BINDER_IMAGE) /home/main/start-notebook.sh --ip=0.0.0.0
+
kernel/target/scala-2.10/$(ASSEMBLY_JAR): VM_WORKDIR=/src/toree-kernel
kernel/target/scala-2.10/$(ASSEMBLY_JAR): ${shell find ./*/src/main/**/*}
kernel/target/scala-2.10/$(ASSEMBLY_JAR): ${shell find ./*/build.sbt}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/463f8ac4/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 188d97e..553a3e2 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
[![Build Status][build-badge]][build-url]
[![License][license-badge]][license-url]
[![Join the chat at https://gitter.im/ibm-et/spark-kernel][gitter-badge]][gitter-url]
+[](http://mybinder.org/repo/apache/incubator-toree)
Apache Toree
============
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/463f8ac4/etc/examples/notebooks/meetup-streaming-toree.ipynb
----------------------------------------------------------------------
diff --git a/etc/examples/notebooks/meetup-streaming-toree.ipynb b/etc/examples/notebooks/meetup-streaming-toree.ipynb
index d40fdec..51898bb 100644
--- a/etc/examples/notebooks/meetup-streaming-toree.ipynb
+++ b/etc/examples/notebooks/meetup-streaming-toree.ipynb
@@ -2,44 +2,21 @@
"cells": [
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "layout": {
- "col": 4,
- "height": 2,
- "row": 0,
- "width": 5
- }
- }
- }
- },
+ "metadata": {},
"source": [
"# Streaming Meetups Dashboard"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"The purpose of this notebook is to give an all-in-one demo of streaming data from the [meetup.com RSVP API](http://www.meetup.com/meetup_api/docs/stream/2/rsvps/#websockets), through a local [Spark Streaming job](http://spark.apache.org/streaming/), and into [declarative widgets](https://github.com/jupyter-incubator/declarativewidgets) in a dashboard layout. You can peek at the meetup.com stream [here](http://stream.meetup.com/2/rsvps)."
]
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"This example is a Scala adapatation of [this](https://github.com/jupyter-incubator/dashboards/blob/master/etc/notebooks/stream_demo/meetup-streaming.ipynb) notebook from [jupyter_dashboards](https://github.com/jupyter-incubator/dashboards).\n",
"\n",
@@ -55,13 +32,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"<div class=\"alert alert-info\" role=\"alert\" style=\"margin-top: 10px\">\n",
"<p><strong>Note</strong><p>\n",
@@ -72,13 +43,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"## Initialize DeclarativeWidgets<span style=\"float: right; font-size: 0.5em\"><a href=\"#Initialize-DeclarativeWidgets\">Top</a></span>\n",
"\n",
@@ -89,12 +54,35 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ "<span id=\"__jar_url__\"></span>\n",
+ "<script>\n",
+ "(function(){\n",
+ " var thisUrl = this.location.toString();\n",
+ " var jarUrl = thisUrl.substring(0,thisUrl.indexOf(\"/notebooks\"))+\"/nbextensions/urth_widgets/urth-widgets.jar\";\n",
+ " document.getElementById(\"__jar_url__\").innerHTML = jarUrl;\n",
+ "})();\n",
+ "</script>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<div class=\"alert alert-info\" role=\"alert\" style=\"margin-top: 10px\">\n",
+ "<p><strong>Note</strong> The output of the cell above is the URL to use below.\n",
+ "</div>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -105,10 +93,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
- "urth": {
- "dashboard": {}
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -118,13 +103,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"## Define the Spark Streaming Application<span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Scala\">Top</a></span>\n",
"\n",
@@ -133,13 +112,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Install external dependencies"
]
@@ -148,12 +121,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -162,13 +130,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Custom WebSocker Receiver\n",
"\n",
@@ -179,10 +141,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
- "urth": {
- "dashboard": {}
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -222,13 +181,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Spark Meetup Application\n",
"\n",
@@ -241,10 +194,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {}
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -478,18 +428,12 @@
" Some((partial._1, partial._2, partial._3 + last_sum.getOrElse((0,0,0))._3))\n",
" }\n",
" }\n",
- "} "
+ "}"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"## Create a Dashboard Interface with Widgets <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>\n",
"\n",
@@ -498,13 +442,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Control to start and stop the Stream\n",
"\n",
@@ -515,10 +453,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {}
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -535,17 +470,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "layout": {
- "col": 10,
- "height": 2,
- "row": 0,
- "width": 2
- }
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -568,13 +493,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Topic Bar Chart\n",
"\n",
@@ -585,17 +504,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "layout": {
- "col": 6,
- "height": 19,
- "row": 9,
- "width": 6
- }
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -609,13 +518,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Topic Filter\n",
"\n",
@@ -628,12 +531,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -646,17 +544,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "layout": {
- "col": 0,
- "height": 3,
- "row": 2,
- "width": 6
- }
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -677,13 +565,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### User Card\n",
"\n",
@@ -694,17 +576,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "layout": {
- "col": 6,
- "height": 7,
- "row": 2,
- "width": 5
- }
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -784,13 +656,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"#### Map Venues\n",
"\n",
@@ -801,17 +667,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "urth": {
- "dashboard": {
- "layout": {
- "col": 0,
- "height": 19,
- "row": 5,
- "width": 6
- }
- }
- }
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -826,27 +682,14 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "collapsed": true,
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"## Arrange the Dashboard Layout <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "urth": {
- "dashboard": {
- "hidden": true
- }
- }
- },
+ "metadata": {},
"source": [
"Before toggling the stream on/off switch, we should switch to dashboard view. Otherwise, we'll need to scroll up and down this notebook to see the widgets updating. For convenience, this notebook already contains metadata to position our widgets in a grid layout.\n",
"\n",
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/463f8ac4/index.ipynb
----------------------------------------------------------------------
diff --git a/index.ipynb b/index.ipynb
new file mode 100644
index 0000000..8335bc0
--- /dev/null
+++ b/index.ipynb
@@ -0,0 +1,48 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Welcome\n",
+ "\n",
+ "This is a try site for [Apache Toree](https://github.com/apache/incubator-toree). A Jupyter Notebook Server was **launched just for you** to try a variety of demos of Apache Toree. Some of the demos are simple to illustrate some of Apache Toree's features, others are more involved and showcase Apache Toree working in conjunction with other projects from the [Jupyter](http://jupyter.org/) Ecosystem.\n",
+ "\n",
+ "A full collection of examples are [here](etc/examples/notebooks)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### [Meetup.com Streaming Demo](etc/examples/notebooks/meetup-streaming-toree.ipynb)\n",
+ "\n",
+ "This demostration includes:\n",
+ "* Live streaming analytics using [Apache Spark Streaming](http://spark.apache.org/streaming/)\n",
+ "* Interactive areas using [jupyter_declarativewidgets](https://github.com/jupyter-incubator/declarativewidgets)\n",
+ "* Dashboard layout using [jupyter_dashboard](https://github.com/jupyter-incubator/dashboards)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.4"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}