You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/02/18 20:43:00 UTC
[2/2] incubator-toree git commit: Initial prototype of example
notebooks.
Initial prototype of example notebooks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/3b0381ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/3b0381ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/3b0381ab
Branch: refs/heads/master
Commit: 3b0381ab76f92693c3a17519f8d7090fe35c75f8
Parents: 116d8bc
Author: Corey A. Stubbs <cs...@us.ibm.com>
Authored: Fri Feb 12 17:00:46 2016 -0600
Committer: Gino Bustelo <lb...@apache.org>
Committed: Thu Feb 18 13:42:17 2016 -0600
----------------------------------------------------------------------
.gitignore | 2 +
Makefile | 3 +-
.../notebooks/meetup-streaming-toree.ipynb | 1469 ++++++++++++++++++
3 files changed, 1473 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/3b0381ab/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index c5eead5..0bd7bb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,8 @@ target/
.vagrant
*.iml
*.ipynb
+.ipynb_checkpoints/
+!etc/**/*.ipynb
scratch/
**/*ivy.xml
test-output/
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/3b0381ab/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 688441e..8904850 100644
--- a/Makefile
+++ b/Makefile
@@ -121,5 +121,6 @@ release: pip-release bin-release
twine upload -r pypi toree-$(VERSION).tar.gz'
jupyter: DOCKER_WORKDIR=/srv/toree/dist
+jupyter: EXTRA_CMD?=pip install jupyter_declarativewidgets==0.4.0; jupyter declarativewidgets install --user; jupyter declarativewidgets activate; apt-get update; apt-get install --yes curl; curl --silent --location https://deb.nodesource.com/setup_0.12 | sudo bash -; apt-get install --yes nodejs; npm install -g bower;
jupyter: pip-release
- @$(DOCKER) -p 8888:8888 --user=root $(IMAGE) bash -c 'pip install toree-$(VERSION).tar.gz && jupyter toree install && cd ~ && jupyter notebook --ip=* --no-browser'
+ @$(DOCKER) -p 8888:8888 --user=root $(IMAGE) bash -c '$(EXTRA_CMD) pip install toree-$(VERSION).tar.gz && jupyter toree install && cd /srv/toree/etc/examples/notebooks && jupyter notebook --ip=* --no-browser'
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/3b0381ab/etc/examples/notebooks/meetup-streaming-toree.ipynb
----------------------------------------------------------------------
diff --git a/etc/examples/notebooks/meetup-streaming-toree.ipynb b/etc/examples/notebooks/meetup-streaming-toree.ipynb
new file mode 100644
index 0000000..a2efdec
--- /dev/null
+++ b/etc/examples/notebooks/meetup-streaming-toree.ipynb
@@ -0,0 +1,1469 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 4,
+ "height": 3,
+ "row": 0,
+ "width": 6
+ }
+ }
+ }
+ },
+ "source": [
+ "# Streaming Meetups Dashboard"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 6,
+ "height": 4,
+ "row": 3,
+ "width": 6
+ }
+ }
+ }
+ },
+ "source": [
+ "The purpose of this notebook is to give an all-in-one demo of streaming data from the [meetup.com RSVP API](http://www.meetup.com/meetup_api/docs/stream/2/rsvps/#websockets), through a local [Spark Streaming job](http://spark.apache.org/streaming/), and into [declarative widgets](https://github.com/jupyter-incubator/declarativewidgets) in a dashboard layout."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "On your first visit to this notebook, we recommend that you execute one cell at a time as you read along. Later, if you just want to see the demo, select *Cell > Run All* from the menu bar. Once you've run all of the cells, select *View > View Dashboard* and then click on the **Stream** toggle to start the data stream.\n",
+ "\n",
+ "**Table of Contents**\n",
+ "\n",
+ "1. [Create the Frontend Widgets](#Create-the-Widgets-Top) <span class=\"text-muted\" style=\"float:right\">topic histogram, filter entry, user card, global heatmap</span>\n",
+ "2. [Define the Spark Streaming Job](#Define-the-Spark-Streaming-Job-Top) <span style=\"float:right\" class=\"text-muted\">filter by topic, top topics, venue metadata</span>\n",
+ "3. [Publish Data to Widget Channels](#Publish-Data-to-Widget-Channels-Top)\n",
+ "4. [Connect to the Data Source](#Connect-to-the-Data-Source-Top) <span class=\"text-muted\" style=\"float:right\">\"custom receiver\", websocket connection, stream toggle</span>\n",
+ "5. [Arrange the Dashboard Layout](#Arrange-the-Dashboard-Layout-Top)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "<div class=\"alert alert-info\" role=\"alert\" style=\"margin-top: 10px\">\n",
+ "<p><strong>Note</strong><p>\n",
+ "\n",
+ "<p>We've condensed all of the demo logic into a single notebook for educational purposes. If you want to turn this into a scalable, multi-tenant dashboard, you'll want to separate the stream processing portions from the dashboard view. That way, multiple dashboard instances can pull from the same processed data stream instead of recomputing it.</p>\n",
+ "</div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Initialize DeclarativeWidgets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Starting download from http://localhost:8888/nbextensions/urth_widgets/urth-widgets.jar\n",
+ "Finished download of urth-widgets.jar\n"
+ ]
+ }
+ ],
+ "source": [
+ "%AddJar http://localhost:8888/nbextensions/urth_widgets/urth-widgets.jar"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "import urth.widgets._\n",
+ "initWidgets"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Adding external dependencies"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ ":: loading settings :: url = jar:file:/usr/local/spark-1.5.1-bin-hadoop2.6/lib/spark-assembly-1.5.1-hadoop2.6.0.jar!/org/apache/ivy/core/settings/ivysettings.xml\n",
+ ":: resolving dependencies :: org.apache.toree#kernel;working\n",
+ "\tconfs: [default]\n",
+ "\tfound eu.piotrbuda#scalawebsocket_2.10;0.1.1 in central\n",
+ "\tfound com.ning#async-http-client;1.7.13 in central\n",
+ "\tfound io.netty#netty;3.6.3.Final in central\n",
+ "\tfound com.typesafe#scalalogging-log4j_2.10;1.0.1 in central\n",
+ "\tfound org.apache.logging.log4j#log4j-api;2.0-beta4 in central\n",
+ "\tfound org.apache.logging.log4j#log4j-core;2.0-beta4 in central\n",
+ "\tfound ch.qos.logback#logback-classic;1.0.9 in central\n",
+ "\tfound ch.qos.logback#logback-core;1.0.9 in central\n",
+ ":: resolution report :: resolve 569ms :: artifacts dl 16ms\n",
+ "\t:: modules in use:\n",
+ "\tch.qos.logback#logback-classic;1.0.9 from central in [default]\n",
+ "\tch.qos.logback#logback-core;1.0.9 from central in [default]\n",
+ "\tcom.ning#async-http-client;1.7.13 from central in [default]\n",
+ "\tcom.typesafe#scalalogging-log4j_2.10;1.0.1 from central in [default]\n",
+ "\teu.piotrbuda#scalawebsocket_2.10;0.1.1 from central in [default]\n",
+ "\tio.netty#netty;3.6.3.Final from central in [default]\n",
+ "\torg.apache.logging.log4j#log4j-api;2.0-beta4 from central in [default]\n",
+ "\torg.apache.logging.log4j#log4j-core;2.0-beta4 from central in [default]\n",
+ "\t:: evicted modules:\n",
+ "\torg.apache.logging.log4j#log4j-api;2.0-beta3 by [org.apache.logging.log4j#log4j-api;2.0-beta4] in [default]\n",
+ "\t---------------------------------------------------------------------\n",
+ "\t| | modules || artifacts |\n",
+ "\t| conf | number| search|dwnlded|evicted|| number|dwnlded|\n",
+ "\t---------------------------------------------------------------------\n",
+ "\t| default | 9 | 0 | 0 | 1 || 8 | 0 |\n",
+ "\t---------------------------------------------------------------------\n",
+ ":: retrieving :: org.apache.toree#kernel\n",
+ "\tconfs: [default]\n",
+ "\t0 artifacts copied, 8 already retrieved (0kB/38ms)\n"
+ ]
+ }
+ ],
+ "source": [
+ "%AddDeps eu.piotrbuda scalawebsocket_2.10 0.1.1 --transitive"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "## Create the Widgets <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>\n",
+ "\n",
+ "We'll start by defining what we want to show in our dashboard, and use that definition to drive the stream processing we'll perform. Just keep in mind that none of the widgets we include here will render anything useful yet.\n",
+ "\n",
+ "### Topic Bar Chart\n",
+ "\n",
+ "Here we insert a `<urth-viz-chart>` to show the top 25 meetup topics by occurrence in the stream. Take note of the `<template>` element. We use it to specify that the HTML within will make use of a `counts` channel. We will put data on the `counts` channel later in this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 6,
+ "height": 19,
+ "row": 10,
+ "width": 6
+ }
+ }
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<link rel=\"import\" href=\"urth_components/urth-viz-chart/urth-viz-chart.html\" is=\"urth-core-import\">\n",
+ "\n",
+ "<template is=\"urth-core-bind\" channel=\"counts\">\n",
+ " <urth-viz-chart type='bar' datarows='[[counts.data]]' columns='[[counts.columns]]' rotatelabels='30'></urth-viz-chart>\n",
+ "</template>"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%html\n",
+ "<link rel=\"import\" href=\"urth_components/urth-viz-chart/urth-viz-chart.html\" is=\"urth-core-import\">\n",
+ "\n",
+ "<template is=\"urth-core-bind\" channel=\"counts\">\n",
+ " <urth-viz-chart type='bar' datarows='[[counts.data]]' columns='[[counts.columns]]' rotatelabels='30'></urth-viz-chart>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Topic Filter\n",
+ "\n",
+ "Next we create an `<urth-core-function>` which that binds the value of a `<paper-input>` widget to a Python function that sets a global variable. The function will set a string that we'll use to filter the incoming events to only pertaining to a certain topic.\n",
+ "\n",
+ "Notice that the `<link>` tag here is different than what we specified above. `<urth-viz-chart>` is already loaded within the notebook, but here we are using a third-party [Polymer](https://www.polymer-project.org/1.0/) element which needs to download first. To handle that automatically, we specify `is=\"urth-core-import\"` and set the [bower](http://bower.io/) package name as the `package` attribute value."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "val set_topic_filter = (value: String) => {\n",
+ " MeetupApp.topic_filter = value\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 6,
+ "height": 3,
+ "row": 7,
+ "width": 6
+ }
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ "<link rel=\"import\" href=\"urth_components/paper-input/paper-input.html\"\n",
+ " is=\"urth-core-import\" package=\"PolymerElements/paper-input\">\n",
+ " \n",
+ "<template is=\"urth-core-bind\" channel=\"filter\" id=\"filter-input\">\n",
+ " <urth-core-function auto\n",
+ " id=\"set_topic_filter\"\n",
+ " ref=\"set_topic_filter\"\n",
+ " arg-value=\"{{topic_filter}}\">\n",
+ " </urth-core-function>\n",
+ " \n",
+ " <paper-input label=\"Filter\" value=\"{{topic_filter}}\"></paper-input>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### User Card\n",
+ "\n",
+ "Now we add a simple `<paper-card>` element showing the name and photo of one user who RSVPed recently in the event stream. We add some custom styling and a bit of custom JavaScript in this case to format the datetime associated with the RSVP event."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 1,
+ "height": 7,
+ "row": 3,
+ "width": 5
+ }
+ }
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<link rel=\"import\" href=\"urth_components/paper-card/paper-card.html\"\n",
+ " is=\"urth-core-import\" package=\"PolymerElements/paper-card\">\n",
+ "\n",
+ "<style is=\"custom-style\">\n",
+ " paper-card.meetups-card {\n",
+ " max-width: 400px;\n",
+ " width: 100%;\n",
+ " \n",
+ " --paper-card-header: {\n",
+ " height: 100px;\n",
+ " border-bottom: 1px solid #e8e8e8;\n",
+ " };\n",
+ "\n",
+ " --paper-card-header-image: {\n",
+ " height: 80px;\n",
+ " width: 80px !important;\n",
+ " margin: 10px;\n",
+ " border-radius: 50px;\n",
+ " width: auto;\n",
+ " border: 10px solid white;\n",
+ " box-shadow: 0 0 1px 1px #e8e8e8;\n",
+ " };\n",
+ " \n",
+ " --paper-card-header-image-text: {\n",
+ " left: auto;\n",
+ " right: 0px;\n",
+ " width: calc(100% - 130px);\n",
+ " text-align: right;\n",
+ " text-overflow: ellipsis;\n",
+ " overflow: hidden;\n",
+ " };\n",
+ " }\n",
+ " \n",
+ " .meetups-card .card-content a {\n",
+ " display: block;\n",
+ " overflow: hidden;\n",
+ " text-overflow: ellipsis;\n",
+ " white-space: nowrap;\n",
+ " }\n",
+ "</style>\n",
+ "\n",
+ "<template is=\"urth-core-bind\" channel=\"meetups\" id=\"meetup-card\">\n",
+ " <paper-card\n",
+ " class=\"meetups-card\"\n",
+ " heading=\"[[meetup.member.member_name]]\"\n",
+ " image=\"[[meetup.member.photo]]\">\n",
+ " <div class=\"card-content\">\n",
+ " <p><a href=\"[[meetup.event.event_url]]\" target=\"_blank\">[[meetup.event.event_name]]</a></p>\n",
+ " <p>[[getPrettyTime(meetup.event.time)]]</p>\n",
+ " </div>\n",
+ " </paper-card>\n",
+ "</template>\n",
+ "\n",
+ "<!-- see https://github.com/PolymerElements/iron-validator-behavior/blob/master/demo/index.html -->\n",
+ "<script>\n",
+ " (function() {\n",
+ " var dateStringOptions = {weekday:'long', year:'numeric', month: 'long', hour:'2-digit', minute:'2-digit', day:'numeric'};\n",
+ " var locale = navigator.language || navigator.browserLanguage || navigator.systemLanguage || navigator.userLanguage;\n",
+ "\n",
+ " var scope = document.querySelector('template#meetup-card');\n",
+ " scope.getPrettyTime = function(timestamp) {\n",
+ " try {\n",
+ " console.log('The date is', timestamp)\n",
+ " var d = new Date(timestamp);\n",
+ " return d.toLocaleDateString(locale, dateStringOptions);\n",
+ " } catch(e){\n",
+ " return ''\n",
+ " }\n",
+ " }\n",
+ " })();\n",
+ "</script>"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%html\n",
+ "<link rel=\"import\" href=\"urth_components/paper-card/paper-card.html\"\n",
+ " is=\"urth-core-import\" package=\"PolymerElements/paper-card\">\n",
+ "\n",
+ "<style is=\"custom-style\">\n",
+ " paper-card.meetups-card {\n",
+ " max-width: 400px;\n",
+ " width: 100%;\n",
+ " \n",
+ " --paper-card-header: {\n",
+ " height: 100px;\n",
+ " border-bottom: 1px solid #e8e8e8;\n",
+ " };\n",
+ "\n",
+ " --paper-card-header-image: {\n",
+ " height: 80px;\n",
+ " width: 80px !important;\n",
+ " margin: 10px;\n",
+ " border-radius: 50px;\n",
+ " width: auto;\n",
+ " border: 10px solid white;\n",
+ " box-shadow: 0 0 1px 1px #e8e8e8;\n",
+ " };\n",
+ " \n",
+ " --paper-card-header-image-text: {\n",
+ " left: auto;\n",
+ " right: 0px;\n",
+ " width: calc(100% - 130px);\n",
+ " text-align: right;\n",
+ " text-overflow: ellipsis;\n",
+ " overflow: hidden;\n",
+ " };\n",
+ " }\n",
+ " \n",
+ " .meetups-card .card-content a {\n",
+ " display: block;\n",
+ " overflow: hidden;\n",
+ " text-overflow: ellipsis;\n",
+ " white-space: nowrap;\n",
+ " }\n",
+ "</style>\n",
+ "\n",
+ "<template is=\"urth-core-bind\" channel=\"meetups\" id=\"meetup-card\">\n",
+ " <paper-card\n",
+ " class=\"meetups-card\"\n",
+ " heading=\"[[meetup.member.member_name]]\"\n",
+ " image=\"[[meetup.member.photo]]\">\n",
+ " <div class=\"card-content\">\n",
+ " <p><a href=\"[[meetup.event.event_url]]\" target=\"_blank\">[[meetup.event.event_name]]</a></p>\n",
+ " <p>[[getPrettyTime(meetup.event.time)]]</p>\n",
+ " </div>\n",
+ " </paper-card>\n",
+ "</template>\n",
+ "\n",
+ "<!-- see https://github.com/PolymerElements/iron-validator-behavior/blob/master/demo/index.html -->\n",
+ "<script>\n",
+ " (function() {\n",
+ " var dateStringOptions = {weekday:'long', year:'numeric', month: 'long', hour:'2-digit', minute:'2-digit', day:'numeric'};\n",
+ " var locale = navigator.language || navigator.browserLanguage || navigator.systemLanguage || navigator.userLanguage;\n",
+ "\n",
+ " var scope = document.querySelector('template#meetup-card');\n",
+ " scope.getPrettyTime = function(timestamp) {\n",
+ " try {\n",
+ " console.log('The date is', timestamp)\n",
+ " var d = new Date(timestamp);\n",
+ " return d.toLocaleDateString(locale, dateStringOptions);\n",
+ " } catch(e){\n",
+ " return ''\n",
+ " }\n",
+ " }\n",
+ " })();\n",
+ "</script>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Map Venues\n",
+ "\n",
+ "Finally, we add a [WebGL globe](https://github.com/dataarts/webgl-globe) showing the location of meetup venues to which users are RSVPing in the stream. On the globe we render bars to represent the number of recent RSVPs in a geographic area."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 0,
+ "height": 19,
+ "row": 10,
+ "width": 6
+ }
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ "<link rel=\"import\" href=\"urth_components/webgl-globe/webgl-globe.html\"\n",
+ " is=\"urth-core-import\" package=\"http://github.com/ibm-et/webgl-globe.git\">\n",
+ "\n",
+ "<template is=\"urth-core-bind\" channel=\"venues\">\n",
+ " <webgl-globe data=[[venue_data]]></webgl-globe>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "## Define the Spark Streaming Job <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>\n",
+ "\n",
+ "With the frontend widgest in mind, we'll now setup our Spark Streaming job to fulfill their data requirements. In this section, we'll define a set of functions that act on a `SparkStreamingContext` or `RDDs` from that context."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Create Spark Contexts\n",
+ "\n",
+ "Here we create a function that will initialize our Spark contexts. We'll use this function at the end of the notebook when we tie Spark to the data source and a final widget controlling the stream flow."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import scalawebsocket.WebSocket\n",
+ "import org.apache.spark.storage.StorageLevel \n",
+ "import org.apache.spark.streaming.receiver.Receiver\n",
+ "\n",
+ "class WebSocketReceiver(url: String, storageLevel: StorageLevel = StorageLevel.MEMORY_ONLY_SER) extends Receiver[play.api.libs.json.JsValue](storageLevel) {\n",
+ " @volatile private var webSocket: WebSocket = _\n",
+ "\n",
+ " def onStart() {\n",
+ " try{\n",
+ " val newWebSocket = WebSocket().open(url).onTextMessage({ msg: String => parseJson(msg) })\n",
+ " setWebSocket(newWebSocket)\n",
+ " } catch {\n",
+ " case e: Exception => restart(\"Error starting WebSocket stream\", e)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " def onStop() {\n",
+ " setWebSocket(null)\n",
+ " }\n",
+ "\n",
+ " private def setWebSocket(newWebSocket: WebSocket) = synchronized {\n",
+ " if (webSocket != null) {\n",
+ " webSocket.shutdown()\n",
+ " }\n",
+ " webSocket = newWebSocket\n",
+ " }\n",
+ "\n",
+ " private def parseJson(jsonStr: String): Unit = {\n",
+ " val json: play.api.libs.json.JsValue = play.api.libs.json.Json.parse(jsonStr)\n",
+ " store(json)\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "case class TopicCount(topic: String, count: Int)\n",
+ "\n",
+ "\n",
+ "object MeetupApp extends Serializable { \n",
+ " import play.api.libs.json._\n",
+ " import org.apache.spark.storage.StorageLevel \n",
+ " import org.apache.spark.Logging\n",
+ " import org.apache.spark.streaming._\n",
+ " import org.apache.spark.sql.functions._\n",
+ " import org.apache.spark.streaming.dstream.DStream\n",
+ " import org.apache.spark.rdd.RDD\n",
+ " import urth.widgets.WidgetChannels.channel\n",
+ " import org.apache.spark.sql.SQLContext\n",
+ " @transient var ssc:StreamingContext = _\n",
+ " @transient val sqlContext:SQLContext = new SQLContext(sc)\n",
+ " \n",
+ " import sqlContext.implicits._\n",
+ " \n",
+ " var topic_filter = \"\"\n",
+ "\n",
+ " def create_streaming_context(sample_rate: Int): StreamingContext = {\n",
+ " return new StreamingContext(sc, Seconds(sample_rate))\n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Creates a websocket client that pumps events into a ring buffer queue. Creates\n",
+ " a SparkStreamContext that reads from the queue. Creates the events, topics, and\n",
+ " venues DStreams, setting the widget channel publishing functions to iterate over\n",
+ " RDDs in each. Starts the stream processing.\n",
+ " */\n",
+ " def start_stream() = {\n",
+ " ssc = create_streaming_context(5)\n",
+ " ssc.checkpoint(\"/tmp/checkpoint.checkpoint\")\n",
+ " val events = get_events(ssc, sample_event)\n",
+ " get_topics(events, get_topic_counts)\n",
+ " ssc.start()\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " Shuts down the websocket, stops the streaming context, and cleans up the file ring.\n",
+ " */\n",
+ " def shutdown_stream() = {\n",
+ " ssc.stop()\n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Parses the events from the queue. Retains only those events that have at\n",
+ " least one topic exactly matching the current topic_filter. Sends event\n",
+ " RDDs to the for_each function. Returns the event DStream.\n",
+ " */\n",
+ " def get_events(ssc: StreamingContext, for_each: (RDD[play.api.libs.json.JsValue]) => Unit): org.apache.spark.streaming.dstream.DStream[play.api.libs.json.JsValue] = {\n",
+ "\n",
+ " val all_events = ssc.receiverStream( new WebSocketReceiver(\"ws://stream.meetup.com/2/rsvps\"))\n",
+ "\n",
+ " // Filter set of event\n",
+ " val events = all_events.filter(retain_event)\n",
+ "\n",
+ " // Send event data to a widget channel. This will be covered below.\n",
+ " events.foreachRDD(for_each)\n",
+ "\n",
+ " return events\n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Returns true if the user defined topic filter is blank or if at least one\n",
+ " group topic in the event exactly matches the user topic filter string.\n",
+ " */\n",
+ " def retain_event(event: play.api.libs.json.JsValue):Boolean = {\n",
+ " val topics = (event \\ \"group\" \\ \"group_topics\").as[play.api.libs.json.JsArray].value\n",
+ " val isEmpty = topic_filter.trim == \"\"\n",
+ " val containsTopic = topics.map(topic => topic_filter.\n",
+ " equals((topic\\\"urlkey\").as[play.api.libs.json.JsString].value)\n",
+ " ).reduce( (a,b) => a || b)\n",
+ "\n",
+ " isEmpty || containsTopic \n",
+ " }\n",
+ " \n",
+ " /*\n",
+ " Takes an RDD from the event DStream. Takes one event from the RDD.\n",
+ " Substitutes a placeholder photo if the member who RSVPed does not\n",
+ " have one. Publishes the event metadata on the meetup channel.\n",
+ " */\n",
+ " def sample_event(rdd: org.apache.spark.rdd.RDD[play.api.libs.json.JsValue]):Unit = {\n",
+ " \n",
+ " try {\n",
+ " val event = rdd.take(1)(0)\n",
+ "\n",
+ " // use a fallback photo for those members without one\n",
+ " val default_event: play.api.libs.json.JsObject = play.api.libs.json.Json.parse(\"\"\"{\n",
+ " \"member\" : {\n",
+ " \"photo\" : \"http://photos4.meetupstatic.com/img/noPhoto_50.png\"\n",
+ " }\n",
+ " }\n",
+ " \"\"\").as[play.api.libs.json.JsObject]\n",
+ " val fixed_event = default_event ++ (event).as[play.api.libs.json.JsObject] \n",
+ "\n",
+ " channel(\"meetups\").set(\"meetup\", fixed_event.value)\n",
+ " } \n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Pulls group topics from meetup events. Counts each one once and updates\n",
+ " the global topic counts seen since stream start. Sends topic count RDDs\n",
+ " to the for_each function. Returns nothing new.\n",
+ " */ \n",
+ " def get_topics(events:DStream[JsValue], for_each: (RDD[((String,String),Int)]) => Unit){\n",
+ " //Extract the group topic url keys and \"namespace\" them with the current topic filter\n",
+ " val topics = events.\n",
+ " flatMap( (event: JsValue) => {\n",
+ " (event \\ \"group\" \\ \"group_topics\").as[JsArray].value\n",
+ " }).map((topic: JsValue) => {\n",
+ " val filter = if(topic_filter.equals(\"\")) {\n",
+ " \"*\"\n",
+ " } else { \n",
+ " topic_filter \n",
+ " }\n",
+ " ((filter, (topic \\ \"urlkey\").as[JsString].value), 1) \n",
+ " })\n",
+ "\n",
+ " val topic_counts = topics.updateStateByKey(update_topic_counts)\n",
+ "\n",
+ " // Send topic data to a widget channel. This will be covered below.\n",
+ " topic_counts.foreachRDD(for_each)\n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Sums the number of times a topic has been seen in the current sampling\n",
+ " window. Then adds that to the number of times the topic has been\n",
+ " seen in the past. Returns the new sum.\n",
+ " */\n",
+ " def update_topic_counts(new_values: Seq[Int], last_sum: Option[Int]): Option[Int] = {\n",
+ " return Some((new_values :+ 0).reduce(_+_) + last_sum.getOrElse(0))\n",
+ " }\n",
+ " \n",
+ " /**\n",
+ " Takes an RDD from the topic DStream. Takes the top 25 topics by occurrence\n",
+ " and publishes them in a pandas DataFrame on the counts channel.\n",
+ " */\n",
+ " def get_topic_counts(rdd: RDD[((String,String),Int)]){\n",
+ " //counts = rdd.takeOrdered(25, key=lambda x: -x[1])\n",
+ " val filterStr = if (topic_filter.equals(\"\")) \"*\" else topic_filter\n",
+ " \n",
+ " /*\n",
+ " keep only those matching current filter\n",
+ " and sort in descending order, taking top 25\n",
+ " */\n",
+ " val countDF = rdd.filter((x:((String,String),Int)) => filterStr.equals(x._1._1)).\n",
+ " map(tuple => TopicCount(tuple._1._2, tuple._2)).toDF().sort($\"count\".desc)\n",
+ "// val counts = countDF.\n",
+ "// takeOrdered(25)(Ordering[Int].reverse.on(x => x.count))\n",
+ " \n",
+ " \n",
+ " channel(\"counts\").set(\"counts\", countDF)\n",
+ "\n",
+ " }\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "val x = Seq[Int]()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "(x :+ 0).reduce(_+_)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "Option(Array()).flatMap(x => \"!\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<template is=\"urth-core-bind\" channel=\"meetups\">\n",
+ " Sample event is <span>{{meetup}}</span>\n",
+ "</template>"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%html\n",
+ "<template is=\"urth-core-bind\" channel=\"meetups\">\n",
+ " Sample event is <span>{{meetup}}</span>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<template is=\"urth-core-bind\" channel=\"counts\">\n",
+ " Sample event is <span>{{counts}}</span>\n",
+ "</template>"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%html\n",
+ "<template is=\"urth-core-bind\" channel=\"counts\">\n",
+ " Sample event is <span>{{counts}}</span>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "MeetupApp.start_stream()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "MeetupApp.shutdown_stream()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Process Events\n",
+ "\n",
+ "Next we create a function, `get_events` to parse the RSVP stream JSON events and optionally filter them by topic. We define a few helper functions as well."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Process Topics\n",
+ "\n",
+ "Now we define `get_topics` to pull the URL key topic from every group sponsoring a meetup event in the stream and update the global topic counts."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def update_topic_counts(new_values, last_sum):\n",
+ " '''\n",
+ " Sums the number of times a topic has been seen in the current sampling\n",
+ " window. Then adds that to the number of times the topic has been\n",
+ " seen in the past. Returns the new sum.\n",
+ " '''\n",
+ " return sum(new_values) + (last_sum or 0)\n",
+ "\n",
+ "def get_topics(events, for_each):\n",
+ " '''\n",
+ " Pulls group topics from meetup events. Counts each one once and updates\n",
+ " the global topic counts seen since stream start. Sends topic count RDDs\n",
+ " to the for_each function. Returns nothing new.\n",
+ " '''\n",
+ " # Extract the group topic url keys and \"namespace\" them with the current topic filter\n",
+ " topics = (events\n",
+ " .flatMap(lambda event: event['group']['group_topics'])\n",
+ " .map(lambda topic: ((topic_filter if topic_filter else '*', topic['urlkey']), 1)))\n",
+ " \n",
+ " topic_counts = topics.updateStateByKey(update_topic_counts)\n",
+ "\n",
+ " # Send topic data to a widget channel. This will be covered below.\n",
+ " topic_counts.foreachRDD(for_each)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "### Process Venues\n",
+ "\n",
+ "Finally, we add `get_venues` to extract and relay venue metadata."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def get_venues(events, for_each):\n",
+ " '''\n",
+ " Pulls venu metadata from meetup events if it exists. Sends venue \n",
+ " dictionaries RDDs to the for_each function. Returns nothing new.\n",
+ " '''\n",
+ " venues = (events\n",
+ " .filter(lambda event: 'venue' in event)\n",
+ " .map(lambda event: event['venue']))\n",
+ " \n",
+ " # Send topic data to a widget channel\n",
+ " venues.foreachRDD(for_each)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "## Publish Data to Widget Channels <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>\n",
+ "\n",
+ "With both the frontend and Spark job definition in hand, we can now begin to link them. Three of our widgets are set to update when they receive data on channels `counts`, `topics`, and `venues`. We'll define functions that publish data on these channels here."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "We'll eventually pass the `sample_event` function as the `for_each` parameter of the `get_events` function that we defined above."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "Likewise, we'll eventually pass the `get_topic_counts` function as the `for_each` parameter value to the `get_topics` function that we defined above."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def get_topic_counts(rdd):\n",
+ " '''\n",
+ " Takes an RDD from the topic DStream. Takes the top 25 topics by occurrence\n",
+ " and publishes them in a pandas DataFrame on the counts channel.\n",
+ " '''\n",
+ " #counts = rdd.takeOrdered(25, key=lambda x: -x[1])\n",
+ " filterStr = topic_filter if topic_filter else '*'\n",
+ " counts = (rdd\n",
+ " .filter(lambda x: x[0][0] == filterStr) # keep only those matching current filter\n",
+ " .takeOrdered(25, key=lambda x: -x[1])) # sort in descending order, taking top 25\n",
+ " if not counts:\n",
+ " # If there are no data, the bar chart will error out. Instead,\n",
+ " # we send a tuple whose count is zero.\n",
+ " counts = [('NO DATA', 0)]\n",
+ " else:\n",
+ " # Drop the topic filter from the tuple\n",
+ " counts = list(map(lambda x: (x[0][1], x[1]), counts))\n",
+ " df = pd.DataFrame(counts)\n",
+ " channel('counts').set('counts', df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "And, finally, we'll pass the `aggregate_venues` function as the `for_each` parameter value to the `get_venues` function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "venue_data = []\n",
+ "lon_bins = np.linspace(-180, 180, 361)\n",
+ "lat_bins = np.linspace(-90, 90, 181)\n",
+ "scale=100\n",
+ "\n",
+ "def aggregate_venues(rdd):\n",
+ " '''\n",
+ " Takes an RDD from the venues DStream. Builds a histogram of events by \n",
+ " latitude and longitude. Publishes the histogram as a list of three-tuples\n",
+ " on the venues channel.\n",
+ " \n",
+ " Note: To improve scalability, this binning should be performed\n",
+ " on the Spark workers, not collected and performed on the driver.\n",
+ " '''\n",
+ " global venue_data\n",
+ "\n",
+ " # create new lists from previous data and new incoming venues\n",
+ " venues = rdd.collect()\n",
+ " lats = [v[0] for v in venue_data] + [x['lat'] for x in venues]\n",
+ " lons = [v[1] for v in venue_data] + [x['lon'] for x in venues]\n",
+ " weights = [v[2] for v in venue_data] + ([1./scale] * len(venues))\n",
+ " \n",
+ " # create histogram from aggregate data\n",
+ " density, _, _ = np.histogram2d(lats, lons, [lat_bins, lon_bins], weights=weights)\n",
+ " venue_data = [[lat-90, lon-180, min(mag,1)]\n",
+ " for lat,dlats in enumerate(density)\n",
+ " for lon,mag in enumerate(dlats)\n",
+ " if mag > 0]\n",
+ " \n",
+ " channel('venues').set('venue_data', venue_data)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "## Connect to the Data Source <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>\n",
+ "\n",
+ "We're finally ready to connect the meetup.com RSVP websocket server to our Spark job. To do so, we want to write a [custom DStream receiver bridging the Websocket to a `SparkStreamingContext`](http://spark.apache.org/docs/latest/streaming-custom-receivers.html). Unfortunately, we can't write a [custom receiver in Python yet](http://spark.apache.org/docs/latest/streaming-programming-guide.html#custom-sources). We'll work around this limitation by dumping incoming Websocket messages to disk in a ring buffer, and using the Spark `textFileStream` API to read them."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "We now define a functions to start and stop the stream processing. We track the websocket client, SparkStreamingContext, and ring buffer between start and stop calls."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def start_stream():\n",
+ " '''\n",
+ " Creates a websocket client that pumps events into a ring buffer queue. Creates\n",
+ " a SparkStreamContext that reads from the queue. Creates the events, topics, and\n",
+ " venues DStreams, setting the widget channel publishing functions to iterate over\n",
+ " RDDs in each. Starts the stream processing.\n",
+ " '''\n",
+ " global conn_future\n",
+ " global ssc\n",
+ " global receiver\n",
+ " \n",
+ " receiver = FileRingReceiver(max_batches=100) \n",
+ " conn_future = websocket_connect('ws://stream.meetup.com/2/rsvps', on_message_callback=receiver.put)\n",
+ " ssc = create_streaming_context(receiver.queue, 5)\n",
+ " events = get_events(ssc, receiver.queue, sample_event)\n",
+ " get_topics(events, get_topic_counts)\n",
+ " get_venues(events, aggregate_venues)\n",
+ " ssc.start()\n",
+ " \n",
+ "def shutdown_stream():\n",
+ " '''\n",
+ " Shuts down the websocket, stops the streaming context, and cleans up the file ring.\n",
+ " '''\n",
+ " global conn_future\n",
+ " global ssc\n",
+ " global receiver\n",
+ " \n",
+ " conn_future.result().close()\n",
+ " ssc.stop()\n",
+ " receiver.destroy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "We give each of these functions a frontend `<urth-core-function>` representation. We bind these functions to a `<paper-toggle-button>` that starts and stops the stream processing."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": false,
+ "urth": {
+ "dashboard": {
+ "layout": {
+ "col": 10,
+ "height": 2,
+ "row": 0,
+ "width": 2
+ }
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ "<link rel=\"import\" href=\"urth_components/paper-toggle-button/paper-toggle-button.html\"\n",
+ " is=\"urth-core-import\" package=\"PolymerElements/paper-toggle-button#v1.0.10\">\n",
+ " \n",
+ "<template is=\"urth-core-bind\">\n",
+ " <urth-core-function id=\"streamFunc\" ref=\"start_stream\"></urth-core-function>\n",
+ " <urth-core-function id=\"shutdownFunc\" ref=\"shutdown_stream\"></urth-core-function>\n",
+ "</template>\n",
+ "\n",
+ "<style is=\"custom-style\">\n",
+ " paper-toggle-button {\n",
+ " --default-primary-color: green;\n",
+ " }\n",
+ " \n",
+ " paper-toggle-button:hover {\n",
+ " cursor: pointer;\n",
+ " }\n",
+ " \n",
+ " .toggle-btn-container {\n",
+ " margin: 1em 0;\n",
+ " text-align: right;\n",
+ " }\n",
+ " \n",
+ " #stream-label {\n",
+ " font-size: larger;\n",
+ " margin: 0;\n",
+ " padding: 0 0.5em;\n",
+ " }\n",
+ "</style>\n",
+ "\n",
+ "<div class=\"toggle-btn-container\">\n",
+ " <paper-toggle-button id=\"stream-btn\"></paper-toggle-button>\n",
+ " <label id=\"stream-label\">Stream</label>\n",
+ "</div>\n",
+ "\n",
+ "<script>\n",
+ " $('#stream-btn').on('change', function() {\n",
+ " if ($(this).attr('checked')) {\n",
+ " // start streaming\n",
+ " console.warn('Starting Spark Streaming');\n",
+ " $('#streamFunc').get(0).invoke();\n",
+ " } else {\n",
+ " // stop streaming\n",
+ " console.warn('Stopping Spark Streaming');\n",
+ " $('#shutdownFunc').get(0).invoke();\n",
+ " }\n",
+ " });\n",
+ "</script>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true,
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "## Arrange the Dashboard Layout <span style=\"float: right; font-size: 0.5em\"><a href=\"#Streaming-Meetups-Dashboard\">Top</a></span>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "urth": {
+ "dashboard": {
+ "hidden": true
+ }
+ }
+ },
+ "source": [
+ "Before toggling the stream on/off switch, we should switch to dashboard view. Otherwise, we'll need to scroll up and down this notebook to see the widgets updating. For convenience, this notebook already contains metadata to position our widgets in a grid layout.\n",
+ "\n",
+ "Select *View > View Dashboard* from the menu bar to see the dashboard view now. Then toggle the stream switch in the top right of the dashboard to begin stream processing. To return to the regular notebook view, select *View > Notebook*.\n",
+ "\n",
+ "If you want to arrange the notebook cells differently, select *View > Layout Dashboard*. Then, hover your mouse over the main notebook / dashboard area. When you do, you'll see icons appear that allow you to:\n",
+ "\n",
+ "- Drag cells to new locations\n",
+ "- Resize cells\n",
+ "- Show / hide cells in the dashboard view\n",
+ "- Flip to editing mode for a cell\n",
+ "\n",
+ "Save the notebook to save your changes to the layout within the notebook file itself.\n",
+ "\n",
+ "<div class=\"alert alert-info\" role=\"alert\" style=\"margin-top: 10px\">\n",
+ "<p><strong>Note</strong><p>\n",
+ "\n",
+ "<p>in a fresh notebook, the dashboard will only show cells with non-empty output. All other cells can be found in the *Hidden* section at the bottom of the dashboard layout page. You can quickly add all cell outputs or remove all cell outputs from the dashboard using the show / hide icons that appear in the notebook toolbar when you are in layout mode.</p>\n",
+ "</div>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# TESTING"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "start_stream()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ "<template is=\"urth-core-bind\" channel=\"meetups\">\n",
+ " Sample event is <span>{{meetup}}</span>\n",
+ "</template>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "val value = play.api.libs.json.Json.parse(\"\"\"{\n",
+ " \"venue\":{\n",
+ " \"venue_name\":\"Ozio Rooftop Lounge \",\n",
+ " \"lon\":0,\n",
+ " \"lat\":0,\n",
+ " \"venue_id\":23666225\n",
+ " },\n",
+ " \"visibility\":\"public\",\n",
+ " \"response\":\"yes\",\n",
+ " \"guests\":0,\n",
+ " \"member\":{\n",
+ " \"member_id\":2846786,\n",
+ " \"photo\":\"http:\\/\\/photos4.meetupstatic.com\\/photos\\/member\\/7\\/4\\/6\\/0\\/thumb_197129792.jpeg\",\n",
+ " \"member_name\":\"Kristy\"\n",
+ " },\n",
+ " \"rsvp_id\":1594217638,\n",
+ " \"mtime\":1455301122999,\n",
+ " \"event\":{\n",
+ " \"event_name\":\"EXPATS 'Saturday' HAPPY HOUR\",\n",
+ " \"event_id\":\"fzzldlyvfbcc\",\n",
+ " \"time\":1455404400000,\n",
+ " \"event_url\":\"http:\\/\\/www.meetup.com\\/Internationals\\/events\\/219588759\\/\"\n",
+ " },\n",
+ " \"group\":{\n",
+ " \"group_topics\":[\n",
+ " {\n",
+ " \"urlkey\":\"language-exchange\",\n",
+ " \"topic_name\":\"Language Exchange\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"international-professionals\",\n",
+ " \"topic_name\":\"International Professionals\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"expat\",\n",
+ " \"topic_name\":\"Expat\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"professional-networking\",\n",
+ " \"topic_name\":\"Professional Networking\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"professional-singles\",\n",
+ " \"topic_name\":\"Single Professionals\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"culture-exchange\",\n",
+ " \"topic_name\":\"Culture Exchange\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"global-oneness\",\n",
+ " \"topic_name\":\"Global Oneness\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"international-friends\",\n",
+ " \"topic_name\":\"International Friends\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"newintown\",\n",
+ " \"topic_name\":\"New In Town\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"international-travel\",\n",
+ " \"topic_name\":\"International Travel\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"intlrel\",\n",
+ " \"topic_name\":\"International Relations\"\n",
+ " },\n",
+ " {\n",
+ " \"urlkey\":\"global-citizenship\",\n",
+ " \"topic_name\":\"Global Citizenship\"\n",
+ " }\n",
+ " ],\n",
+ " \"group_city\":\"Washington\",\n",
+ " \"group_country\":\"us\",\n",
+ " \"group_id\":1304426,\n",
+ " \"group_name\":\"DC International Professionals\\/Expats Meetup\",\n",
+ " \"group_lon\":-77.02,\n",
+ " \"group_urlname\":\"Internationals\",\n",
+ " \"group_state\":\"DC\",\n",
+ " \"group_lat\":38.91\n",
+ " }\n",
+ "}\n",
+ "\"\"\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "shutdown_stream()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Toree",
+ "language": "",
+ "name": "toree"
+ },
+ "language_info": {
+ "name": "scala"
+ },
+ "urth": {
+ "dashboard": {
+ "cellMargin": 10,
+ "defaultCellHeight": 20,
+ "maxColumns": 12
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}