You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/08 07:22:15 UTC
[13/13] git commit: Merge pull request #313 from tdas/project-refactor
Merge pull request #313 from tdas/project-refactor
Refactored the streaming project to separate external libraries like Twitter, Kafka, Flume, etc.
At a high level, these are the following changes.
1. All the external code was put in `SPARK_HOME/external/` as separate SBT projects and Maven modules. Their artifact names are `spark-streaming-twitter`, `spark-streaming-kafka`, etc. Both SparkBuild.scala and pom.xml files have been updated. References to external libraries and repositories have been removed from the settings of root and streaming projects/modules.
2. To avail the external functionality (say, creating a Twitter stream), the developer has to `import org.apache.spark.streaming.twitter._` . For Scala API, the developer has to call `TwitterUtils.createStream(streamingContext, ...)`. For the Java API, the developer has to call `TwitterUtils.createStream(javaStreamingContext, ...)`.
3. Each external project has its own scala and java unit tests. Note the unit tests of each external library use classes of the streaming unit tests (`TestSuiteBase`, `LocalJavaStreamingContext`, etc.). To enable this code sharing among test classes, `dependsOn(streaming % "compile->compile,test->test")` was used in the SparkBuild.scala . In the streaming/pom.xml, an additional `maven-jar-plugin` was necessary to capture this dependency (see comment inside the pom.xml for more information).
4. Jars of the external projects have been added to examples project but not to the assembly project.
5. In some files, imports have been rearrange to conform to the Spark coding guidelines.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c0f0155e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c0f0155e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c0f0155e
Branch: refs/heads/master
Commit: c0f0155eca6405d0768a476f0be00594e478fce0
Parents: f5f12dc 8f02f1c
Author: Patrick Wendell <pw...@gmail.com>
Authored: Tue Jan 7 22:21:52 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Jan 7 22:21:52 2014 -0800
----------------------------------------------------------------------
bin/run-example | 7 +-
examples/pom.xml | 42 ++--
.../streaming/examples/JavaFlumeEventCount.java | 10 +-
.../streaming/examples/JavaKafkaWordCount.java | 7 +-
.../streaming/examples/FlumeEventCount.scala | 3 +-
.../streaming/examples/KafkaWordCount.scala | 3 +-
.../streaming/examples/MQTTWordCount.scala | 12 +-
.../streaming/examples/TwitterAlgebirdCMS.scala | 6 +-
.../streaming/examples/TwitterAlgebirdHLL.scala | 4 +-
.../streaming/examples/TwitterPopularTags.scala | 3 +-
.../streaming/examples/ZeroMQWordCount.scala | 9 +-
external/flume/pom.xml | 93 +++++++
.../streaming/flume/FlumeInputDStream.scala | 155 ++++++++++++
.../spark/streaming/flume/FlumeUtils.scala | 70 ++++++
.../streaming/flume/JavaFlumeStreamSuite.java | 34 +++
.../flume/src/test/resources/log4j.properties | 29 +++
.../streaming/flume/FlumeStreamSuite.scala | 86 +++++++
external/kafka/pom.xml | 97 ++++++++
.../streaming/kafka/KafkaInputDStream.scala | 154 ++++++++++++
.../spark/streaming/kafka/KafkaUtils.scala | 153 ++++++++++++
.../streaming/kafka/JavaKafkaStreamSuite.java | 45 ++++
.../kafka/src/test/resources/log4j.properties | 29 +++
.../streaming/kafka/KafkaStreamSuite.scala | 39 +++
external/mqtt/pom.xml | 108 +++++++++
.../spark/streaming/mqtt/MQTTInputDStream.scala | 110 +++++++++
.../apache/spark/streaming/mqtt/MQTTUtils.scala | 75 ++++++
.../streaming/mqtt/JavaMQTTStreamSuite.java | 37 +++
.../mqtt/src/test/resources/log4j.properties | 29 +++
.../spark/streaming/mqtt/MQTTStreamSuite.scala | 36 +++
external/twitter/pom.xml | 89 +++++++
.../streaming/twitter/TwitterInputDStream.scala | 100 ++++++++
.../spark/streaming/twitter/TwitterUtils.scala | 126 ++++++++++
.../twitter/JavaTwitterStreamSuite.java | 46 ++++
.../twitter/src/test/resources/log4j.properties | 29 +++
.../streaming/twitter/TwitterStreamSuite.scala | 43 ++++
external/zeromq/pom.xml | 89 +++++++
.../spark/streaming/zeromq/ZeroMQReceiver.scala | 54 +++++
.../spark/streaming/zeromq/ZeroMQUtils.scala | 126 ++++++++++
.../streaming/zeromq/JavaZeroMQStreamSuite.java | 50 ++++
.../zeromq/src/test/resources/log4j.properties | 29 +++
.../streaming/zeromq/ZeroMQStreamSuite.scala | 44 ++++
pom.xml | 29 +--
project/SparkBuild.scala | 93 +++++--
streaming/pom.xml | 83 +++----
.../spark/streaming/StreamingContext.scala | 145 +----------
.../api/java/JavaStreamingContext.scala | 243 +------------------
.../streaming/dstream/FlumeInputDStream.scala | 154 ------------
.../streaming/dstream/KafkaInputDStream.scala | 153 ------------
.../streaming/dstream/MQTTInputDStream.scala | 110 ---------
.../streaming/dstream/TwitterInputDStream.scala | 99 --------
.../streaming/receivers/ZeroMQReceiver.scala | 53 ----
.../scheduler/NetworkInputTracker.scala | 4 +-
.../apache/spark/streaming/JavaAPISuite.java | 104 +-------
.../streaming/LocalJavaStreamingContext.java | 46 ++++
.../spark/streaming/InputStreamsSuite.scala | 82 +------
.../apache/spark/streaming/TestSuiteBase.scala | 20 +-
56 files changed, 2448 insertions(+), 1280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/bin/run-example
----------------------------------------------------------------------
diff --cc bin/run-example
index 6c5d4a6,a94913d..2e9d514
--- a/bin/run-example
+++ b/bin/run-example
@@@ -45,17 -45,11 +45,11 @@@ f
EXAMPLES_DIR="$FWDIR"/examples
SPARK_EXAMPLES_JAR=""
if [ -e "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar ]; then
- # Use the JAR from the SBT build
export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar`
fi
- if [ -e "$EXAMPLES_DIR"/target/spark-examples*[0-9Tg].jar ]; then
- # Use the JAR from the Maven build
- # TODO: this also needs to become an assembly!
- export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR"/target/spark-examples*[0-9Tg].jar`
- fi
if [[ -z $SPARK_EXAMPLES_JAR ]]; then
echo "Failed to find Spark examples assembly in $FWDIR/examples/target" >&2
- echo "You need to build Spark with sbt assembly before running this program" >&2
+ echo "You need to build Spark with sbt/sbt assembly before running this program" >&2
exit 1
fi
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/examples/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c0f0155e/project/SparkBuild.scala
----------------------------------------------------------------------