You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/11 01:26:02 UTC
[45/50] git commit: Merge github.com:apache/incubator-spark
Merge github.com:apache/incubator-spark
Conflicts:
core/src/main/scala/org/apache/spark/SparkEnv.scala
streaming/src/test/java/org/apache/spark/streaming/JavaAPISuite.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/d76e1f90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/d76e1f90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/d76e1f90
Branch: refs/heads/master
Commit: d76e1f90a84755e9e68a27c9fda59f2175a73819
Parents: 80ba9f8 300eaa9
Author: Andrew Or <an...@gmail.com>
Authored: Thu Jan 9 21:38:48 2014 -0800
Committer: Andrew Or <an...@gmail.com>
Committed: Thu Jan 9 21:38:48 2014 -0800
----------------------------------------------------------------------
.gitignore | 3 +
README.md | 20 +-
assembly/lib/PY4J_LICENSE.txt | 27 -
assembly/lib/PY4J_VERSION.txt | 1 -
assembly/lib/net/sf/py4j/py4j/0.7/py4j-0.7.jar | Bin 103286 -> 0 bytes
assembly/lib/net/sf/py4j/py4j/0.7/py4j-0.7.pom | 9 -
.../net/sf/py4j/py4j/maven-metadata-local.xml | 12 -
assembly/pom.xml | 20 +-
assembly/src/main/assembly/assembly.xml | 11 +-
bin/compute-classpath.cmd | 2 +-
bin/compute-classpath.sh | 2 +-
bin/pyspark | 65 ++
bin/pyspark.cmd | 23 +
bin/pyspark2.cmd | 55 +
bin/run-example | 86 ++
bin/run-example.cmd | 23 +
bin/run-example2.cmd | 61 ++
bin/slaves.sh | 91 --
bin/spark-class | 154 +++
bin/spark-class.cmd | 23 +
bin/spark-class2.cmd | 85 ++
bin/spark-config.sh | 36 -
bin/spark-daemon.sh | 183 ----
bin/spark-daemons.sh | 35 -
bin/spark-shell | 102 ++
bin/spark-shell.cmd | 23 +
bin/start-all.sh | 34 -
bin/start-master.sh | 52 -
bin/start-slave.sh | 35 -
bin/start-slaves.sh | 48 -
bin/stop-all.sh | 32 -
bin/stop-master.sh | 27 -
bin/stop-slaves.sh | 35 -
conf/log4j.properties.template | 1 +
conf/spark-env.sh.template | 2 +-
core/pom.xml | 10 +
.../apache/spark/network/netty/FileClient.java | 5 +-
.../netty/FileClientChannelInitializer.java | 2 +-
.../apache/spark/network/netty/FileServer.java | 8 +-
.../netty/FileServerChannelInitializer.java | 4 +-
.../spark/network/netty/FileServerHandler.java | 6 +-
.../spark/network/netty/PathResolver.java | 52 +-
.../org/apache/spark/default-log4j.properties | 8 -
.../org/apache/spark/log4j-defaults.properties | 9 +
.../main/scala/org/apache/spark/Logging.scala | 10 +-
.../org/apache/spark/MapOutputTracker.scala | 14 +-
.../scala/org/apache/spark/Partitioner.scala | 7 +-
.../main/scala/org/apache/spark/SparkConf.scala | 13 +-
.../scala/org/apache/spark/SparkContext.scala | 58 +-
.../main/scala/org/apache/spark/SparkEnv.scala | 15 +-
.../org/apache/spark/SparkHadoopWriter.scala | 4 -
.../apache/spark/api/java/JavaDoubleRDD.scala | 12 +-
.../org/apache/spark/api/java/JavaPairRDD.scala | 6 +
.../org/apache/spark/api/java/JavaRDD.scala | 6 +
.../org/apache/spark/api/java/JavaRDDLike.scala | 6 +
.../spark/api/java/JavaSparkContext.scala | 58 +
.../org/apache/spark/api/python/PythonRDD.scala | 4 +-
.../apache/spark/broadcast/HttpBroadcast.scala | 4 +-
.../spark/broadcast/TorrentBroadcast.scala | 2 +-
.../spark/deploy/ApplicationDescription.scala | 2 +-
.../scala/org/apache/spark/deploy/Client.scala | 151 +++
.../apache/spark/deploy/ClientArguments.scala | 117 ++
.../org/apache/spark/deploy/DeployMessage.scala | 52 +-
.../apache/spark/deploy/DriverDescription.scala | 29 +
.../apache/spark/deploy/client/AppClient.scala | 201 ++++
.../spark/deploy/client/AppClientListener.scala | 39 +
.../org/apache/spark/deploy/client/Client.scala | 190 ----
.../spark/deploy/client/ClientListener.scala | 39 -
.../apache/spark/deploy/client/TestClient.scala | 6 +-
.../spark/deploy/master/ApplicationInfo.scala | 7 +-
.../apache/spark/deploy/master/DriverInfo.scala | 36 +
.../spark/deploy/master/DriverState.scala | 33 +
.../master/FileSystemPersistenceEngine.scala | 17 +-
.../org/apache/spark/deploy/master/Master.scala | 206 +++-
.../spark/deploy/master/PersistenceEngine.scala | 11 +-
.../apache/spark/deploy/master/WorkerInfo.scala | 20 +-
.../master/ZooKeeperPersistenceEngine.scala | 14 +-
.../spark/deploy/master/ui/IndexPage.scala | 56 +-
.../spark/deploy/worker/CommandUtils.scala | 63 ++
.../spark/deploy/worker/DriverRunner.scala | 234 ++++
.../spark/deploy/worker/DriverWrapper.scala | 31 +
.../spark/deploy/worker/ExecutorRunner.scala | 67 +-
.../org/apache/spark/deploy/worker/Worker.scala | 65 +-
.../spark/deploy/worker/WorkerWatcher.scala | 55 +
.../spark/deploy/worker/ui/IndexPage.scala | 65 +-
.../spark/deploy/worker/ui/WorkerWebUI.scala | 43 +-
.../executor/CoarseGrainedExecutorBackend.scala | 27 +-
.../org/apache/spark/io/CompressionCodec.scala | 2 +-
.../spark/network/ConnectionManager.scala | 18 +-
.../spark/network/netty/ShuffleCopier.scala | 2 +-
.../org/apache/spark/rdd/CheckpointRDD.scala | 4 +-
.../org/apache/spark/rdd/NewHadoopRDD.scala | 2 +-
.../org/apache/spark/rdd/PairRDDFunctions.scala | 59 +-
.../main/scala/org/apache/spark/rdd/RDD.scala | 32 +-
.../spark/scheduler/TaskResultGetter.scala | 2 +-
.../spark/scheduler/TaskSchedulerImpl.scala | 8 +-
.../apache/spark/scheduler/TaskSetManager.scala | 8 +-
.../cluster/CoarseGrainedSchedulerBackend.scala | 5 +-
.../cluster/SimrSchedulerBackend.scala | 2 +-
.../cluster/SparkDeploySchedulerBackend.scala | 12 +-
.../mesos/CoarseMesosSchedulerBackend.scala | 6 +-
.../cluster/mesos/MesosSchedulerBackend.scala | 6 +-
.../spark/serializer/KryoSerializer.scala | 4 +-
.../apache/spark/serializer/Serializer.scala | 3 +
.../spark/serializer/SerializerManager.scala | 15 +-
.../spark/storage/BlockFetcherIterator.scala | 2 +-
.../org/apache/spark/storage/BlockManager.scala | 22 +-
.../spark/storage/BlockManagerMaster.scala | 12 +-
.../spark/storage/BlockManagerMasterActor.scala | 7 +-
.../apache/spark/storage/DiskBlockManager.scala | 2 +-
.../spark/storage/ShuffleBlockManager.scala | 4 +-
.../apache/spark/storage/ThreadingTest.scala | 2 +-
.../apache/spark/ui/UIWorkloadGenerator.scala | 4 +-
.../spark/ui/jobs/JobProgressListener.scala | 2 +-
.../scala/org/apache/spark/util/AkkaUtils.scala | 43 +-
.../org/apache/spark/util/MetadataCleaner.scala | 2 +-
.../org/apache/spark/util/XORShiftRandom.scala | 2 +-
.../test/resources/uncommons-maths-1.2.2.jar | Bin 49019 -> 0 bytes
.../scala/org/apache/spark/DriverSuite.scala | 8 +-
.../org/apache/spark/FileServerSuite.scala | 108 +-
.../apache/spark/MapOutputTrackerSuite.scala | 17 +-
.../apache/spark/deploy/JsonProtocolSuite.scala | 45 +-
.../spark/deploy/worker/DriverRunnerTest.scala | 131 +++
.../deploy/worker/ExecutorRunnerTest.scala | 10 +-
.../deploy/worker/WorkerWatcherSuite.scala | 32 +
.../spark/scheduler/TaskSetManagerSuite.scala | 2 +-
.../spark/storage/BlockManagerSuite.scala | 12 +-
.../spark/storage/DiskBlockManagerSuite.scala | 4 +-
.../apache/spark/util/XORShiftRandomSuite.scala | 2 +-
data/kmeans_data.txt | 6 +
data/lr_data.txt | 1000 ++++++++++++++++++
data/pagerank_data.txt | 6 +
docs/bagel-programming-guide.md | 4 +-
docs/building-with-maven.md | 14 +-
docs/configuration.md | 49 +-
docs/css/bootstrap.min.css | 2 +-
docs/index.md | 10 +-
docs/java-programming-guide.md | 4 +-
docs/job-scheduling.md | 5 +-
docs/mllib-guide.md | 330 +++++-
docs/python-programming-guide.md | 31 +-
docs/quick-start.md | 14 +-
docs/running-on-yarn.md | 17 +-
docs/scala-programming-guide.md | 14 +-
docs/spark-debugger.md | 2 +-
docs/spark-standalone.md | 68 +-
docs/streaming-programming-guide.md | 4 +-
ec2/spark_ec2.py | 2 +-
examples/pom.xml | 45 +-
.../org/apache/spark/examples/JavaHdfsLR.java | 31 +-
.../org/apache/spark/examples/JavaKMeans.java | 25 +-
.../org/apache/spark/examples/JavaLogQuery.java | 22 +-
.../org/apache/spark/examples/JavaPageRank.java | 13 +-
.../org/apache/spark/examples/JavaSparkPi.java | 12 +-
.../java/org/apache/spark/examples/JavaTC.java | 18 +-
.../apache/spark/examples/JavaWordCount.java | 12 +-
.../apache/spark/mllib/examples/JavaALS.java | 20 +-
.../apache/spark/mllib/examples/JavaKMeans.java | 18 +-
.../org/apache/spark/mllib/examples/JavaLR.java | 21 +-
.../streaming/examples/JavaFlumeEventCount.java | 18 +-
.../streaming/examples/JavaKafkaWordCount.java | 28 +-
.../examples/JavaNetworkWordCount.java | 18 +-
.../streaming/examples/JavaQueueStream.java | 13 +-
.../apache/spark/examples/BroadcastTest.scala | 2 +-
.../spark/examples/DriverSubmissionTest.scala | 46 +
.../spark/examples/ExceptionHandlingTest.scala | 2 +-
.../org/apache/spark/examples/GroupByTest.scala | 2 +-
.../org/apache/spark/examples/HBaseTest.scala | 2 +-
.../org/apache/spark/examples/HdfsTest.scala | 2 +-
.../org/apache/spark/examples/LogQuery.scala | 2 +-
.../spark/examples/MultiBroadcastTest.scala | 2 +-
.../examples/SimpleSkewedGroupByTest.scala | 2 +-
.../spark/examples/SkewedGroupByTest.scala | 2 +-
.../org/apache/spark/examples/SparkALS.scala | 2 +-
.../org/apache/spark/examples/SparkHdfsLR.scala | 2 +-
.../org/apache/spark/examples/SparkKMeans.scala | 2 +-
.../org/apache/spark/examples/SparkLR.scala | 2 +-
.../apache/spark/examples/SparkPageRank.scala | 2 +-
.../org/apache/spark/examples/SparkPi.scala | 2 +-
.../org/apache/spark/examples/SparkTC.scala | 2 +-
.../streaming/examples/ActorWordCount.scala | 6 +-
.../streaming/examples/FlumeEventCount.scala | 5 +-
.../streaming/examples/HdfsWordCount.scala | 4 +-
.../streaming/examples/KafkaWordCount.scala | 7 +-
.../streaming/examples/MQTTWordCount.scala | 18 +-
.../streaming/examples/NetworkWordCount.scala | 4 +-
.../spark/streaming/examples/QueueStream.scala | 2 +-
.../streaming/examples/RawNetworkGrep.scala | 2 +-
.../examples/StatefulNetworkWordCount.scala | 4 +-
.../streaming/examples/TwitterAlgebirdCMS.scala | 8 +-
.../streaming/examples/TwitterAlgebirdHLL.scala | 6 +-
.../streaming/examples/TwitterPopularTags.scala | 5 +-
.../streaming/examples/ZeroMQWordCount.scala | 15 +-
.../clickstream/PageViewGenerator.scala | 4 +-
.../examples/clickstream/PageViewStream.scala | 6 +-
external/flume/pom.xml | 93 ++
.../streaming/flume/FlumeInputDStream.scala | 155 +++
.../spark/streaming/flume/FlumeUtils.scala | 70 ++
.../streaming/flume/JavaFlumeStreamSuite.java | 34 +
.../flume/src/test/resources/log4j.properties | 29 +
.../streaming/flume/FlumeStreamSuite.scala | 86 ++
external/kafka/pom.xml | 97 ++
.../streaming/kafka/KafkaInputDStream.scala | 154 +++
.../spark/streaming/kafka/KafkaUtils.scala | 153 +++
.../streaming/kafka/JavaKafkaStreamSuite.java | 45 +
.../kafka/src/test/resources/log4j.properties | 29 +
.../streaming/kafka/KafkaStreamSuite.scala | 39 +
external/mqtt/pom.xml | 108 ++
.../spark/streaming/mqtt/MQTTInputDStream.scala | 110 ++
.../apache/spark/streaming/mqtt/MQTTUtils.scala | 75 ++
.../streaming/mqtt/JavaMQTTStreamSuite.java | 37 +
.../mqtt/src/test/resources/log4j.properties | 29 +
.../spark/streaming/mqtt/MQTTStreamSuite.scala | 36 +
external/twitter/pom.xml | 89 ++
.../streaming/twitter/TwitterInputDStream.scala | 100 ++
.../spark/streaming/twitter/TwitterUtils.scala | 126 +++
.../twitter/JavaTwitterStreamSuite.java | 46 +
.../twitter/src/test/resources/log4j.properties | 29 +
.../streaming/twitter/TwitterStreamSuite.scala | 43 +
external/zeromq/pom.xml | 89 ++
.../spark/streaming/zeromq/ZeroMQReceiver.scala | 54 +
.../spark/streaming/zeromq/ZeroMQUtils.scala | 126 +++
.../streaming/zeromq/JavaZeroMQStreamSuite.java | 50 +
.../zeromq/src/test/resources/log4j.properties | 29 +
.../streaming/zeromq/ZeroMQStreamSuite.scala | 44 +
kmeans_data.txt | 6 -
lr_data.txt | 1000 ------------------
make-distribution.sh | 27 +-
.../spark/mllib/api/python/PythonMLLibAPI.scala | 30 +
.../spark/mllib/classification/NaiveBayes.scala | 119 +++
.../spark/mllib/optimization/Gradient.scala | 4 +-
.../MatrixFactorizationModel.scala | 38 +-
.../mllib/classification/NaiveBayesSuite.scala | 108 ++
.../optimization/GradientDescentSuite.scala | 116 ++
.../spark/mllib/recommendation/ALSSuite.scala | 33 +-
new-yarn/pom.xml | 161 ---
.../spark/deploy/yarn/ApplicationMaster.scala | 428 --------
.../yarn/ApplicationMasterArguments.scala | 94 --
.../org/apache/spark/deploy/yarn/Client.scala | 523 ---------
.../spark/deploy/yarn/ClientArguments.scala | 150 ---
.../yarn/ClientDistributedCacheManager.scala | 228 ----
.../spark/deploy/yarn/WorkerLauncher.scala | 225 ----
.../spark/deploy/yarn/WorkerRunnable.scala | 209 ----
.../deploy/yarn/YarnAllocationHandler.scala | 694 ------------
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 43 -
.../cluster/YarnClientClusterScheduler.scala | 48 -
.../cluster/YarnClientSchedulerBackend.scala | 110 --
.../cluster/YarnClusterScheduler.scala | 56 -
.../ClientDistributedCacheManagerSuite.scala | 220 ----
pagerank_data.txt | 6 -
pom.xml | 89 +-
project/SparkBuild.scala | 142 ++-
project/build.properties | 1 -
pyspark | 70 --
pyspark.cmd | 23 -
pyspark2.cmd | 55 -
python/lib/py4j-0.8.1-src.zip | Bin 0 -> 37662 bytes
python/lib/py4j0.7.egg | Bin 191756 -> 0 bytes
python/pyspark/__init__.py | 2 +-
python/pyspark/java_gateway.py | 2 +-
python/pyspark/mllib/_common.py | 25 +
python/pyspark/mllib/recommendation.py | 12 +-
python/pyspark/shell.py | 2 +-
python/run-tests | 2 +-
repl-bin/src/deb/bin/run | 3 +-
repl/pom.xml | 1 -
.../apache/spark/repl/SparkCommandLine.scala | 37 +
.../org/apache/spark/repl/SparkILoop.scala | 11 +-
.../apache/spark/repl/SparkRunnerSettings.scala | 32 +
run-example | 91 --
run-example.cmd | 23 -
run-example2.cmd | 61 --
sbin/slaves.sh | 91 ++
sbin/spark-config.sh | 36 +
sbin/spark-daemon.sh | 183 ++++
sbin/spark-daemons.sh | 35 +
sbin/spark-executor | 23 +
sbin/start-all.sh | 34 +
sbin/start-master.sh | 52 +
sbin/start-slave.sh | 35 +
sbin/start-slaves.sh | 48 +
sbin/stop-all.sh | 32 +
sbin/stop-master.sh | 27 +
sbin/stop-slaves.sh | 35 +
sbt/sbt | 53 +-
sbt/sbt-launch-0.11.3-2.jar | Bin 1096763 -> 0 bytes
sbt/sbt.cmd | 25 -
spark-class | 154 ---
spark-class.cmd | 23 -
spark-class2.cmd | 85 --
spark-executor | 22 -
spark-shell | 102 --
spark-shell.cmd | 22 -
streaming/pom.xml | 83 +-
.../spark/streaming/StreamingContext.scala | 151 +--
.../api/java/JavaStreamingContext.scala | 260 +----
.../streaming/dstream/FlumeInputDStream.scala | 154 ---
.../streaming/dstream/KafkaInputDStream.scala | 153 ---
.../streaming/dstream/MQTTInputDStream.scala | 110 --
.../streaming/dstream/NetworkInputDStream.scala | 4 +-
.../streaming/dstream/TwitterInputDStream.scala | 99 --
.../streaming/receivers/ZeroMQReceiver.scala | 53 -
.../apache/spark/streaming/scheduler/Job.scala | 2 +-
.../streaming/scheduler/JobGenerator.scala | 2 +-
.../streaming/scheduler/JobScheduler.scala | 2 +-
.../scheduler/NetworkInputTracker.scala | 4 +-
.../apache/spark/streaming/JavaAPISuite.java | 99 +-
.../streaming/LocalJavaStreamingContext.java | 46 +
.../spark/streaming/InputStreamsSuite.scala | 82 +-
.../apache/spark/streaming/TestSuiteBase.scala | 20 +-
yarn/README.md | 12 +
yarn/alpha/pom.xml | 32 +
.../spark/deploy/yarn/ApplicationMaster.scala | 464 ++++++++
.../org/apache/spark/deploy/yarn/Client.scala | 509 +++++++++
.../spark/deploy/yarn/WorkerLauncher.scala | 250 +++++
.../spark/deploy/yarn/WorkerRunnable.scala | 236 +++++
.../deploy/yarn/YarnAllocationHandler.scala | 680 ++++++++++++
.../yarn/ApplicationMasterArguments.scala | 94 ++
.../spark/deploy/yarn/ClientArguments.scala | 150 +++
.../yarn/ClientDistributedCacheManager.scala | 228 ++++
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 43 +
.../cluster/YarnClientClusterScheduler.scala | 48 +
.../cluster/YarnClientSchedulerBackend.scala | 112 ++
.../cluster/YarnClusterScheduler.scala | 56 +
.../ClientDistributedCacheManagerSuite.scala | 220 ++++
yarn/pom.xml | 84 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 458 --------
.../yarn/ApplicationMasterArguments.scala | 94 --
.../org/apache/spark/deploy/yarn/Client.scala | 505 ---------
.../spark/deploy/yarn/ClientArguments.scala | 147 ---
.../yarn/ClientDistributedCacheManager.scala | 228 ----
.../spark/deploy/yarn/WorkerLauncher.scala | 247 -----
.../spark/deploy/yarn/WorkerRunnable.scala | 235 ----
.../deploy/yarn/YarnAllocationHandler.scala | 680 ------------
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 43 -
.../cluster/YarnClientClusterScheduler.scala | 48 -
.../cluster/YarnClientSchedulerBackend.scala | 110 --
.../cluster/YarnClusterScheduler.scala | 59 --
.../ClientDistributedCacheManagerSuite.scala | 220 ----
yarn/stable/pom.xml | 32 +
.../spark/deploy/yarn/ApplicationMaster.scala | 432 ++++++++
.../org/apache/spark/deploy/yarn/Client.scala | 525 +++++++++
.../spark/deploy/yarn/WorkerLauncher.scala | 230 ++++
.../spark/deploy/yarn/WorkerRunnable.scala | 210 ++++
.../deploy/yarn/YarnAllocationHandler.scala | 695 ++++++++++++
345 files changed, 13678 insertions(+), 11095 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/d76e1f90/core/src/main/scala/org/apache/spark/SparkEnv.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/SparkEnv.scala
index b581c7b,e093e2f..584261d
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@@ -17,12 -17,10 +17,12 @@@
package org.apache.spark
+import java.util.concurrent.atomic.AtomicInteger
+
import scala.collection.mutable
+ import scala.concurrent.Await
import akka.actor._
- import akka.remote.RemoteActorRefProvider
import org.apache.spark.broadcast.BroadcastManager
import org.apache.spark.metrics.MetricsSystem
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/d76e1f90/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/d76e1f90/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/d76e1f90/streaming/src/test/java/org/apache/spark/streaming/JavaAPISuite.java
----------------------------------------------------------------------
diff --cc streaming/src/test/java/org/apache/spark/streaming/JavaAPISuite.java
index 354f04f,0d2145d..be93799
--- a/streaming/src/test/java/org/apache/spark/streaming/JavaAPISuite.java
+++ b/streaming/src/test/java/org/apache/spark/streaming/JavaAPISuite.java
@@@ -17,25 -17,19 +17,27 @@@
package org.apache.spark.streaming;
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+
+import kafka.serializer.StringDecoder;
+
- import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
- import org.apache.spark.SparkConf;
- import org.apache.spark.streaming.api.java.JavaDStreamLike;
+ import scala.Tuple2;
+
import org.junit.After;
import org.junit.Assert;
- import org.junit.Before;
import org.junit.Test;
+ import java.io.*;
+ import java.util.*;
- import scala.Tuple2;
- import twitter4j.Status;
+ import com.google.common.base.Optional;
+ import com.google.common.collect.Lists;
+ import com.google.common.io.Files;
+ import org.apache.spark.SparkConf;
import org.apache.spark.HashPartitioner;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;