You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/09/01 23:59:35 UTC
[51/69] [abbrv] [partial] git commit: Initial work to rename package
to org.apache.spark
Initial work to rename package to org.apache.spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/46eecd11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/46eecd11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/46eecd11
Branch: refs/heads/branch-0.8
Commit: 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef
Parents: a30fac1
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Sat Aug 31 19:27:07 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Sun Sep 1 14:13:13 2013 -0700
----------------------------------------------------------------------
README.md | 2 +-
assembly/pom.xml | 16 +-
assembly/src/main/assembly/assembly.xml | 10 +-
bagel/pom.xml | 6 +-
.../scala/org/apache/spark/bagel/Bagel.scala | 293 ++++
bagel/src/main/scala/spark/bagel/Bagel.scala | 294 ----
bagel/src/test/scala/bagel/BagelSuite.scala | 118 --
.../org/apache/spark/bagel/BagelSuite.scala | 116 ++
bin/start-master.sh | 2 +-
bin/start-slave.sh | 2 +-
bin/stop-master.sh | 2 +-
bin/stop-slaves.sh | 4 +-
core/pom.xml | 4 +-
.../apache/spark/network/netty/FileClient.java | 89 ++
.../netty/FileClientChannelInitializer.java | 41 +
.../spark/network/netty/FileClientHandler.java | 60 +
.../apache/spark/network/netty/FileServer.java | 103 ++
.../netty/FileServerChannelInitializer.java | 42 +
.../spark/network/netty/FileServerHandler.java | 82 ++
.../spark/network/netty/PathResolver.java | 29 +
.../java/spark/network/netty/FileClient.java | 89 --
.../netty/FileClientChannelInitializer.java | 41 -
.../spark/network/netty/FileClientHandler.java | 60 -
.../java/spark/network/netty/FileServer.java | 103 --
.../netty/FileServerChannelInitializer.java | 42 -
.../spark/network/netty/FileServerHandler.java | 82 --
.../java/spark/network/netty/PathResolver.java | 29 -
.../apache/spark/ui/static/bootstrap.min.css | 874 ++++++++++++
.../org/apache/spark/ui/static/sorttable.js | 495 +++++++
.../spark/ui/static/spark-logo-77x50px-hd.png | Bin 0 -> 3536 bytes
.../org/apache/spark/ui/static/spark_logo.png | Bin 0 -> 14233 bytes
.../org/apache/spark/ui/static/webui.css | 63 +
.../resources/spark/ui/static/bootstrap.min.css | 874 ------------
.../main/resources/spark/ui/static/sorttable.js | 495 -------
.../spark/ui/static/spark-logo-77x50px-hd.png | Bin 3536 -> 0 bytes
.../resources/spark/ui/static/spark_logo.png | Bin 14233 -> 0 bytes
.../main/resources/spark/ui/static/webui.css | 63 -
.../scala/org/apache/spark/Accumulators.scala | 256 ++++
.../scala/org/apache/spark/Aggregator.scala | 61 +
.../apache/spark/BlockStoreShuffleFetcher.scala | 89 ++
.../scala/org/apache/spark/CacheManager.scala | 82 ++
.../scala/org/apache/spark/ClosureCleaner.scala | 231 ++++
.../scala/org/apache/spark/Dependency.scala | 81 ++
.../org/apache/spark/DoubleRDDFunctions.scala | 78 ++
.../org/apache/spark/FetchFailedException.scala | 44 +
.../scala/org/apache/spark/HttpFileServer.scala | 62 +
.../scala/org/apache/spark/HttpServer.scala | 88 ++
.../scala/org/apache/spark/JavaSerializer.scala | 83 ++
.../scala/org/apache/spark/KryoSerializer.scala | 156 +++
.../main/scala/org/apache/spark/Logging.scala | 95 ++
.../org/apache/spark/MapOutputTracker.scala | 338 +++++
.../org/apache/spark/PairRDDFunctions.scala | 703 ++++++++++
.../main/scala/org/apache/spark/Partition.scala | 31 +
.../scala/org/apache/spark/Partitioner.scala | 135 ++
core/src/main/scala/org/apache/spark/RDD.scala | 957 +++++++++++++
.../org/apache/spark/RDDCheckpointData.scala | 130 ++
.../apache/spark/SequenceFileRDDFunctions.scala | 107 ++
.../org/apache/spark/SerializableWritable.scala | 42 +
.../scala/org/apache/spark/ShuffleFetcher.scala | 35 +
.../scala/org/apache/spark/SizeEstimator.scala | 283 ++++
.../scala/org/apache/spark/SparkContext.scala | 995 +++++++++++++
.../main/scala/org/apache/spark/SparkEnv.scala | 240 ++++
.../scala/org/apache/spark/SparkException.scala | 24 +
.../main/scala/org/apache/spark/SparkFiles.java | 42 +
.../org/apache/spark/SparkHadoopWriter.scala | 201 +++
.../scala/org/apache/spark/TaskContext.scala | 41 +
.../scala/org/apache/spark/TaskEndReason.scala | 51 +
.../main/scala/org/apache/spark/TaskState.scala | 51 +
.../src/main/scala/org/apache/spark/Utils.scala | 780 +++++++++++
.../apache/spark/api/java/JavaDoubleRDD.scala | 167 +++
.../org/apache/spark/api/java/JavaPairRDD.scala | 601 ++++++++
.../org/apache/spark/api/java/JavaRDD.scala | 114 ++
.../org/apache/spark/api/java/JavaRDDLike.scala | 426 ++++++
.../spark/api/java/JavaSparkContext.scala | 418 ++++++
.../java/JavaSparkContextVarargsWorkaround.java | 64 +
.../org/apache/spark/api/java/JavaUtils.scala | 28 +
.../apache/spark/api/java/StorageLevels.java | 48 +
.../java/function/DoubleFlatMapFunction.java | 37 +
.../spark/api/java/function/DoubleFunction.java | 34 +
.../api/java/function/FlatMapFunction.scala | 28 +
.../api/java/function/FlatMapFunction2.scala | 28 +
.../spark/api/java/function/Function.java | 39 +
.../spark/api/java/function/Function2.java | 38 +
.../api/java/function/PairFlatMapFunction.java | 46 +
.../spark/api/java/function/PairFunction.java | 45 +
.../spark/api/java/function/VoidFunction.scala | 33 +
.../api/java/function/WrappedFunction1.scala | 32 +
.../api/java/function/WrappedFunction2.scala | 32 +
.../spark/api/python/PythonPartitioner.scala | 50 +
.../org/apache/spark/api/python/PythonRDD.scala | 344 +++++
.../spark/api/python/PythonWorkerFactory.scala | 132 ++
.../spark/broadcast/BitTorrentBroadcast.scala | 1057 ++++++++++++++
.../org/apache/spark/broadcast/Broadcast.scala | 70 +
.../spark/broadcast/BroadcastFactory.scala | 30 +
.../apache/spark/broadcast/HttpBroadcast.scala | 171 +++
.../apache/spark/broadcast/MultiTracker.scala | 409 ++++++
.../org/apache/spark/broadcast/SourceInfo.scala | 54 +
.../apache/spark/broadcast/TreeBroadcast.scala | 602 ++++++++
.../spark/deploy/ApplicationDescription.scala | 32 +
.../scala/org/apache/spark/deploy/Command.scala | 26 +
.../org/apache/spark/deploy/DeployMessage.scala | 130 ++
.../org/apache/spark/deploy/ExecutorState.scala | 28 +
.../org/apache/spark/deploy/JsonProtocol.scala | 86 ++
.../apache/spark/deploy/LocalSparkCluster.scala | 69 +
.../apache/spark/deploy/SparkHadoopUtil.scala | 36 +
.../scala/org/apache/spark/deploy/WebUI.scala | 47 +
.../org/apache/spark/deploy/client/Client.scala | 145 ++
.../spark/deploy/client/ClientListener.scala | 35 +
.../apache/spark/deploy/client/TestClient.scala | 51 +
.../spark/deploy/client/TestExecutor.scala | 27 +
.../spark/deploy/master/ApplicationInfo.scala | 85 ++
.../spark/deploy/master/ApplicationSource.scala | 24 +
.../spark/deploy/master/ApplicationState.scala | 28 +
.../spark/deploy/master/ExecutorInfo.scala | 32 +
.../org/apache/spark/deploy/master/Master.scala | 386 ++++++
.../spark/deploy/master/MasterArguments.scala | 89 ++
.../spark/deploy/master/MasterSource.scala | 25 +
.../apache/spark/deploy/master/WorkerInfo.scala | 77 ++
.../spark/deploy/master/WorkerState.scala | 24 +
.../deploy/master/ui/ApplicationPage.scala | 118 ++
.../spark/deploy/master/ui/IndexPage.scala | 141 ++
.../spark/deploy/master/ui/MasterWebUI.scala | 80 ++
.../spark/deploy/worker/ExecutorRunner.scala | 199 +++
.../org/apache/spark/deploy/worker/Worker.scala | 213 +++
.../spark/deploy/worker/WorkerArguments.scala | 153 ++
.../spark/deploy/worker/WorkerSource.scala | 34 +
.../spark/deploy/worker/ui/IndexPage.scala | 115 ++
.../spark/deploy/worker/ui/WorkerWebUI.scala | 190 +++
.../org/apache/spark/executor/Executor.scala | 269 ++++
.../apache/spark/executor/ExecutorBackend.scala | 28 +
.../spark/executor/ExecutorExitCode.scala | 60 +
.../apache/spark/executor/ExecutorSource.scala | 55 +
.../spark/executor/ExecutorURLClassLoader.scala | 31 +
.../spark/executor/MesosExecutorBackend.scala | 95 ++
.../executor/StandaloneExecutorBackend.scala | 107 ++
.../org/apache/spark/executor/TaskMetrics.scala | 105 ++
.../org/apache/spark/io/CompressionCodec.scala | 82 ++
.../apache/spark/metrics/MetricsConfig.scala | 100 ++
.../apache/spark/metrics/MetricsSystem.scala | 163 +++
.../apache/spark/metrics/sink/ConsoleSink.scala | 59 +
.../org/apache/spark/metrics/sink/CsvSink.scala | 68 +
.../org/apache/spark/metrics/sink/JmxSink.scala | 35 +
.../spark/metrics/sink/MetricsServlet.scala | 55 +
.../org/apache/spark/metrics/sink/Sink.scala | 23 +
.../apache/spark/metrics/source/JvmSource.scala | 32 +
.../apache/spark/metrics/source/Source.scala | 25 +
.../apache/spark/network/BufferMessage.scala | 111 ++
.../org/apache/spark/network/Connection.scala | 586 ++++++++
.../spark/network/ConnectionManager.scala | 720 ++++++++++
.../spark/network/ConnectionManagerId.scala | 38 +
.../spark/network/ConnectionManagerTest.scala | 102 ++
.../org/apache/spark/network/Message.scala | 93 ++
.../org/apache/spark/network/MessageChunk.scala | 42 +
.../spark/network/MessageChunkHeader.scala | 75 +
.../org/apache/spark/network/ReceiverTest.scala | 37 +
.../org/apache/spark/network/SenderTest.scala | 70 +
.../apache/spark/network/netty/FileHeader.scala | 74 +
.../spark/network/netty/ShuffleCopier.scala | 118 ++
.../spark/network/netty/ShuffleSender.scala | 70 +
.../main/scala/org/apache/spark/package.scala | 32 +
.../partial/ApproximateActionListener.scala | 87 ++
.../spark/partial/ApproximateEvaluator.scala | 27 +
.../apache/spark/partial/BoundedDouble.scala | 25 +
.../apache/spark/partial/CountEvaluator.scala | 55 +
.../spark/partial/GroupedCountEvaluator.scala | 79 ++
.../spark/partial/GroupedMeanEvaluator.scala | 82 ++
.../spark/partial/GroupedSumEvaluator.scala | 89 ++
.../apache/spark/partial/MeanEvaluator.scala | 58 +
.../apache/spark/partial/PartialResult.scala | 137 ++
.../apache/spark/partial/StudentTCacher.scala | 43 +
.../org/apache/spark/partial/SumEvaluator.scala | 68 +
.../scala/org/apache/spark/rdd/BlockRDD.scala | 51 +
.../org/apache/spark/rdd/CartesianRDD.scala | 90 ++
.../org/apache/spark/rdd/CheckpointRDD.scala | 155 +++
.../org/apache/spark/rdd/CoGroupedRDD.scala | 144 ++
.../org/apache/spark/rdd/CoalescedRDD.scala | 342 +++++
.../scala/org/apache/spark/rdd/EmptyRDD.scala | 33 +
.../org/apache/spark/rdd/FilteredRDD.scala | 33 +
.../org/apache/spark/rdd/FlatMappedRDD.scala | 33 +
.../apache/spark/rdd/FlatMappedValuesRDD.scala | 36 +
.../scala/org/apache/spark/rdd/GlommedRDD.scala | 29 +
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 137 ++
.../scala/org/apache/spark/rdd/JdbcRDD.scala | 120 ++
.../org/apache/spark/rdd/MapPartitionsRDD.scala | 37 +
.../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 +
.../scala/org/apache/spark/rdd/MappedRDD.scala | 30 +
.../org/apache/spark/rdd/MappedValuesRDD.scala | 34 +
.../org/apache/spark/rdd/NewHadoopRDD.scala | 126 ++
.../apache/spark/rdd/OrderedRDDFunctions.scala | 51 +
.../spark/rdd/ParallelCollectionRDD.scala | 151 ++
.../apache/spark/rdd/PartitionPruningRDD.scala | 72 +
.../scala/org/apache/spark/rdd/PipedRDD.scala | 125 ++
.../scala/org/apache/spark/rdd/SampledRDD.scala | 66 +
.../org/apache/spark/rdd/ShuffledRDD.scala | 67 +
.../org/apache/spark/rdd/SubtractedRDD.scala | 129 ++
.../scala/org/apache/spark/rdd/UnionRDD.scala | 73 +
.../apache/spark/rdd/ZippedPartitionsRDD.scala | 143 ++
.../scala/org/apache/spark/rdd/ZippedRDD.scala | 85 ++
.../org/apache/spark/scheduler/ActiveJob.scala | 39 +
.../apache/spark/scheduler/DAGScheduler.scala | 849 ++++++++++++
.../spark/scheduler/DAGSchedulerEvent.scala | 63 +
.../spark/scheduler/DAGSchedulerSource.scala | 30 +
.../spark/scheduler/InputFormatInfo.scala | 178 +++
.../apache/spark/scheduler/JobListener.scala | 28 +
.../org/apache/spark/scheduler/JobLogger.scala | 292 ++++
.../org/apache/spark/scheduler/JobResult.scala | 26 +
.../org/apache/spark/scheduler/JobWaiter.scala | 66 +
.../org/apache/spark/scheduler/MapStatus.scala | 44 +
.../org/apache/spark/scheduler/ResultTask.scala | 134 ++
.../apache/spark/scheduler/ShuffleMapTask.scala | 189 +++
.../apache/spark/scheduler/SparkListener.scala | 204 +++
.../spark/scheduler/SparkListenerBus.scala | 74 +
.../org/apache/spark/scheduler/SplitInfo.scala | 78 ++
.../org/apache/spark/scheduler/Stage.scala | 112 ++
.../org/apache/spark/scheduler/StageInfo.scala | 29 +
.../scala/org/apache/spark/scheduler/Task.scala | 115 ++
.../apache/spark/scheduler/TaskLocation.scala | 34 +
.../org/apache/spark/scheduler/TaskResult.scala | 72 +
.../apache/spark/scheduler/TaskScheduler.scala | 52 +
.../spark/scheduler/TaskSchedulerListener.scala | 45 +
.../org/apache/spark/scheduler/TaskSet.scala | 35 +
.../scheduler/cluster/ClusterScheduler.scala | 440 ++++++
.../cluster/ClusterTaskSetManager.scala | 712 ++++++++++
.../scheduler/cluster/ExecutorLossReason.scala | 38 +
.../apache/spark/scheduler/cluster/Pool.scala | 121 ++
.../spark/scheduler/cluster/Schedulable.scala | 48 +
.../scheduler/cluster/SchedulableBuilder.scala | 137 ++
.../scheduler/cluster/SchedulerBackend.scala | 37 +
.../scheduler/cluster/SchedulingAlgorithm.scala | 81 ++
.../scheduler/cluster/SchedulingMode.scala | 29 +
.../cluster/SparkDeploySchedulerBackend.scala | 91 ++
.../cluster/StandaloneClusterMessage.scala | 63 +
.../cluster/StandaloneSchedulerBackend.scala | 198 +++
.../scheduler/cluster/TaskDescription.scala | 37 +
.../spark/scheduler/cluster/TaskInfo.scala | 72 +
.../spark/scheduler/cluster/TaskLocality.scala | 32 +
.../scheduler/cluster/TaskSetManager.scala | 51 +
.../spark/scheduler/cluster/WorkerOffer.scala | 24 +
.../spark/scheduler/local/LocalScheduler.scala | 272 ++++
.../scheduler/local/LocalTaskSetManager.scala | 194 +++
.../mesos/CoarseMesosSchedulerBackend.scala | 286 ++++
.../scheduler/mesos/MesosSchedulerBackend.scala | 342 +++++
.../apache/spark/serializer/Serializer.scala | 112 ++
.../spark/serializer/SerializerManager.scala | 62 +
.../apache/spark/storage/BlockException.scala | 22 +
.../spark/storage/BlockFetchTracker.scala | 27 +
.../spark/storage/BlockFetcherIterator.scala | 348 +++++
.../org/apache/spark/storage/BlockManager.scala | 1046 ++++++++++++++
.../apache/spark/storage/BlockManagerId.scala | 118 ++
.../spark/storage/BlockManagerMaster.scala | 178 +++
.../spark/storage/BlockManagerMasterActor.scala | 404 ++++++
.../spark/storage/BlockManagerMessages.scala | 110 ++
.../spark/storage/BlockManagerSlaveActor.scala | 39 +
.../spark/storage/BlockManagerSource.scala | 48 +
.../spark/storage/BlockManagerWorker.scala | 139 ++
.../org/apache/spark/storage/BlockMessage.scala | 223 +++
.../spark/storage/BlockMessageArray.scala | 159 +++
.../spark/storage/BlockObjectWriter.scala | 65 +
.../org/apache/spark/storage/BlockStore.scala | 61 +
.../org/apache/spark/storage/DiskStore.scala | 329 +++++
.../org/apache/spark/storage/MemoryStore.scala | 257 ++++
.../org/apache/spark/storage/PutResult.scala | 26 +
.../spark/storage/ShuffleBlockManager.scala | 67 +
.../org/apache/spark/storage/StorageLevel.scala | 146 ++
.../org/apache/spark/storage/StorageUtils.scala | 115 ++
.../apache/spark/storage/ThreadingTest.scala | 113 ++
.../scala/org/apache/spark/ui/JettyUtils.scala | 132 ++
.../main/scala/org/apache/spark/ui/Page.scala | 22 +
.../scala/org/apache/spark/ui/SparkUI.scala | 87 ++
.../scala/org/apache/spark/ui/UIUtils.scala | 131 ++
.../apache/spark/ui/UIWorkloadGenerator.scala | 105 ++
.../org/apache/spark/ui/env/EnvironmentUI.scala | 91 ++
.../org/apache/spark/ui/exec/ExecutorsUI.scala | 136 ++
.../org/apache/spark/ui/jobs/IndexPage.scala | 90 ++
.../spark/ui/jobs/JobProgressListener.scala | 156 +++
.../apache/spark/ui/jobs/JobProgressUI.scala | 60 +
.../org/apache/spark/ui/jobs/PoolPage.scala | 32 +
.../org/apache/spark/ui/jobs/PoolTable.scala | 55 +
.../org/apache/spark/ui/jobs/StagePage.scala | 183 +++
.../org/apache/spark/ui/jobs/StageTable.scala | 107 ++
.../spark/ui/storage/BlockManagerUI.scala | 41 +
.../org/apache/spark/ui/storage/IndexPage.scala | 65 +
.../org/apache/spark/ui/storage/RDDPage.scala | 132 ++
.../scala/org/apache/spark/util/AkkaUtils.scala | 72 +
.../spark/util/BoundedPriorityQueue.scala | 62 +
.../spark/util/ByteBufferInputStream.scala | 80 ++
.../scala/org/apache/spark/util/Clock.scala | 29 +
.../apache/spark/util/CompletionIterator.scala | 42 +
.../org/apache/spark/util/Distribution.scala | 82 ++
.../org/apache/spark/util/IdGenerator.scala | 31 +
.../scala/org/apache/spark/util/IntParam.scala | 31 +
.../org/apache/spark/util/MemoryParam.scala | 34 +
.../org/apache/spark/util/MetadataCleaner.scala | 61 +
.../org/apache/spark/util/MutablePair.scala | 36 +
.../org/apache/spark/util/NextIterator.scala | 88 ++
.../spark/util/RateLimitedOutputStream.scala | 79 ++
.../apache/spark/util/SerializableBuffer.scala | 54 +
.../org/apache/spark/util/StatCounter.scala | 131 ++
.../apache/spark/util/TimeStampedHashMap.scala | 122 ++
.../apache/spark/util/TimeStampedHashSet.scala | 86 ++
.../scala/org/apache/spark/util/Vector.scala | 139 ++
core/src/main/scala/spark/Accumulators.scala | 256 ----
core/src/main/scala/spark/Aggregator.scala | 61 -
.../scala/spark/BlockStoreShuffleFetcher.scala | 89 --
core/src/main/scala/spark/CacheManager.scala | 82 --
core/src/main/scala/spark/ClosureCleaner.scala | 231 ----
core/src/main/scala/spark/Dependency.scala | 81 --
.../main/scala/spark/DoubleRDDFunctions.scala | 78 --
.../main/scala/spark/FetchFailedException.scala | 44 -
core/src/main/scala/spark/HttpFileServer.scala | 62 -
core/src/main/scala/spark/HttpServer.scala | 88 --
core/src/main/scala/spark/JavaSerializer.scala | 83 --
core/src/main/scala/spark/KryoSerializer.scala | 156 ---
core/src/main/scala/spark/Logging.scala | 95 --
.../src/main/scala/spark/MapOutputTracker.scala | 338 -----
.../src/main/scala/spark/PairRDDFunctions.scala | 703 ----------
core/src/main/scala/spark/Partition.scala | 31 -
core/src/main/scala/spark/Partitioner.scala | 135 --
core/src/main/scala/spark/RDD.scala | 957 -------------
.../main/scala/spark/RDDCheckpointData.scala | 130 --
.../scala/spark/SequenceFileRDDFunctions.scala | 107 --
.../main/scala/spark/SerializableWritable.scala | 42 -
core/src/main/scala/spark/ShuffleFetcher.scala | 35 -
core/src/main/scala/spark/SizeEstimator.scala | 283 ----
core/src/main/scala/spark/SparkContext.scala | 995 -------------
core/src/main/scala/spark/SparkEnv.scala | 241 ----
core/src/main/scala/spark/SparkException.scala | 24 -
core/src/main/scala/spark/SparkFiles.java | 42 -
.../main/scala/spark/SparkHadoopWriter.scala | 201 ---
core/src/main/scala/spark/TaskContext.scala | 41 -
core/src/main/scala/spark/TaskEndReason.scala | 51 -
core/src/main/scala/spark/TaskState.scala | 51 -
core/src/main/scala/spark/Utils.scala | 780 -----------
.../scala/spark/api/java/JavaDoubleRDD.scala | 167 ---
.../main/scala/spark/api/java/JavaPairRDD.scala | 601 --------
.../src/main/scala/spark/api/java/JavaRDD.scala | 114 --
.../main/scala/spark/api/java/JavaRDDLike.scala | 426 ------
.../scala/spark/api/java/JavaSparkContext.scala | 418 ------
.../java/JavaSparkContextVarargsWorkaround.java | 64 -
.../main/scala/spark/api/java/JavaUtils.scala | 28 -
.../scala/spark/api/java/StorageLevels.java | 48 -
.../java/function/DoubleFlatMapFunction.java | 37 -
.../spark/api/java/function/DoubleFunction.java | 34 -
.../api/java/function/FlatMapFunction.scala | 28 -
.../api/java/function/FlatMapFunction2.scala | 28 -
.../scala/spark/api/java/function/Function.java | 39 -
.../spark/api/java/function/Function2.java | 38 -
.../api/java/function/PairFlatMapFunction.java | 46 -
.../spark/api/java/function/PairFunction.java | 45 -
.../spark/api/java/function/VoidFunction.scala | 33 -
.../api/java/function/WrappedFunction1.scala | 32 -
.../api/java/function/WrappedFunction2.scala | 32 -
.../spark/api/python/PythonPartitioner.scala | 50 -
.../main/scala/spark/api/python/PythonRDD.scala | 344 -----
.../spark/api/python/PythonWorkerFactory.scala | 132 --
.../spark/broadcast/BitTorrentBroadcast.scala | 1057 --------------
.../main/scala/spark/broadcast/Broadcast.scala | 70 -
.../spark/broadcast/BroadcastFactory.scala | 30 -
.../scala/spark/broadcast/HttpBroadcast.scala | 171 ---
.../scala/spark/broadcast/MultiTracker.scala | 409 ------
.../main/scala/spark/broadcast/SourceInfo.scala | 54 -
.../scala/spark/broadcast/TreeBroadcast.scala | 602 --------
.../spark/deploy/ApplicationDescription.scala | 32 -
core/src/main/scala/spark/deploy/Command.scala | 26 -
.../main/scala/spark/deploy/DeployMessage.scala | 130 --
.../main/scala/spark/deploy/ExecutorState.scala | 28 -
.../main/scala/spark/deploy/JsonProtocol.scala | 86 --
.../scala/spark/deploy/LocalSparkCluster.scala | 69 -
.../scala/spark/deploy/SparkHadoopUtil.scala | 36 -
core/src/main/scala/spark/deploy/WebUI.scala | 47 -
.../main/scala/spark/deploy/client/Client.scala | 145 --
.../spark/deploy/client/ClientListener.scala | 35 -
.../scala/spark/deploy/client/TestClient.scala | 51 -
.../spark/deploy/client/TestExecutor.scala | 27 -
.../spark/deploy/master/ApplicationInfo.scala | 85 --
.../spark/deploy/master/ApplicationSource.scala | 24 -
.../spark/deploy/master/ApplicationState.scala | 28 -
.../spark/deploy/master/ExecutorInfo.scala | 32 -
.../main/scala/spark/deploy/master/Master.scala | 386 ------
.../spark/deploy/master/MasterArguments.scala | 89 --
.../spark/deploy/master/MasterSource.scala | 25 -
.../scala/spark/deploy/master/WorkerInfo.scala | 77 --
.../scala/spark/deploy/master/WorkerState.scala | 24 -
.../deploy/master/ui/ApplicationPage.scala | 118 --
.../spark/deploy/master/ui/IndexPage.scala | 141 --
.../spark/deploy/master/ui/MasterWebUI.scala | 80 --
.../spark/deploy/worker/ExecutorRunner.scala | 199 ---
.../main/scala/spark/deploy/worker/Worker.scala | 213 ---
.../spark/deploy/worker/WorkerArguments.scala | 153 --
.../spark/deploy/worker/WorkerSource.scala | 34 -
.../spark/deploy/worker/ui/IndexPage.scala | 115 --
.../spark/deploy/worker/ui/WorkerWebUI.scala | 190 ---
.../main/scala/spark/executor/Executor.scala | 269 ----
.../scala/spark/executor/ExecutorBackend.scala | 28 -
.../scala/spark/executor/ExecutorExitCode.scala | 60 -
.../scala/spark/executor/ExecutorSource.scala | 55 -
.../spark/executor/ExecutorURLClassLoader.scala | 31 -
.../spark/executor/MesosExecutorBackend.scala | 95 --
.../executor/StandaloneExecutorBackend.scala | 107 --
.../main/scala/spark/executor/TaskMetrics.scala | 105 --
.../main/scala/spark/io/CompressionCodec.scala | 82 --
.../scala/spark/metrics/MetricsConfig.scala | 100 --
.../scala/spark/metrics/MetricsSystem.scala | 163 ---
.../scala/spark/metrics/sink/ConsoleSink.scala | 59 -
.../main/scala/spark/metrics/sink/CsvSink.scala | 68 -
.../main/scala/spark/metrics/sink/JmxSink.scala | 35 -
.../spark/metrics/sink/MetricsServlet.scala | 55 -
.../main/scala/spark/metrics/sink/Sink.scala | 23 -
.../scala/spark/metrics/source/JvmSource.scala | 32 -
.../scala/spark/metrics/source/Source.scala | 25 -
.../scala/spark/network/BufferMessage.scala | 111 --
.../main/scala/spark/network/Connection.scala | 586 --------
.../scala/spark/network/ConnectionManager.scala | 720 ----------
.../spark/network/ConnectionManagerId.scala | 38 -
.../spark/network/ConnectionManagerTest.scala | 102 --
core/src/main/scala/spark/network/Message.scala | 93 --
.../main/scala/spark/network/MessageChunk.scala | 42 -
.../spark/network/MessageChunkHeader.scala | 75 -
.../main/scala/spark/network/ReceiverTest.scala | 37 -
.../main/scala/spark/network/SenderTest.scala | 70 -
.../scala/spark/network/netty/FileHeader.scala | 74 -
.../spark/network/netty/ShuffleCopier.scala | 118 --
.../spark/network/netty/ShuffleSender.scala | 70 -
core/src/main/scala/spark/package.scala | 32 -
.../partial/ApproximateActionListener.scala | 87 --
.../spark/partial/ApproximateEvaluator.scala | 27 -
.../scala/spark/partial/BoundedDouble.scala | 25 -
.../scala/spark/partial/CountEvaluator.scala | 55 -
.../spark/partial/GroupedCountEvaluator.scala | 79 --
.../spark/partial/GroupedMeanEvaluator.scala | 82 --
.../spark/partial/GroupedSumEvaluator.scala | 89 --
.../scala/spark/partial/MeanEvaluator.scala | 58 -
.../scala/spark/partial/PartialResult.scala | 137 --
.../scala/spark/partial/StudentTCacher.scala | 43 -
.../main/scala/spark/partial/SumEvaluator.scala | 68 -
core/src/main/scala/spark/rdd/BlockRDD.scala | 51 -
.../src/main/scala/spark/rdd/CartesianRDD.scala | 90 --
.../main/scala/spark/rdd/CheckpointRDD.scala | 155 ---
.../src/main/scala/spark/rdd/CoGroupedRDD.scala | 144 --
.../src/main/scala/spark/rdd/CoalescedRDD.scala | 342 -----
core/src/main/scala/spark/rdd/EmptyRDD.scala | 33 -
core/src/main/scala/spark/rdd/FilteredRDD.scala | 33 -
.../main/scala/spark/rdd/FlatMappedRDD.scala | 33 -
.../scala/spark/rdd/FlatMappedValuesRDD.scala | 36 -
core/src/main/scala/spark/rdd/GlommedRDD.scala | 29 -
core/src/main/scala/spark/rdd/HadoopRDD.scala | 137 --
core/src/main/scala/spark/rdd/JdbcRDD.scala | 120 --
.../main/scala/spark/rdd/MapPartitionsRDD.scala | 37 -
.../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 -
core/src/main/scala/spark/rdd/MappedRDD.scala | 30 -
.../main/scala/spark/rdd/MappedValuesRDD.scala | 34 -
.../src/main/scala/spark/rdd/NewHadoopRDD.scala | 126 --
.../scala/spark/rdd/OrderedRDDFunctions.scala | 51 -
.../scala/spark/rdd/ParallelCollectionRDD.scala | 151 --
.../scala/spark/rdd/PartitionPruningRDD.scala | 72 -
core/src/main/scala/spark/rdd/PipedRDD.scala | 125 --
core/src/main/scala/spark/rdd/SampledRDD.scala | 66 -
core/src/main/scala/spark/rdd/ShuffledRDD.scala | 67 -
.../main/scala/spark/rdd/SubtractedRDD.scala | 129 --
core/src/main/scala/spark/rdd/UnionRDD.scala | 73 -
.../scala/spark/rdd/ZippedPartitionsRDD.scala | 143 --
core/src/main/scala/spark/rdd/ZippedRDD.scala | 85 --
.../main/scala/spark/scheduler/ActiveJob.scala | 39 -
.../scala/spark/scheduler/DAGScheduler.scala | 849 ------------
.../spark/scheduler/DAGSchedulerEvent.scala | 63 -
.../spark/scheduler/DAGSchedulerSource.scala | 30 -
.../scala/spark/scheduler/InputFormatInfo.scala | 178 ---
.../scala/spark/scheduler/JobListener.scala | 28 -
.../main/scala/spark/scheduler/JobLogger.scala | 292 ----
.../main/scala/spark/scheduler/JobResult.scala | 26 -
.../main/scala/spark/scheduler/JobWaiter.scala | 66 -
.../main/scala/spark/scheduler/MapStatus.scala | 44 -
.../main/scala/spark/scheduler/ResultTask.scala | 134 --
.../scala/spark/scheduler/ShuffleMapTask.scala | 189 ---
.../scala/spark/scheduler/SparkListener.scala | 204 ---
.../spark/scheduler/SparkListenerBus.scala | 74 -
.../main/scala/spark/scheduler/SplitInfo.scala | 78 --
core/src/main/scala/spark/scheduler/Stage.scala | 112 --
.../main/scala/spark/scheduler/StageInfo.scala | 29 -
core/src/main/scala/spark/scheduler/Task.scala | 115 --
.../scala/spark/scheduler/TaskLocation.scala | 34 -
.../main/scala/spark/scheduler/TaskResult.scala | 72 -
.../scala/spark/scheduler/TaskScheduler.scala | 52 -
.../spark/scheduler/TaskSchedulerListener.scala | 45 -
.../main/scala/spark/scheduler/TaskSet.scala | 35 -
.../scheduler/cluster/ClusterScheduler.scala | 440 ------
.../cluster/ClusterTaskSetManager.scala | 712 ----------
.../scheduler/cluster/ExecutorLossReason.scala | 38 -
.../scala/spark/scheduler/cluster/Pool.scala | 121 --
.../spark/scheduler/cluster/Schedulable.scala | 48 -
.../scheduler/cluster/SchedulableBuilder.scala | 137 --
.../scheduler/cluster/SchedulerBackend.scala | 37 -
.../scheduler/cluster/SchedulingAlgorithm.scala | 81 --
.../scheduler/cluster/SchedulingMode.scala | 29 -
.../cluster/SparkDeploySchedulerBackend.scala | 90 --
.../cluster/StandaloneClusterMessage.scala | 63 -
.../cluster/StandaloneSchedulerBackend.scala | 198 ---
.../scheduler/cluster/TaskDescription.scala | 37 -
.../spark/scheduler/cluster/TaskInfo.scala | 72 -
.../spark/scheduler/cluster/TaskLocality.scala | 32 -
.../scheduler/cluster/TaskSetManager.scala | 51 -
.../spark/scheduler/cluster/WorkerOffer.scala | 24 -
.../spark/scheduler/local/LocalScheduler.scala | 272 ----
.../scheduler/local/LocalTaskSetManager.scala | 194 ---
.../mesos/CoarseMesosSchedulerBackend.scala | 284 ----
.../scheduler/mesos/MesosSchedulerBackend.scala | 342 -----
.../scala/spark/serializer/Serializer.scala | 112 --
.../spark/serializer/SerializerManager.scala | 62 -
.../scala/spark/storage/BlockException.scala | 22 -
.../scala/spark/storage/BlockFetchTracker.scala | 27 -
.../spark/storage/BlockFetcherIterator.scala | 348 -----
.../main/scala/spark/storage/BlockManager.scala | 1046 --------------
.../scala/spark/storage/BlockManagerId.scala | 118 --
.../spark/storage/BlockManagerMaster.scala | 178 ---
.../spark/storage/BlockManagerMasterActor.scala | 404 ------
.../spark/storage/BlockManagerMessages.scala | 110 --
.../spark/storage/BlockManagerSlaveActor.scala | 39 -
.../spark/storage/BlockManagerSource.scala | 48 -
.../spark/storage/BlockManagerWorker.scala | 139 --
.../main/scala/spark/storage/BlockMessage.scala | 223 ---
.../scala/spark/storage/BlockMessageArray.scala | 159 ---
.../scala/spark/storage/BlockObjectWriter.scala | 65 -
.../main/scala/spark/storage/BlockStore.scala | 61 -
.../main/scala/spark/storage/DiskStore.scala | 329 -----
.../main/scala/spark/storage/MemoryStore.scala | 257 ----
.../main/scala/spark/storage/PutResult.scala | 26 -
.../spark/storage/ShuffleBlockManager.scala | 67 -
.../main/scala/spark/storage/StorageLevel.scala | 146 --
.../main/scala/spark/storage/StorageUtils.scala | 115 --
.../scala/spark/storage/ThreadingTest.scala | 113 --
core/src/main/scala/spark/ui/JettyUtils.scala | 132 --
core/src/main/scala/spark/ui/Page.scala | 22 -
core/src/main/scala/spark/ui/SparkUI.scala | 87 --
core/src/main/scala/spark/ui/UIUtils.scala | 131 --
.../scala/spark/ui/UIWorkloadGenerator.scala | 105 --
.../main/scala/spark/ui/env/EnvironmentUI.scala | 91 --
.../main/scala/spark/ui/exec/ExecutorsUI.scala | 136 --
.../main/scala/spark/ui/jobs/IndexPage.scala | 90 --
.../spark/ui/jobs/JobProgressListener.scala | 156 ---
.../scala/spark/ui/jobs/JobProgressUI.scala | 60 -
.../src/main/scala/spark/ui/jobs/PoolPage.scala | 32 -
.../main/scala/spark/ui/jobs/PoolTable.scala | 55 -
.../main/scala/spark/ui/jobs/StagePage.scala | 183 ---
.../main/scala/spark/ui/jobs/StageTable.scala | 107 --
.../scala/spark/ui/storage/BlockManagerUI.scala | 41 -
.../main/scala/spark/ui/storage/IndexPage.scala | 65 -
.../main/scala/spark/ui/storage/RDDPage.scala | 132 --
core/src/main/scala/spark/util/AkkaUtils.scala | 72 -
.../scala/spark/util/BoundedPriorityQueue.scala | 62 -
.../spark/util/ByteBufferInputStream.scala | 80 --
core/src/main/scala/spark/util/Clock.scala | 29 -
.../scala/spark/util/CompletionIterator.scala | 42 -
.../main/scala/spark/util/Distribution.scala | 82 --
.../src/main/scala/spark/util/IdGenerator.scala | 31 -
core/src/main/scala/spark/util/IntParam.scala | 31 -
.../src/main/scala/spark/util/MemoryParam.scala | 34 -
.../main/scala/spark/util/MetadataCleaner.scala | 61 -
.../src/main/scala/spark/util/MutablePair.scala | 36 -
.../main/scala/spark/util/NextIterator.scala | 88 --
.../spark/util/RateLimitedOutputStream.scala | 79 --
.../scala/spark/util/SerializableBuffer.scala | 54 -
.../src/main/scala/spark/util/StatCounter.scala | 131 --
.../scala/spark/util/TimeStampedHashMap.scala | 121 --
.../scala/spark/util/TimeStampedHashSet.scala | 86 --
core/src/main/scala/spark/util/Vector.scala | 139 --
.../resources/test_metrics_config.properties | 2 +-
.../resources/test_metrics_system.properties | 6 +-
.../org/apache/spark/AccumulatorSuite.scala | 143 ++
.../scala/org/apache/spark/BroadcastSuite.scala | 39 +
.../org/apache/spark/CheckpointSuite.scala | 392 ++++++
.../org/apache/spark/ClosureCleanerSuite.scala | 146 ++
.../org/apache/spark/DistributedSuite.scala | 362 +++++
.../scala/org/apache/spark/DriverSuite.scala | 54 +
.../scala/org/apache/spark/FailureSuite.scala | 127 ++
.../org/apache/spark/FileServerSuite.scala | 123 ++
.../test/scala/org/apache/spark/FileSuite.scala | 212 +++
.../scala/org/apache/spark/JavaAPISuite.java | 865 ++++++++++++
.../org/apache/spark/KryoSerializerSuite.scala | 208 +++
.../org/apache/spark/LocalSparkContext.scala | 68 +
.../apache/spark/MapOutputTrackerSuite.scala | 136 ++
.../apache/spark/PairRDDFunctionsSuite.scala | 299 ++++
.../apache/spark/PartitionPruningRDDSuite.scala | 28 +
.../org/apache/spark/PartitioningSuite.scala | 150 ++
.../scala/org/apache/spark/PipedRDDSuite.scala | 93 ++
.../test/scala/org/apache/spark/RDDSuite.scala | 389 ++++++
.../org/apache/spark/SharedSparkContext.scala | 42 +
.../org/apache/spark/ShuffleNettySuite.scala | 34 +
.../scala/org/apache/spark/ShuffleSuite.scala | 210 +++
.../org/apache/spark/SizeEstimatorSuite.scala | 164 +++
.../scala/org/apache/spark/SortingSuite.scala | 123 ++
.../apache/spark/SparkContextInfoSuite.scala | 60 +
.../scala/org/apache/spark/ThreadingSuite.scala | 152 ++
.../scala/org/apache/spark/UnpersistSuite.scala | 47 +
.../scala/org/apache/spark/UtilsSuite.scala | 139 ++
.../apache/spark/ZippedPartitionsSuite.scala | 50 +
.../apache/spark/io/CompressionCodecSuite.scala | 62 +
.../spark/metrics/MetricsConfigSuite.scala | 89 ++
.../spark/metrics/MetricsSystemSuite.scala | 54 +
.../org/apache/spark/rdd/JdbcRDDSuite.scala | 73 +
.../rdd/ParallelCollectionSplitSuite.scala | 212 +++
.../spark/scheduler/DAGSchedulerSuite.scala | 421 ++++++
.../apache/spark/scheduler/JobLoggerSuite.scala | 121 ++
.../spark/scheduler/SparkListenerSuite.scala | 102 ++
.../spark/scheduler/TaskContextSuite.scala | 49 +
.../cluster/ClusterSchedulerSuite.scala | 266 ++++
.../cluster/ClusterTaskSetManagerSuite.scala | 273 ++++
.../spark/scheduler/cluster/FakeTask.scala | 26 +
.../scheduler/local/LocalSchedulerSuite.scala | 223 +++
.../spark/storage/BlockManagerSuite.scala | 666 +++++++++
.../scala/org/apache/spark/ui/UISuite.scala | 47 +
.../apache/spark/util/DistributionSuite.scala | 42 +
.../scala/org/apache/spark/util/FakeClock.scala | 26 +
.../apache/spark/util/NextIteratorSuite.scala | 85 ++
.../util/RateLimitedOutputStreamSuite.scala | 40 +
.../src/test/scala/spark/AccumulatorSuite.scala | 143 --
core/src/test/scala/spark/BroadcastSuite.scala | 39 -
core/src/test/scala/spark/CheckpointSuite.scala | 392 ------
.../test/scala/spark/ClosureCleanerSuite.scala | 146 --
.../src/test/scala/spark/DistributedSuite.scala | 362 -----
core/src/test/scala/spark/DriverSuite.scala | 54 -
core/src/test/scala/spark/FailureSuite.scala | 127 --
core/src/test/scala/spark/FileServerSuite.scala | 123 --
core/src/test/scala/spark/FileSuite.scala | 212 ---
core/src/test/scala/spark/JavaAPISuite.java | 865 ------------
.../test/scala/spark/KryoSerializerSuite.scala | 208 ---
.../test/scala/spark/LocalSparkContext.scala | 68 -
.../scala/spark/MapOutputTrackerSuite.scala | 136 --
.../scala/spark/PairRDDFunctionsSuite.scala | 299 ----
.../scala/spark/PartitionPruningRDDSuite.scala | 28 -
.../test/scala/spark/PartitioningSuite.scala | 150 --
core/src/test/scala/spark/PipedRDDSuite.scala | 93 --
core/src/test/scala/spark/RDDSuite.scala | 389 ------
.../test/scala/spark/SharedSparkContext.scala | 42 -
.../test/scala/spark/ShuffleNettySuite.scala | 34 -
core/src/test/scala/spark/ShuffleSuite.scala | 210 ---
.../test/scala/spark/SizeEstimatorSuite.scala | 164 ---
core/src/test/scala/spark/SortingSuite.scala | 123 --
.../scala/spark/SparkContextInfoSuite.scala | 60 -
core/src/test/scala/spark/ThreadingSuite.scala | 152 --
core/src/test/scala/spark/UnpersistSuite.scala | 47 -
core/src/test/scala/spark/UtilsSuite.scala | 139 --
.../scala/spark/ZippedPartitionsSuite.scala | 50 -
.../scala/spark/io/CompressionCodecSuite.scala | 62 -
.../spark/metrics/MetricsConfigSuite.scala | 89 --
.../spark/metrics/MetricsSystemSuite.scala | 53 -
.../src/test/scala/spark/rdd/JdbcRDDSuite.scala | 73 -
.../rdd/ParallelCollectionSplitSuite.scala | 212 ---
.../spark/scheduler/DAGSchedulerSuite.scala | 421 ------
.../scala/spark/scheduler/JobLoggerSuite.scala | 121 --
.../spark/scheduler/SparkListenerSuite.scala | 102 --
.../spark/scheduler/TaskContextSuite.scala | 49 -
.../cluster/ClusterSchedulerSuite.scala | 266 ----
.../cluster/ClusterTaskSetManagerSuite.scala | 273 ----
.../spark/scheduler/cluster/FakeTask.scala | 26 -
.../scheduler/local/LocalSchedulerSuite.scala | 223 ---
.../scala/spark/storage/BlockManagerSuite.scala | 665 ---------
core/src/test/scala/spark/ui/UISuite.scala | 47 -
.../scala/spark/util/DistributionSuite.scala | 42 -
core/src/test/scala/spark/util/FakeClock.scala | 26 -
.../scala/spark/util/NextIteratorSuite.scala | 85 --
.../util/RateLimitedOutputStreamSuite.scala | 40 -
docs/_layouts/global.html | 2 +-
examples/pom.xml | 14 +-
.../org/apache/spark/examples/JavaHdfsLR.java | 140 ++
.../org/apache/spark/examples/JavaKMeans.java | 131 ++
.../org/apache/spark/examples/JavaLogQuery.java | 131 ++
.../org/apache/spark/examples/JavaPageRank.java | 115 ++
.../org/apache/spark/examples/JavaSparkPi.java | 65 +
.../java/org/apache/spark/examples/JavaTC.java | 97 ++
.../apache/spark/examples/JavaWordCount.java | 66 +
.../apache/spark/mllib/examples/JavaALS.java | 87 ++
.../apache/spark/mllib/examples/JavaKMeans.java | 81 ++
.../org/apache/spark/mllib/examples/JavaLR.java | 85 ++
.../streaming/examples/JavaFlumeEventCount.java | 68 +
.../examples/JavaNetworkWordCount.java | 79 ++
.../streaming/examples/JavaQueueStream.java | 80 ++
.../main/java/spark/examples/JavaHdfsLR.java | 140 --
.../main/java/spark/examples/JavaKMeans.java | 131 --
.../main/java/spark/examples/JavaLogQuery.java | 131 --
.../main/java/spark/examples/JavaPageRank.java | 115 --
.../main/java/spark/examples/JavaSparkPi.java | 65 -
.../src/main/java/spark/examples/JavaTC.java | 97 --
.../main/java/spark/examples/JavaWordCount.java | 66 -
.../main/java/spark/mllib/examples/JavaALS.java | 87 --
.../java/spark/mllib/examples/JavaKMeans.java | 81 --
.../main/java/spark/mllib/examples/JavaLR.java | 85 --
.../streaming/examples/JavaFlumeEventCount.java | 68 -
.../examples/JavaNetworkWordCount.java | 79 --
.../streaming/examples/JavaQueueStream.java | 80 --
.../apache/spark/examples/BroadcastTest.scala | 50 +
.../apache/spark/examples/CassandraTest.scala | 213 +++
.../spark/examples/ExceptionHandlingTest.scala | 38 +
.../org/apache/spark/examples/GroupByTest.scala | 57 +
.../org/apache/spark/examples/HBaseTest.scala | 52 +
.../org/apache/spark/examples/HdfsTest.scala | 37 +
.../org/apache/spark/examples/LocalALS.scala | 140 ++
.../org/apache/spark/examples/LocalFileLR.scala | 55 +
.../org/apache/spark/examples/LocalKMeans.scala | 99 ++
.../org/apache/spark/examples/LocalLR.scala | 63 +
.../org/apache/spark/examples/LocalPi.scala | 34 +
.../org/apache/spark/examples/LogQuery.scala | 85 ++
.../spark/examples/MultiBroadcastTest.scala | 53 +
.../examples/SimpleSkewedGroupByTest.scala | 71 +
.../spark/examples/SkewedGroupByTest.scala | 61 +
.../org/apache/spark/examples/SparkALS.scala | 143 ++
.../org/apache/spark/examples/SparkHdfsLR.scala | 78 ++
.../org/apache/spark/examples/SparkKMeans.scala | 91 ++
.../org/apache/spark/examples/SparkLR.scala | 71 +
.../apache/spark/examples/SparkPageRank.scala | 46 +
.../org/apache/spark/examples/SparkPi.scala | 43 +
.../org/apache/spark/examples/SparkTC.scala | 75 +
.../spark/examples/bagel/PageRankUtils.scala | 123 ++
.../examples/bagel/WikipediaPageRank.scala | 101 ++
.../bagel/WikipediaPageRankStandalone.scala | 223 +++
.../streaming/examples/ActorWordCount.scala | 175 +++
.../streaming/examples/FlumeEventCount.scala | 61 +
.../streaming/examples/HdfsWordCount.scala | 54 +
.../streaming/examples/KafkaWordCount.scala | 98 ++
.../streaming/examples/NetworkWordCount.scala | 54 +
.../spark/streaming/examples/QueueStream.scala | 57 +
.../streaming/examples/RawNetworkGrep.scala | 64 +
.../examples/StatefulNetworkWordCount.scala | 67 +
.../streaming/examples/TwitterAlgebirdCMS.scala | 110 ++
.../streaming/examples/TwitterAlgebirdHLL.scala | 88 ++
.../streaming/examples/TwitterPopularTags.scala | 70 +
.../streaming/examples/ZeroMQWordCount.scala | 91 ++
.../clickstream/PageViewGenerator.scala | 102 ++
.../examples/clickstream/PageViewStream.scala | 101 ++
.../scala/spark/examples/BroadcastTest.scala | 50 -
.../scala/spark/examples/CassandraTest.scala | 213 ---
.../spark/examples/ExceptionHandlingTest.scala | 38 -
.../main/scala/spark/examples/GroupByTest.scala | 57 -
.../main/scala/spark/examples/HBaseTest.scala | 52 -
.../main/scala/spark/examples/HdfsTest.scala | 37 -
.../main/scala/spark/examples/LocalALS.scala | 140 --
.../main/scala/spark/examples/LocalFileLR.scala | 55 -
.../main/scala/spark/examples/LocalKMeans.scala | 99 --
.../src/main/scala/spark/examples/LocalLR.scala | 63 -
.../src/main/scala/spark/examples/LocalPi.scala | 34 -
.../main/scala/spark/examples/LogQuery.scala | 85 --
.../spark/examples/MultiBroadcastTest.scala | 53 -
.../examples/SimpleSkewedGroupByTest.scala | 71 -
.../spark/examples/SkewedGroupByTest.scala | 61 -
.../main/scala/spark/examples/SparkALS.scala | 143 --
.../main/scala/spark/examples/SparkHdfsLR.scala | 78 --
.../main/scala/spark/examples/SparkKMeans.scala | 91 --
.../src/main/scala/spark/examples/SparkLR.scala | 71 -
.../scala/spark/examples/SparkPageRank.scala | 46 -
.../src/main/scala/spark/examples/SparkPi.scala | 43 -
.../src/main/scala/spark/examples/SparkTC.scala | 75 -
.../spark/examples/bagel/PageRankUtils.scala | 123 --
.../examples/bagel/WikipediaPageRank.scala | 101 --
.../bagel/WikipediaPageRankStandalone.scala | 223 ---
.../streaming/examples/ActorWordCount.scala | 175 ---
.../streaming/examples/FlumeEventCount.scala | 61 -
.../streaming/examples/HdfsWordCount.scala | 54 -
.../streaming/examples/KafkaWordCount.scala | 98 --
.../streaming/examples/NetworkWordCount.scala | 54 -
.../spark/streaming/examples/QueueStream.scala | 57 -
.../streaming/examples/RawNetworkGrep.scala | 64 -
.../examples/StatefulNetworkWordCount.scala | 67 -
.../streaming/examples/TwitterAlgebirdCMS.scala | 110 --
.../streaming/examples/TwitterAlgebirdHLL.scala | 88 --
.../streaming/examples/TwitterPopularTags.scala | 70 -
.../streaming/examples/ZeroMQWordCount.scala | 91 --
.../clickstream/PageViewGenerator.scala | 102 --
.../examples/clickstream/PageViewStream.scala | 101 --
mllib/pom.xml | 6 +-
.../classification/ClassificationModel.scala | 21 +
.../classification/LogisticRegression.scala | 188 +++
.../apache/spark/mllib/classification/SVM.scala | 187 +++
.../apache/spark/mllib/clustering/KMeans.scala | 335 +++++
.../spark/mllib/clustering/KMeansModel.scala | 44 +
.../spark/mllib/clustering/LocalKMeans.scala | 105 ++
.../spark/mllib/optimization/Gradient.scala | 98 ++
.../mllib/optimization/GradientDescent.scala | 166 +++
.../spark/mllib/optimization/Optimizer.scala | 29 +
.../spark/mllib/optimization/Updater.scala | 99 ++
.../apache/spark/mllib/recommendation/ALS.scala | 453 ++++++
.../MatrixFactorizationModel.scala | 49 +
.../regression/GeneralizedLinearAlgorithm.scala | 159 +++
.../spark/mllib/regression/LabeledPoint.scala | 26 +
.../apache/spark/mllib/regression/Lasso.scala | 210 +++
.../mllib/regression/LinearRegression.scala | 167 +++
.../mllib/regression/RegressionModel.scala | 38 +
.../mllib/regression/RidgeRegression.scala | 213 +++
.../spark/mllib/util/DataValidators.scala | 42 +
.../spark/mllib/util/KMeansDataGenerator.scala | 84 ++
.../spark/mllib/util/LinearDataGenerator.scala | 132 ++
.../util/LogisticRegressionDataGenerator.scala | 81 ++
.../spark/mllib/util/MFDataGenerator.scala | 113 ++
.../org/apache/spark/mllib/util/MLUtils.scala | 122 ++
.../spark/mllib/util/SVMDataGenerator.scala | 50 +
.../classification/ClassificationModel.scala | 21 -
.../classification/LogisticRegression.scala | 188 ---
.../scala/spark/mllib/classification/SVM.scala | 187 ---
.../scala/spark/mllib/clustering/KMeans.scala | 335 -----
.../spark/mllib/clustering/KMeansModel.scala | 44 -
.../spark/mllib/clustering/LocalKMeans.scala | 105 --
.../spark/mllib/optimization/Gradient.scala | 98 --
.../mllib/optimization/GradientDescent.scala | 166 ---
.../spark/mllib/optimization/Optimizer.scala | 29 -
.../spark/mllib/optimization/Updater.scala | 99 --
.../scala/spark/mllib/recommendation/ALS.scala | 453 ------
.../MatrixFactorizationModel.scala | 49 -
.../regression/GeneralizedLinearAlgorithm.scala | 159 ---
.../spark/mllib/regression/LabeledPoint.scala | 26 -
.../scala/spark/mllib/regression/Lasso.scala | 210 ---
.../mllib/regression/LinearRegression.scala | 167 ---
.../mllib/regression/RegressionModel.scala | 38 -
.../mllib/regression/RidgeRegression.scala | 213 ---
.../scala/spark/mllib/util/DataValidators.scala | 42 -
.../spark/mllib/util/KMeansDataGenerator.scala | 84 --
.../spark/mllib/util/LinearDataGenerator.scala | 132 --
.../util/LogisticRegressionDataGenerator.scala | 81 --
.../spark/mllib/util/MFDataGenerator.scala | 113 --
.../main/scala/spark/mllib/util/MLUtils.scala | 122 --
.../spark/mllib/util/SVMDataGenerator.scala | 50 -
.../JavaLogisticRegressionSuite.java | 98 ++
.../mllib/classification/JavaSVMSuite.java | 98 ++
.../spark/mllib/clustering/JavaKMeansSuite.java | 115 ++
.../mllib/recommendation/JavaALSSuite.java | 110 ++
.../spark/mllib/regression/JavaLassoSuite.java | 97 ++
.../regression/JavaLinearRegressionSuite.java | 94 ++
.../regression/JavaRidgeRegressionSuite.java | 110 ++
.../JavaLogisticRegressionSuite.java | 98 --
.../mllib/classification/JavaSVMSuite.java | 98 --
.../spark/mllib/clustering/JavaKMeansSuite.java | 115 --
.../mllib/recommendation/JavaALSSuite.java | 110 --
.../spark/mllib/regression/JavaLassoSuite.java | 97 --
.../regression/JavaLinearRegressionSuite.java | 94 --
.../regression/JavaRidgeRegressionSuite.java | 110 --
.../LogisticRegressionSuite.scala | 150 ++
.../spark/mllib/classification/SVMSuite.scala | 169 +++
.../spark/mllib/clustering/KMeansSuite.scala | 173 +++
.../spark/mllib/recommendation/ALSSuite.scala | 125 ++
.../spark/mllib/regression/LassoSuite.scala | 121 ++
.../regression/LinearRegressionSuite.scala | 72 +
.../mllib/regression/RidgeRegressionSuite.scala | 90 ++
.../LogisticRegressionSuite.scala | 150 --
.../spark/mllib/classification/SVMSuite.scala | 169 ---
.../spark/mllib/clustering/KMeansSuite.scala | 173 ---
.../spark/mllib/recommendation/ALSSuite.scala | 125 --
.../spark/mllib/regression/LassoSuite.scala | 121 --
.../regression/LinearRegressionSuite.scala | 72 -
.../mllib/regression/RidgeRegressionSuite.scala | 90 --
pom.xml | 14 +-
project/SparkBuild.scala | 12 +-
python/pyspark/context.py | 4 +-
python/pyspark/files.py | 2 +-
python/pyspark/java_gateway.py | 4 +-
repl-bin/pom.xml | 12 +-
repl/pom.xml | 12 +-
.../apache/spark/repl/ExecutorClassLoader.scala | 124 ++
.../main/scala/org/apache/spark/repl/Main.scala | 33 +
.../org/apache/spark/repl/SparkHelper.scala | 5 +
.../org/apache/spark/repl/SparkILoop.scala | 1008 ++++++++++++++
.../org/apache/spark/repl/SparkIMain.scala | 1160 ++++++++++++++++
.../org/apache/spark/repl/SparkISettings.scala | 63 +
.../org/apache/spark/repl/SparkImports.scala | 214 +++
.../spark/repl/SparkJLineCompletion.scala | 379 +++++
.../apache/spark/repl/SparkJLineReader.scala | 79 ++
.../apache/spark/repl/SparkMemberHandlers.scala | 207 +++
.../scala/spark/repl/ExecutorClassLoader.scala | 124 --
repl/src/main/scala/spark/repl/Main.scala | 33 -
.../src/main/scala/spark/repl/SparkHelper.scala | 5 -
repl/src/main/scala/spark/repl/SparkILoop.scala | 1008 --------------
repl/src/main/scala/spark/repl/SparkIMain.scala | 1160 ----------------
.../main/scala/spark/repl/SparkISettings.scala | 63 -
.../main/scala/spark/repl/SparkImports.scala | 214 ---
.../scala/spark/repl/SparkJLineCompletion.scala | 379 -----
.../scala/spark/repl/SparkJLineReader.scala | 79 --
.../scala/spark/repl/SparkMemberHandlers.scala | 207 ---
.../scala/org/apache/spark/repl/ReplSuite.scala | 207 +++
repl/src/test/scala/spark/repl/ReplSuite.scala | 207 ---
spark-executor | 2 +-
spark-shell | 2 +-
spark-shell.cmd | 2 +-
streaming/pom.xml | 6 +-
.../org/apache/spark/streaming/Checkpoint.scala | 190 +++
.../org/apache/spark/streaming/DStream.scala | 702 ++++++++++
.../spark/streaming/DStreamCheckpointData.scala | 110 ++
.../apache/spark/streaming/DStreamGraph.scala | 167 +++
.../org/apache/spark/streaming/Duration.scala | 83 ++
.../org/apache/spark/streaming/Interval.scala | 59 +
.../scala/org/apache/spark/streaming/Job.scala | 41 +
.../org/apache/spark/streaming/JobManager.scala | 88 ++
.../spark/streaming/NetworkInputTracker.scala | 173 +++
.../spark/streaming/PairDStreamFunctions.scala | 534 +++++++
.../org/apache/spark/streaming/Scheduler.scala | 131 ++
.../spark/streaming/StreamingContext.scala | 563 ++++++++
.../scala/org/apache/spark/streaming/Time.scala | 72 +
.../spark/streaming/api/java/JavaDStream.scala | 102 ++
.../streaming/api/java/JavaDStreamLike.scala | 316 +++++
.../streaming/api/java/JavaPairDStream.scala | 613 ++++++++
.../api/java/JavaStreamingContext.scala | 614 +++++++++
.../streaming/dstream/CoGroupedDStream.scala | 57 +
.../dstream/ConstantInputDStream.scala | 36 +
.../streaming/dstream/FileInputDStream.scala | 199 +++
.../streaming/dstream/FilteredDStream.scala | 38 +
.../dstream/FlatMapValuedDStream.scala | 37 +
.../streaming/dstream/FlatMappedDStream.scala | 37 +
.../streaming/dstream/FlumeInputDStream.scala | 154 +++
.../streaming/dstream/ForEachDStream.scala | 45 +
.../streaming/dstream/GlommedDStream.scala | 34 +
.../spark/streaming/dstream/InputDStream.scala | 70 +
.../streaming/dstream/KafkaInputDStream.scala | 141 ++
.../dstream/MapPartitionedDStream.scala | 38 +
.../streaming/dstream/MapValuedDStream.scala | 38 +
.../spark/streaming/dstream/MappedDStream.scala | 37 +
.../streaming/dstream/NetworkInputDStream.scala | 272 ++++
.../dstream/PluggableInputDStream.scala | 30 +
.../streaming/dstream/QueueInputDStream.scala | 59 +
.../streaming/dstream/RawInputDStream.scala | 108 ++
.../dstream/ReducedWindowedDStream.scala | 174 +++
.../streaming/dstream/ShuffledDStream.scala | 44 +
.../streaming/dstream/SocketInputDStream.scala | 94 ++
.../spark/streaming/dstream/StateDStream.scala | 109 ++
.../streaming/dstream/TransformedDStream.scala | 36 +
.../streaming/dstream/TwitterInputDStream.scala | 99 ++
.../spark/streaming/dstream/UnionDStream.scala | 57 +
.../streaming/dstream/WindowedDStream.scala | 57 +
.../streaming/receivers/ActorReceiver.scala | 175 +++
.../streaming/receivers/ZeroMQReceiver.scala | 50 +
.../org/apache/spark/streaming/util/Clock.scala | 101 ++
.../streaming/util/MasterFailureTest.scala | 414 ++++++
.../spark/streaming/util/RawTextHelper.scala | 115 ++
.../spark/streaming/util/RawTextSender.scala | 77 ++
.../spark/streaming/util/RecurringTimer.scala | 94 ++
.../main/scala/spark/streaming/Checkpoint.scala | 190 ---
.../main/scala/spark/streaming/DStream.scala | 700 ----------
.../spark/streaming/DStreamCheckpointData.scala | 110 --
.../scala/spark/streaming/DStreamGraph.scala | 167 ---
.../main/scala/spark/streaming/Duration.scala | 83 --
.../main/scala/spark/streaming/Interval.scala | 59 -
.../src/main/scala/spark/streaming/Job.scala | 41 -
.../main/scala/spark/streaming/JobManager.scala | 88 --
.../spark/streaming/NetworkInputTracker.scala | 173 ---
.../spark/streaming/PairDStreamFunctions.scala | 534 -------
.../main/scala/spark/streaming/Scheduler.scala | 130 --
.../spark/streaming/StreamingContext.scala | 563 --------
.../src/main/scala/spark/streaming/Time.scala | 72 -
.../spark/streaming/api/java/JavaDStream.scala | 102 --
.../streaming/api/java/JavaDStreamLike.scala | 316 -----
.../streaming/api/java/JavaPairDStream.scala | 613 --------
.../api/java/JavaStreamingContext.scala | 613 --------
.../streaming/dstream/CoGroupedDStream.scala | 57 -
.../dstream/ConstantInputDStream.scala | 36 -
.../streaming/dstream/FileInputDStream.scala | 199 ---
.../streaming/dstream/FilteredDStream.scala | 38 -
.../dstream/FlatMapValuedDStream.scala | 37 -
.../streaming/dstream/FlatMappedDStream.scala | 37 -
.../streaming/dstream/FlumeInputDStream.scala | 154 ---
.../streaming/dstream/ForEachDStream.scala | 45 -
.../streaming/dstream/GlommedDStream.scala | 34 -
.../spark/streaming/dstream/InputDStream.scala | 70 -
.../streaming/dstream/KafkaInputDStream.scala | 141 --
.../dstream/MapPartitionedDStream.scala | 38 -
.../streaming/dstream/MapValuedDStream.scala | 38 -
.../spark/streaming/dstream/MappedDStream.scala | 37 -
.../streaming/dstream/NetworkInputDStream.scala | 272 ----
.../dstream/PluggableInputDStream.scala | 30 -
.../streaming/dstream/QueueInputDStream.scala | 59 -
.../streaming/dstream/RawInputDStream.scala | 108 --
.../dstream/ReducedWindowedDStream.scala | 174 ---
.../streaming/dstream/ShuffledDStream.scala | 44 -
.../streaming/dstream/SocketInputDStream.scala | 94 --
.../spark/streaming/dstream/StateDStream.scala | 109 --
.../streaming/dstream/TransformedDStream.scala | 36 -
.../streaming/dstream/TwitterInputDStream.scala | 99 --
.../spark/streaming/dstream/UnionDStream.scala | 57 -
.../streaming/dstream/WindowedDStream.scala | 57 -
.../streaming/receivers/ActorReceiver.scala | 175 ---
.../streaming/receivers/ZeroMQReceiver.scala | 50 -
.../main/scala/spark/streaming/util/Clock.scala | 101 --
.../streaming/util/MasterFailureTest.scala | 414 ------
.../spark/streaming/util/RawTextHelper.scala | 115 --
.../spark/streaming/util/RawTextSender.scala | 77 --
.../spark/streaming/util/RecurringTimer.scala | 94 --
.../apache/spark/streaming/JavaAPISuite.java | 1304 ++++++++++++++++++
.../apache/spark/streaming/JavaTestUtils.scala | 85 ++
.../test/java/spark/streaming/JavaAPISuite.java | 1304 ------------------
.../java/spark/streaming/JavaTestUtils.scala | 84 --
.../spark/streaming/BasicOperationsSuite.scala | 322 +++++
.../spark/streaming/CheckpointSuite.scala | 372 +++++
.../apache/spark/streaming/FailureSuite.scala | 57 +
.../spark/streaming/InputStreamsSuite.scala | 349 +++++
.../apache/spark/streaming/TestSuiteBase.scala | 314 +++++
.../spark/streaming/WindowOperationsSuite.scala | 340 +++++
.../spark/streaming/BasicOperationsSuite.scala | 322 -----
.../scala/spark/streaming/CheckpointSuite.scala | 372 -----
.../scala/spark/streaming/FailureSuite.scala | 57 -
.../spark/streaming/InputStreamsSuite.scala | 349 -----
.../scala/spark/streaming/TestSuiteBase.scala | 314 -----
.../spark/streaming/WindowOperationsSuite.scala | 340 -----
tools/pom.xml | 8 +-
.../tools/JavaAPICompletenessChecker.scala | 360 +++++
.../tools/JavaAPICompletenessChecker.scala | 360 -----
yarn/pom.xml | 6 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 371 +++++
.../yarn/ApplicationMasterArguments.scala | 94 ++
.../org/apache/spark/deploy/yarn/Client.scala | 336 +++++
.../spark/deploy/yarn/ClientArguments.scala | 116 ++
.../spark/deploy/yarn/WorkerRunnable.scala | 224 +++
.../deploy/yarn/YarnAllocationHandler.scala | 564 ++++++++
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 46 +
.../cluster/YarnClusterScheduler.scala | 52 +
.../spark/deploy/yarn/ApplicationMaster.scala | 371 -----
.../yarn/ApplicationMasterArguments.scala | 94 --
.../main/scala/spark/deploy/yarn/Client.scala | 336 -----
.../spark/deploy/yarn/ClientArguments.scala | 116 --
.../spark/deploy/yarn/WorkerRunnable.scala | 224 ---
.../deploy/yarn/YarnAllocationHandler.scala | 564 --------
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 46 -
.../cluster/YarnClusterScheduler.scala | 52 -
1015 files changed, 70352 insertions(+), 70341 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 2ddfe86..c417065 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ Or, for the Python API, the Python shell (`./pyspark`).
Spark also comes with several sample programs in the `examples` directory.
To run one of them, use `./run-example <class> <params>`. For example:
- ./run-example spark.examples.SparkLR local[2]
+ ./run-example org.apache.spark.examples.SparkLR local[2]
will run the Logistic Regression example locally on 2 CPUs.
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 74990b6..dc63811 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,13 +19,13 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-parent</artifactId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-assembly</artifactId>
<name>Spark Project Assembly</name>
<url>http://spark-project.org/</url>
@@ -40,27 +40,27 @@
<dependencies>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-bagel</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-mllib</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-repl</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-streaming</artifactId>
<version>${project.version}</version>
</dependency>
@@ -121,7 +121,7 @@
<id>hadoop2-yarn</id>
<dependencies>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-yarn</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/assembly/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml
index 4543b52..47d3fa9 100644
--- a/assembly/src/main/assembly/assembly.xml
+++ b/assembly/src/main/assembly/assembly.xml
@@ -30,9 +30,9 @@
</fileSet>
<fileSet>
<directory>
- ${project.parent.basedir}/core/src/main/resources/spark/ui/static/
+ ${project.parent.basedir}/core/src/main/resources/org/apache/spark/ui/static/
</directory>
- <outputDirectory>/ui-resources/spark/ui/static</outputDirectory>
+ <outputDirectory>/ui-resources/org/apache/spark/ui/static</outputDirectory>
<includes>
<include>**/*</include>
</includes>
@@ -63,10 +63,10 @@
<dependencySets>
<dependencySet>
<includes>
- <include>org.spark-project:*:jar</include>
+ <include>org.apache.spark:*:jar</include>
</includes>
<excludes>
- <exclude>org.spark-project:spark-assembly:jar</exclude>
+ <exclude>org.apache.spark:spark-assembly:jar</exclude>
</excludes>
</dependencySet>
<dependencySet>
@@ -77,7 +77,7 @@
<useProjectArtifact>false</useProjectArtifact>
<excludes>
<exclude>org.apache.hadoop:*:jar</exclude>
- <exclude>org.spark-project:*:jar</exclude>
+ <exclude>org.apache.spark:*:jar</exclude>
</excludes>
</dependencySet>
</dependencySets>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/bagel/pom.xml
----------------------------------------------------------------------
diff --git a/bagel/pom.xml b/bagel/pom.xml
index cbcf8d1..9340991 100644
--- a/bagel/pom.xml
+++ b/bagel/pom.xml
@@ -19,13 +19,13 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-parent</artifactId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-bagel</artifactId>
<packaging>jar</packaging>
<name>Spark Project Bagel</name>
@@ -33,7 +33,7 @@
<dependencies>
<dependency>
- <groupId>org.spark-project</groupId>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala
----------------------------------------------------------------------
diff --git a/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala b/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala
new file mode 100644
index 0000000..fec8737
--- /dev/null
+++ b/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.bagel
+
+import org.apache.spark._
+import org.apache.spark.SparkContext._
+
+import org.apache.spark.storage.StorageLevel
+
+object Bagel extends Logging {
+ val DEFAULT_STORAGE_LEVEL = StorageLevel.MEMORY_AND_DISK
+
+ /**
+ * Runs a Bagel program.
+ * @param sc [[org.apache.spark.SparkContext]] to use for the program.
+ * @param vertices vertices of the graph represented as an RDD of (Key, Vertex) pairs. Often the Key will be
+ * the vertex id.
+ * @param messages initial set of messages represented as an RDD of (Key, Message) pairs. Often this will be an
+ * empty array, i.e. sc.parallelize(Array[K, Message]()).
+ * @param combiner [[org.apache.spark.bagel.Combiner]] combines multiple individual messages to a given vertex into one
+ * message before sending (which often involves network I/O).
+ * @param aggregator [[org.apache.spark.bagel.Aggregator]] performs a reduce across all vertices after each superstep,
+ * and provides the result to each vertex in the next superstep.
+ * @param partitioner [[org.apache.spark.Partitioner]] partitions values by key
+ * @param numPartitions number of partitions across which to split the graph.
+ * Default is the default parallelism of the SparkContext
+ * @param storageLevel [[org.apache.spark.storage.StorageLevel]] to use for caching of intermediate RDDs in each superstep.
+ * Defaults to caching in memory.
+ * @param compute function that takes a Vertex, optional set of (possibly combined) messages to the Vertex,
+ * optional Aggregator and the current superstep,
+ * and returns a set of (Vertex, outgoing Messages) pairs
+ * @tparam K key
+ * @tparam V vertex type
+ * @tparam M message type
+ * @tparam C combiner
+ * @tparam A aggregator
+ * @return an RDD of (K, V) pairs representing the graph after completion of the program
+ */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest,
+ C: Manifest, A: Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ combiner: Combiner[M, C],
+ aggregator: Option[Aggregator[V, A]],
+ partitioner: Partitioner,
+ numPartitions: Int,
+ storageLevel: StorageLevel = DEFAULT_STORAGE_LEVEL
+ )(
+ compute: (V, Option[C], Option[A], Int) => (V, Array[M])
+ ): RDD[(K, V)] = {
+ val splits = if (numPartitions != 0) numPartitions else sc.defaultParallelism
+
+ var superstep = 0
+ var verts = vertices
+ var msgs = messages
+ var noActivity = false
+ do {
+ logInfo("Starting superstep "+superstep+".")
+ val startTime = System.currentTimeMillis
+
+ val aggregated = agg(verts, aggregator)
+ val combinedMsgs = msgs.combineByKey(
+ combiner.createCombiner _, combiner.mergeMsg _, combiner.mergeCombiners _, partitioner)
+ val grouped = combinedMsgs.groupWith(verts)
+ val superstep_ = superstep // Create a read-only copy of superstep for capture in closure
+ val (processed, numMsgs, numActiveVerts) =
+ comp[K, V, M, C](sc, grouped, compute(_, _, aggregated, superstep_), storageLevel)
+
+ val timeTaken = System.currentTimeMillis - startTime
+ logInfo("Superstep %d took %d s".format(superstep, timeTaken / 1000))
+
+ verts = processed.mapValues { case (vert, msgs) => vert }
+ msgs = processed.flatMap {
+ case (id, (vert, msgs)) => msgs.map(m => (m.targetId, m))
+ }
+ superstep += 1
+
+ noActivity = numMsgs == 0 && numActiveVerts == 0
+ } while (!noActivity)
+
+ verts
+ }
+
+ /** Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]] and the default storage level */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ combiner: Combiner[M, C],
+ partitioner: Partitioner,
+ numPartitions: Int
+ )(
+ compute: (V, Option[C], Int) => (V, Array[M])
+ ): RDD[(K, V)] = run(sc, vertices, messages, combiner, numPartitions, DEFAULT_STORAGE_LEVEL)(compute)
+
+ /** Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]] */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ combiner: Combiner[M, C],
+ partitioner: Partitioner,
+ numPartitions: Int,
+ storageLevel: StorageLevel
+ )(
+ compute: (V, Option[C], Int) => (V, Array[M])
+ ): RDD[(K, V)] = {
+ run[K, V, M, C, Nothing](
+ sc, vertices, messages, combiner, None, partitioner, numPartitions, storageLevel)(
+ addAggregatorArg[K, V, M, C](compute))
+ }
+
+ /**
+ * Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]], default [[org.apache.spark.HashPartitioner]]
+ * and default storage level
+ */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ combiner: Combiner[M, C],
+ numPartitions: Int
+ )(
+ compute: (V, Option[C], Int) => (V, Array[M])
+ ): RDD[(K, V)] = run(sc, vertices, messages, combiner, numPartitions, DEFAULT_STORAGE_LEVEL)(compute)
+
+ /** Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]] and the default [[org.apache.spark.HashPartitioner]]*/
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ combiner: Combiner[M, C],
+ numPartitions: Int,
+ storageLevel: StorageLevel
+ )(
+ compute: (V, Option[C], Int) => (V, Array[M])
+ ): RDD[(K, V)] = {
+ val part = new HashPartitioner(numPartitions)
+ run[K, V, M, C, Nothing](
+ sc, vertices, messages, combiner, None, part, numPartitions, storageLevel)(
+ addAggregatorArg[K, V, M, C](compute))
+ }
+
+ /**
+ * Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]], default [[org.apache.spark.HashPartitioner]],
+ * [[org.apache.spark.bagel.DefaultCombiner]] and the default storage level
+ */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ numPartitions: Int
+ )(
+ compute: (V, Option[Array[M]], Int) => (V, Array[M])
+ ): RDD[(K, V)] = run(sc, vertices, messages, numPartitions, DEFAULT_STORAGE_LEVEL)(compute)
+
+ /**
+ * Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]], the default [[org.apache.spark.HashPartitioner]]
+ * and [[org.apache.spark.bagel.DefaultCombiner]]
+ */
+ def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest](
+ sc: SparkContext,
+ vertices: RDD[(K, V)],
+ messages: RDD[(K, M)],
+ numPartitions: Int,
+ storageLevel: StorageLevel
+ )(
+ compute: (V, Option[Array[M]], Int) => (V, Array[M])
+ ): RDD[(K, V)] = {
+ val part = new HashPartitioner(numPartitions)
+ run[K, V, M, Array[M], Nothing](
+ sc, vertices, messages, new DefaultCombiner(), None, part, numPartitions, storageLevel)(
+ addAggregatorArg[K, V, M, Array[M]](compute))
+ }
+
+ /**
+ * Aggregates the given vertices using the given aggregator, if it
+ * is specified.
+ */
+ private def agg[K, V <: Vertex, A: Manifest](
+ verts: RDD[(K, V)],
+ aggregator: Option[Aggregator[V, A]]
+ ): Option[A] = aggregator match {
+ case Some(a) =>
+ Some(verts.map {
+ case (id, vert) => a.createAggregator(vert)
+ }.reduce(a.mergeAggregators(_, _)))
+ case None => None
+ }
+
+ /**
+ * Processes the given vertex-message RDD using the compute
+ * function. Returns the processed RDD, the number of messages
+ * created, and the number of active vertices.
+ */
+ private def comp[K: Manifest, V <: Vertex, M <: Message[K], C](
+ sc: SparkContext,
+ grouped: RDD[(K, (Seq[C], Seq[V]))],
+ compute: (V, Option[C]) => (V, Array[M]),
+ storageLevel: StorageLevel
+ ): (RDD[(K, (V, Array[M]))], Int, Int) = {
+ var numMsgs = sc.accumulator(0)
+ var numActiveVerts = sc.accumulator(0)
+ val processed = grouped.flatMapValues {
+ case (_, vs) if vs.size == 0 => None
+ case (c, vs) =>
+ val (newVert, newMsgs) =
+ compute(vs(0), c match {
+ case Seq(comb) => Some(comb)
+ case Seq() => None
+ })
+
+ numMsgs += newMsgs.size
+ if (newVert.active)
+ numActiveVerts += 1
+
+ Some((newVert, newMsgs))
+ }.persist(storageLevel)
+
+ // Force evaluation of processed RDD for accurate performance measurements
+ processed.foreach(x => {})
+
+ (processed, numMsgs.value, numActiveVerts.value)
+ }
+
+ /**
+ * Converts a compute function that doesn't take an aggregator to
+ * one that does, so it can be passed to Bagel.run.
+ */
+ private def addAggregatorArg[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C](
+ compute: (V, Option[C], Int) => (V, Array[M])
+ ): (V, Option[C], Option[Nothing], Int) => (V, Array[M]) = {
+ (vert: V, msgs: Option[C], aggregated: Option[Nothing], superstep: Int) =>
+ compute(vert, msgs, superstep)
+ }
+}
+
+trait Combiner[M, C] {
+ def createCombiner(msg: M): C
+ def mergeMsg(combiner: C, msg: M): C
+ def mergeCombiners(a: C, b: C): C
+}
+
+trait Aggregator[V, A] {
+ def createAggregator(vert: V): A
+ def mergeAggregators(a: A, b: A): A
+}
+
+/** Default combiner that simply appends messages together (i.e. performs no aggregation) */
+class DefaultCombiner[M: Manifest] extends Combiner[M, Array[M]] with Serializable {
+ def createCombiner(msg: M): Array[M] =
+ Array(msg)
+ def mergeMsg(combiner: Array[M], msg: M): Array[M] =
+ combiner :+ msg
+ def mergeCombiners(a: Array[M], b: Array[M]): Array[M] =
+ a ++ b
+}
+
+/**
+ * Represents a Bagel vertex.
+ *
+ * Subclasses may store state along with each vertex and must
+ * inherit from java.io.Serializable or scala.Serializable.
+ */
+trait Vertex {
+ def active: Boolean
+}
+
+/**
+ * Represents a Bagel message to a target vertex.
+ *
+ * Subclasses may contain a payload to deliver to the target vertex
+ * and must inherit from java.io.Serializable or scala.Serializable.
+ */
+trait Message[K] {
+ def targetId: K
+}