You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/09/02 00:01:15 UTC
[69/69] [abbrv] git commit: Merge remote-tracking branch 'old/master'
Merge remote-tracking branch 'old/master'
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/2ce200bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/2ce200bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/2ce200bf
Branch: refs/heads/master
Commit: 2ce200bf7f7a38afbcacf3303ca2418e49bdbe2a
Parents: 87d586e f957c26
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Sun Sep 1 14:57:27 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Sun Sep 1 14:57:27 2013 -0700
----------------------------------------------------------------------
.gitignore | 3 +
README.md | 77 +-
assembly/README | 13 +
assembly/lib/PY4J_LICENSE.txt | 27 +
assembly/lib/PY4J_VERSION.txt | 1 +
assembly/lib/net/sf/py4j/py4j/0.7/py4j-0.7.jar | Bin 0 -> 103286 bytes
assembly/lib/net/sf/py4j/py4j/0.7/py4j-0.7.pom | 9 +
.../net/sf/py4j/py4j/maven-metadata-local.xml | 12 +
assembly/pom.xml | 159 +++
assembly/src/main/assembly/assembly.xml | 85 ++
bagel/pom.xml | 111 +-
.../scala/org/apache/spark/bagel/Bagel.scala | 293 ++++
bagel/src/main/scala/spark/bagel/Bagel.scala | 294 ----
.../spark/bagel/examples/PageRankUtils.scala | 123 --
.../bagel/examples/WikipediaPageRank.scala | 101 --
.../examples/WikipediaPageRankStandalone.scala | 223 ---
bagel/src/test/scala/bagel/BagelSuite.scala | 118 --
.../org/apache/spark/bagel/BagelSuite.scala | 116 ++
bin/compute-classpath.cmd | 4 +
bin/compute-classpath.sh | 90 +-
bin/slaves.sh | 4 +-
bin/spark-daemon.sh | 24 +-
bin/start-master.sh | 2 +-
bin/start-slave.sh | 2 +-
bin/start-slaves.sh | 2 -
bin/stop-all.sh | 1 +
bin/stop-master.sh | 2 +-
bin/stop-slaves.sh | 4 +-
conf/metrics.properties.template | 90 ++
conf/slaves | 2 +-
conf/spark-env.sh.template | 20 +-
core/pom.xml | 228 +--
.../apache/hadoop/mapred/HadoopMapRedUtil.scala | 27 -
.../hadoop/mapreduce/HadoopMapReduceUtil.scala | 29 -
.../scala/spark/deploy/SparkHadoopUtil.scala | 44 -
.../apache/hadoop/mapred/HadoopMapRedUtil.scala | 30 -
.../hadoop/mapreduce/HadoopMapReduceUtil.scala | 30 -
.../scala/spark/deploy/SparkHadoopUtil.scala | 76 -
.../spark/deploy/yarn/ApplicationMaster.scala | 351 -----
.../yarn/ApplicationMasterArguments.scala | 94 --
.../scala/spark/deploy/yarn/Client.scala | 327 -----
.../spark/deploy/yarn/ClientArguments.scala | 116 --
.../spark/deploy/yarn/WorkerRunnable.scala | 217 ---
.../deploy/yarn/YarnAllocationHandler.scala | 564 --------
.../cluster/YarnClusterScheduler.scala | 59 -
.../apache/hadoop/mapred/HadoopMapRedUtil.scala | 27 -
.../hadoop/mapreduce/HadoopMapReduceUtil.scala | 30 -
.../scala/spark/deploy/SparkHadoopUtil.scala | 44 -
.../apache/spark/network/netty/FileClient.java | 89 ++
.../netty/FileClientChannelInitializer.java | 41 +
.../spark/network/netty/FileClientHandler.java | 60 +
.../apache/spark/network/netty/FileServer.java | 103 ++
.../netty/FileServerChannelInitializer.java | 42 +
.../spark/network/netty/FileServerHandler.java | 82 ++
.../spark/network/netty/PathResolver.java | 29 +
.../java/spark/network/netty/FileClient.java | 89 --
.../netty/FileClientChannelInitializer.java | 41 -
.../spark/network/netty/FileClientHandler.java | 60 -
.../java/spark/network/netty/FileServer.java | 103 --
.../netty/FileServerChannelInitializer.java | 42 -
.../spark/network/netty/FileServerHandler.java | 82 --
.../java/spark/network/netty/PathResolver.java | 29 -
.../apache/spark/ui/static/bootstrap.min.css | 874 ++++++++++++
.../org/apache/spark/ui/static/sorttable.js | 495 +++++++
.../spark/ui/static/spark-logo-77x50px-hd.png | Bin 0 -> 3536 bytes
.../org/apache/spark/ui/static/spark_logo.png | Bin 0 -> 14233 bytes
.../org/apache/spark/ui/static/webui.css | 63 +
.../ui/static/bootstrap-responsive.min.css | 9 -
.../resources/spark/ui/static/bootstrap.min.css | 9 -
.../main/resources/spark/ui/static/sorttable.js | 495 -------
.../spark/ui/static/spark-logo-77x50px-hd.png | Bin 3536 -> 0 bytes
.../resources/spark/ui/static/spark_logo.png | Bin 14233 -> 0 bytes
.../main/resources/spark/ui/static/webui.css | 49 -
.../hadoop/mapred/SparkHadoopMapRedUtil.scala | 45 +
.../mapreduce/SparkHadoopMapReduceUtil.scala | 69 +
.../scala/org/apache/spark/Accumulators.scala | 257 ++++
.../scala/org/apache/spark/Aggregator.scala | 61 +
.../apache/spark/BlockStoreShuffleFetcher.scala | 89 ++
.../scala/org/apache/spark/CacheManager.scala | 83 ++
.../scala/org/apache/spark/Dependency.scala | 83 ++
.../org/apache/spark/FetchFailedException.scala | 44 +
.../scala/org/apache/spark/HttpFileServer.scala | 63 +
.../scala/org/apache/spark/HttpServer.scala | 89 ++
.../main/scala/org/apache/spark/Logging.scala | 95 ++
.../org/apache/spark/MapOutputTracker.scala | 338 +++++
.../main/scala/org/apache/spark/Partition.scala | 31 +
.../scala/org/apache/spark/Partitioner.scala | 138 ++
.../org/apache/spark/SerializableWritable.scala | 42 +
.../scala/org/apache/spark/ShuffleFetcher.scala | 35 +
.../scala/org/apache/spark/SparkContext.scala | 993 +++++++++++++
.../main/scala/org/apache/spark/SparkEnv.scala | 240 ++++
.../scala/org/apache/spark/SparkException.scala | 24 +
.../main/scala/org/apache/spark/SparkFiles.java | 42 +
.../org/apache/spark/SparkHadoopWriter.scala | 201 +++
.../scala/org/apache/spark/TaskContext.scala | 41 +
.../scala/org/apache/spark/TaskEndReason.scala | 51 +
.../main/scala/org/apache/spark/TaskState.scala | 51 +
.../apache/spark/api/java/JavaDoubleRDD.scala | 167 +++
.../org/apache/spark/api/java/JavaPairRDD.scala | 601 ++++++++
.../org/apache/spark/api/java/JavaRDD.scala | 115 ++
.../org/apache/spark/api/java/JavaRDDLike.scala | 428 ++++++
.../spark/api/java/JavaSparkContext.scala | 418 ++++++
.../java/JavaSparkContextVarargsWorkaround.java | 64 +
.../org/apache/spark/api/java/JavaUtils.scala | 28 +
.../apache/spark/api/java/StorageLevels.java | 48 +
.../java/function/DoubleFlatMapFunction.java | 37 +
.../spark/api/java/function/DoubleFunction.java | 34 +
.../api/java/function/FlatMapFunction.scala | 28 +
.../api/java/function/FlatMapFunction2.scala | 28 +
.../spark/api/java/function/Function.java | 39 +
.../spark/api/java/function/Function2.java | 38 +
.../api/java/function/PairFlatMapFunction.java | 46 +
.../spark/api/java/function/PairFunction.java | 45 +
.../spark/api/java/function/VoidFunction.scala | 33 +
.../api/java/function/WrappedFunction1.scala | 32 +
.../api/java/function/WrappedFunction2.scala | 32 +
.../spark/api/python/PythonPartitioner.scala | 50 +
.../org/apache/spark/api/python/PythonRDD.scala | 346 +++++
.../spark/api/python/PythonWorkerFactory.scala | 132 ++
.../spark/broadcast/BitTorrentBroadcast.scala | 1058 ++++++++++++++
.../org/apache/spark/broadcast/Broadcast.scala | 70 +
.../spark/broadcast/BroadcastFactory.scala | 30 +
.../apache/spark/broadcast/HttpBroadcast.scala | 171 +++
.../apache/spark/broadcast/MultiTracker.scala | 410 ++++++
.../org/apache/spark/broadcast/SourceInfo.scala | 54 +
.../apache/spark/broadcast/TreeBroadcast.scala | 603 ++++++++
.../spark/deploy/ApplicationDescription.scala | 32 +
.../scala/org/apache/spark/deploy/Command.scala | 26 +
.../org/apache/spark/deploy/DeployMessage.scala | 130 ++
.../org/apache/spark/deploy/ExecutorState.scala | 28 +
.../org/apache/spark/deploy/JsonProtocol.scala | 86 ++
.../apache/spark/deploy/LocalSparkCluster.scala | 69 +
.../apache/spark/deploy/SparkHadoopUtil.scala | 36 +
.../scala/org/apache/spark/deploy/WebUI.scala | 47 +
.../org/apache/spark/deploy/client/Client.scala | 145 ++
.../spark/deploy/client/ClientListener.scala | 35 +
.../apache/spark/deploy/client/TestClient.scala | 51 +
.../spark/deploy/client/TestExecutor.scala | 27 +
.../spark/deploy/master/ApplicationInfo.scala | 85 ++
.../spark/deploy/master/ApplicationSource.scala | 24 +
.../spark/deploy/master/ApplicationState.scala | 28 +
.../spark/deploy/master/ExecutorInfo.scala | 32 +
.../org/apache/spark/deploy/master/Master.scala | 386 ++++++
.../spark/deploy/master/MasterArguments.scala | 88 ++
.../spark/deploy/master/MasterSource.scala | 25 +
.../apache/spark/deploy/master/WorkerInfo.scala | 77 ++
.../spark/deploy/master/WorkerState.scala | 24 +
.../deploy/master/ui/ApplicationPage.scala | 118 ++
.../spark/deploy/master/ui/IndexPage.scala | 141 ++
.../spark/deploy/master/ui/MasterWebUI.scala | 81 ++
.../spark/deploy/worker/ExecutorRunner.scala | 200 +++
.../org/apache/spark/deploy/worker/Worker.scala | 213 +++
.../spark/deploy/worker/WorkerArguments.scala | 151 ++
.../spark/deploy/worker/WorkerSource.scala | 34 +
.../spark/deploy/worker/ui/IndexPage.scala | 115 ++
.../spark/deploy/worker/ui/WorkerWebUI.scala | 191 +++
.../org/apache/spark/executor/Executor.scala | 270 ++++
.../apache/spark/executor/ExecutorBackend.scala | 28 +
.../spark/executor/ExecutorExitCode.scala | 60 +
.../apache/spark/executor/ExecutorSource.scala | 55 +
.../spark/executor/ExecutorURLClassLoader.scala | 31 +
.../spark/executor/MesosExecutorBackend.scala | 96 ++
.../executor/StandaloneExecutorBackend.scala | 107 ++
.../org/apache/spark/executor/TaskMetrics.scala | 105 ++
.../org/apache/spark/io/CompressionCodec.scala | 82 ++
.../apache/spark/metrics/MetricsConfig.scala | 100 ++
.../apache/spark/metrics/MetricsSystem.scala | 163 +++
.../apache/spark/metrics/sink/ConsoleSink.scala | 59 +
.../org/apache/spark/metrics/sink/CsvSink.scala | 68 +
.../org/apache/spark/metrics/sink/JmxSink.scala | 35 +
.../spark/metrics/sink/MetricsServlet.scala | 55 +
.../org/apache/spark/metrics/sink/Sink.scala | 23 +
.../apache/spark/metrics/source/JvmSource.scala | 32 +
.../apache/spark/metrics/source/Source.scala | 25 +
.../apache/spark/network/BufferMessage.scala | 111 ++
.../org/apache/spark/network/Connection.scala | 586 ++++++++
.../spark/network/ConnectionManager.scala | 721 ++++++++++
.../spark/network/ConnectionManagerId.scala | 38 +
.../spark/network/ConnectionManagerTest.scala | 102 ++
.../org/apache/spark/network/Message.scala | 93 ++
.../org/apache/spark/network/MessageChunk.scala | 42 +
.../spark/network/MessageChunkHeader.scala | 75 +
.../org/apache/spark/network/ReceiverTest.scala | 37 +
.../org/apache/spark/network/SenderTest.scala | 70 +
.../apache/spark/network/netty/FileHeader.scala | 74 +
.../spark/network/netty/ShuffleCopier.scala | 118 ++
.../spark/network/netty/ShuffleSender.scala | 70 +
.../main/scala/org/apache/spark/package.scala | 35 +
.../partial/ApproximateActionListener.scala | 88 ++
.../spark/partial/ApproximateEvaluator.scala | 27 +
.../apache/spark/partial/BoundedDouble.scala | 25 +
.../apache/spark/partial/CountEvaluator.scala | 55 +
.../spark/partial/GroupedCountEvaluator.scala | 79 ++
.../spark/partial/GroupedMeanEvaluator.scala | 82 ++
.../spark/partial/GroupedSumEvaluator.scala | 89 ++
.../apache/spark/partial/MeanEvaluator.scala | 58 +
.../apache/spark/partial/PartialResult.scala | 137 ++
.../apache/spark/partial/StudentTCacher.scala | 43 +
.../org/apache/spark/partial/SumEvaluator.scala | 68 +
.../scala/org/apache/spark/rdd/BlockRDD.scala | 51 +
.../org/apache/spark/rdd/CartesianRDD.scala | 90 ++
.../org/apache/spark/rdd/CheckpointRDD.scala | 155 +++
.../org/apache/spark/rdd/CoGroupedRDD.scala | 144 ++
.../org/apache/spark/rdd/CoalescedRDD.scala | 342 +++++
.../apache/spark/rdd/DoubleRDDFunctions.scala | 79 ++
.../scala/org/apache/spark/rdd/EmptyRDD.scala | 33 +
.../org/apache/spark/rdd/FilteredRDD.scala | 33 +
.../org/apache/spark/rdd/FlatMappedRDD.scala | 33 +
.../apache/spark/rdd/FlatMappedValuesRDD.scala | 36 +
.../scala/org/apache/spark/rdd/GlommedRDD.scala | 29 +
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 131 ++
.../scala/org/apache/spark/rdd/JdbcRDD.scala | 120 ++
.../org/apache/spark/rdd/MapPartitionsRDD.scala | 37 +
.../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 +
.../scala/org/apache/spark/rdd/MappedRDD.scala | 30 +
.../org/apache/spark/rdd/MappedValuesRDD.scala | 34 +
.../org/apache/spark/rdd/NewHadoopRDD.scala | 126 ++
.../apache/spark/rdd/OrderedRDDFunctions.scala | 52 +
.../org/apache/spark/rdd/PairRDDFunctions.scala | 702 ++++++++++
.../spark/rdd/ParallelCollectionRDD.scala | 153 ++
.../apache/spark/rdd/PartitionPruningRDD.scala | 72 +
.../scala/org/apache/spark/rdd/PipedRDD.scala | 125 ++
.../main/scala/org/apache/spark/rdd/RDD.scala | 942 +++++++++++++
.../apache/spark/rdd/RDDCheckpointData.scala | 131 ++
.../scala/org/apache/spark/rdd/SampledRDD.scala | 66 +
.../spark/rdd/SequenceFileRDDFunctions.scala | 89 ++
.../org/apache/spark/rdd/ShuffledRDD.scala | 67 +
.../org/apache/spark/rdd/SubtractedRDD.scala | 128 ++
.../scala/org/apache/spark/rdd/UnionRDD.scala | 73 +
.../apache/spark/rdd/ZippedPartitionsRDD.scala | 143 ++
.../scala/org/apache/spark/rdd/ZippedRDD.scala | 85 ++
.../org/apache/spark/scheduler/ActiveJob.scala | 39 +
.../apache/spark/scheduler/DAGScheduler.scala | 850 ++++++++++++
.../spark/scheduler/DAGSchedulerEvent.scala | 64 +
.../spark/scheduler/DAGSchedulerSource.scala | 30 +
.../spark/scheduler/InputFormatInfo.scala | 178 +++
.../apache/spark/scheduler/JobListener.scala | 28 +
.../org/apache/spark/scheduler/JobLogger.scala | 293 ++++
.../org/apache/spark/scheduler/JobResult.scala | 26 +
.../org/apache/spark/scheduler/JobWaiter.scala | 66 +
.../org/apache/spark/scheduler/MapStatus.scala | 44 +
.../org/apache/spark/scheduler/ResultTask.scala | 137 ++
.../apache/spark/scheduler/ShuffleMapTask.scala | 191 +++
.../apache/spark/scheduler/SparkListener.scala | 204 +++
.../spark/scheduler/SparkListenerBus.scala | 74 +
.../org/apache/spark/scheduler/SplitInfo.scala | 78 ++
.../org/apache/spark/scheduler/Stage.scala | 111 ++
.../org/apache/spark/scheduler/StageInfo.scala | 29 +
.../scala/org/apache/spark/scheduler/Task.scala | 115 ++
.../apache/spark/scheduler/TaskLocation.scala | 34 +
.../org/apache/spark/scheduler/TaskResult.scala | 73 +
.../apache/spark/scheduler/TaskScheduler.scala | 52 +
.../spark/scheduler/TaskSchedulerListener.scala | 45 +
.../org/apache/spark/scheduler/TaskSet.scala | 35 +
.../scheduler/cluster/ClusterScheduler.scala | 440 ++++++
.../cluster/ClusterTaskSetManager.scala | 712 ++++++++++
.../scheduler/cluster/ExecutorLossReason.scala | 38 +
.../apache/spark/scheduler/cluster/Pool.scala | 121 ++
.../spark/scheduler/cluster/Schedulable.scala | 48 +
.../scheduler/cluster/SchedulableBuilder.scala | 137 ++
.../scheduler/cluster/SchedulerBackend.scala | 37 +
.../scheduler/cluster/SchedulingAlgorithm.scala | 81 ++
.../scheduler/cluster/SchedulingMode.scala | 29 +
.../cluster/SparkDeploySchedulerBackend.scala | 92 ++
.../cluster/StandaloneClusterMessage.scala | 62 +
.../cluster/StandaloneSchedulerBackend.scala | 199 +++
.../scheduler/cluster/TaskDescription.scala | 37 +
.../spark/scheduler/cluster/TaskInfo.scala | 72 +
.../spark/scheduler/cluster/TaskLocality.scala | 32 +
.../scheduler/cluster/TaskSetManager.scala | 51 +
.../spark/scheduler/cluster/WorkerOffer.scala | 24 +
.../spark/scheduler/local/LocalScheduler.scala | 273 ++++
.../scheduler/local/LocalTaskSetManager.scala | 194 +++
.../mesos/CoarseMesosSchedulerBackend.scala | 286 ++++
.../scheduler/mesos/MesosSchedulerBackend.scala | 343 +++++
.../spark/serializer/JavaSerializer.scala | 82 ++
.../spark/serializer/KryoSerializer.scala | 159 +++
.../apache/spark/serializer/Serializer.scala | 112 ++
.../spark/serializer/SerializerManager.scala | 62 +
.../apache/spark/storage/BlockException.scala | 22 +
.../spark/storage/BlockFetchTracker.scala | 27 +
.../spark/storage/BlockFetcherIterator.scala | 348 +++++
.../org/apache/spark/storage/BlockManager.scala | 1046 ++++++++++++++
.../apache/spark/storage/BlockManagerId.scala | 118 ++
.../spark/storage/BlockManagerMaster.scala | 178 +++
.../spark/storage/BlockManagerMasterActor.scala | 405 ++++++
.../spark/storage/BlockManagerMessages.scala | 110 ++
.../spark/storage/BlockManagerSlaveActor.scala | 39 +
.../spark/storage/BlockManagerSource.scala | 48 +
.../spark/storage/BlockManagerWorker.scala | 140 ++
.../org/apache/spark/storage/BlockMessage.scala | 223 +++
.../spark/storage/BlockMessageArray.scala | 159 +++
.../spark/storage/BlockObjectWriter.scala | 65 +
.../org/apache/spark/storage/BlockStore.scala | 61 +
.../org/apache/spark/storage/DiskStore.scala | 329 +++++
.../org/apache/spark/storage/MemoryStore.scala | 257 ++++
.../org/apache/spark/storage/PutResult.scala | 26 +
.../spark/storage/ShuffleBlockManager.scala | 67 +
.../org/apache/spark/storage/StorageLevel.scala | 146 ++
.../org/apache/spark/storage/StorageUtils.scala | 116 ++
.../apache/spark/storage/ThreadingTest.scala | 113 ++
.../scala/org/apache/spark/ui/JettyUtils.scala | 131 ++
.../main/scala/org/apache/spark/ui/Page.scala | 22 +
.../scala/org/apache/spark/ui/SparkUI.scala | 88 ++
.../scala/org/apache/spark/ui/UIUtils.scala | 131 ++
.../apache/spark/ui/UIWorkloadGenerator.scala | 105 ++
.../org/apache/spark/ui/env/EnvironmentUI.scala | 91 ++
.../org/apache/spark/ui/exec/ExecutorsUI.scala | 137 ++
.../org/apache/spark/ui/jobs/IndexPage.scala | 90 ++
.../spark/ui/jobs/JobProgressListener.scala | 156 +++
.../apache/spark/ui/jobs/JobProgressUI.scala | 61 +
.../org/apache/spark/ui/jobs/PoolPage.scala | 32 +
.../org/apache/spark/ui/jobs/PoolTable.scala | 55 +
.../org/apache/spark/ui/jobs/StagePage.scala | 183 +++
.../org/apache/spark/ui/jobs/StageTable.scala | 107 ++
.../spark/ui/storage/BlockManagerUI.scala | 41 +
.../org/apache/spark/ui/storage/IndexPage.scala | 65 +
.../org/apache/spark/ui/storage/RDDPage.scala | 132 ++
.../scala/org/apache/spark/util/AkkaUtils.scala | 72 +
.../spark/util/BoundedPriorityQueue.scala | 62 +
.../spark/util/ByteBufferInputStream.scala | 80 ++
.../scala/org/apache/spark/util/Clock.scala | 29 +
.../org/apache/spark/util/ClosureCleaner.scala | 232 ++++
.../apache/spark/util/CompletionIterator.scala | 42 +
.../org/apache/spark/util/Distribution.scala | 82 ++
.../org/apache/spark/util/IdGenerator.scala | 31 +
.../scala/org/apache/spark/util/IntParam.scala | 31 +
.../org/apache/spark/util/MemoryParam.scala | 32 +
.../org/apache/spark/util/MetadataCleaner.scala | 61 +
.../org/apache/spark/util/MutablePair.scala | 36 +
.../org/apache/spark/util/NextIterator.scala | 88 ++
.../spark/util/RateLimitedOutputStream.scala | 79 ++
.../apache/spark/util/SerializableBuffer.scala | 54 +
.../org/apache/spark/util/SizeEstimator.scala | 284 ++++
.../org/apache/spark/util/StatCounter.scala | 131 ++
.../apache/spark/util/TimeStampedHashMap.scala | 122 ++
.../apache/spark/util/TimeStampedHashSet.scala | 86 ++
.../scala/org/apache/spark/util/Utils.scala | 781 +++++++++++
.../scala/org/apache/spark/util/Vector.scala | 139 ++
core/src/main/scala/spark/Accumulators.scala | 256 ----
core/src/main/scala/spark/Aggregator.scala | 61 -
.../scala/spark/BlockStoreShuffleFetcher.scala | 88 --
core/src/main/scala/spark/Cache.scala | 80 --
core/src/main/scala/spark/CacheManager.scala | 82 --
core/src/main/scala/spark/ClosureCleaner.scala | 231 ----
core/src/main/scala/spark/Dependency.scala | 82 --
.../main/scala/spark/DoubleRDDFunctions.scala | 72 -
.../main/scala/spark/FetchFailedException.scala | 44 -
core/src/main/scala/spark/HadoopWriter.scala | 201 ---
core/src/main/scala/spark/HttpFileServer.scala | 62 -
core/src/main/scala/spark/HttpServer.scala | 88 --
core/src/main/scala/spark/JavaSerializer.scala | 83 --
core/src/main/scala/spark/KryoSerializer.scala | 241 ----
core/src/main/scala/spark/Logging.scala | 95 --
.../src/main/scala/spark/MapOutputTracker.scala | 338 -----
.../src/main/scala/spark/PairRDDFunctions.scala | 774 -----------
core/src/main/scala/spark/Partition.scala | 31 -
core/src/main/scala/spark/Partitioner.scala | 143 --
core/src/main/scala/spark/RDD.scala | 953 -------------
.../main/scala/spark/RDDCheckpointData.scala | 130 --
.../scala/spark/SequenceFileRDDFunctions.scala | 107 --
.../main/scala/spark/SerializableWritable.scala | 42 -
core/src/main/scala/spark/ShuffleFetcher.scala | 34 -
core/src/main/scala/spark/SizeEstimator.scala | 283 ----
.../main/scala/spark/SoftReferenceCache.scala | 35 -
core/src/main/scala/spark/SparkContext.scala | 925 -------------
core/src/main/scala/spark/SparkEnv.scala | 218 ---
core/src/main/scala/spark/SparkException.scala | 24 -
core/src/main/scala/spark/SparkFiles.java | 42 -
core/src/main/scala/spark/TaskContext.scala | 41 -
core/src/main/scala/spark/TaskEndReason.scala | 51 -
core/src/main/scala/spark/TaskState.scala | 49 -
core/src/main/scala/spark/Utils.scala | 759 ----------
.../scala/spark/api/java/JavaDoubleRDD.scala | 152 --
.../main/scala/spark/api/java/JavaPairRDD.scala | 591 --------
.../src/main/scala/spark/api/java/JavaRDD.scala | 114 --
.../main/scala/spark/api/java/JavaRDDLike.scala | 429 ------
.../scala/spark/api/java/JavaSparkContext.scala | 416 ------
.../java/JavaSparkContextVarargsWorkaround.java | 64 -
.../scala/spark/api/java/StorageLevels.java | 48 -
.../java/function/DoubleFlatMapFunction.java | 37 -
.../spark/api/java/function/DoubleFunction.java | 34 -
.../api/java/function/FlatMapFunction.scala | 28 -
.../api/java/function/FlatMapFunction2.scala | 28 -
.../scala/spark/api/java/function/Function.java | 39 -
.../spark/api/java/function/Function2.java | 38 -
.../api/java/function/PairFlatMapFunction.java | 46 -
.../spark/api/java/function/PairFunction.java | 45 -
.../spark/api/java/function/VoidFunction.scala | 33 -
.../api/java/function/WrappedFunction1.scala | 32 -
.../api/java/function/WrappedFunction2.scala | 32 -
.../spark/api/python/PythonPartitioner.scala | 65 -
.../main/scala/spark/api/python/PythonRDD.scala | 329 -----
.../spark/api/python/PythonWorkerFactory.scala | 130 --
.../spark/broadcast/BitTorrentBroadcast.scala | 1057 --------------
.../main/scala/spark/broadcast/Broadcast.scala | 70 -
.../spark/broadcast/BroadcastFactory.scala | 30 -
.../scala/spark/broadcast/HttpBroadcast.scala | 167 ---
.../scala/spark/broadcast/MultiTracker.scala | 409 ------
.../main/scala/spark/broadcast/SourceInfo.scala | 54 -
.../scala/spark/broadcast/TreeBroadcast.scala | 602 --------
.../spark/deploy/ApplicationDescription.scala | 32 -
core/src/main/scala/spark/deploy/Command.scala | 26 -
.../main/scala/spark/deploy/DeployMessage.scala | 125 --
.../main/scala/spark/deploy/ExecutorState.scala | 28 -
.../main/scala/spark/deploy/JsonProtocol.scala | 82 --
.../scala/spark/deploy/LocalSparkCluster.scala | 69 -
core/src/main/scala/spark/deploy/WebUI.scala | 47 -
.../main/scala/spark/deploy/client/Client.scala | 142 --
.../spark/deploy/client/ClientListener.scala | 35 -
.../scala/spark/deploy/client/TestClient.scala | 51 -
.../spark/deploy/client/TestExecutor.scala | 27 -
.../spark/deploy/master/ApplicationInfo.scala | 82 --
.../spark/deploy/master/ApplicationState.scala | 28 -
.../spark/deploy/master/ExecutorInfo.scala | 32 -
.../main/scala/spark/deploy/master/Master.scala | 347 -----
.../spark/deploy/master/MasterArguments.scala | 86 --
.../scala/spark/deploy/master/WorkerInfo.scala | 77 --
.../scala/spark/deploy/master/WorkerState.scala | 24 -
.../deploy/master/ui/ApplicationPage.scala | 117 --
.../spark/deploy/master/ui/IndexPage.scala | 138 --
.../spark/deploy/master/ui/MasterWebUI.scala | 76 -
.../spark/deploy/worker/ExecutorRunner.scala | 188 ---
.../main/scala/spark/deploy/worker/Worker.scala | 202 ---
.../spark/deploy/worker/WorkerArguments.scala | 153 --
.../spark/deploy/worker/ui/IndexPage.scala | 117 --
.../spark/deploy/worker/ui/WorkerWebUI.scala | 182 ---
.../main/scala/spark/executor/Executor.scala | 233 ----
.../scala/spark/executor/ExecutorBackend.scala | 28 -
.../scala/spark/executor/ExecutorExitCode.scala | 60 -
.../spark/executor/ExecutorURLClassLoader.scala | 31 -
.../spark/executor/MesosExecutorBackend.scala | 95 --
.../executor/StandaloneExecutorBackend.scala | 124 --
.../main/scala/spark/executor/TaskMetrics.scala | 100 --
.../scala/spark/network/BufferMessage.scala | 111 --
.../main/scala/spark/network/Connection.scala | 552 --------
.../scala/spark/network/ConnectionManager.scala | 719 ----------
.../spark/network/ConnectionManagerId.scala | 38 -
.../spark/network/ConnectionManagerTest.scala | 102 --
core/src/main/scala/spark/network/Message.scala | 93 --
.../main/scala/spark/network/MessageChunk.scala | 42 -
.../spark/network/MessageChunkHeader.scala | 75 -
.../main/scala/spark/network/ReceiverTest.scala | 37 -
.../main/scala/spark/network/SenderTest.scala | 70 -
.../scala/spark/network/netty/FileHeader.scala | 74 -
.../spark/network/netty/ShuffleCopier.scala | 118 --
.../spark/network/netty/ShuffleSender.scala | 70 -
core/src/main/scala/spark/package.scala | 32 -
.../partial/ApproximateActionListener.scala | 87 --
.../spark/partial/ApproximateEvaluator.scala | 27 -
.../scala/spark/partial/BoundedDouble.scala | 25 -
.../scala/spark/partial/CountEvaluator.scala | 55 -
.../spark/partial/GroupedCountEvaluator.scala | 79 --
.../spark/partial/GroupedMeanEvaluator.scala | 82 --
.../spark/partial/GroupedSumEvaluator.scala | 89 --
.../scala/spark/partial/MeanEvaluator.scala | 58 -
.../scala/spark/partial/PartialResult.scala | 137 --
.../scala/spark/partial/StudentTCacher.scala | 43 -
.../main/scala/spark/partial/SumEvaluator.scala | 68 -
core/src/main/scala/spark/rdd/BlockRDD.scala | 52 -
.../src/main/scala/spark/rdd/CartesianRDD.scala | 90 --
.../main/scala/spark/rdd/CheckpointRDD.scala | 153 --
.../src/main/scala/spark/rdd/CoGroupedRDD.scala | 168 ---
.../src/main/scala/spark/rdd/CoalescedRDD.scala | 81 --
core/src/main/scala/spark/rdd/EmptyRDD.scala | 33 -
core/src/main/scala/spark/rdd/FilteredRDD.scala | 33 -
.../main/scala/spark/rdd/FlatMappedRDD.scala | 33 -
core/src/main/scala/spark/rdd/GlommedRDD.scala | 29 -
core/src/main/scala/spark/rdd/HadoopRDD.scala | 134 --
core/src/main/scala/spark/rdd/JdbcRDD.scala | 120 --
.../main/scala/spark/rdd/MapPartitionsRDD.scala | 37 -
.../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 -
core/src/main/scala/spark/rdd/MappedRDD.scala | 30 -
.../src/main/scala/spark/rdd/NewHadoopRDD.scala | 122 --
.../scala/spark/rdd/ParallelCollectionRDD.scala | 114 --
.../scala/spark/rdd/PartitionPruningRDD.scala | 71 -
core/src/main/scala/spark/rdd/PipedRDD.scala | 125 --
core/src/main/scala/spark/rdd/SampledRDD.scala | 66 -
core/src/main/scala/spark/rdd/ShuffledRDD.scala | 54 -
.../main/scala/spark/rdd/SubtractedRDD.scala | 125 --
core/src/main/scala/spark/rdd/UnionRDD.scala | 73 -
.../scala/spark/rdd/ZippedPartitionsRDD.scala | 155 ---
core/src/main/scala/spark/rdd/ZippedRDD.scala | 96 --
.../main/scala/spark/scheduler/ActiveJob.scala | 39 -
.../scala/spark/scheduler/DAGScheduler.scala | 800 -----------
.../spark/scheduler/DAGSchedulerEvent.scala | 63 -
.../scala/spark/scheduler/InputFormatInfo.scala | 177 ---
.../scala/spark/scheduler/JobListener.scala | 28 -
.../main/scala/spark/scheduler/JobLogger.scala | 323 -----
.../main/scala/spark/scheduler/JobResult.scala | 26 -
.../main/scala/spark/scheduler/JobWaiter.scala | 66 -
.../main/scala/spark/scheduler/MapStatus.scala | 44 -
.../main/scala/spark/scheduler/ResultTask.scala | 137 --
.../scala/spark/scheduler/ShuffleMapTask.scala | 203 ---
.../scala/spark/scheduler/SparkListener.scala | 196 ---
.../main/scala/spark/scheduler/SplitInfo.scala | 78 --
core/src/main/scala/spark/scheduler/Stage.scala | 111 --
.../main/scala/spark/scheduler/StageInfo.scala | 29 -
core/src/main/scala/spark/scheduler/Task.scala | 115 --
.../main/scala/spark/scheduler/TaskResult.scala | 55 -
.../scala/spark/scheduler/TaskScheduler.scala | 45 -
.../spark/scheduler/TaskSchedulerListener.scala | 42 -
.../main/scala/spark/scheduler/TaskSet.scala | 35 -
.../scheduler/cluster/ClusterScheduler.scala | 631 ---------
.../cluster/ClusterTaskSetManager.scala | 765 ----------
.../scheduler/cluster/ExecutorLossReason.scala | 38 -
.../scala/spark/scheduler/cluster/Pool.scala | 121 --
.../spark/scheduler/cluster/Schedulable.scala | 44 -
.../scheduler/cluster/SchedulableBuilder.scala | 132 --
.../scheduler/cluster/SchedulerBackend.scala | 37 -
.../scheduler/cluster/SchedulingAlgorithm.scala | 81 --
.../scheduler/cluster/SchedulingMode.scala | 24 -
.../cluster/SparkDeploySchedulerBackend.scala | 91 --
.../cluster/StandaloneClusterMessage.scala | 62 -
.../cluster/StandaloneSchedulerBackend.scala | 193 ---
.../scheduler/cluster/TaskDescription.scala | 34 -
.../spark/scheduler/cluster/TaskInfo.scala | 63 -
.../scheduler/cluster/TaskSetManager.scala | 34 -
.../spark/scheduler/cluster/WorkerOffer.scala | 25 -
.../spark/scheduler/local/LocalScheduler.scala | 254 ----
.../scheduler/local/LocalTaskSetManager.scala | 190 ---
.../mesos/CoarseMesosSchedulerBackend.scala | 271 ----
.../scheduler/mesos/MesosSchedulerBackend.scala | 335 -----
.../scala/spark/serializer/Serializer.scala | 112 --
.../spark/serializer/SerializerManager.scala | 62 -
.../scala/spark/storage/BlockException.scala | 22 -
.../scala/spark/storage/BlockFetchTracker.scala | 27 -
.../spark/storage/BlockFetcherIterator.scala | 347 -----
.../main/scala/spark/storage/BlockManager.scala | 1044 --------------
.../scala/spark/storage/BlockManagerId.scala | 118 --
.../spark/storage/BlockManagerMaster.scala | 177 ---
.../spark/storage/BlockManagerMasterActor.scala | 402 ------
.../spark/storage/BlockManagerMessages.scala | 123 --
.../spark/storage/BlockManagerSlaveActor.scala | 39 -
.../spark/storage/BlockManagerWorker.scala | 139 --
.../main/scala/spark/storage/BlockMessage.scala | 224 ---
.../scala/spark/storage/BlockMessageArray.scala | 160 ---
.../scala/spark/storage/BlockObjectWriter.scala | 67 -
.../main/scala/spark/storage/BlockStore.scala | 61 -
.../main/scala/spark/storage/DiskStore.scala | 330 -----
.../main/scala/spark/storage/MemoryStore.scala | 257 ----
.../main/scala/spark/storage/PutResult.scala | 26 -
.../spark/storage/ShuffleBlockManager.scala | 67 -
.../main/scala/spark/storage/StorageLevel.scala | 146 --
.../main/scala/spark/storage/StorageUtils.scala | 115 --
.../scala/spark/storage/ThreadingTest.scala | 113 --
core/src/main/scala/spark/ui/JettyUtils.scala | 132 --
core/src/main/scala/spark/ui/Page.scala | 20 -
core/src/main/scala/spark/ui/SparkUI.scala | 79 --
core/src/main/scala/spark/ui/UIUtils.scala | 135 --
.../scala/spark/ui/UIWorkloadGenerator.scala | 88 --
.../main/scala/spark/ui/env/EnvironmentUI.scala | 86 --
.../main/scala/spark/ui/jobs/IndexPage.scala | 129 --
.../scala/spark/ui/jobs/JobProgressUI.scala | 144 --
.../main/scala/spark/ui/jobs/StagePage.scala | 131 --
.../scala/spark/ui/storage/BlockManagerUI.scala | 41 -
.../main/scala/spark/ui/storage/IndexPage.scala | 81 --
.../main/scala/spark/ui/storage/RDDPage.scala | 130 --
core/src/main/scala/spark/util/AkkaUtils.scala | 72 -
.../scala/spark/util/BoundedPriorityQueue.scala | 62 -
.../spark/util/ByteBufferInputStream.scala | 80 --
.../scala/spark/util/CompletionIterator.scala | 42 -
.../main/scala/spark/util/Distribution.scala | 82 --
.../src/main/scala/spark/util/IdGenerator.scala | 31 -
core/src/main/scala/spark/util/IntParam.scala | 31 -
.../src/main/scala/spark/util/MemoryParam.scala | 34 -
.../main/scala/spark/util/MetadataCleaner.scala | 61 -
.../main/scala/spark/util/NextIterator.scala | 88 --
.../spark/util/RateLimitedOutputStream.scala | 79 --
.../scala/spark/util/SerializableBuffer.scala | 54 -
.../src/main/scala/spark/util/StatCounter.scala | 131 --
.../scala/spark/util/TimeStampedHashMap.scala | 118 --
.../scala/spark/util/TimeStampedHashSet.scala | 86 --
core/src/main/scala/spark/util/Vector.scala | 142 --
.../resources/test_metrics_config.properties | 6 +
.../resources/test_metrics_system.properties | 7 +
.../org/apache/spark/AccumulatorSuite.scala | 143 ++
.../scala/org/apache/spark/BroadcastSuite.scala | 39 +
.../org/apache/spark/CheckpointSuite.scala | 393 ++++++
.../org/apache/spark/DistributedSuite.scala | 362 +++++
.../scala/org/apache/spark/DriverSuite.scala | 55 +
.../scala/org/apache/spark/FailureSuite.scala | 128 ++
.../org/apache/spark/FileServerSuite.scala | 123 ++
.../test/scala/org/apache/spark/FileSuite.scala | 212 +++
.../scala/org/apache/spark/JavaAPISuite.java | 865 ++++++++++++
.../org/apache/spark/LocalSparkContext.scala | 68 +
.../apache/spark/MapOutputTrackerSuite.scala | 136 ++
.../apache/spark/PartitionPruningRDDSuite.scala | 28 +
.../org/apache/spark/PartitioningSuite.scala | 153 ++
.../scala/org/apache/spark/PipedRDDSuite.scala | 93 ++
.../org/apache/spark/SharedSparkContext.scala | 42 +
.../org/apache/spark/ShuffleNettySuite.scala | 34 +
.../scala/org/apache/spark/ShuffleSuite.scala | 211 +++
.../apache/spark/SparkContextInfoSuite.scala | 60 +
.../scala/org/apache/spark/ThreadingSuite.scala | 152 ++
.../scala/org/apache/spark/UnpersistSuite.scala | 47 +
.../apache/spark/ZippedPartitionsSuite.scala | 50 +
.../apache/spark/io/CompressionCodecSuite.scala | 62 +
.../spark/metrics/MetricsConfigSuite.scala | 89 ++
.../spark/metrics/MetricsSystemSuite.scala | 54 +
.../org/apache/spark/rdd/JdbcRDDSuite.scala | 73 +
.../spark/rdd/PairRDDFunctionsSuite.scala | 300 ++++
.../rdd/ParallelCollectionSplitSuite.scala | 212 +++
.../scala/org/apache/spark/rdd/RDDSuite.scala | 391 ++++++
.../org/apache/spark/rdd/SortingSuite.scala | 125 ++
.../spark/scheduler/DAGSchedulerSuite.scala | 421 ++++++
.../apache/spark/scheduler/JobLoggerSuite.scala | 125 ++
.../spark/scheduler/SparkListenerSuite.scala | 102 ++
.../spark/scheduler/TaskContextSuite.scala | 49 +
.../cluster/ClusterSchedulerSuite.scala | 266 ++++
.../cluster/ClusterTaskSetManagerSuite.scala | 273 ++++
.../spark/scheduler/cluster/FakeTask.scala | 26 +
.../scheduler/local/LocalSchedulerSuite.scala | 223 +++
.../spark/serializer/KryoSerializerSuite.scala | 208 +++
.../spark/storage/BlockManagerSuite.scala | 662 +++++++++
.../scala/org/apache/spark/ui/UISuite.scala | 47 +
.../apache/spark/util/ClosureCleanerSuite.scala | 147 ++
.../apache/spark/util/DistributionSuite.scala | 42 +
.../scala/org/apache/spark/util/FakeClock.scala | 26 +
.../apache/spark/util/NextIteratorSuite.scala | 85 ++
.../util/RateLimitedOutputStreamSuite.scala | 40 +
.../apache/spark/util/SizeEstimatorSuite.scala | 164 +++
.../org/apache/spark/util/UtilsSuite.scala | 139 ++
.../src/test/scala/spark/AccumulatorSuite.scala | 143 --
core/src/test/scala/spark/BroadcastSuite.scala | 39 -
core/src/test/scala/spark/CheckpointSuite.scala | 392 ------
.../test/scala/spark/ClosureCleanerSuite.scala | 146 --
.../src/test/scala/spark/DistributedSuite.scala | 362 -----
core/src/test/scala/spark/DriverSuite.scala | 54 -
core/src/test/scala/spark/FailureSuite.scala | 102 --
core/src/test/scala/spark/FileServerSuite.scala | 123 --
core/src/test/scala/spark/FileSuite.scala | 212 ---
core/src/test/scala/spark/JavaAPISuite.java | 833 -----------
.../test/scala/spark/KryoSerializerSuite.scala | 157 ---
.../test/scala/spark/LocalSparkContext.scala | 68 -
.../scala/spark/MapOutputTrackerSuite.scala | 136 --
.../scala/spark/PairRDDFunctionsSuite.scala | 304 ----
.../test/scala/spark/PartitioningSuite.scala | 150 --
core/src/test/scala/spark/PipedRDDSuite.scala | 93 --
core/src/test/scala/spark/RDDSuite.scala | 328 -----
.../test/scala/spark/SharedSparkContext.scala | 42 -
.../test/scala/spark/ShuffleNettySuite.scala | 34 -
core/src/test/scala/spark/ShuffleSuite.scala | 148 --
.../test/scala/spark/SizeEstimatorSuite.scala | 164 ---
core/src/test/scala/spark/SortingSuite.scala | 123 --
core/src/test/scala/spark/ThreadingSuite.scala | 152 --
core/src/test/scala/spark/UnpersistSuite.scala | 47 -
core/src/test/scala/spark/UtilsSuite.scala | 139 --
.../scala/spark/ZippedPartitionsSuite.scala | 50 -
.../src/test/scala/spark/rdd/JdbcRDDSuite.scala | 73 -
.../rdd/ParallelCollectionSplitSuite.scala | 212 ---
.../spark/scheduler/ClusterSchedulerSuite.scala | 267 ----
.../spark/scheduler/DAGSchedulerSuite.scala | 420 ------
.../scala/spark/scheduler/JobLoggerSuite.scala | 121 --
.../spark/scheduler/LocalSchedulerSuite.scala | 223 ---
.../spark/scheduler/SparkListenerSuite.scala | 102 --
.../spark/scheduler/TaskContextSuite.scala | 49 -
.../scala/spark/storage/BlockManagerSuite.scala | 665 ---------
core/src/test/scala/spark/ui/UISuite.scala | 46 -
.../scala/spark/util/DistributionSuite.scala | 42 -
.../scala/spark/util/NextIteratorSuite.scala | 85 --
.../util/RateLimitedOutputStreamSuite.scala | 40 -
docs/README.md | 2 +-
docs/_layouts/global.html | 34 +-
docs/_plugins/copy_api_dirs.rb | 4 +-
docs/bagel-programming-guide.md | 37 +-
docs/building-with-maven.md | 58 +-
docs/configuration.md | 152 +-
docs/css/bootstrap.min.css | 2 +-
docs/ec2-scripts.md | 32 +-
docs/hardware-provisioning.md | 69 +
docs/img/incubator-logo.png | Bin 0 -> 11651 bytes
docs/index.md | 77 +-
docs/java-programming-guide.md | 33 +-
docs/mllib-guide.md | 6 +
docs/python-programming-guide.md | 47 +-
docs/quick-start.md | 79 +-
docs/running-on-mesos.md | 3 +-
docs/running-on-yarn.md | 55 +-
docs/scala-programming-guide.md | 36 +-
docs/spark-debugger.md | 4 +-
docs/spark-simple-tutorial.md | 41 -
docs/spark-standalone.md | 58 +-
docs/streaming-custom-receivers.md | 61 +-
docs/streaming-programming-guide.md | 45 +-
docs/tuning.md | 13 +-
.../root/mesos-ec2/ec2-variables.sh | 9 -
.../root/spark-ec2/ec2-variables.sh | 18 +-
ec2/spark_ec2.py | 277 ++--
examples/pom.xml | 293 ++--
.../org/apache/spark/examples/JavaHdfsLR.java | 140 ++
.../org/apache/spark/examples/JavaKMeans.java | 131 ++
.../org/apache/spark/examples/JavaLogQuery.java | 131 ++
.../org/apache/spark/examples/JavaPageRank.java | 115 ++
.../org/apache/spark/examples/JavaSparkPi.java | 65 +
.../java/org/apache/spark/examples/JavaTC.java | 97 ++
.../apache/spark/examples/JavaWordCount.java | 66 +
.../apache/spark/mllib/examples/JavaALS.java | 87 ++
.../apache/spark/mllib/examples/JavaKMeans.java | 81 ++
.../org/apache/spark/mllib/examples/JavaLR.java | 85 ++
.../streaming/examples/JavaFlumeEventCount.java | 68 +
.../examples/JavaNetworkWordCount.java | 79 ++
.../streaming/examples/JavaQueueStream.java | 80 ++
.../main/java/spark/examples/JavaHdfsLR.java | 140 --
.../main/java/spark/examples/JavaKMeans.java | 131 --
.../main/java/spark/examples/JavaLogQuery.java | 131 --
.../main/java/spark/examples/JavaSparkPi.java | 65 -
.../src/main/java/spark/examples/JavaTC.java | 97 --
.../main/java/spark/examples/JavaWordCount.java | 66 -
.../streaming/examples/JavaFlumeEventCount.java | 68 -
.../examples/JavaNetworkWordCount.java | 79 --
.../streaming/examples/JavaQueueStream.java | 80 --
.../apache/spark/examples/BroadcastTest.scala | 50 +
.../apache/spark/examples/CassandraTest.scala | 213 +++
.../spark/examples/ExceptionHandlingTest.scala | 38 +
.../org/apache/spark/examples/GroupByTest.scala | 57 +
.../org/apache/spark/examples/HBaseTest.scala | 52 +
.../org/apache/spark/examples/HdfsTest.scala | 37 +
.../org/apache/spark/examples/LocalALS.scala | 140 ++
.../org/apache/spark/examples/LocalFileLR.scala | 55 +
.../org/apache/spark/examples/LocalKMeans.scala | 99 ++
.../org/apache/spark/examples/LocalLR.scala | 63 +
.../org/apache/spark/examples/LocalPi.scala | 34 +
.../org/apache/spark/examples/LogQuery.scala | 85 ++
.../spark/examples/MultiBroadcastTest.scala | 53 +
.../examples/SimpleSkewedGroupByTest.scala | 71 +
.../spark/examples/SkewedGroupByTest.scala | 61 +
.../org/apache/spark/examples/SparkALS.scala | 143 ++
.../org/apache/spark/examples/SparkHdfsLR.scala | 78 ++
.../org/apache/spark/examples/SparkKMeans.scala | 91 ++
.../org/apache/spark/examples/SparkLR.scala | 71 +
.../apache/spark/examples/SparkPageRank.scala | 46 +
.../org/apache/spark/examples/SparkPi.scala | 43 +
.../org/apache/spark/examples/SparkTC.scala | 75 +
.../spark/examples/bagel/PageRankUtils.scala | 124 ++
.../examples/bagel/WikipediaPageRank.scala | 101 ++
.../bagel/WikipediaPageRankStandalone.scala | 220 +++
.../streaming/examples/ActorWordCount.scala | 175 +++
.../streaming/examples/FlumeEventCount.scala | 61 +
.../streaming/examples/HdfsWordCount.scala | 54 +
.../streaming/examples/KafkaWordCount.scala | 98 ++
.../streaming/examples/NetworkWordCount.scala | 54 +
.../spark/streaming/examples/QueueStream.scala | 57 +
.../streaming/examples/RawNetworkGrep.scala | 64 +
.../examples/StatefulNetworkWordCount.scala | 67 +
.../streaming/examples/TwitterAlgebirdCMS.scala | 110 ++
.../streaming/examples/TwitterAlgebirdHLL.scala | 88 ++
.../streaming/examples/TwitterPopularTags.scala | 70 +
.../streaming/examples/ZeroMQWordCount.scala | 91 ++
.../clickstream/PageViewGenerator.scala | 102 ++
.../examples/clickstream/PageViewStream.scala | 101 ++
.../scala/spark/examples/BroadcastTest.scala | 50 -
.../scala/spark/examples/CassandraTest.scala | 213 ---
.../spark/examples/ExceptionHandlingTest.scala | 38 -
.../main/scala/spark/examples/GroupByTest.scala | 57 -
.../main/scala/spark/examples/HBaseTest.scala | 52 -
.../main/scala/spark/examples/HdfsTest.scala | 37 -
.../main/scala/spark/examples/LocalALS.scala | 140 --
.../main/scala/spark/examples/LocalFileLR.scala | 55 -
.../main/scala/spark/examples/LocalKMeans.scala | 99 --
.../src/main/scala/spark/examples/LocalLR.scala | 63 -
.../src/main/scala/spark/examples/LocalPi.scala | 34 -
.../main/scala/spark/examples/LogQuery.scala | 85 --
.../spark/examples/MultiBroadcastTest.scala | 53 -
.../examples/SimpleSkewedGroupByTest.scala | 71 -
.../spark/examples/SkewedGroupByTest.scala | 61 -
.../main/scala/spark/examples/SparkALS.scala | 143 --
.../main/scala/spark/examples/SparkHdfsLR.scala | 79 --
.../main/scala/spark/examples/SparkKMeans.scala | 91 --
.../src/main/scala/spark/examples/SparkLR.scala | 71 -
.../src/main/scala/spark/examples/SparkPi.scala | 43 -
.../src/main/scala/spark/examples/SparkTC.scala | 75 -
.../streaming/examples/ActorWordCount.scala | 175 ---
.../streaming/examples/FlumeEventCount.scala | 61 -
.../streaming/examples/HdfsWordCount.scala | 54 -
.../streaming/examples/KafkaWordCount.scala | 98 --
.../streaming/examples/NetworkWordCount.scala | 54 -
.../spark/streaming/examples/QueueStream.scala | 57 -
.../streaming/examples/RawNetworkGrep.scala | 64 -
.../examples/StatefulNetworkWordCount.scala | 67 -
.../streaming/examples/TwitterAlgebirdCMS.scala | 110 --
.../streaming/examples/TwitterAlgebirdHLL.scala | 88 --
.../streaming/examples/TwitterPopularTags.scala | 70 -
.../streaming/examples/ZeroMQWordCount.scala | 91 --
.../clickstream/PageViewGenerator.scala | 102 --
.../examples/clickstream/PageViewStream.scala | 101 --
make-distribution.sh | 70 +-
mllib/pom.xml | 75 +
.../classification/ClassificationModel.scala | 21 +
.../classification/LogisticRegression.scala | 189 +++
.../apache/spark/mllib/classification/SVM.scala | 188 +++
.../apache/spark/mllib/clustering/KMeans.scala | 336 +++++
.../spark/mllib/clustering/KMeansModel.scala | 44 +
.../spark/mllib/clustering/LocalKMeans.scala | 105 ++
.../spark/mllib/optimization/Gradient.scala | 98 ++
.../mllib/optimization/GradientDescent.scala | 168 +++
.../spark/mllib/optimization/Optimizer.scala | 29 +
.../spark/mllib/optimization/Updater.scala | 99 ++
.../apache/spark/mllib/recommendation/ALS.scala | 454 ++++++
.../MatrixFactorizationModel.scala | 49 +
.../regression/GeneralizedLinearAlgorithm.scala | 160 +++
.../spark/mllib/regression/LabeledPoint.scala | 26 +
.../apache/spark/mllib/regression/Lasso.scala | 211 +++
.../mllib/regression/LinearRegression.scala | 168 +++
.../mllib/regression/RegressionModel.scala | 38 +
.../mllib/regression/RidgeRegression.scala | 214 +++
.../spark/mllib/util/DataValidators.scala | 43 +
.../spark/mllib/util/KMeansDataGenerator.scala | 85 ++
.../spark/mllib/util/LinearDataGenerator.scala | 133 ++
.../util/LogisticRegressionDataGenerator.scala | 82 ++
.../spark/mllib/util/MFDataGenerator.scala | 114 ++
.../org/apache/spark/mllib/util/MLUtils.scala | 123 ++
.../spark/mllib/util/SVMDataGenerator.scala | 51 +
.../scala/spark/mllib/clustering/KMeans.scala | 334 -----
.../spark/mllib/clustering/KMeansModel.scala | 44 -
.../spark/mllib/clustering/LocalKMeans.scala | 105 --
.../spark/mllib/optimization/Gradient.scala | 50 -
.../mllib/optimization/GradientDescent.scala | 79 --
.../spark/mllib/optimization/Updater.scala | 44 -
.../scala/spark/mllib/recommendation/ALS.scala | 436 ------
.../MatrixFactorizationModel.scala | 40 -
.../mllib/regression/LogisticRegression.scala | 175 ---
.../LogisticRegressionGenerator.scala | 58 -
.../spark/mllib/regression/Regression.scala | 38 -
.../mllib/regression/RidgeRegression.scala | 211 ---
.../regression/RidgeRegressionGenerator.scala | 72 -
.../main/scala/spark/mllib/util/MLUtils.scala | 112 --
.../JavaLogisticRegressionSuite.java | 98 ++
.../mllib/classification/JavaSVMSuite.java | 98 ++
.../spark/mllib/clustering/JavaKMeansSuite.java | 115 ++
.../mllib/recommendation/JavaALSSuite.java | 110 ++
.../spark/mllib/regression/JavaLassoSuite.java | 97 ++
.../regression/JavaLinearRegressionSuite.java | 94 ++
.../regression/JavaRidgeRegressionSuite.java | 110 ++
mllib/src/test/resources/log4j.properties | 2 +-
.../LogisticRegressionSuite.scala | 150 ++
.../spark/mllib/classification/SVMSuite.scala | 169 +++
.../spark/mllib/clustering/KMeansSuite.scala | 173 +++
.../spark/mllib/recommendation/ALSSuite.scala | 125 ++
.../spark/mllib/regression/LassoSuite.scala | 121 ++
.../regression/LinearRegressionSuite.scala | 72 +
.../mllib/regression/RidgeRegressionSuite.scala | 90 ++
.../spark/mllib/clustering/KMeansSuite.scala | 170 ---
.../spark/mllib/recommendation/ALSSuite.scala | 97 --
.../regression/LogisticRegressionSuite.scala | 74 -
.../mllib/regression/RidgeRegressionSuite.scala | 64 -
pagerank_data.txt | 6 +
pom.xml | 316 +++--
project/SparkBuild.scala | 176 ++-
project/build.properties | 2 +-
project/plugins.sbt | 6 +-
pyspark | 25 +-
python/examples/als.py | 5 +-
python/examples/kmeans.py | 3 +-
python/examples/logistic_regression.py | 54 +-
python/examples/pagerank.py | 70 +
python/examples/pi.py | 3 +-
python/examples/transitive_closure.py | 5 +-
python/examples/wordcount.py | 5 +-
python/lib/py4j0.7.jar | Bin 103286 -> 0 bytes
python/pyspark/context.py | 29 +-
python/pyspark/files.py | 2 +-
python/pyspark/java_gateway.py | 12 +-
python/pyspark/rdd.py | 191 ++-
python/pyspark/rddsampler.py | 112 ++
python/pyspark/shell.py | 20 +-
python/pyspark/statcounter.py | 109 ++
python/pyspark/tests.py | 24 +-
python/pyspark/worker.py | 13 +-
python/run-tests | 25 +-
python/test_support/userlib-0.1-py2.7.egg | Bin 0 -> 1945 bytes
repl-bin/pom.xml | 168 +--
repl-bin/src/deb/bin/run | 2 +-
repl-bin/src/deb/bin/spark-executor | 2 +-
repl-bin/src/deb/bin/spark-shell | 2 +-
repl/pom.xml | 223 +--
.../apache/spark/repl/ExecutorClassLoader.scala | 124 ++
.../main/scala/org/apache/spark/repl/Main.scala | 33 +
.../org/apache/spark/repl/SparkHelper.scala | 5 +
.../org/apache/spark/repl/SparkILoop.scala | 1008 ++++++++++++++
.../org/apache/spark/repl/SparkIMain.scala | 1160 ++++++++++++++++
.../org/apache/spark/repl/SparkISettings.scala | 63 +
.../org/apache/spark/repl/SparkImports.scala | 214 +++
.../spark/repl/SparkJLineCompletion.scala | 379 +++++
.../apache/spark/repl/SparkJLineReader.scala | 79 ++
.../apache/spark/repl/SparkMemberHandlers.scala | 207 +++
.../scala/spark/repl/ExecutorClassLoader.scala | 124 --
repl/src/main/scala/spark/repl/Main.scala | 33 -
.../src/main/scala/spark/repl/SparkHelper.scala | 5 -
repl/src/main/scala/spark/repl/SparkILoop.scala | 1004 --------------
repl/src/main/scala/spark/repl/SparkIMain.scala | 1160 ----------------
.../main/scala/spark/repl/SparkISettings.scala | 63 -
.../main/scala/spark/repl/SparkImports.scala | 214 ---
.../scala/spark/repl/SparkJLineCompletion.scala | 379 -----
.../scala/spark/repl/SparkJLineReader.scala | 79 --
.../scala/spark/repl/SparkMemberHandlers.scala | 207 ---
.../scala/org/apache/spark/repl/ReplSuite.scala | 207 +++
repl/src/test/scala/spark/repl/ReplSuite.scala | 207 ---
run | 151 --
run-example | 81 ++
sbt/sbt | 6 +-
spark-class | 117 ++
spark-executor | 2 +-
spark-shell | 7 +-
spark-shell.cmd | 2 +-
streaming/pom.xml | 110 +-
.../org/apache/spark/streaming/Checkpoint.scala | 190 +++
.../org/apache/spark/streaming/DStream.scala | 703 ++++++++++
.../spark/streaming/DStreamCheckpointData.scala | 110 ++
.../apache/spark/streaming/DStreamGraph.scala | 167 +++
.../org/apache/spark/streaming/Duration.scala | 83 ++
.../org/apache/spark/streaming/Interval.scala | 59 +
.../scala/org/apache/spark/streaming/Job.scala | 41 +
.../org/apache/spark/streaming/JobManager.scala | 88 ++
.../spark/streaming/NetworkInputTracker.scala | 173 +++
.../spark/streaming/PairDStreamFunctions.scala | 535 +++++++
.../org/apache/spark/streaming/Scheduler.scala | 131 ++
.../spark/streaming/StreamingContext.scala | 564 ++++++++
.../scala/org/apache/spark/streaming/Time.scala | 72 +
.../spark/streaming/api/java/JavaDStream.scala | 102 ++
.../streaming/api/java/JavaDStreamLike.scala | 316 +++++
.../streaming/api/java/JavaPairDStream.scala | 614 +++++++++
.../api/java/JavaStreamingContext.scala | 617 +++++++++
.../streaming/dstream/CoGroupedDStream.scala | 58 +
.../dstream/ConstantInputDStream.scala | 36 +
.../streaming/dstream/FileInputDStream.scala | 199 +++
.../streaming/dstream/FilteredDStream.scala | 38 +
.../dstream/FlatMapValuedDStream.scala | 37 +
.../streaming/dstream/FlatMappedDStream.scala | 37 +
.../streaming/dstream/FlumeInputDStream.scala | 153 ++
.../streaming/dstream/ForEachDStream.scala | 45 +
.../streaming/dstream/GlommedDStream.scala | 34 +
.../spark/streaming/dstream/InputDStream.scala | 70 +
.../streaming/dstream/KafkaInputDStream.scala | 141 ++
.../dstream/MapPartitionedDStream.scala | 38 +
.../streaming/dstream/MapValuedDStream.scala | 38 +
.../spark/streaming/dstream/MappedDStream.scala | 37 +
.../streaming/dstream/NetworkInputDStream.scala | 271 ++++
.../dstream/PluggableInputDStream.scala | 30 +
.../streaming/dstream/QueueInputDStream.scala | 59 +
.../streaming/dstream/RawInputDStream.scala | 108 ++
.../dstream/ReducedWindowedDStream.scala | 174 +++
.../streaming/dstream/ShuffledDStream.scala | 45 +
.../streaming/dstream/SocketInputDStream.scala | 94 ++
.../spark/streaming/dstream/StateDStream.scala | 109 ++
.../streaming/dstream/TransformedDStream.scala | 36 +
.../streaming/dstream/TwitterInputDStream.scala | 99 ++
.../spark/streaming/dstream/UnionDStream.scala | 57 +
.../streaming/dstream/WindowedDStream.scala | 57 +
.../streaming/receivers/ActorReceiver.scala | 175 +++
.../streaming/receivers/ZeroMQReceiver.scala | 50 +
.../org/apache/spark/streaming/util/Clock.scala | 101 ++
.../streaming/util/MasterFailureTest.scala | 415 ++++++
.../spark/streaming/util/RawTextHelper.scala | 115 ++
.../spark/streaming/util/RawTextSender.scala | 78 ++
.../spark/streaming/util/RecurringTimer.scala | 94 ++
.../main/scala/spark/streaming/Checkpoint.scala | 182 ---
.../main/scala/spark/streaming/DStream.scala | 700 ----------
.../spark/streaming/DStreamCheckpointData.scala | 110 --
.../scala/spark/streaming/DStreamGraph.scala | 167 ---
.../main/scala/spark/streaming/Duration.scala | 83 --
.../main/scala/spark/streaming/Interval.scala | 59 -
.../src/main/scala/spark/streaming/Job.scala | 41 -
.../main/scala/spark/streaming/JobManager.scala | 88 --
.../spark/streaming/NetworkInputTracker.scala | 173 ---
.../spark/streaming/PairDStreamFunctions.scala | 534 -------
.../main/scala/spark/streaming/Scheduler.scala | 130 --
.../spark/streaming/StreamingContext.scala | 561 --------
.../src/main/scala/spark/streaming/Time.scala | 72 -
.../spark/streaming/api/java/JavaDStream.scala | 102 --
.../streaming/api/java/JavaDStreamLike.scala | 316 -----
.../streaming/api/java/JavaPairDStream.scala | 616 ---------
.../api/java/JavaStreamingContext.scala | 613 --------
.../streaming/dstream/CoGroupedDStream.scala | 57 -
.../dstream/ConstantInputDStream.scala | 36 -
.../streaming/dstream/FileInputDStream.scala | 199 ---
.../streaming/dstream/FilteredDStream.scala | 38 -
.../dstream/FlatMapValuedDStream.scala | 37 -
.../streaming/dstream/FlatMappedDStream.scala | 37 -
.../streaming/dstream/FlumeInputDStream.scala | 154 ---
.../streaming/dstream/ForEachDStream.scala | 45 -
.../streaming/dstream/GlommedDStream.scala | 34 -
.../spark/streaming/dstream/InputDStream.scala | 70 -
.../streaming/dstream/KafkaInputDStream.scala | 141 --
.../dstream/MapPartitionedDStream.scala | 38 -
.../streaming/dstream/MapValuedDStream.scala | 38 -
.../spark/streaming/dstream/MappedDStream.scala | 37 -
.../streaming/dstream/NetworkInputDStream.scala | 272 ----
.../dstream/PluggableInputDStream.scala | 30 -
.../streaming/dstream/QueueInputDStream.scala | 59 -
.../streaming/dstream/RawInputDStream.scala | 108 --
.../dstream/ReducedWindowedDStream.scala | 174 ---
.../streaming/dstream/ShuffledDStream.scala | 44 -
.../streaming/dstream/SocketInputDStream.scala | 94 --
.../spark/streaming/dstream/StateDStream.scala | 109 --
.../streaming/dstream/TransformedDStream.scala | 36 -
.../streaming/dstream/TwitterInputDStream.scala | 99 --
.../spark/streaming/dstream/UnionDStream.scala | 57 -
.../streaming/dstream/WindowedDStream.scala | 57 -
.../streaming/receivers/ActorReceiver.scala | 173 ---
.../streaming/receivers/ZeroMQReceiver.scala | 50 -
.../main/scala/spark/streaming/util/Clock.scala | 101 --
.../streaming/util/MasterFailureTest.scala | 414 ------
.../spark/streaming/util/RawTextHelper.scala | 115 --
.../spark/streaming/util/RawTextSender.scala | 77 --
.../spark/streaming/util/RecurringTimer.scala | 94 --
.../apache/spark/streaming/JavaAPISuite.java | 1304 ++++++++++++++++++
.../apache/spark/streaming/JavaTestUtils.scala | 85 ++
.../test/java/spark/streaming/JavaAPISuite.java | 1304 ------------------
.../java/spark/streaming/JavaTestUtils.scala | 84 --
.../spark/streaming/BasicOperationsSuite.scala | 322 +++++
.../spark/streaming/CheckpointSuite.scala | 372 +++++
.../apache/spark/streaming/FailureSuite.scala | 57 +
.../spark/streaming/InputStreamsSuite.scala | 349 +++++
.../apache/spark/streaming/TestSuiteBase.scala | 315 +++++
.../spark/streaming/WindowOperationsSuite.scala | 340 +++++
.../spark/streaming/BasicOperationsSuite.scala | 322 -----
.../scala/spark/streaming/CheckpointSuite.scala | 372 -----
.../scala/spark/streaming/FailureSuite.scala | 57 -
.../spark/streaming/InputStreamsSuite.scala | 349 -----
.../scala/spark/streaming/TestSuiteBase.scala | 314 -----
.../spark/streaming/WindowOperationsSuite.scala | 340 -----
tools/pom.xml | 69 +
.../tools/JavaAPICompletenessChecker.scala | 360 +++++
yarn/pom.xml | 111 ++
.../spark/deploy/yarn/ApplicationMaster.scala | 371 +++++
.../yarn/ApplicationMasterArguments.scala | 94 ++
.../org/apache/spark/deploy/yarn/Client.scala | 336 +++++
.../spark/deploy/yarn/ClientArguments.scala | 116 ++
.../spark/deploy/yarn/WorkerRunnable.scala | 224 +++
.../deploy/yarn/YarnAllocationHandler.scala | 564 ++++++++
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 46 +
.../cluster/YarnClusterScheduler.scala | 52 +
1033 files changed, 73399 insertions(+), 65507 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/2ce200bf/README.md
----------------------------------------------------------------------