You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/11/04 05:43:35 UTC
[2/5] git commit: Merge branch 'master' into hash1
Merge branch 'master' into hash1
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/da6bb0ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/da6bb0ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/da6bb0ae
Branch: refs/heads/master
Commit: da6bb0aedd5121d9e3b92031dcc0884a9682da05
Parents: 3e7df8f 41ead7a
Author: Reynold Xin <rx...@apache.org>
Authored: Sat Nov 2 22:45:15 2013 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Sat Nov 2 22:45:15 2013 -0700
----------------------------------------------------------------------
README.md | 2 +-
bin/compute-classpath.sh | 22 +-
bin/slaves.sh | 19 +-
bin/spark-daemon.sh | 21 +-
bin/spark-daemons.sh | 2 +-
.../spark/network/netty/FileServerHandler.java | 19 +-
.../spark/network/netty/PathResolver.java | 11 +-
.../hadoop/mapred/SparkHadoopMapRedUtil.scala | 17 +-
.../mapreduce/SparkHadoopMapReduceUtil.scala | 33 +-
.../org/apache/spark/MapOutputTracker.scala | 168 +--
.../scala/org/apache/spark/SparkContext.scala | 132 ++-
.../main/scala/org/apache/spark/SparkEnv.scala | 25 +-
.../org/apache/spark/SparkHadoopWriter.scala | 21 +-
.../scala/org/apache/spark/TaskContext.scala | 2 +-
.../apache/spark/api/java/JavaDoubleRDD.scala | 24 +
.../org/apache/spark/api/java/JavaPairRDD.scala | 35 +
.../org/apache/spark/api/java/JavaRDD.scala | 19 +
.../java/function/DoubleFlatMapFunction.java | 10 +-
.../spark/api/java/function/DoubleFunction.java | 5 +-
.../api/java/function/FlatMapFunction.scala | 3 -
.../api/java/function/FlatMapFunction2.scala | 3 -
.../spark/api/java/function/Function.java | 3 -
.../spark/api/java/function/Function2.java | 3 -
.../spark/api/java/function/Function3.java | 36 +
.../api/java/function/PairFlatMapFunction.java | 3 -
.../spark/api/java/function/PairFunction.java | 6 +-
.../api/java/function/WrappedFunction3.scala | 34 +
.../org/apache/spark/api/python/PythonRDD.scala | 2 +-
.../spark/broadcast/BitTorrentBroadcast.scala | 1058 ------------------
.../apache/spark/broadcast/MultiTracker.scala | 410 -------
.../org/apache/spark/broadcast/SourceInfo.scala | 54 -
.../spark/broadcast/TorrentBroadcast.scala | 247 ++++
.../apache/spark/broadcast/TreeBroadcast.scala | 601 ----------
.../apache/spark/deploy/SparkHadoopUtil.scala | 41 +-
.../executor/CoarseGrainedExecutorBackend.scala | 122 ++
.../org/apache/spark/executor/Executor.scala | 46 +-
.../executor/StandaloneExecutorBackend.scala | 115 --
.../org/apache/spark/executor/TaskMetrics.scala | 5 +
.../spark/network/ConnectionManager.scala | 3 +-
.../spark/network/netty/ShuffleSender.scala | 7 +-
.../main/scala/org/apache/spark/package.scala | 2 +
.../org/apache/spark/rdd/CheckpointRDD.scala | 7 +-
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 7 +-
.../main/scala/org/apache/spark/rdd/RDD.scala | 13 +
.../apache/spark/scheduler/DAGScheduler.scala | 77 +-
.../spark/scheduler/DAGSchedulerEvent.scala | 5 +
.../spark/scheduler/InputFormatInfo.scala | 7 +-
.../org/apache/spark/scheduler/JobLogger.scala | 128 +--
.../apache/spark/scheduler/ShuffleMapTask.scala | 7 +-
.../apache/spark/scheduler/SparkListener.scala | 19 +-
.../spark/scheduler/SparkListenerBus.scala | 2 +
.../org/apache/spark/scheduler/Stage.scala | 6 +-
.../org/apache/spark/scheduler/StageInfo.scala | 13 +-
.../scala/org/apache/spark/scheduler/Task.scala | 2 +-
.../org/apache/spark/scheduler/TaskInfo.scala | 20 +
.../apache/spark/scheduler/TaskScheduler.scala | 7 +-
.../spark/scheduler/TaskSchedulerListener.scala | 44 -
.../scheduler/cluster/ClusterScheduler.scala | 18 +-
.../cluster/ClusterTaskSetManager.scala | 27 +-
.../cluster/CoarseGrainedClusterMessage.scala | 69 ++
.../cluster/CoarseGrainedSchedulerBackend.scala | 230 ++++
.../cluster/SimrSchedulerBackend.scala | 66 ++
.../cluster/SparkDeploySchedulerBackend.scala | 6 +-
.../cluster/StandaloneClusterMessage.scala | 65 --
.../cluster/StandaloneSchedulerBackend.scala | 207 ----
.../scheduler/cluster/TaskResultGetter.scala | 26 +-
.../mesos/CoarseMesosSchedulerBackend.scala | 15 +-
.../spark/scheduler/local/LocalScheduler.scala | 6 +-
.../scheduler/local/LocalTaskSetManager.scala | 13 +-
.../org/apache/spark/storage/BlockId.scala | 9 +-
.../org/apache/spark/storage/BlockInfo.scala | 97 ++
.../org/apache/spark/storage/BlockManager.scala | 556 ++++-----
.../spark/storage/BlockManagerMasterActor.scala | 4 +-
.../spark/storage/BlockManagerSlaveActor.scala | 1 +
.../spark/storage/BlockObjectWriter.scala | 133 ++-
.../apache/spark/storage/DiskBlockManager.scala | 184 +++
.../org/apache/spark/storage/DiskStore.scala | 263 +----
.../org/apache/spark/storage/FileSegment.scala | 28 +
.../spark/storage/ShuffleBlockManager.scala | 59 +-
.../spark/storage/StoragePerfTester.scala | 86 ++
.../org/apache/spark/ui/jobs/IndexPage.scala | 2 +-
.../spark/ui/jobs/JobProgressListener.scala | 105 +-
.../org/apache/spark/ui/jobs/PoolTable.scala | 8 +-
.../org/apache/spark/ui/jobs/StagePage.scala | 21 +-
.../org/apache/spark/ui/jobs/StageTable.scala | 26 +-
.../org/apache/spark/util/MetadataCleaner.scala | 5 +-
.../scala/org/apache/spark/util/Utils.scala | 32 +-
.../scala/org/apache/spark/BroadcastSuite.scala | 52 +-
.../org/apache/spark/FileServerSuite.scala | 16 +
.../scala/org/apache/spark/JavaAPISuite.java | 21 +
.../org/apache/spark/JobCancellationSuite.scala | 32 +
.../apache/spark/MapOutputTrackerSuite.scala | 20 +-
.../scala/org/apache/spark/rdd/RDDSuite.scala | 20 +
.../spark/scheduler/DAGSchedulerSuite.scala | 8 +-
.../apache/spark/scheduler/JobLoggerSuite.scala | 11 +-
.../spark/scheduler/SparkListenerSuite.scala | 130 ++-
.../cluster/ClusterTaskSetManagerSuite.scala | 49 +-
docker/spark-test/README.md | 7 +-
docs/cluster-overview.md | 14 +-
docs/configuration.md | 10 +-
docs/ec2-scripts.md | 2 +-
docs/python-programming-guide.md | 11 +
docs/scala-programming-guide.md | 6 +-
docs/streaming-programming-guide.md | 4 +
ec2/spark_ec2.py | 2 +-
examples/pom.xml | 36 +-
.../streaming/examples/JavaKafkaWordCount.java | 98 ++
.../apache/spark/examples/BroadcastTest.scala | 15 +-
.../org/apache/spark/examples/SparkHdfsLR.scala | 3 +-
.../org/apache/spark/examples/SparkPi.scala | 2 +-
.../streaming/examples/KafkaWordCount.scala | 28 +-
.../streaming/examples/MQTTWordCount.scala | 107 ++
.../clickstream/PageViewGenerator.scala | 13 +-
pom.xml | 85 +-
project/SparkBuild.scala | 30 +-
python/pyspark/accumulators.py | 13 +-
python/pyspark/context.py | 50 +-
.../org/apache/spark/repl/SparkILoop.scala | 23 +-
spark-class | 15 +-
.../kafka/0.7.2-spark/kafka-0.7.2-spark.jar | Bin 1358063 -> 0 bytes
.../kafka/0.7.2-spark/kafka-0.7.2-spark.jar.md5 | 1 -
.../0.7.2-spark/kafka-0.7.2-spark.jar.sha1 | 1 -
.../kafka/0.7.2-spark/kafka-0.7.2-spark.pom | 9 -
.../kafka/0.7.2-spark/kafka-0.7.2-spark.pom.md5 | 1 -
.../0.7.2-spark/kafka-0.7.2-spark.pom.sha1 | 1 -
.../apache/kafka/kafka/maven-metadata-local.xml | 12 -
.../kafka/kafka/maven-metadata-local.xml.md5 | 1 -
.../kafka/kafka/maven-metadata-local.xml.sha1 | 1 -
streaming/pom.xml | 57 +-
.../org/apache/spark/streaming/Checkpoint.scala | 2 +
.../org/apache/spark/streaming/DStream.scala | 53 +-
.../spark/streaming/PairDStreamFunctions.scala | 152 ++-
.../spark/streaming/StreamingContext.scala | 52 +-
.../spark/streaming/api/java/JavaDStream.scala | 6 +
.../streaming/api/java/JavaDStreamLike.scala | 97 +-
.../streaming/api/java/JavaPairDStream.scala | 184 ++-
.../api/java/JavaStreamingContext.scala | 108 +-
.../streaming/dstream/CoGroupedDStream.scala | 58 -
.../streaming/dstream/KafkaInputDStream.scala | 61 +-
.../streaming/dstream/MQTTInputDStream.scala | 109 ++
.../streaming/dstream/TransformedDStream.scala | 20 +-
.../apache/spark/streaming/JavaAPISuite.java | 425 ++++++-
.../apache/spark/streaming/JavaTestUtils.scala | 36 +-
.../spark/streaming/BasicOperationsSuite.scala | 141 ++-
.../spark/streaming/CheckpointSuite.scala | 4 +-
.../spark/streaming/InputStreamsSuite.scala | 8 +-
.../apache/spark/streaming/TestSuiteBase.scala | 61 +-
.../org/apache/spark/deploy/yarn/Client.scala | 32 +-
.../spark/deploy/yarn/WorkerRunnable.scala | 7 +-
.../deploy/yarn/YarnAllocationHandler.scala | 4 +-
150 files changed, 4622 insertions(+), 4137 deletions(-)
----------------------------------------------------------------------