You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/10/17 22:21:14 UTC
[2/6] git commit: Merge branch 'master' of
https://github.com/apache/incubator-spark
Merge branch 'master' of https://github.com/apache/incubator-spark
Updating local branch
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/6c6b146f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/6c6b146f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/6c6b146f
Branch: refs/heads/master
Commit: 6c6b146fc20f2ba85f1cf1bb9e35283ee195f59a
Parents: d2c86e7 e33b183
Author: KarthikTunga <ka...@gmail.com>
Authored: Tue Oct 15 00:46:35 2013 -0700
Committer: KarthikTunga <ka...@gmail.com>
Committed: Tue Oct 15 00:46:35 2013 -0700
----------------------------------------------------------------------
bin/stop-slaves.sh | 2 -
core/pom.xml | 4 +
.../spark/network/netty/FileClientHandler.java | 3 +-
.../spark/network/netty/FileServerHandler.java | 6 +-
.../scala/org/apache/spark/Aggregator.scala | 49 ++-
.../apache/spark/BlockStoreShuffleFetcher.scala | 23 +-
.../scala/org/apache/spark/CacheManager.scala | 27 +-
.../scala/org/apache/spark/FutureAction.scala | 250 +++++++++++
.../apache/spark/InterruptibleIterator.scala | 30 ++
.../org/apache/spark/MapOutputTracker.scala | 4 +-
.../scala/org/apache/spark/ShuffleFetcher.scala | 5 +-
.../scala/org/apache/spark/SparkContext.scala | 102 +++--
.../scala/org/apache/spark/TaskContext.scala | 23 +-
.../scala/org/apache/spark/TaskEndReason.scala | 2 +
.../spark/broadcast/BitTorrentBroadcast.scala | 4 +-
.../apache/spark/broadcast/HttpBroadcast.scala | 13 +-
.../apache/spark/broadcast/TreeBroadcast.scala | 8 +-
.../org/apache/spark/deploy/DeployMessage.scala | 29 +-
.../spark/deploy/ExecutorDescription.scala | 34 ++
.../spark/deploy/FaultToleranceTest.scala | 420 +++++++++++++++++++
.../org/apache/spark/deploy/JsonProtocol.scala | 3 +-
.../apache/spark/deploy/LocalSparkCluster.scala | 7 +-
.../org/apache/spark/deploy/client/Client.scala | 84 +++-
.../spark/deploy/client/ClientListener.scala | 4 +
.../apache/spark/deploy/client/TestClient.scala | 7 +-
.../spark/deploy/master/ApplicationInfo.scala | 53 ++-
.../spark/deploy/master/ApplicationSource.scala | 2 +-
.../spark/deploy/master/ApplicationState.scala | 4 +-
.../spark/deploy/master/ExecutorInfo.scala | 7 +-
.../master/FileSystemPersistenceEngine.scala | 90 ++++
.../deploy/master/LeaderElectionAgent.scala | 45 ++
.../org/apache/spark/deploy/master/Master.scala | 228 ++++++++--
.../spark/deploy/master/MasterMessages.scala | 46 ++
.../spark/deploy/master/MasterSource.scala | 6 +-
.../spark/deploy/master/PersistenceEngine.scala | 53 +++
.../spark/deploy/master/RecoveryState.scala | 26 ++
.../deploy/master/SparkZooKeeperSession.scala | 203 +++++++++
.../apache/spark/deploy/master/WorkerInfo.scala | 42 +-
.../spark/deploy/master/WorkerState.scala | 6 +-
.../master/ZooKeeperLeaderElectionAgent.scala | 136 ++++++
.../master/ZooKeeperPersistenceEngine.scala | 85 ++++
.../spark/deploy/worker/ExecutorRunner.scala | 13 +-
.../org/apache/spark/deploy/worker/Worker.scala | 175 +++++---
.../spark/deploy/worker/WorkerArguments.scala | 8 +-
.../spark/deploy/worker/WorkerSource.scala | 10 +-
.../spark/deploy/worker/ui/WorkerWebUI.scala | 2 +-
.../org/apache/spark/executor/Executor.scala | 122 ++++--
.../apache/spark/executor/ExecutorSource.scala | 18 +-
.../spark/executor/MesosExecutorBackend.scala | 18 +-
.../executor/StandaloneExecutorBackend.scala | 10 +-
.../apache/spark/network/netty/FileHeader.scala | 22 +-
.../spark/network/netty/ShuffleCopier.scala | 27 +-
.../spark/network/netty/ShuffleSender.scala | 8 +-
.../org/apache/spark/rdd/AsyncRDDActions.scala | 122 ++++++
.../scala/org/apache/spark/rdd/BlockRDD.scala | 6 +-
.../org/apache/spark/rdd/CheckpointRDD.scala | 2 +-
.../org/apache/spark/rdd/CoGroupedRDD.scala | 26 +-
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 116 +++--
.../spark/rdd/MapPartitionsWithContextRDD.scala | 41 ++
.../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 --
.../org/apache/spark/rdd/NewHadoopRDD.scala | 79 ++--
.../org/apache/spark/rdd/PairRDDFunctions.scala | 16 +-
.../spark/rdd/ParallelCollectionRDD.scala | 5 +-
.../main/scala/org/apache/spark/rdd/RDD.scala | 95 +++--
.../org/apache/spark/rdd/ShuffledRDD.scala | 2 +-
.../org/apache/spark/rdd/SubtractedRDD.scala | 2 +-
.../apache/spark/scheduler/DAGScheduler.scala | 128 +++---
.../spark/scheduler/DAGSchedulerEvent.scala | 24 +-
.../spark/scheduler/DAGSchedulerSource.scala | 12 +-
.../org/apache/spark/scheduler/JobWaiter.scala | 62 +--
.../scala/org/apache/spark/scheduler/Pool.scala | 5 +-
.../org/apache/spark/scheduler/ResultTask.scala | 48 ++-
.../spark/scheduler/SchedulableBuilder.scala | 3 +
.../apache/spark/scheduler/ShuffleMapTask.scala | 50 ++-
.../apache/spark/scheduler/SparkListener.scala | 2 +-
.../spark/scheduler/SparkListenerBus.scala | 18 +
.../scala/org/apache/spark/scheduler/Task.scala | 63 ++-
.../org/apache/spark/scheduler/TaskResult.scala | 3 +-
.../apache/spark/scheduler/TaskScheduler.scala | 3 +
.../org/apache/spark/scheduler/TaskSet.scala | 4 +
.../scheduler/cluster/ClusterScheduler.scala | 37 +-
.../cluster/ClusterTaskSetManager.scala | 85 ++--
.../scheduler/cluster/SchedulerBackend.scala | 6 +-
.../cluster/SparkDeploySchedulerBackend.scala | 14 +-
.../cluster/StandaloneClusterMessage.scala | 2 +
.../cluster/StandaloneSchedulerBackend.scala | 7 +
.../spark/scheduler/local/LocalScheduler.scala | 190 +++------
.../scheduler/local/LocalTaskSetManager.scala | 13 +-
.../spark/serializer/KryoSerializer.scala | 30 +-
.../apache/spark/storage/BlockException.scala | 2 +-
.../spark/storage/BlockFetcherIterator.scala | 24 +-
.../org/apache/spark/storage/BlockId.scala | 96 +++++
.../org/apache/spark/storage/BlockManager.scala | 130 +++---
.../spark/storage/BlockManagerMaster.scala | 8 +-
.../spark/storage/BlockManagerMasterActor.scala | 21 +-
.../spark/storage/BlockManagerMessages.scala | 16 +-
.../spark/storage/BlockManagerSource.scala | 8 +-
.../spark/storage/BlockManagerWorker.scala | 4 +-
.../org/apache/spark/storage/BlockMessage.scala | 38 +-
.../spark/storage/BlockMessageArray.scala | 7 +-
.../spark/storage/BlockObjectWriter.scala | 2 +-
.../org/apache/spark/storage/BlockStore.scala | 14 +-
.../org/apache/spark/storage/DiskStore.scala | 37 +-
.../org/apache/spark/storage/MemoryStore.scala | 34 +-
.../spark/storage/ShuffleBlockManager.scala | 16 +-
.../org/apache/spark/storage/StorageUtils.scala | 47 +--
.../apache/spark/storage/ThreadingTest.scala | 6 +-
.../apache/spark/ui/UIWorkloadGenerator.scala | 2 +-
.../org/apache/spark/ui/storage/RDDPage.scala | 23 +-
.../org/apache/spark/util/AppendOnlyMap.scala | 230 ++++++++++
.../org/apache/spark/util/MetadataCleaner.scala | 36 +-
.../org/apache/spark/CacheManagerSuite.scala | 21 +-
.../org/apache/spark/CheckpointSuite.scala | 10 +-
.../org/apache/spark/DistributedSuite.scala | 16 +-
.../scala/org/apache/spark/JavaAPISuite.java | 2 +-
.../org/apache/spark/JobCancellationSuite.scala | 177 ++++++++
.../apache/spark/deploy/JsonProtocolSuite.scala | 7 +-
.../apache/spark/rdd/AsyncRDDActionsSuite.scala | 176 ++++++++
.../spark/rdd/PairRDDFunctionsSuite.scala | 2 +-
.../spark/scheduler/DAGSchedulerSuite.scala | 27 +-
.../spark/scheduler/SparkListenerSuite.scala | 36 +-
.../spark/scheduler/cluster/FakeTask.scala | 5 +-
.../cluster/TaskResultGetterSuite.scala | 3 +-
.../scheduler/local/LocalSchedulerSuite.scala | 28 +-
.../spark/serializer/KryoSerializerSuite.scala | 21 +
.../org/apache/spark/storage/BlockIdSuite.scala | 114 +++++
.../spark/storage/BlockManagerSuite.scala | 102 ++---
.../scala/org/apache/spark/ui/UISuite.scala | 7 +-
.../apache/spark/util/AppendOnlyMapSuite.scala | 154 +++++++
docker/README.md | 5 +
docker/build | 22 +
docker/spark-test/README.md | 10 +
docker/spark-test/base/Dockerfile | 38 ++
docker/spark-test/build | 22 +
docker/spark-test/master/Dockerfile | 21 +
docker/spark-test/master/default_cmd | 22 +
docker/spark-test/worker/Dockerfile | 22 +
docker/spark-test/worker/default_cmd | 22 +
docs/mllib-guide.md | 24 +-
docs/python-programming-guide.md | 2 +-
docs/running-on-yarn.md | 9 +-
docs/spark-standalone.md | 75 ++++
docs/streaming-programming-guide.md | 5 +-
docs/tuning.md | 2 +-
ec2/spark_ec2.py | 2 +-
.../org/apache/spark/examples/SparkKMeans.scala | 2 -
.../apache/spark/mllib/recommendation/ALS.scala | 199 +++++++--
.../mllib/recommendation/JavaALSSuite.java | 85 +++-
.../spark/mllib/recommendation/ALSSuite.scala | 75 +++-
pom.xml | 11 +
project/SparkBuild.scala | 14 +-
python/pyspark/rdd.py | 60 ++-
.../spark/streaming/NetworkInputTracker.scala | 11 +-
.../streaming/dstream/NetworkInputDStream.scala | 14 +-
.../streaming/dstream/RawInputDStream.scala | 4 +-
.../streaming/receivers/ActorReceiver.scala | 4 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 55 ++-
.../org/apache/spark/deploy/yarn/Client.scala | 160 ++++++-
.../spark/deploy/yarn/ClientArguments.scala | 25 +-
.../spark/deploy/yarn/WorkerRunnable.scala | 58 ++-
160 files changed, 5357 insertions(+), 1402 deletions(-)
----------------------------------------------------------------------