You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/04 08:50:43 UTC
[05/10] git commit: Merge branch 'master' into spark-1002-remove-jars
Merge branch 'master' into spark-1002-remove-jars
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b4bb8000
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b4bb8000
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b4bb8000
Branch: refs/heads/master
Commit: b4bb80002bbf0ac3642c78ae9e5c260b5da4a4cc
Parents: 08ec10d 498a5f0
Author: Prashant Sharma <pr...@imaginea.com>
Authored: Fri Jan 3 12:12:04 2014 +0530
Committer: Prashant Sharma <pr...@imaginea.com>
Committed: Fri Jan 3 12:12:04 2014 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/Accumulators.scala | 8 +-
.../org/apache/spark/MapOutputTracker.scala | 11 +-
.../scala/org/apache/spark/Partitioner.scala | 4 +-
.../main/scala/org/apache/spark/SparkConf.scala | 189 ++++++++++
.../scala/org/apache/spark/SparkContext.scala | 267 +++++++++-----
.../main/scala/org/apache/spark/SparkEnv.scala | 54 +--
.../spark/api/java/JavaSparkContext.scala | 35 +-
.../org/apache/spark/api/python/PythonRDD.scala | 4 +-
.../org/apache/spark/broadcast/Broadcast.scala | 8 +-
.../spark/broadcast/BroadcastFactory.scala | 4 +-
.../apache/spark/broadcast/HttpBroadcast.scala | 43 ++-
.../spark/broadcast/TorrentBroadcast.scala | 45 +--
.../spark/deploy/FaultToleranceTest.scala | 4 +-
.../apache/spark/deploy/LocalSparkCluster.scala | 7 +-
.../apache/spark/deploy/SparkHadoopUtil.scala | 14 +-
.../org/apache/spark/deploy/client/Client.scala | 13 +-
.../apache/spark/deploy/client/TestClient.scala | 10 +-
.../org/apache/spark/deploy/master/Master.scala | 41 ++-
.../spark/deploy/master/MasterArguments.scala | 11 +-
.../deploy/master/SparkZooKeeperSession.scala | 7 +-
.../master/ZooKeeperLeaderElectionAgent.scala | 9 +-
.../master/ZooKeeperPersistenceEngine.scala | 8 +-
.../spark/deploy/master/ui/MasterWebUI.scala | 2 +-
.../org/apache/spark/deploy/worker/Worker.scala | 34 +-
.../spark/deploy/worker/ui/WorkerWebUI.scala | 6 +-
.../executor/CoarseGrainedExecutorBackend.scala | 6 +-
.../org/apache/spark/executor/Executor.scala | 45 +--
.../org/apache/spark/io/CompressionCodec.scala | 19 +-
.../apache/spark/metrics/MetricsSystem.scala | 10 +-
.../spark/network/ConnectionManager.scala | 22 +-
.../org/apache/spark/network/ReceiverTest.scala | 12 +-
.../org/apache/spark/network/SenderTest.scala | 16 +-
.../spark/network/netty/ShuffleCopier.scala | 10 +-
.../org/apache/spark/rdd/CheckpointRDD.scala | 5 +-
.../org/apache/spark/rdd/CoGroupedRDD.scala | 2 +-
.../spark/rdd/PartitionerAwareUnionRDD.scala | 110 ++++++
.../main/scala/org/apache/spark/rdd/RDD.scala | 3 +-
.../apache/spark/rdd/RDDCheckpointData.scala | 2 +-
.../org/apache/spark/rdd/ShuffledRDD.scala | 2 +-
.../org/apache/spark/rdd/SubtractedRDD.scala | 2 +-
.../apache/spark/scheduler/DAGScheduler.scala | 3 +-
.../spark/scheduler/InputFormatInfo.scala | 14 +-
.../org/apache/spark/scheduler/ResultTask.scala | 4 +-
.../spark/scheduler/SchedulableBuilder.scala | 6 +-
.../spark/scheduler/SchedulerBackend.scala | 3 -
.../apache/spark/scheduler/ShuffleMapTask.scala | 6 +-
.../spark/scheduler/TaskResultGetter.scala | 3 +-
.../spark/scheduler/TaskSchedulerImpl.scala | 25 +-
.../apache/spark/scheduler/TaskSetManager.scala | 23 +-
.../cluster/CoarseGrainedSchedulerBackend.scala | 20 +-
.../cluster/SimrSchedulerBackend.scala | 4 +-
.../cluster/SparkDeploySchedulerBackend.scala | 8 +-
.../mesos/CoarseMesosSchedulerBackend.scala | 14 +-
.../cluster/mesos/MesosSchedulerBackend.scala | 8 +-
.../spark/scheduler/local/LocalBackend.scala | 3 +-
.../spark/serializer/JavaSerializer.scala | 3 +-
.../spark/serializer/KryoSerializer.scala | 14 +-
.../spark/serializer/SerializerManager.scala | 12 +-
.../spark/storage/BlockFetcherIterator.scala | 4 +-
.../org/apache/spark/storage/BlockManager.scala | 58 +--
.../spark/storage/BlockManagerMaster.scala | 11 +-
.../spark/storage/BlockManagerMasterActor.scala | 14 +-
.../spark/storage/BlockObjectWriter.scala | 5 +-
.../apache/spark/storage/DiskBlockManager.scala | 2 +-
.../spark/storage/ShuffleBlockManager.scala | 10 +-
.../spark/storage/StoragePerfTester.scala | 2 +-
.../apache/spark/storage/ThreadingTest.scala | 8 +-
.../scala/org/apache/spark/ui/SparkUI.scala | 4 +-
.../apache/spark/ui/UIWorkloadGenerator.scala | 17 +-
.../org/apache/spark/ui/env/EnvironmentUI.scala | 15 +-
.../spark/ui/jobs/JobProgressListener.scala | 4 +-
.../scala/org/apache/spark/util/AkkaUtils.scala | 25 +-
.../org/apache/spark/util/MetadataCleaner.scala | 35 +-
.../org/apache/spark/util/SizeEstimator.scala | 14 +-
.../scala/org/apache/spark/util/Utils.scala | 25 +-
core/src/test/resources/spark.conf | 8 +
.../org/apache/spark/CheckpointSuite.scala | 361 +++++++++++--------
.../apache/spark/MapOutputTrackerSuite.scala | 16 +-
.../org/apache/spark/SharedSparkContext.scala | 4 +-
.../scala/org/apache/spark/SparkConfSuite.scala | 110 ++++++
.../apache/spark/io/CompressionCodecSuite.scala | 8 +-
.../spark/metrics/MetricsSystemSuite.scala | 8 +-
.../scala/org/apache/spark/rdd/RDDSuite.scala | 27 ++
.../spark/scheduler/ClusterSchedulerSuite.scala | 2 +-
.../spark/scheduler/DAGSchedulerSuite.scala | 23 +-
.../apache/spark/scheduler/JobLoggerSuite.scala | 2 +-
.../spark/scheduler/TaskResultGetterSuite.scala | 6 +-
.../spark/scheduler/TaskSetManagerSuite.scala | 4 +-
.../spark/serializer/KryoSerializerSuite.scala | 29 +-
.../spark/storage/BlockManagerSuite.scala | 97 ++---
.../spark/storage/DiskBlockManagerSuite.scala | 18 +-
.../apache/spark/util/SizeEstimatorSuite.scala | 2 +-
docs/_config.yml | 2 +-
docs/configuration.md | 71 +++-
docs/css/bootstrap.min.css | 2 +-
docs/job-scheduling.md | 21 +-
docs/monitoring.md | 3 +-
docs/python-programming-guide.md | 15 +-
docs/quick-start.md | 52 ++-
docs/running-on-mesos.md | 19 +-
docs/scala-programming-guide.md | 4 +-
docs/spark-standalone.md | 15 +-
docs/streaming-programming-guide.md | 4 +-
docs/tuning.md | 21 +-
.../examples/bagel/WikipediaPageRank.scala | 10 +-
.../bagel/WikipediaPageRankStandalone.scala | 8 +-
.../streaming/examples/ActorWordCount.scala | 3 +-
.../apache/spark/mllib/recommendation/ALS.scala | 13 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 84 ++---
.../org/apache/spark/deploy/yarn/Client.scala | 42 ++-
.../spark/deploy/yarn/ClientArguments.scala | 3 +-
.../spark/deploy/yarn/WorkerLauncher.scala | 13 +-
.../deploy/yarn/YarnAllocationHandler.scala | 27 +-
.../cluster/YarnClientSchedulerBackend.scala | 4 +-
project/SparkBuild.scala | 1 +
python/epydoc.conf | 2 +-
python/pyspark/__init__.py | 32 +-
python/pyspark/broadcast.py | 11 +
python/pyspark/conf.py | 171 +++++++++
python/pyspark/context.py | 59 ++-
python/pyspark/java_gateway.py | 1 +
python/pyspark/rdd.py | 66 +++-
python/run-tests | 3 +-
.../org/apache/spark/repl/SparkILoop.scala | 17 +-
.../org/apache/spark/repl/SparkIMain.scala | 7 +-
.../org/apache/spark/streaming/Checkpoint.scala | 18 +-
.../org/apache/spark/streaming/DStream.scala | 2 +-
.../spark/streaming/PairDStreamFunctions.scala | 13 +-
.../spark/streaming/StreamingContext.scala | 55 ++-
.../streaming/api/java/JavaPairDStream.scala | 18 +-
.../api/java/JavaStreamingContext.scala | 9 +
.../streaming/dstream/NetworkInputDStream.scala | 6 +-
.../streaming/dstream/ShuffledDStream.scala | 9 +-
.../streaming/dstream/WindowedDStream.scala | 16 +-
.../streaming/scheduler/JobGenerator.scala | 8 +-
.../streaming/scheduler/JobScheduler.scala | 4 +-
.../spark/streaming/util/RawTextSender.scala | 4 +-
.../apache/spark/streaming/JavaAPISuite.java | 10 +-
.../spark/streaming/BasicOperationsSuite.scala | 8 +-
.../spark/streaming/CheckpointSuite.scala | 15 +-
.../spark/streaming/InputStreamsSuite.scala | 18 +-
.../apache/spark/streaming/TestSuiteBase.scala | 34 +-
.../spark/streaming/WindowOperationsSuite.scala | 5 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 97 ++---
.../org/apache/spark/deploy/yarn/Client.scala | 54 +--
.../spark/deploy/yarn/ClientArguments.scala | 3 +-
.../spark/deploy/yarn/WorkerLauncher.scala | 16 +-
.../deploy/yarn/YarnAllocationHandler.scala | 25 +-
.../cluster/YarnClientSchedulerBackend.scala | 4 +-
149 files changed, 2291 insertions(+), 1221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b4bb8000/docs/python-programming-guide.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b4bb8000/docs/quick-start.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b4bb8000/docs/scala-programming-guide.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b4bb8000/project/SparkBuild.scala
----------------------------------------------------------------------