You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/12/05 08:33:19 UTC
[08/14] git commit: Merge remote-tracking branch 'origin/master' into
yarn-2.2
Merge remote-tracking branch 'origin/master' into yarn-2.2
Conflicts:
yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/afe4fe7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/afe4fe7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/afe4fe7f
Branch: refs/heads/master
Commit: afe4fe7f5ed9e5b82b641e72bbfc14f2d952c3be
Parents: a1a1c62 cb976df
Author: Harvey Feng <ha...@databricks.com>
Authored: Tue Nov 26 15:03:03 2013 -0800
Committer: Harvey Feng <ha...@databricks.com>
Committed: Tue Nov 26 15:03:03 2013 -0800
----------------------------------------------------------------------
conf/metrics.properties.template | 8 +
core/pom.xml | 4 +
.../scala/org/apache/spark/SparkContext.scala | 25 ++
.../apache/spark/api/java/JavaDoubleRDD.scala | 40 +++
.../org/apache/spark/executor/TaskMetrics.scala | 23 +-
.../spark/metrics/sink/GraphiteSink.scala | 82 ++++++
.../apache/spark/rdd/DoubleRDDFunctions.scala | 126 +++++++++
.../org/apache/spark/rdd/MapPartitionsRDD.scala | 10 +-
.../spark/rdd/MapPartitionsWithContextRDD.scala | 41 ---
.../main/scala/org/apache/spark/rdd/RDD.scala | 60 ++--
.../apache/spark/rdd/ZippedPartitionsRDD.scala | 21 +-
.../apache/spark/scheduler/ShuffleMapTask.scala | 9 +-
.../spark/storage/BlockObjectWriter.scala | 2 +
.../org/apache/spark/util/AppendOnlyMap.scala | 93 ++++---
.../scala/org/apache/spark/util/Utils.scala | 24 ++
.../org/apache/spark/util/XORShiftRandom.scala | 94 +++++++
.../spark/util/collection/OpenHashSet.scala | 107 ++++----
.../org/apache/spark/CheckpointSuite.scala | 2 -
.../scala/org/apache/spark/JavaAPISuite.java | 14 +
.../org/apache/spark/rdd/DoubleRDDSuite.scala | 271 +++++++++++++++++++
.../apache/spark/util/XORShiftRandomSuite.scala | 76 ++++++
.../util/collection/OpenHashMapSuite.scala | 16 +-
.../util/collection/OpenHashSetSuite.scala | 20 +-
.../PrimitiveKeyOpenHashMapSuite.scala | 102 +++++++
.../PrimitiveKeyOpenHashSetSuite.scala | 90 ------
docs/monitoring.md | 1 +
docs/running-on-yarn.md | 27 +-
docs/tuning.md | 5 +-
.../apache/spark/mllib/clustering/KMeans.scala | 11 +-
pom.xml | 5 +
project/SparkBuild.scala | 1 +
.../org/apache/spark/deploy/yarn/Client.scala | 10 +-
.../spark/deploy/yarn/ClientArguments.scala | 40 +--
.../spark/deploy/yarn/WorkerLauncher.scala | 246 +++++++++++++++++
.../cluster/YarnClientClusterScheduler.scala | 47 ++++
.../cluster/YarnClientSchedulerBackend.scala | 109 ++++++++
36 files changed, 1555 insertions(+), 307 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/afe4fe7f/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/afe4fe7f/project/SparkBuild.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/afe4fe7f/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --cc yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index b3a6d2b,bb73f6d..79dd038
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@@ -55,12 -51,13 +55,14 @@@ class Client(conf: Configuration, args
private val SPARK_STAGING: String = ".sparkStaging"
private val distCacheMgr = new ClientDistributedCacheManager()
- // staging directory is private! -> rwx--------
+ // Staging directory is private! -> rwx--------
val STAGING_DIR_PERMISSION: FsPermission = FsPermission.createImmutable(0700:Short)
- // app files are world-wide readable and owner writable -> rw-r--r--
- val APP_FILE_PERMISSION: FsPermission = FsPermission.createImmutable(0644:Short)
++
+ // App files are world-wide readable and owner writable -> rw-r--r--
+ val APP_FILE_PERMISSION: FsPermission = FsPermission.createImmutable(0644:Short)
- def run() {
+ // for client user who want to monitor app status by itself.
+ def runApp() = {
validateArgs()
init(yarnConf)