You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2014/09/15 15:54:04 UTC
git commit: [SPARK-3410] The priority of shutdownhook for
ApplicationMaster should not be integer literal
Repository: spark
Updated Branches:
refs/heads/master f493f7982 -> cc1464446
[SPARK-3410] The priority of shutdownhook for ApplicationMaster should not be integer literal
I think, it need to keep the priority of shutdown hook for ApplicationMaster than the priority of shutdown hook for o.a.h.FileSystem depending on changing the priority for FileSystem.
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Closes #2283 from sarutak/SPARK-3410 and squashes the following commits:
1d44fef [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3410
bd6cc53 [Kousuke Saruta] Modified style
ee6f1aa [Kousuke Saruta] Added constant "SHUTDOWN_HOOK_PRIORITY" to ApplicationMaster
54eb68f [Kousuke Saruta] Changed Shutdown hook priority to 20
2f0aee3 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3410
4c5cb93 [Kousuke Saruta] Modified the priority for AM's shutdown hook
217d1a4 [Kousuke Saruta] Removed unused import statements
717aba2 [Kousuke Saruta] Modified ApplicationMaster to make to keep the priority of shutdown hook for ApplicationMaster higher than the priority of shutdown hook for HDFS
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc146444
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc146444
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc146444
Branch: refs/heads/master
Commit: cc14644460872efb344e8d895859d70213a40840
Parents: f493f79
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Authored: Mon Sep 15 08:53:58 2014 -0500
Committer: Thomas Graves <tg...@apache.org>
Committed: Mon Sep 15 08:53:58 2014 -0500
----------------------------------------------------------------------
.../apache/spark/deploy/yarn/ApplicationMaster.scala | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cc146444/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 735d772..cde5fff 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -21,12 +21,8 @@ import java.io.IOException
import java.net.Socket
import java.util.concurrent.atomic.AtomicReference
-import scala.collection.JavaConversions._
-import scala.util.Try
-
import akka.actor._
import akka.remote._
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.util.ShutdownHookManager
import org.apache.hadoop.yarn.api._
@@ -107,8 +103,11 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments,
}
}
}
- // Use priority 30 as it's higher than HDFS. It's the same priority MapReduce is using.
- ShutdownHookManager.get().addShutdownHook(cleanupHook, 30)
+
+ // Use higher priority than FileSystem.
+ assert(ApplicationMaster.SHUTDOWN_HOOK_PRIORITY > FileSystem.SHUTDOWN_HOOK_PRIORITY)
+ ShutdownHookManager
+ .get().addShutdownHook(cleanupHook, ApplicationMaster.SHUTDOWN_HOOK_PRIORITY)
// Call this to force generation of secret so it gets populated into the
// Hadoop UGI. This has to happen before the startUserClass which does a
@@ -407,6 +406,8 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments,
object ApplicationMaster extends Logging {
+ val SHUTDOWN_HOOK_PRIORITY: Int = 30
+
private var master: ApplicationMaster = _
def main(args: Array[String]) = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org