You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/12/18 21:56:12 UTC
spark git commit: Revert "[SPARK-12365][CORE] Use ShutdownHookManager
where Runtime.getRuntime.addShutdownHook() is called"
Repository: spark
Updated Branches:
refs/heads/branch-1.6 1dc71ec77 -> 3b903e44b
Revert "[SPARK-12365][CORE] Use ShutdownHookManager where Runtime.getRuntime.addShutdownHook() is called"
This reverts commit 4af64385b085002d94c54d11bbd144f9f026bbd8.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3b903e44
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3b903e44
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3b903e44
Branch: refs/heads/branch-1.6
Commit: 3b903e44b912cd36ec26e9e95444656eee7b0c46
Parents: 1dc71ec
Author: Andrew Or <an...@databricks.com>
Authored: Fri Dec 18 12:56:03 2015 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Fri Dec 18 12:56:03 2015 -0800
----------------------------------------------------------------------
.../spark/deploy/ExternalShuffleService.scala | 18 ++++++++++++------
.../deploy/mesos/MesosClusterDispatcher.scala | 13 ++++++++-----
.../apache/spark/util/ShutdownHookManager.scala | 4 ----
scalastyle-config.xml | 12 ------------
4 files changed, 20 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
index 7fc96e4..e8a1e35 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -28,7 +28,7 @@ import org.apache.spark.network.sasl.SaslServerBootstrap
import org.apache.spark.network.server.{TransportServerBootstrap, TransportServer}
import org.apache.spark.network.shuffle.ExternalShuffleBlockHandler
import org.apache.spark.network.util.TransportConf
-import org.apache.spark.util.{ShutdownHookManager, Utils}
+import org.apache.spark.util.Utils
/**
* Provides a server from which Executors can read shuffle files (rather than reading directly from
@@ -118,13 +118,19 @@ object ExternalShuffleService extends Logging {
server = newShuffleService(sparkConf, securityManager)
server.start()
- ShutdownHookManager.addShutdownHook { () =>
- logInfo("Shutting down shuffle service.")
- server.stop()
- barrier.countDown()
- }
+ installShutdownHook()
// keep running until the process is terminated
barrier.await()
}
+
+ private def installShutdownHook(): Unit = {
+ Runtime.getRuntime.addShutdownHook(new Thread("External Shuffle Service shutdown thread") {
+ override def run() {
+ logInfo("Shutting down shuffle service.")
+ server.stop()
+ barrier.countDown()
+ }
+ })
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala
index 389eff5..5d4e5b8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.CountDownLatch
import org.apache.spark.deploy.mesos.ui.MesosClusterUI
import org.apache.spark.deploy.rest.mesos.MesosRestServer
import org.apache.spark.scheduler.cluster.mesos._
-import org.apache.spark.util.{ShutdownHookManager, SignalLogger}
+import org.apache.spark.util.SignalLogger
import org.apache.spark.{Logging, SecurityManager, SparkConf}
/*
@@ -103,11 +103,14 @@ private[mesos] object MesosClusterDispatcher extends Logging {
}
val dispatcher = new MesosClusterDispatcher(dispatcherArgs, conf)
dispatcher.start()
- ShutdownHookManager.addShutdownHook { () =>
- logInfo("Shutdown hook is shutting down dispatcher")
- dispatcher.stop()
- dispatcher.awaitShutdown()
+ val shutdownHook = new Thread() {
+ override def run() {
+ logInfo("Shutdown hook is shutting down dispatcher")
+ dispatcher.stop()
+ dispatcher.awaitShutdown()
+ }
}
+ Runtime.getRuntime.addShutdownHook(shutdownHook)
dispatcher.awaitShutdown()
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
index 1a0f3b4..620f226 100644
--- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
+++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
@@ -162,9 +162,7 @@ private[spark] object ShutdownHookManager extends Logging {
val hook = new Thread {
override def run() {}
}
- // scalastyle:off runtimeaddshutdownhook
Runtime.getRuntime.addShutdownHook(hook)
- // scalastyle:on runtimeaddshutdownhook
Runtime.getRuntime.removeShutdownHook(hook)
} catch {
case ise: IllegalStateException => return true
@@ -230,9 +228,7 @@ private [util] class SparkShutdownHookManager {
.invoke(shm, hookTask, Integer.valueOf(fsPriority + 30))
case Failure(_) =>
- // scalastyle:off runtimeaddshutdownhook
Runtime.getRuntime.addShutdownHook(new Thread(hookTask, "Spark Shutdown Hook"));
- // scalastyle:on runtimeaddshutdownhook
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/scalastyle-config.xml
----------------------------------------------------------------------
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 4af6dea..050c3f3 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -157,18 +157,6 @@ This file is divided into 3 sections:
]]></customMessage>
</check>
- <check customId="runtimeaddshutdownhook" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
- <parameters><parameter name="regex">Runtime\.getRuntime\.addShutdownHook</parameter></parameters>
- <customMessage><![CDATA[
- Are you sure that you want to use Runtime.getRuntime.addShutdownHook? In most cases, you should use
- ShutdownHookManager.addShutdownHook instead.
- If you must use Runtime.getRuntime.addShutdownHook, wrap the code block with
- // scalastyle:off runtimeaddshutdownhook
- Runtime.getRuntime.addShutdownHook(...)
- // scalastyle:on runtimeaddshutdownhook
- ]]></customMessage>
- </check>
-
<check customId="classforname" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters><parameter name="regex">Class\.forName</parameter></parameters>
<customMessage><![CDATA[
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org