You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/04/22 18:37:02 UTC
spark git commit: [SPARK-10001] Consolidate Signaling and
SignalLogger.
Repository: spark
Updated Branches:
refs/heads/master 056883e07 -> c089c6f4e
[SPARK-10001] Consolidate Signaling and SignalLogger.
## What changes were proposed in this pull request?
This is a follow-up to #12557, with the following changes:
1. Fixes some of the style issues.
2. Merges Signaling and SignalLogger into a new class called SignalUtils. It was pretty confusing to have Signaling and Signal in one file, and it was also confusing to have two classes named Signaling and one called the other.
3. Made logging registration idempotent.
## How was this patch tested?
N/A.
Author: Reynold Xin <rx...@databricks.com>
Closes #12605 from rxin/SPARK-10001.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c089c6f4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c089c6f4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c089c6f4
Branch: refs/heads/master
Commit: c089c6f4e83d85e622b8d13f466a656c2852702b
Parents: 056883e
Author: Reynold Xin <rx...@databricks.com>
Authored: Fri Apr 22 09:36:59 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Apr 22 09:36:59 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/util/SignalLogger.scala | 36 ------
.../org/apache/spark/util/SignalUtils.scala | 116 +++++++++++++++++++
.../scala/org/apache/spark/util/Signaling.scala | 99 ----------------
.../scala/org/apache/spark/util/Utils.scala | 2 +-
.../scala/org/apache/spark/repl/Signaling.scala | 4 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
6 files changed, 120 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/core/src/main/scala/org/apache/spark/util/SignalLogger.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/SignalLogger.scala b/core/src/main/scala/org/apache/spark/util/SignalLogger.scala
deleted file mode 100644
index a793c91..0000000
--- a/core/src/main/scala/org/apache/spark/util/SignalLogger.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-import org.slf4j.Logger
-
-/**
- * Used to log signals received. This can be very useful in debugging crashes or kills.
- */
-private[spark] object SignalLogger {
-
- private var registered = false
-
- /** Register a signal handler to log signals on UNIX-like systems. */
- def register(log: Logger): Unit = Seq("TERM", "HUP", "INT").foreach{ sig =>
- Signaling.register(sig) {
- log.error("RECEIVED SIGNAL " + sig)
- false
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/SignalUtils.scala b/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
new file mode 100644
index 0000000..9479d8f
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
+import org.apache.commons.lang3.SystemUtils
+import org.slf4j.Logger
+import sun.misc.{Signal, SignalHandler}
+
+import org.apache.spark.internal.Logging
+
+/**
+ * Contains utilities for working with posix signals.
+ */
+private[spark] object SignalUtils extends Logging {
+
+ /** A flag to make sure we only register the logger once. */
+ private var loggerRegistered = false
+
+ /** Register a signal handler to log signals on UNIX-like systems. */
+ def registerLogger(log: Logger): Unit = synchronized {
+ if (!loggerRegistered) {
+ Seq("TERM", "HUP", "INT").foreach { sig =>
+ SignalUtils.register(sig) {
+ log.error("RECEIVED SIGNAL " + sig)
+ false
+ }
+ }
+ loggerRegistered = true
+ }
+ }
+
+ /**
+ * Adds an action to be run when a given signal is received by this process.
+ *
+ * Note that signals are only supported on unix-like operating systems and work on a best-effort
+ * basis: if a signal is not available or cannot be intercepted, only a warning is emitted.
+ *
+ * All actions for a given signal are run in a separate thread.
+ */
+ def register(signal: String)(action: => Boolean): Unit = synchronized {
+ if (SystemUtils.IS_OS_UNIX) {
+ try {
+ val handler = handlers.getOrElseUpdate(signal, {
+ logInfo("Registered signal handler for " + signal)
+ new ActionHandler(new Signal(signal))
+ })
+ handler.register(action)
+ } catch {
+ case ex: Exception => logWarning(s"Failed to register signal handler for " + signal, ex)
+ }
+ }
+ }
+
+ /**
+ * A handler for the given signal that runs a collection of actions.
+ */
+ private class ActionHandler(signal: Signal) extends SignalHandler {
+
+ /**
+ * List of actions upon the signal; the callbacks should return true if the signal is "handled",
+ * i.e. should not escalate to the next callback.
+ */
+ private val actions = Collections.synchronizedList(new java.util.LinkedList[() => Boolean])
+
+ // original signal handler, before this handler was attached
+ private val prevHandler: SignalHandler = Signal.handle(signal, this)
+
+ /**
+ * Called when this handler's signal is received. Note that if the same signal is received
+ * before this method returns, it is escalated to the previous handler.
+ */
+ override def handle(sig: Signal): Unit = {
+ // register old handler, will receive incoming signals while this handler is running
+ Signal.handle(signal, prevHandler)
+
+ // run all actions, escalate to parent handler if no action catches the signal
+ // (i.e. all actions return false)
+ val escalate = actions.asScala.forall { action => !action() }
+ if (escalate) {
+ prevHandler.handle(sig)
+ }
+
+ // re-register this handler
+ Signal.handle(signal, this)
+ }
+
+ /**
+ * Adds an action to be run by this handler.
+ * @param action An action to be run when a signal is received. Return true if the signal
+ * should be stopped with this handler, false if it should be escalated.
+ */
+ def register(action: => Boolean): Unit = actions.add(() => action)
+ }
+
+ /** Mapping from signal to their respective handlers. */
+ private val handlers = new scala.collection.mutable.HashMap[String, ActionHandler]
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/core/src/main/scala/org/apache/spark/util/Signaling.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Signaling.scala b/core/src/main/scala/org/apache/spark/util/Signaling.scala
deleted file mode 100644
index 2075cc4..0000000
--- a/core/src/main/scala/org/apache/spark/util/Signaling.scala
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-import java.util.{Collections, LinkedList}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.HashMap
-
-import org.apache.commons.lang3.SystemUtils
-import sun.misc.{Signal, SignalHandler}
-
-import org.apache.spark.internal.Logging
-
-
-/**
- * Contains utilities for working with posix signals.
- */
-private[spark] object Signaling extends Logging {
-
- /**
- * A handler for the given signal that runs a collection of actions.
- */
- private class ActionHandler(signal: Signal) extends SignalHandler {
-
- private val actions = Collections.synchronizedList(new LinkedList[() => Boolean])
-
- // original signal handler, before this handler was attached
- private val prevHandler: SignalHandler = Signal.handle(signal, this)
-
- /**
- * Called when this handler's signal is received. Note that if the same signal is received
- * before this method returns, it is escalated to the previous handler.
- */
- override def handle(sig: Signal): Unit = {
- // register old handler, will receive incoming signals while this handler is running
- Signal.handle(signal, prevHandler)
-
- val escalate = actions.asScala forall { action =>
- !action()
- }
-
- if(escalate) {
- prevHandler.handle(sig)
- }
-
- // re-register this handler
- Signal.handle(signal, this)
- }
-
- /**
- * Add an action to be run by this handler.
- * @param action An action to be run when a signal is received. Return true if the signal
- * should be stopped with this handler, false if it should be escalated.
- */
- def register(action: => Boolean): Unit = actions.add(() => action)
-
- }
-
- // contains association of signals to their respective handlers
- private val handlers = new HashMap[String, ActionHandler]
-
- /**
- * Adds an action to be run when a given signal is received by this process.
- *
- * Note that signals are only supported on unix-like operating systems and work on a best-effort
- * basis: if a signal is not available or cannot be intercepted, only a warning is emitted.
- *
- * All actions for a given signal are run in a separate thread.
- */
- def register(signal: String)(action: => Boolean): Unit = synchronized {
- if (SystemUtils.IS_OS_UNIX) try {
- val handler = handlers.getOrElseUpdate(signal, {
- val h = new ActionHandler(new Signal(signal))
- logInfo("Registered signal handler for " + signal)
- h
- })
- handler.register(action)
- } catch {
- case ex: Exception => logWarning(s"Failed to register signal handler for " + signal, ex)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 848f7d7..ea49991 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2284,7 +2284,7 @@ private[spark] object Utils extends Logging {
*/
def initDaemon(log: Logger): Unit = {
log.info(s"Started daemon with process name: ${Utils.getProcessName()}")
- SignalLogger.register(log)
+ SignalUtils.registerLogger(log)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/repl/src/main/scala/org/apache/spark/repl/Signaling.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/spark/repl/Signaling.scala b/repl/src/main/scala/org/apache/spark/repl/Signaling.scala
index c305ed5..202febf 100644
--- a/repl/src/main/scala/org/apache/spark/repl/Signaling.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/Signaling.scala
@@ -19,7 +19,7 @@ package org.apache.spark.repl
import org.apache.spark.SparkContext
import org.apache.spark.internal.Logging
-import org.apache.spark.util.{Signaling => USignaling}
+import org.apache.spark.util.SignalUtils
private[repl] object Signaling extends Logging {
@@ -28,7 +28,7 @@ private[repl] object Signaling extends Logging {
* when no jobs are currently running.
* This makes it possible to interrupt a running shell job by pressing Ctrl+C.
*/
- def cancelOnInterrupt(ctx: SparkContext): Unit = USignaling.register("INT") {
+ def cancelOnInterrupt(ctx: SparkContext): Unit = SignalUtils.register("INT") {
if (!ctx.statusTracker.getActiveJobIds().isEmpty) {
logWarning("Cancelling all active jobs, this can take a while. " +
"Press Ctrl+C again to exit now.")
http://git-wip-us.apache.org/repos/asf/spark/blob/c089c6f4/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 5bb6350..4df90d7 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -716,7 +716,7 @@ object ApplicationMaster extends Logging {
private var master: ApplicationMaster = _
def main(args: Array[String]): Unit = {
- SignalLogger.register(log)
+ SignalUtils.registerLogger(log)
val amArgs = new ApplicationMasterArguments(args)
SparkHadoopUtil.get.runAsSparkUser { () =>
master = new ApplicationMaster(amArgs, new YarnRMClient)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org