You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/03/12 10:40:16 UTC
asterixdb git commit: [NO ISSUE] Refactor shutdown watchdog to
ExitUtil
Repository: asterixdb
Updated Branches:
refs/heads/master bb49e5e39 -> 12009bd40
[NO ISSUE] Refactor shutdown watchdog to ExitUtil
Change-Id: Id27635f90587c64ee29ab1a4ac2b1a6182042900
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2470
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/12009bd4
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/12009bd4
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/12009bd4
Branch: refs/heads/master
Commit: 12009bd40701d44b206e4c65fe0561279f554235
Parents: bb49e5e
Author: Michael Blow <mb...@apache.org>
Authored: Sat Mar 10 14:09:44 2018 -0800
Committer: Michael Blow <mb...@apache.org>
Committed: Sat Mar 10 23:26:20 2018 -0800
----------------------------------------------------------------------
.../RegistrationTasksResponseMessage.java | 3 +-
.../bootstrap/GlobalRecoveryManager.java | 3 +-
.../management/service/logging/LogManager.java | 4 +-
.../lifecycle/LifeCycleComponentManager.java | 4 +-
.../control/cc/work/ClusterShutdownWork.java | 5 +-
.../hyracks/control/nc/NCShutdownHook.java | 28 +--------
.../hyracks/control/nc/task/ShutdownTask.java | 5 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 63 ++++++++++++++++++--
8 files changed, 76 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
index a6f10ca..868c2ad 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
@@ -28,7 +28,6 @@ import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.replication.INCLifecycleMessage;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.service.IControllerService;
-import org.apache.hyracks.control.nc.NCShutdownHook;
import org.apache.hyracks.util.ExitUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -80,7 +79,7 @@ public class RegistrationTasksResponseMessage extends CcIdentifiedMessage
} finally {
if (!success) {
// stop NC so that it can be started again
- ExitUtil.exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE);
+ ExitUtil.exit(ExitUtil.EC_FAILED_TO_STARTUP);
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index e2f1eaf..3d9b822 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -46,7 +46,6 @@ import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.control.nc.NCShutdownHook;
import org.apache.hyracks.util.ExitUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -99,7 +98,7 @@ public class GlobalRecoveryManager implements IGlobalRecoveryManager {
recover(appCtx);
} catch (HyracksDataException e) {
LOGGER.log(Level.ERROR, "Global recovery failed. Shutting down...", e);
- ExitUtil.exit(NCShutdownHook.FAILED_TO_RECOVER_EXIT_CODE);
+ ExitUtil.exit(ExitUtil.EC_FAILED_TO_RECOVER);
}
});
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
index ce65de0..6208cef 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.transaction.management.service.logging;
+import static org.apache.hyracks.util.ExitUtil.EC_IMMEDIATE_HALT;
+
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -716,7 +718,7 @@ class LogFlusher implements Callable<Boolean> {
}
} catch (Exception e) {
LOGGER.log(Level.ERROR, "LogFlusher is terminating abnormally. System is in unusable state; halting", e);
- ExitUtil.halt(44);
+ ExitUtil.halt(EC_IMMEDIATE_HALT);
throw new AssertionError("not reachable");
} finally {
if (interrupted) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
index 6d5d246..a9b7a97 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.api.lifecycle;
+import static org.apache.hyracks.util.ExitUtil.EC_UNHANDLED_EXCEPTION;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -57,7 +59,7 @@ public class LifeCycleComponentManager implements ILifeCycleComponentManager {
try {
LOGGER.log(Level.ERROR, "Uncaught Exception from thread " + t.getName() + ". Calling shutdown hook", e);
} finally {
- ExitUtil.exit(99);
+ ExitUtil.exit(EC_UNHANDLED_EXCEPTION);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
index a7c3c2f..194d27f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
@@ -19,6 +19,9 @@
package org.apache.hyracks.control.cc.work;
+import static org.apache.hyracks.util.ExitUtil.EC_ABNORMAL_TERMINATION;
+import static org.apache.hyracks.util.ExitUtil.EC_NORMAL_TERMINATION;
+
import java.util.Collection;
import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -82,7 +85,7 @@ public class ClusterShutdownWork extends SynchronizableWork {
}
callback.setValue(cleanShutdown);
ccs.stop(terminateNCService);
- ExitUtil.exit(cleanShutdown ? 0 : 1);
+ ExitUtil.exit(cleanShutdown ? EC_NORMAL_TERMINATION : EC_ABNORMAL_TERMINATION);
} catch (Exception e) {
callback.setException(e);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
index ccac00e..9a19f8e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.control.nc;
-import org.apache.hyracks.util.ExitUtil;
import org.apache.hyracks.util.ThreadDumpUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -31,35 +30,12 @@ import org.apache.logging.log4j.Logger;
*/
public class NCShutdownHook extends Thread {
- public static final int FAILED_TO_STARTUP_EXIT_CODE = 2;
- public static final int FAILED_TO_RECOVER_EXIT_CODE = 3;
private static final Logger LOGGER = LogManager.getLogger();
- private static final long SHUTDOWN_WAIT_TIME = 10 * 60 * 1000L;
- private final Thread watchDog;
private final NodeControllerService nodeControllerService;
- private volatile Thread shutdownHookThread;
- public NCShutdownHook(NodeControllerService nodeControllerService) {
+ NCShutdownHook(NodeControllerService nodeControllerService) {
super("ShutdownHook-" + nodeControllerService.getId());
this.nodeControllerService = nodeControllerService;
- watchDog = new Thread(watch(), "ShutdownHookWatchDog-" + nodeControllerService.getId());
- }
-
- private Runnable watch() {
- return () -> {
- try {
- shutdownHookThread.join(SHUTDOWN_WAIT_TIME); // 10 min
- if (shutdownHookThread.isAlive()) {
- try {
- LOGGER.info("Watchdog is angry. Killing shutdown hook");
- } finally {
- ExitUtil.halt(ExitUtil.EXIT_CODE_SHUTDOWN_TIMED_OUT);
- }
- }
- } catch (Throwable th) { // NOSONAR must catch them all
- ExitUtil.halt(ExitUtil.EXIT_CODE_WATCHDOG_FAILED);
- }
- };
}
@Override
@@ -69,8 +45,6 @@ public class NCShutdownHook extends Thread {
LOGGER.info("Shutdown hook called");
} catch (Throwable th) {//NOSONAR
}
- shutdownHookThread = Thread.currentThread();
- watchDog.start();
LOGGER.log(Level.INFO, () -> "Thread dump at shutdown: " + ThreadDumpUtil.takeDumpString());
nodeControllerService.stop();
} catch (Throwable th) { // NOSONAR... This is fine since this is shutdown hook
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
index 4dd57f2..ba8c3da 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
@@ -19,6 +19,9 @@
package org.apache.hyracks.control.nc.task;
+import static org.apache.hyracks.util.ExitUtil.EC_NORMAL_TERMINATION;
+import static org.apache.hyracks.util.ExitUtil.EC_TERMINATE_NC_SERVICE_DIRECTIVE;
+
import org.apache.hyracks.util.ExitUtil;
public class ShutdownTask implements Runnable {
@@ -30,7 +33,7 @@ public class ShutdownTask implements Runnable {
@Override
public void run() {
- ExitUtil.exit(terminateNCService ? 99 : 0);
+ ExitUtil.exit(terminateNCService ? EC_TERMINATE_NC_SERVICE_DIRECTIVE : EC_NORMAL_TERMINATION);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/12009bd4/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index 523480a..ec6587e 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.util;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -26,10 +29,26 @@ public class ExitUtil {
private static final Logger LOGGER = LogManager.getLogger();
+ public static final int EC_NORMAL_TERMINATION = 0;
+ public static final int EC_ABNORMAL_TERMINATION = 1;
+ public static final int EC_FAILED_TO_STARTUP = 2;
+ public static final int EC_FAILED_TO_RECOVER = 3;
+ public static final int EC_UNHANDLED_EXCEPTION = 11;
+ public static final int EC_IMMEDIATE_HALT = 33;
+ public static final int EC_HALT_ABNORMAL_RESERVED_44 = 44;
+ public static final int EC_HALT_ABNORMAL_RESERVED_55 = 55;
+ public static final int EC_HALT_SHUTDOWN_TIMED_OUT = 66;
+ public static final int EC_HALT_WATCHDOG_FAILED = 77;
+ public static final int EC_HALT_ABNORMAL_RESERVED_88 = 88;
+ public static final int EC_TERMINATE_NC_SERVICE_DIRECTIVE = 99;
+
private static final ExitThread exitThread = new ExitThread();
+ private static final ShutdownWatchdog watchdogThread = new ShutdownWatchdog();
+ private static final MutableLong shutdownHaltDelay = new MutableLong(10 * 60 * 1000L); // 10 minutes default
- public static final int EXIT_CODE_SHUTDOWN_TIMED_OUT = 66;
- public static final int EXIT_CODE_WATCHDOG_FAILED = 77;
+ static {
+ Runtime.getRuntime().addShutdownHook(new Thread(watchdogThread::start));
+ }
private ExitUtil() {
}
@@ -39,8 +58,20 @@ public class ExitUtil {
}
public static void exit(int status) {
- exitThread.setStatus(status);
- exitThread.start();
+ synchronized (exitThread) {
+ if (exitThread.isAlive()) {
+ LOGGER.warn("ignoring duplicate request to exit with status " + status
+ + "; already exiting with status " + exitThread.status + "...");
+ } else {
+ exitThread.setStatus(status);
+ exitThread.start();
+ }
+ }
+ }
+
+ public static void exit(int status, long timeBeforeHalt, TimeUnit timeBeforeHaltUnit) {
+ shutdownHaltDelay.setValue(timeBeforeHaltUnit.toMillis(timeBeforeHalt));
+ exit(status);
}
@SuppressWarnings("squid:S2142") // catch interrupted
@@ -55,6 +86,30 @@ public class ExitUtil {
Runtime.getRuntime().halt(status);
}
+ private static class ShutdownWatchdog extends Thread {
+
+ private ShutdownWatchdog() {
+ super("ShutdownWatchdog");
+ setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ exitThread.join(shutdownHaltDelay.getValue()); // 10 min
+ if (exitThread.isAlive()) {
+ try {
+ LOGGER.info("Watchdog is angry. Killing shutdown hook");
+ } finally {
+ ExitUtil.halt(EC_HALT_SHUTDOWN_TIMED_OUT);
+ }
+ }
+ } catch (Throwable th) { // NOSONAR must catch them all
+ ExitUtil.halt(EC_HALT_WATCHDOG_FAILED);
+ }
+ }
+ }
+
private static class ExitThread extends Thread {
private int status;