You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/12/10 01:53:38 UTC
[3/3] hbase git commit: HBASE-12467 Master joins cluster but never
completes initialization
HBASE-12467 Master joins cluster but never completes initialization
Signed-off-by: Andrew Purtell <ap...@apache.org>
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b9e6595a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b9e6595a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b9e6595a
Branch: refs/heads/0.98
Commit: b9e6595ad4ce29e172c656d73e328ca9ad6019c8
Parents: cb88d78
Author: Nick Dimiduk <nd...@apache.org>
Authored: Mon Nov 17 09:23:09 2014 +0100
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Dec 9 16:33:40 2014 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 59 ++++++++++++++++++++
1 file changed, 59 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b9e6595a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index eb220ea..8cdb144 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
@@ -286,6 +288,59 @@ RegionServerStatusProtos.RegionServerStatusService.BlockingInterface,
MasterServices, Server {
private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
+ /**
+ * Protection against zombie master. Started once Master accepts active responsibility and
+ * starts taking over responsibilities. Allows a finite time window before giving up ownership.
+ */
+ private static class InitializationMonitor extends HasThread {
+ /** The amount of time in milliseconds to sleep before checking initialization status. */
+ public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
+ public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
+
+ /**
+ * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
+ * true, do nothing otherwise.
+ */
+ public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
+ public static final boolean HALT_DEFAULT = false;
+
+ private final HMaster master;
+ private final long timeout;
+ private final boolean haltOnTimeout;
+
+ /** Creates a Thread that monitors the {@link #isInitialized()} state. */
+ InitializationMonitor(HMaster master) {
+ super("MasterInitializationMonitor");
+ this.master = master;
+ this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
+ this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
+ this.setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (!master.isStopped() && master.isActiveMaster()) {
+ Thread.sleep(timeout);
+ if (master.isInitialized()) {
+ LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
+ } else {
+ LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
+ + " consider submitting a bug report including a thread dump of this process.");
+ if (haltOnTimeout) {
+ LOG.error("Zombie Master exiting. Thread dump to stdout");
+ org.apache.hadoop.util.ReflectionUtils.printThreadInfo(
+ new PrintWriter(System.out), "Zombie HMaster");
+ System.exit(-1);
+ }
+ }
+ }
+ } catch (InterruptedException ie) {
+ LOG.trace("InitMonitor thread interrupted. Existing.");
+ }
+ }
+ }
+
// MASTER is name of the webapp and the attribute name used stuffing this
//instance into web context.
public static final String MASTER = "master";
@@ -811,6 +866,8 @@ MasterServices, Server {
throws IOException, InterruptedException, KeeperException {
isActiveMaster = true;
+ Thread zombieDetector = new Thread(new InitializationMonitor(this));
+ zombieDetector.start();
/*
* We are active master now... go initialize components we need to run.
@@ -984,6 +1041,8 @@ MasterServices, Server {
}
}
}
+
+ zombieDetector.interrupt();
}
/**