You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2012/04/01 05:33:40 UTC
svn commit: r1308013 - in /hbase/branches/0.90: ./
src/main/java/org/apache/hadoop/hbase/master/
src/test/java/org/apache/hadoop/hbase/master/
Author: jmhsieh
Date: Sun Apr 1 03:33:40 2012
New Revision: 1308013
URL: http://svn.apache.org/viewvc?rev=1308013&view=rev
Log:
HBASE-5213 "hbase master stop" does not bring down backup masters (Gregory)
Added:
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1308013&r1=1308012&r2=1308013&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Sun Apr 1 03:33:40 2012
@@ -1,4 +1,4 @@
-HBase Change Log
+SHBase Change Log
Release 0.90.7 - Unreleased
BUG FIXES
@@ -12,6 +12,7 @@ Release 0.90.7 - Unreleased
HBASE-5615 The master never does balance because of balancing the parent region (Xufeng)
HBASE-5637 Fix failing 0.90 TestHMsg testcase introduced by HBASE-5563
HBASE-5638 Backport HBASE-5633: NPE reading ZK config in HBase (Matteo Bertozzi)
+ HBASE-5213 "hbase master stop" does not bring down backup masters (Gregory)
IMPROVEMENT
HBASE-5588 Deprecate/remove AssignmentManager#clearRegionFromTransition
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java?rev=1308013&r1=1308012&r2=1308013&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java Sun Apr 1 03:33:40 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
/**
* Handles everything on master-side related to master election.
@@ -118,7 +119,8 @@ class ActiveMasterManager extends ZooKee
* master was running or if some other problem (zookeeper, stop flag has been
* set on this Master)
*/
- boolean blockUntilBecomingActiveMaster() {
+ boolean blockUntilBecomingActiveMaster(
+ ClusterStatusTracker clusterStatusTracker) {
boolean cleanSetOfActiveMaster = true;
// Try to become the active master, watch if there is another master
try {
@@ -158,11 +160,14 @@ class ActiveMasterManager extends ZooKee
LOG.debug("Interrupted waiting for master to die", e);
}
}
+ if (!clusterStatusTracker.isClusterUp()) {
+ this.master.stop("Cluster went down before this master became active");
+ }
if (this.master.isStopped()) {
return cleanSetOfActiveMaster;
}
// Try to become active master again now that there is no active master
- blockUntilBecomingActiveMaster();
+ blockUntilBecomingActiveMaster(clusterStatusTracker);
}
return cleanSetOfActiveMaster;
}
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1308013&r1=1308012&r2=1308013&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Sun Apr 1 03:33:40 2012
@@ -275,7 +275,15 @@ implements HMasterInterface, HMasterRegi
this.activeMasterManager = new ActiveMasterManager(zooKeeper, address, this);
this.zooKeeper.registerListener(activeMasterManager);
stallIfBackupMaster(this.conf, this.activeMasterManager);
- this.activeMasterManager.blockUntilBecomingActiveMaster();
+
+ // The ClusterStatusTracker is setup before the other
+ // ZKBasedSystemTrackers because it's needed by the activeMasterManager
+ // to check if the cluster should be shutdown.
+ this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
+ this.clusterStatusTracker.start();
+ this.activeMasterManager.blockUntilBecomingActiveMaster(
+ this.clusterStatusTracker);
+
// We are either the active master or we were asked to shutdown
if (!this.stopped) {
finishInitialization();
@@ -361,8 +369,6 @@ implements HMasterInterface, HMasterRegi
// Set the cluster as up. If new RSs, they'll be waiting on this before
// going ahead with their startup.
- this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
- this.clusterStatusTracker.start();
boolean wasUp = this.clusterStatusTracker.isClusterUp();
if (!wasUp) this.clusterStatusTracker.setClusterUp();
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java?rev=1308013&r1=1308012&r2=1308013&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java Sun Apr 1 03:33:40 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.catalog.C
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -64,29 +65,32 @@ public class TestActiveMasterManager {
ZKUtil.createAndFailSilent(zk, zk.baseZNode);
try {
ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+ ZKUtil.deleteNode(zk, zk.clusterStateZNode);
} catch(KeeperException.NoNodeException nne) {}
// Create the master node with a dummy address
HServerAddress master = new HServerAddress("localhost", 1);
// Should not have a master yet
- DummyMaster dummyMaster = new DummyMaster();
- ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
- master, dummyMaster);
- zk.registerListener(activeMasterManager);
+ DummyMaster dummyMaster = new DummyMaster(zk,master);
+ ClusterStatusTracker clusterStatusTracker =
+ dummyMaster.getClusterStatusTracker();
+ ActiveMasterManager activeMasterManager =
+ dummyMaster.getActiveMasterManager();
assertFalse(activeMasterManager.clusterHasActiveMaster.get());
// First test becoming the active master uninterrupted
- activeMasterManager.blockUntilBecomingActiveMaster();
+ clusterStatusTracker.setClusterUp();
+
+ activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker);
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, master);
// Now pretend master restart
- DummyMaster secondDummyMaster = new DummyMaster();
- ActiveMasterManager secondActiveMasterManager = new ActiveMasterManager(zk,
- master, secondDummyMaster);
- zk.registerListener(secondActiveMasterManager);
+ DummyMaster secondDummyMaster = new DummyMaster(zk,master);
+ ActiveMasterManager secondActiveMasterManager =
+ secondDummyMaster.getActiveMasterManager();
assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
- activeMasterManager.blockUntilBecomingActiveMaster();
+ activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker);
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, master);
}
@@ -103,6 +107,7 @@ public class TestActiveMasterManager {
ZKUtil.createAndFailSilent(zk, zk.baseZNode);
try {
ZKUtil.deleteNode(zk, zk.masterAddressZNode);
+ ZKUtil.deleteNode(zk, zk.clusterStateZNode);
} catch(KeeperException.NoNodeException nne) {}
// Create the master node with a dummy address
@@ -110,20 +115,21 @@ public class TestActiveMasterManager {
HServerAddress secondMasterAddress = new HServerAddress("localhost", 2);
// Should not have a master yet
- DummyMaster ms1 = new DummyMaster();
- ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
- firstMasterAddress, ms1);
- zk.registerListener(activeMasterManager);
+ DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
+ ActiveMasterManager activeMasterManager =
+ ms1.getActiveMasterManager();
assertFalse(activeMasterManager.clusterHasActiveMaster.get());
// First test becoming the active master uninterrupted
- activeMasterManager.blockUntilBecomingActiveMaster();
+ ClusterStatusTracker clusterStatusTracker =
+ ms1.getClusterStatusTracker();
+ clusterStatusTracker.setClusterUp();
+ activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker);
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, firstMasterAddress);
// New manager will now try to become the active master in another thread
WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
- zk.registerListener(t.manager);
t.start();
// Wait for this guy to figure out there is another active master
// Wait for 1 second at most
@@ -186,18 +192,20 @@ public class TestActiveMasterManager {
public static class WaitToBeMasterThread extends Thread {
ActiveMasterManager manager;
+ DummyMaster dummyMaster;
boolean isActiveMaster;
public WaitToBeMasterThread(ZooKeeperWatcher zk,
HServerAddress address) {
- this.manager = new ActiveMasterManager(zk, address,
- new DummyMaster());
+ this.dummyMaster = new DummyMaster(zk,address);
+ this.manager = this.dummyMaster.getActiveMasterManager();
isActiveMaster = false;
}
@Override
public void run() {
- manager.blockUntilBecomingActiveMaster();
+ manager.blockUntilBecomingActiveMaster(
+ this.dummyMaster.getClusterStatusTracker());
LOG.info("Second master has become the active master!");
isActiveMaster = true;
}
@@ -233,6 +241,18 @@ public class TestActiveMasterManager {
*/
public static class DummyMaster implements Server {
private volatile boolean stopped;
+ private ClusterStatusTracker clusterStatusTracker;
+ private ActiveMasterManager activeMasterManager;
+
+ public DummyMaster(ZooKeeperWatcher zk, HServerAddress master) {
+ this.clusterStatusTracker =
+ new ClusterStatusTracker(zk, this);
+ clusterStatusTracker.start();
+
+ this.activeMasterManager =
+ new ActiveMasterManager(zk, master, this);
+ zk.registerListener(activeMasterManager);
+ }
@Override
public void abort(final String msg, final Throwable t) {}
@@ -266,5 +286,13 @@ public class TestActiveMasterManager {
public CatalogTracker getCatalogTracker() {
return null;
}
+
+ public ClusterStatusTracker getClusterStatusTracker() {
+ return clusterStatusTracker;
+ }
+
+ public ActiveMasterManager getActiveMasterManager() {
+ return activeMasterManager;
+ }
}
}
Added: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java?rev=1308013&view=auto
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java (added)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java Sun Apr 1 03:33:40 2012
@@ -0,0 +1,90 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+public class TestMasterShutdown {
+ private static final Log LOG = LogFactory.getLog(TestMasterShutdown.class);
+
+ /**
+ * Simple test of shutdown.
+ * <p>
+ * Starts with three masters. Tells the active master to shutdown the cluster.
+ * Verifies that all masters are properly shutdown.
+ * @throws Exception
+ */
+ @Test (timeout=240000)
+ public void testMasterShutdown() throws Exception {
+
+ final int NUM_MASTERS = 3;
+ final int NUM_RS = 3;
+
+ // Create config to use for this cluster
+ Configuration conf = HBaseConfiguration.create();
+
+ // Start the cluster
+ HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+ TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+
+ // get all the master threads
+ List<MasterThread> masterThreads = cluster.getMasterThreads();
+
+ // wait for each to come online
+ for (MasterThread mt : masterThreads) {
+ assertTrue(mt.isAlive());
+ }
+
+ // find the active master
+ HMaster active = null;
+ for (int i = 0; i < masterThreads.size(); i++) {
+ if (masterThreads.get(i).getMaster().isActiveMaster()) {
+ active = masterThreads.get(i).getMaster();
+ break;
+ }
+ }
+ assertNotNull(active);
+
+ // tell the active master to shutdown the cluster
+ active.shutdown();
+
+ for (int i = NUM_MASTERS - 1; i >= 0 ;--i) {
+ cluster.waitOnMaster(i);
+ }
+ // make sure all the masters properly shutdown
+ assertEquals(0,masterThreads.size());
+
+ TEST_UTIL.shutdownMiniCluster();
+ }
+}
+