You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2012/06/05 18:39:04 UTC
svn commit: r1346460 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/master/ test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/master/
Author: ramkrishna
Date: Tue Jun 5 16:39:04 2012
New Revision: 1346460
URL: http://svn.apache.org/viewvc?rev=1346460&view=rev
Log:
HBASE-6046 Master retry on ZK session expiry causes inconsistent region assignments. (Asutosh)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1346460&r1=1346459&r2=1346460&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Jun 5 16:39:04 2012
@@ -406,7 +406,7 @@ Server {
// We are either the active master or we were asked to shutdown
if (!this.stopped) {
- finishInitialization(startupStatus);
+ finishInitialization(startupStatus, false);
loop();
}
} catch (Throwable t) {
@@ -549,9 +549,9 @@ Server {
}
}
-/**
+ /**
* Finish initialization of HMaster after becoming the primary master.
- *
+ *
* <ol>
* <li>Initialize master components - file system manager, server manager,
* assignment manager, region server tracker, catalog tracker, etc</li>
@@ -563,12 +563,14 @@ Server {
* <li>Ensure assignment of root and meta regions<li>
* <li>Handle either fresh cluster start or master failover</li>
* </ol>
- *
+ *
+ * @param masterRecovery
+ *
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
- private void finishInitialization(MonitoredTask status)
+ private void finishInitialization(MonitoredTask status, boolean masterRecovery)
throws IOException, InterruptedException, KeeperException {
isActiveMaster = true;
@@ -582,7 +584,7 @@ Server {
status.setStatus("Initializing Master file system");
this.masterActiveTime = System.currentTimeMillis();
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
- this.fileSystemManager = new MasterFileSystem(this, this, metrics);
+ this.fileSystemManager = new MasterFileSystem(this, this, metrics, masterRecovery);
this.tableDescriptors =
new FSTableDescriptors(this.fileSystemManager.getFileSystem(),
@@ -592,20 +594,23 @@ Server {
status.setStatus("Publishing Cluster ID in ZooKeeper");
ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
- this.executorService = new ExecutorService(getServerName().toString());
-
- this.serverManager = createServerManager(this, this);
+ if (!masterRecovery) {
+ this.executorService = new ExecutorService(getServerName().toString());
+ this.serverManager = createServerManager(this, this);
+ }
status.setStatus("Initializing ZK system trackers");
initializeZKBasedSystemTrackers();
- // initialize master side coprocessors before we start handling requests
- status.setStatus("Initializing master coprocessors");
- this.cpHost = new MasterCoprocessorHost(this, this.conf);
-
- // start up all service threads.
- status.setStatus("Initializing master service threads");
- startServiceThreads();
+ if (!masterRecovery) {
+ // initialize master side coprocessors before we start handling requests
+ status.setStatus("Initializing master coprocessors");
+ this.cpHost = new MasterCoprocessorHost(this, this.conf);
+
+ // start up all service threads.
+ status.setStatus("Initializing master service threads");
+ startServiceThreads();
+ }
// Wait for region servers to report in.
this.serverManager.waitForRegionServers(status);
@@ -619,7 +624,9 @@ Server {
}
}
- this.assignmentManager.startTimeOutMonitor();
+ if (!masterRecovery) {
+ this.assignmentManager.startTimeOutMonitor();
+ }
// TODO: Should do this in background rather than block master startup
status.setStatus("Splitting logs after master startup");
splitLogAfterStartup(this.fileSystemManager);
@@ -647,14 +654,16 @@ Server {
status.setStatus("Fixing up missing daughters");
fixupDaughters(status);
- // Start balancer and meta catalog janitor after meta and regions have
- // been assigned.
- status.setStatus("Starting balancer and catalog janitor");
- this.balancerChore = getAndStartBalancerChore(this);
- this.catalogJanitorChore = new CatalogJanitor(this, this);
- startCatalogJanitorChore();
-
- registerMBean();
+ if (!masterRecovery) {
+ // Start balancer and meta catalog janitor after meta and regions have
+ // been assigned.
+ status.setStatus("Starting balancer and catalog janitor");
+ this.balancerChore = getAndStartBalancerChore(this);
+ this.catalogJanitorChore = new CatalogJanitor(this, this);
+ startCatalogJanitorChore();
+
+ registerMBean();
+ }
status.markComplete("Initialization successful");
LOG.info("Master has completed initialization");
@@ -664,12 +673,14 @@ Server {
// master initialization. See HBASE-5916.
this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
- if (this.cpHost != null) {
- // don't let cp initialization errors kill the master
- try {
- this.cpHost.postStartMaster();
- } catch (IOException ioe) {
- LOG.error("Coprocessor postStartMaster() hook failed", ioe);
+ if (!masterRecovery) {
+ if (this.cpHost != null) {
+ // don't let cp initialization errors kill the master
+ try {
+ this.cpHost.postStartMaster();
+ } catch (IOException ioe) {
+ LOG.error("Coprocessor postStartMaster() hook failed", ioe);
+ }
}
}
}
@@ -1750,13 +1761,9 @@ Server {
if (!becomeActiveMaster(status)) {
return Boolean.FALSE;
}
- initializeZKBasedSystemTrackers();
- // Update in-memory structures to reflect our earlier Root/Meta assignment.
- assignRootAndMeta(status);
- // process RIT if any
- // TODO: Why does this not call AssignmentManager.joinCluster? Otherwise
- // we are not processing dead servers if any.
- assignmentManager.processDeadServersAndRegionsInTransition();
+ serverShutdownHandlerEnabled = false;
+ initialized = false;
+ finishInitialization(status, true);
return Boolean.TRUE;
} finally {
status.cleanup();
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1346460&r1=1346459&r2=1346460&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Tue Jun 5 16:39:04 2012
@@ -85,7 +85,7 @@ public class MasterFileSystem {
private final MasterServices services;
public MasterFileSystem(Server master, MasterServices services,
- MasterMetrics metrics)
+ MasterMetrics metrics, boolean masterRecovery)
throws IOException {
this.conf = master.getConfiguration();
this.master = master;
@@ -107,7 +107,7 @@ public class MasterFileSystem {
if (this.distributedLogSplitting) {
this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
master.getConfiguration(), master, master.getServerName());
- this.splitLogManager.finishInitialization();
+ this.splitLogManager.finishInitialization(masterRecovery);
} else {
this.splitLogManager = null;
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1346460&r1=1346459&r2=1346460&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Tue Jun 5 16:39:04 2012
@@ -182,9 +182,11 @@ public class SplitLogManager extends Zoo
new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper);
}
- public void finishInitialization() {
- Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName +
- ".splitLogManagerTimeoutMonitor");
+ public void finishInitialization(boolean masterRecovery) {
+ if (!masterRecovery) {
+ Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName
+ + ".splitLogManagerTimeoutMonitor");
+ }
// Watcher can be null during tests with Mock'd servers.
if (this.watcher != null) {
this.watcher.registerListener(this);
@@ -1207,4 +1209,12 @@ public class SplitLogManager extends Zoo
return statusMsg;
}
}
+
+ /**
+ * Completes the initialization
+ */
+ public void finishInitialization() {
+ finishInitialization(false);
+ }
+
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java?rev=1346460&r1=1346459&r2=1346460&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java Tue Jun 5 16:39:04 2012
@@ -28,6 +28,9 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Map;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,9 +41,15 @@ import org.apache.hadoop.hbase.client.HC
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.master.balancer.DefaultLoadBalancer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -56,6 +65,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+
+
@Category(LargeTests.class)
public class TestZooKeeper {
private final Log LOG = LogFactory.getLog(this.getClass());
@@ -73,6 +84,8 @@ public class TestZooKeeper {
TEST_UTIL.startMiniZKCluster();
conf.setBoolean("dfs.support.append", true);
conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
+ conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
+ LoadBalancer.class);
TEST_UTIL.startMiniCluster(2);
}
@@ -362,6 +375,94 @@ public class TestZooKeeper {
ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
}
+ /**
+ * Tests that the master does not call retainAssignment after recovery from expired zookeeper
+ * session. Without the HBASE-6046 fix master always tries to assign all the user regions by
+ * calling retainAssignment.
+ */
+ @Test
+ public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ cluster.startRegionServer();
+ HMaster m = cluster.getMaster();
+ // now the cluster is up. So assign some regions.
+ HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
+ Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
+ Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
+
+ String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
+ admin.createTable(new HTableDescriptor(tableName), SPLIT_KEYS);
+ ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
+ ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
+ m.getZooKeeperWatcher().close();
+ MockLoadBalancer.retainAssignCalled = false;
+ m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException());
+ assertFalse(m.isStopped());
+ // The recovered master should not call retainAssignment, as it is not a
+ // clean startup.
+ assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
+ }
+
+ /**
+ * Tests whether the logs are split when master recovers from a expired zookeeper session and an
+ * RS goes down.
+ */
+ @Test(timeout = 60000)
+ public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
+ KeeperException, InterruptedException {
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ cluster.startRegionServer();
+ HMaster m = cluster.getMaster();
+ // now the cluster is up. So assign some regions.
+ HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
+ Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
+
+ String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ HColumnDescriptor hcd = new HColumnDescriptor("col");
+ htd.addFamily(hcd);
+ admin.createTable(htd, SPLIT_KEYS);
+ ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
+ ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
+ HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+
+ Put p = null;
+ int numberOfPuts = 0;
+ for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
+ p = new Put(Bytes.toBytes(numberOfPuts));
+ p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
+ table.put(p);
+ }
+ m.getZooKeeperWatcher().close();
+ m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException());
+ assertFalse(m.isStopped());
+ cluster.getRegionServer(0).abort("Aborting");
+ // Without patch for HBASE-6046 this test case will always timeout
+ // with patch the test case should pass.
+ Scan scan = new Scan();
+ int numberOfRows = 0;
+ ResultScanner scanner = table.getScanner(scan);
+ Result[] result = scanner.next(1);
+ while (result != null && result.length > 0) {
+ numberOfRows++;
+ result = scanner.next(1);
+ }
+ assertEquals("Number of rows should be equal to number of puts.", numberOfPuts, numberOfRows);
+ }
+
+ static class MockLoadBalancer extends DefaultLoadBalancer {
+ static boolean retainAssignCalled = false;
+
+ @Override
+ public Map<ServerName, List<HRegionInfo>> retainAssignment(
+ Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
+ retainAssignCalled = true;
+ return super.retainAssignment(regions, servers);
+ }
+ }
+
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java?rev=1346460&r1=1346459&r2=1346460&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java Tue Jun 5 16:39:04 2012
@@ -165,7 +165,7 @@ public class TestCatalogJanitor {
private final AssignmentManager asm;
MockMasterServices(final Server server) throws IOException {
- this.mfs = new MasterFileSystem(server, this, null);
+ this.mfs = new MasterFileSystem(server, this, null, false);
this.asm = Mockito.mock(AssignmentManager.class);
}