You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 21:13:31 UTC
svn commit: r1182039 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/
Author: nspiegelberg
Date: Tue Oct 11 19:13:31 2011
New Revision: 1182039
URL: http://svn.apache.org/viewvc?rev=1182039&view=rev
Log:
Initial support for testing multiple masters in hbase-89
Summary: Porting some plumbing from Apache HBase trunk to hbase-89 that would
allow to test multiple masters (e.g. various failure scenarios). To do that we
need to maintain a list of thread objects, each holding a reference to an
HMaster, instead of just a single HMaster reference. No multi-master tests are
being added yet -- before that is done, master znode wait needs to be moved out
from the constructor. However, I'd like to get this in as a first step towards
testing HBase 89 master more thoroughly. Most of the code in this diff comes
directly from Apache HBase trunk.
Test Plan: Existing unit tests pass.
Reviewers: kranganathan, jgray, nspiegelberg
Reviewed By: kranganathan
CC: hbase-eng@lists, kranganathan, mbautin
Differential Revision: 336813
Revert Plan: OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1182039&r1=1182038&r2=1182039&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Oct 11 19:13:31 2011
@@ -129,6 +129,9 @@ public final class HConstants {
/** Default region server interface class name. */
public static final String DEFAULT_REGION_SERVER_CLASS = HRegionInterface.class.getName();
+ /** Parameter name for what master implementation to use. */
+ public static final String MASTER_IMPL = "hbase.master.impl";
+
/** Parameter name for how often threads should wake up */
public static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=1182039&r1=1182038&r2=1182039&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Tue Oct 11 19:13:31 2011
@@ -56,14 +56,17 @@ import org.apache.hadoop.hbase.util.JVMC
*/
public class LocalHBaseCluster {
static final Log LOG = LogFactory.getLog(LocalHBaseCluster.class);
- private final HMaster master;
- private final List<JVMClusterUtil.RegionServerThread> regionThreads;
+ private final List<JVMClusterUtil.MasterThread> masterThreads =
+ new CopyOnWriteArrayList<JVMClusterUtil.MasterThread>();
+ private final List<JVMClusterUtil.RegionServerThread> regionThreads =
+ new CopyOnWriteArrayList<JVMClusterUtil.RegionServerThread>();
private final static int DEFAULT_NO = 1;
/** local mode */
public static final String LOCAL = "local";
/** 'local:' */
public static final String LOCAL_COLON = LOCAL + ":";
private final Configuration conf;
+ private final Class<? extends HMaster> masterClass;
private final Class<? extends HRegionServer> regionServerClass;
/**
@@ -85,7 +88,8 @@ public class LocalHBaseCluster {
*/
public LocalHBaseCluster(final Configuration conf, final int noRegionServers)
throws IOException {
- this(conf, noRegionServers, HMaster.class, getRegionServerImplementation(conf));
+ this(conf, 1, noRegionServers, HMaster.class,
+ getRegionServerImplementation(conf));
}
@SuppressWarnings("unchecked")
@@ -98,27 +102,34 @@ public class LocalHBaseCluster {
* Constructor.
* @param conf Configuration to use. Post construction has the master's
* address.
+ * @param noMasters Count of masters to start.
* @param noRegionServers Count of regionservers to start.
- * @param masterClass
+ * @param masterClass master implementation to use if not specified by conf
+ * @param regionServerClass RS implementation to use if not specified by conf
* @throws IOException
*/
@SuppressWarnings("unchecked")
- public LocalHBaseCluster(final Configuration conf,
+ public LocalHBaseCluster(final Configuration conf, final int noMasters,
final int noRegionServers, final Class<? extends HMaster> masterClass,
final Class<? extends HRegionServer> regionServerClass)
throws IOException {
this.conf = conf;
- // Create the master
- this.master = HMaster.constructMaster(masterClass, conf);
- // Start the HRegionServers. Always have region servers come up on
- // port '0' so there won't be clashes over default port as unit tests
- // start/stop ports at different times during the life of the test.
+ // Always have masters and regionservers come up on port '0' so we don't
+ // clash over default ports.
+ conf.set(HConstants.MASTER_PORT, "0");
conf.set(HConstants.REGIONSERVER_PORT, "0");
- this.regionThreads =
- new CopyOnWriteArrayList<JVMClusterUtil.RegionServerThread>();
+ // Start the HMasters.
+ this.masterClass =
+ (Class<? extends HMaster>)conf.getClass(HConstants.MASTER_IMPL,
+ masterClass);
+ for (int i = 0; i < noMasters; i++) {
+ addMaster(new Configuration(conf), i);
+ }
+ // Start the HRegionServers.
this.regionServerClass =
(Class<? extends HRegionServer>)conf.getClass(HConstants.REGION_SERVER_IMPL,
regionServerClass);
+
for (int i = 0; i < noRegionServers; i++) {
addRegionServer(i);
}
@@ -136,6 +147,22 @@ public class LocalHBaseCluster {
return rst;
}
+ public JVMClusterUtil.MasterThread addMaster() throws IOException {
+ return addMaster(new Configuration(conf), this.masterThreads.size());
+ }
+
+ public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index)
+ throws IOException {
+ // Create each master with its own Configuration instance so each has
+ // its HConnection instance rather than share (see HBASE_INSTANCES down in
+ // the guts of HConnectionManager.
+ JVMClusterUtil.MasterThread mt =
+ JVMClusterUtil.createMasterThread(c,
+ this.masterClass, index);
+ this.masterThreads.add(mt);
+ return mt;
+ }
+
/**
* @param serverNumber
* @return region server
@@ -148,7 +175,10 @@ public class LocalHBaseCluster {
* @return the HMaster thread
*/
public HMaster getMaster() {
- return this.master;
+ if (masterThreads.size() != 1) {
+ throw new AssertionError("one master expected");
+ }
+ return this.masterThreads.get(0).getMaster();
}
/**
@@ -211,11 +241,15 @@ public class LocalHBaseCluster {
}
}
}
- if (this.master != null && this.master.isAlive()) {
- try {
- this.master.join();
- } catch(InterruptedException e) {
- // continue
+ if (this.masterThreads != null) {
+ for (Thread t : this.masterThreads) {
+ if (t.isAlive()) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ // continue
+ }
+ }
}
}
}
@@ -223,15 +257,15 @@ public class LocalHBaseCluster {
/**
* Start the cluster.
*/
- public void startup() {
- JVMClusterUtil.startup(this.master, this.regionThreads);
+ public void startup() throws IOException {
+ JVMClusterUtil.startup(this.masterThreads, this.regionThreads);
}
/**
* Shut down the mini HBase cluster
*/
public void shutdown() {
- JVMClusterUtil.shutdown(this.master, this.regionThreads);
+ JVMClusterUtil.shutdown(this.masterThreads, this.regionThreads);
}
/**
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1182039&r1=1182038&r2=1182039&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Oct 11 19:13:31 2011
@@ -185,6 +185,9 @@ public class HMaster extends Thread impl
private long applyPreferredAssignmentPeriod = 0l;
private long holdRegionForBestLocalityPeriod = 0l;
+ // flag set after we become the active master (used for testing)
+ private volatile boolean isActiveMaster = false;
+
/**
* Constructor
* @param conf configuration
@@ -268,6 +271,7 @@ public class HMaster extends Thread impl
}
this.zkMasterAddressWatcher.writeAddressToZooKeeper(this.address, true);
+ isActiveMaster = true;
this.regionServerOperationQueue =
new RegionServerOperationQueue(this.conf, this.closed);
@@ -489,7 +493,7 @@ public class HMaster extends Thread impl
return this.fs;
}
- AtomicBoolean getShutdownRequested() {
+ public AtomicBoolean getShutdownRequested() {
return this.shutdownRequested;
}
@@ -497,7 +501,7 @@ public class HMaster extends Thread impl
return this.closed;
}
- boolean isClosed() {
+ public boolean isClosed() {
return this.closed.get();
}
@@ -1646,7 +1650,7 @@ public class HMaster extends Thread impl
Integer.toString(clientPort));
// Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out.
- LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1,
+ LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
LocalHMaster.class, HRegionServer.class);
((LocalHMaster)cluster.getMaster()).setZKCluster(zooKeeperCluster);
cluster.startup();
@@ -1705,6 +1709,22 @@ public class HMaster extends Thread impl
}
/**
+ * Report whether this master is currently the active master or not.
+ * If not active master, we are parked on ZK waiting to become active.
+ *
+ * This method is used for testing.
+ *
+ * @return true if active master, false if not.
+ */
+ public boolean isActiveMaster() {
+ return isActiveMaster;
+ }
+
+ public String getServerName() {
+ return address.toString();
+ }
+
+ /**
* Main program
* @param args
*/
@@ -1717,4 +1737,9 @@ public class HMaster extends Thread impl
this.regionManager.clearFromInTransition(region.getRegionName());
LOG.info("Cleared region " + region + " from transition map");
}
+
+ public void stopMaster() {
+ closed.set(true);
+ }
+
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java?rev=1182039&r1=1182038&r2=1182039&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java Tue Oct 11 19:13:31 2011
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.util;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -75,12 +76,13 @@ public class JVMClusterUtil {
* Call 'start' on the returned thread to make it run.
* @param c Configuration to use.
* @param hrsc Class to create.
- * @param index Used distingushing the object returned.
+ * @param index Used distinguishing the object returned.
* @throws IOException
* @return Region server added.
*/
- public static JVMClusterUtil.RegionServerThread createRegionServerThread(final Configuration c,
- final Class<? extends HRegionServer> hrsc, final int index)
+ public static JVMClusterUtil.RegionServerThread createRegionServerThread(
+ final Configuration c, final Class<? extends HRegionServer> hrsc,
+ final int index)
throws IOException {
HRegionServer server;
try {
@@ -94,57 +96,134 @@ public class JVMClusterUtil {
}
/**
+ * Datastructure to hold Master Thread and Master instance
+ */
+ public static class MasterThread extends Thread {
+ private final HMaster master;
+
+ public MasterThread(final HMaster m, final int index) {
+ super(m, "Master:" + index);
+ this.master = m;
+ }
+
+ /** @return the master */
+ public HMaster getMaster() {
+ return this.master;
+ }
+
+ /**
+ * Block until the master has come online, indicating it is ready
+ * to be used.
+ */
+ public void waitForServerOnline() {
+ // The server is marked online after init begins but before race to become
+ // the active master.
+ while (!this.master.isMasterRunning() &&
+ !this.master.getShutdownRequested().get()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // continue waiting
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates a {@link MasterThread}.
+ * Call 'start' on the returned thread to make it run.
+ * @param c Configuration to use.
+ * @param hmc Class to create.
+ * @param index Used distinguishing the object returned.
+ * @throws IOException
+ * @return Master added.
+ */
+ public static JVMClusterUtil.MasterThread createMasterThread(
+ final Configuration c, final Class<? extends HMaster> hmc,
+ final int index)
+ throws IOException {
+ HMaster server;
+ try {
+ server = hmc.getConstructor(Configuration.class).newInstance(c);
+ } catch (InvocationTargetException ite) {
+ Throwable target = ite.getTargetException();
+ throw new RuntimeException("Failed construction of Master: " +
+ hmc.toString() + ((target.getCause() != null)?
+ target.getCause().getMessage(): ""), target);
+ } catch (Exception e) {
+ IOException ioe = new IOException();
+ ioe.initCause(e);
+ throw ioe;
+ }
+ return new JVMClusterUtil.MasterThread(server, index);
+ }
+
+ /**
* Start the cluster.
* @param m
* @param regionServers
* @return Address to use contacting master.
*/
- public static String startup(final HMaster m,
- final List<JVMClusterUtil.RegionServerThread> regionservers) {
- if (m != null) m.start();
+ public static String startup(final List<JVMClusterUtil.MasterThread> masters,
+ final List<JVMClusterUtil.RegionServerThread> regionservers) throws IOException {
+ if (masters != null) {
+ for (JVMClusterUtil.MasterThread t : masters) {
+ t.start();
+ }
+ }
if (regionservers != null) {
for (JVMClusterUtil.RegionServerThread t: regionservers) {
t.start();
}
}
- return m == null? null: m.getMasterAddress().toString();
+ if (masters == null || masters.isEmpty()) {
+ return null;
+ }
+ // Wait for an active master
+ while (true) {
+ for (JVMClusterUtil.MasterThread t : masters) {
+ if (t.master.isActiveMaster()) {
+ return t.master.getServerName().toString();
+ }
+ }
+ try {
+ Thread.sleep(1000);
+ } catch(InterruptedException e) {
+ // Keep waiting
+ }
+ }
}
/**
* @param master
* @param regionservers
*/
- public static void shutdown(final HMaster master,
+ public static void shutdown(final List<MasterThread> masters,
final List<RegionServerThread> regionservers) {
LOG.debug("Shutting down HBase Cluster");
- if (master != null) {
- master.shutdown();
+ if (masters != null) {
+ for (JVMClusterUtil.MasterThread t : masters) {
+ if (t.master.isActiveMaster()) {
+ t.master.shutdown();
+ } else {
+ t.master.stopMaster();
+ }
+ }
}
// regionServerThreads can never be null because they are initialized when
// the class is constructed.
- for(Thread t: regionservers) {
- if (t.isAlive()) {
- try {
- t.join();
- } catch (InterruptedException e) {
- // continue
- }
- }
- }
- if (master != null) {
- while (master.isAlive()) {
+ for(Thread t: regionservers) {
+ if (t.isAlive()) {
try {
- // The below has been replaced to debug sometime hangs on end of
- // tests.
- // this.master.join():
- Threads.threadDumpingIsAlive(master);
- } catch(InterruptedException e) {
+ t.join();
+ } catch (InterruptedException e) {
// continue
}
}
}
- LOG.info("Shutdown " +
- ((regionservers != null)? master.getName(): "0 masters") +
- " " + regionservers.size() + " region server(s)");
+ LOG.info("Shutdown of " +
+ ((masters != null) ? masters.size() : "0") + " master(s) and " +
+ ((regionservers != null) ? regionservers.size() : "0") +
+ " regionserver(s) complete");
}
}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=1182039&r1=1182038&r2=1182039&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Tue Oct 11 19:13:31 2011
@@ -67,14 +67,27 @@ public class MiniHBaseCluster {
* @throws IOException
*/
public MiniHBaseCluster(Configuration conf, int numRegionServers)
- throws IOException {
+ throws IOException, InterruptedException {
+ this(conf, 1, numRegionServers);
+ }
+
+ /**
+ * Start a MiniHBaseCluster.
+ * @param conf Configuration to be used for cluster
+ * @param numMasters initial number of masters to start.
+ * @param numRegionServers initial number of region servers to start.
+ * @throws IOException
+ */
+ public MiniHBaseCluster(Configuration conf, int numMasters,
+ int numRegionServers)
+ throws IOException, InterruptedException {
this.conf = conf;
conf.set(HConstants.MASTER_PORT, "0");
conf.setLong("hbase.master.applyPreferredAssignment.period",
PREFERRED_ASSIGNMENT);
conf.setLong("hbase.master.holdRegionForBestLocality.period",
PREFERRED_ASSIGNMENT / 5);
- init(numRegionServers);
+ init(numMasters, numRegionServers);
}
/**
@@ -232,10 +245,11 @@ public class MiniHBaseCluster {
}
}
- private void init(final int nRegionNodes) throws IOException {
+ private void init(final int nMasterNodes, final int nRegionNodes)
+ throws IOException {
try {
// start up a LocalHBaseCluster
- hbaseCluster = new LocalHBaseCluster(conf, nRegionNodes,
+ hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, nRegionNodes,
MiniHBaseCluster.MiniHBaseClusterMaster.class,
MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
hbaseCluster.startup();