You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2013/09/04 18:04:04 UTC
svn commit: r1520058 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/handler/
test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/regionserver/handler/
Author: tedyu
Date: Wed Sep 4 16:04:04 2013
New Revision: 1520058
URL: http://svn.apache.org/r1520058
Log:
HBASE-9387 Region could get lost during assignment (Ted Yu)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1520058&r1=1520057&r2=1520058&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java Wed Sep 4 16:04:04 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.zookeeper.KeeperException;
/**
@@ -359,7 +360,7 @@ public class OpenRegionHandler extends E
* @return whether znode is successfully transitioned to OPENED state.
* @throws IOException
*/
- private boolean transitionToOpened(final HRegion r) throws IOException {
+ boolean transitionToOpened(final HRegion r) throws IOException {
boolean result = false;
HRegionInfo hri = r.getRegionInfo();
final String name = hri.getRegionNameAsString();
@@ -367,11 +368,20 @@ public class OpenRegionHandler extends E
try {
if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
this.server.getServerName(), this.version) == -1) {
- LOG.warn("Completed the OPEN of region " + name +
- " but when transitioning from " +
- " OPENING to OPENED got a version mismatch, someone else clashed " +
- "so now unassigning -- closing region on server: " +
- this.server.getServerName());
+ String warnMsg = "Completed the OPEN of region " + name +
+ " but when transitioning from " + " OPENING to OPENED ";
+ try {
+ String node = ZKAssign.getNodeName(this.server.getZooKeeper(), hri.getEncodedName());
+ if (ZKUtil.checkExists(this.server.getZooKeeper(), node) < 0) {
+ // if the znode
+ rsServices.abort(warnMsg + "the znode disappeared", null);
+ } else {
+ LOG.warn(warnMsg + "got a version mismatch, someone else clashed; " +
+ "so now unassigning -- closing region on server: " + this.server.getServerName());
+ }
+ } catch (KeeperException ke) {
+ rsServices.abort(warnMsg, ke);
+ }
} else {
LOG.debug("Transitioned " + r.getRegionInfo().getEncodedName() +
" to OPENED in zk on " + this.server.getServerName());
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java?rev=1520058&r1=1520057&r2=1520058&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java Wed Sep 4 16:04:04 2013
@@ -55,6 +55,7 @@ class MockRegionServerServices implement
private ZooKeeperWatcher zkw = null;
private ServerName serverName = null;
private RpcServerInterface rpcServer = null;
+ private volatile boolean abortRequested;
MockRegionServerServices(ZooKeeperWatcher zkw) {
this.zkw = zkw;
@@ -154,7 +155,8 @@ class MockRegionServerServices implement
@Override
public void abort(String why, Throwable e) {
- //no-op
+ this.abortRequested = true;
+ stop(why);
}
@Override
@@ -169,7 +171,7 @@ class MockRegionServerServices implement
@Override
public boolean isAborted() {
- return false;
+ return this.abortRequested;
}
@Override
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=1520058&r1=1520057&r2=1520058&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java Wed Sep 4 16:04:04 2013
@@ -55,7 +55,7 @@ public class TestFullLogReconstruction {
c.setInt("ipc.client.connect.max.retries", 1);
c.setInt("dfs.client.block.recovery.retries", 1);
c.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
- TEST_UTIL.startMiniCluster(2);
+ TEST_UTIL.startMiniCluster(3);
}
/**
@@ -109,4 +109,4 @@ public class TestFullLogReconstruction {
assertEquals(count, newCount);
table.close();
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java?rev=1520058&r1=1520057&r2=1520058&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java Wed Sep 4 16:04:04 2013
@@ -129,6 +129,62 @@ public class TestOpenRegionHandler {
}
}
+ /**
+ * Test the openregionhandler can deal with perceived failure of transitioning to OPENED state
+ * due to intermittent zookeeper malfunctioning.
+ * @see <a href="https://issues.apache.org/jira/browse/HBASE-9387">HBASE-9387</a>
+ * @throws IOException
+ * @throws NodeExistsException
+ * @throws KeeperException
+ */
+ @Test
+ public void testRegionServerAbortionDueToFailureTransitioningToOpened()
+ throws IOException, NodeExistsException, KeeperException {
+ final Server server = new MockServer(HTU);
+ final RegionServerServices rss = HTU.createMockRegionServerService();
+
+ HTableDescriptor htd = TEST_HTD;
+ final HRegionInfo hri = TEST_HRI;
+ HRegion region =
+ HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU
+ .getConfiguration(), htd);
+ assertNotNull(region);
+ try {
+ OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd) {
+ boolean transitionToOpened(final HRegion r) throws IOException {
+ // remove znode simulating intermittent zookeeper connection issue
+ ZooKeeperWatcher zkw = this.server.getZooKeeper();
+ String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
+ try {
+ ZKUtil.deleteNodeFailSilent(zkw, node);
+ } catch (KeeperException e) {
+ throw new RuntimeException("Ugh failed delete of " + node, e);
+ }
+ // then try to transition to OPENED
+ return super.transitionToOpened(r);
+ }
+ };
+ rss.getRegionsInTransitionInRS().put(
+ hri.getEncodedNameAsBytes(), Boolean.TRUE);
+ // Call process without first creating OFFLINE region in zk, see if
+ // exception or just quiet return (expected).
+ handler.process();
+ rss.getRegionsInTransitionInRS().put(
+ hri.getEncodedNameAsBytes(), Boolean.TRUE);
+ ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
+ // Call process again but this time yank the zk znode out from under it
+ // post OPENING; again will expect it to come back w/o NPE or exception.
+ handler.process();
+ } catch (IOException ioe) {
+ } finally {
+ HRegion.closeHRegion(region);
+ }
+ // Region server is expected to abort due to OpenRegionHandler perceiving transitioning
+ // to OPENED as failed
+ // This was corresponding to the second handler.process() call above.
+ assertTrue("region server should have aborted", rss.isAborted());
+ }
+
@Test
public void testFailedOpenRegion() throws Exception {
Server server = new MockServer(HTU);