You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2014/09/19 20:44:38 UTC
git commit: HBASE-12005 Split/merge fails if master restarts before
PONR
Repository: hbase
Updated Branches:
refs/heads/master 74c6b33e8 -> b4c07e9a8
HBASE-12005 Split/merge fails if master restarts before PONR
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b4c07e9a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b4c07e9a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b4c07e9a
Branch: refs/heads/master
Commit: b4c07e9a80aecdfd829670e513daa8b6469973e7
Parents: 74c6b33
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Wed Sep 17 17:39:27 2014 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Fri Sep 19 11:44:07 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/master/RegionState.java | 48 ++++++++
.../hadoop/hbase/master/AssignmentManager.java | 91 ++++++++-------
.../hadoop/hbase/master/RegionStates.java | 2 +-
.../TestRegionMergeTransactionOnCluster.java | 79 ++++++++++++-
.../TestSplitTransactionOnCluster.java | 110 ++++++++++++++++---
5 files changed, 273 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4c07e9a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
index e7af336..e844964 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
@@ -272,6 +272,54 @@ public class RegionState {
return serverName != null && serverName.equals(sn);
}
+ public boolean isMergingOnServer(final ServerName sn) {
+ return isOnServer(sn) && isMerging();
+ }
+
+ public boolean isMergingNewOnServer(final ServerName sn) {
+ return isOnServer(sn) && isMergingNew();
+ }
+
+ public boolean isMergingNewOrOpenedOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isMergingNew() || isOpened());
+ }
+
+ public boolean isMergingNewOrOfflineOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isMergingNew() || isOffline());
+ }
+
+ public boolean isSplittingOnServer(final ServerName sn) {
+ return isOnServer(sn) && isSplitting();
+ }
+
+ public boolean isSplittingNewOnServer(final ServerName sn) {
+ return isOnServer(sn) && isSplittingNew();
+ }
+
+ public boolean isSplittingOrOpenedOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isSplitting() || isOpened());
+ }
+
+ public boolean isSplittingOrSplitOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isSplitting() || isSplit());
+ }
+
+ public boolean isClosingOnServer(final ServerName sn) {
+ return isOnServer(sn) && isClosing();
+ }
+
+ public boolean isOpeningOrFailedOpenOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isOpening() || isFailedOpen());
+ }
+
+ public boolean isOpeningOrOpenedOnServer(final ServerName sn) {
+ return isOnServer(sn) && (isOpening() || isOpened());
+ }
+
+ public boolean isOpenedOnServer(final ServerName sn) {
+ return isOnServer(sn) && isOpened();
+ }
+
/**
* Check if a region state can transition to offline
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4c07e9a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index f184ba0..e289040 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -2051,11 +2051,11 @@ public class AssignmentManager {
// The region must be opening on this server.
// If current state is failed_open on the same server,
// it could be a reportRegionTransition RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isOpening() || current.isFailedOpen())) {
+ if (current == null || !current.isOpeningOrFailedOpenOnServer(serverName)) {
return hri.getShortNameToLog() + " is not opening on " + serverName;
}
+ // Just return in case of retrying
if (current.isFailedOpen()) {
return null;
}
@@ -2106,11 +2106,11 @@ public class AssignmentManager {
// The region must be opening on this server.
// If current state is already opened on the same server,
// it could be a reportRegionTransition RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isOpening() || current.isOpened())) {
+ if (current == null || !current.isOpeningOrOpenedOnServer(serverName)) {
return hri.getShortNameToLog() + " is not opening on " + serverName;
}
+ // Just return in case of retrying
if (current.isOpened()) {
return null;
}
@@ -2140,7 +2140,7 @@ public class AssignmentManager {
// already have moved away from closed state. 2. On the region server side, we
// don't care much about the response for this transition. We only make sure
// master has got and processed this report, either successfully or not.
- if (current == null || !current.isOnServer(serverName) || !current.isClosing()) {
+ if (current == null || !current.isClosingOnServer(serverName)) {
return hri.getShortNameToLog() + " is not closing on " + serverName;
}
if (getTableStateManager().isTableState(hri.getTable(), TableState.State.DISABLED,
@@ -2162,11 +2162,11 @@ public class AssignmentManager {
// The region must be opened on this server.
// If current state is already splitting on the same server,
// it could be a reportRegionTransition RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isOpened() || current.isSplitting())) {
+ if (current == null || !current.isSplittingOrOpenedOnServer(serverName)) {
return hri.getShortNameToLog() + " is not opening on " + serverName;
}
+ // Just return in case of retrying
if (current.isSplitting()) {
return null;
}
@@ -2194,7 +2194,7 @@ public class AssignmentManager {
final ServerName serverName, final RegionStateTransition transition) {
// The region must be splitting on this server, and the daughters must be in
// splitting_new state. To check RPC retry, we use server holding info.
- if (current == null || !current.isOnServer(serverName) || !current.isSplitting()) {
+ if (current == null || !current.isSplittingOnServer(serverName)) {
return hri.getShortNameToLog() + " is not splitting on " + serverName;
}
@@ -2202,13 +2202,23 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isSplittingNew()
- || !rs_b.isSplittingNew()) {
+
+ // Master could have restarted and lost the new region
+ // states, if so, they must be lost together
+ if (rs_a == null && rs_b == null) {
+ rs_a = regionStates.createRegionState(
+ a, State.SPLITTING_NEW, serverName, null);
+ rs_b = regionStates.createRegionState(
+ b, State.SPLITTING_NEW, serverName, null);
+ }
+
+ if (rs_a == null || !rs_a.isSplittingNewOnServer(serverName)
+ || rs_b == null || !rs_b.isSplittingNewOnServer(serverName)) {
return "Some daughter is not known to be splitting on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
+ // Just return in case of retrying
if (!regionStates.isRegionOnServer(hri, serverName)) {
return null;
}
@@ -2228,11 +2238,11 @@ public class AssignmentManager {
// splitting_new state.
// If current state is already split on the same server,
// it could be a reportRegionTransition RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isSplitting() || current.isSplit())) {
+ if (current == null || !current.isSplittingOrSplitOnServer(serverName)) {
return hri.getShortNameToLog() + " is not splitting on " + serverName;
}
+ // Just return in case of retrying
if (current.isSplit()) {
return null;
}
@@ -2241,9 +2251,8 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isSplittingNew()
- || !rs_b.isSplittingNew()) {
+ if (rs_a == null || !rs_a.isSplittingNewOnServer(serverName)
+ || rs_b == null || !rs_b.isSplittingNewOnServer(serverName)) {
return "Some daughter is not known to be splitting on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
@@ -2278,11 +2287,11 @@ public class AssignmentManager {
// The region must be splitting on this server, and the daughters must be in
// splitting_new state.
// If the region is in open state, it could be an RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isSplitting() || current.isOpened())) {
+ if (current == null || !current.isSplittingOrOpenedOnServer(serverName)) {
return hri.getShortNameToLog() + " is not splitting on " + serverName;
}
+ // Just return in case of retrying
if (current.isOpened()) {
return null;
}
@@ -2291,9 +2300,8 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isSplittingNew()
- || !rs_b.isSplittingNew()) {
+ if (rs_a == null || !rs_a.isSplittingNewOnServer(serverName)
+ || rs_b == null || !rs_b.isSplittingNewOnServer(serverName)) {
return "Some daughter is not known to be splitting on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
@@ -2312,11 +2320,11 @@ public class AssignmentManager {
final ServerName serverName, final RegionStateTransition transition) {
// The region must be new, and the daughters must be open on this server.
// If the region is in merge_new state, it could be an RPC retry.
- if (current != null && (!current.isOnServer(serverName)
- || !current.isMergingNew())) {
+ if (current != null && !current.isMergingNewOnServer(serverName)) {
return "Merging daughter region already exists, p=" + current;
}
+ // Just return in case of retrying
if (current != null) {
return null;
}
@@ -2330,9 +2338,8 @@ public class AssignmentManager {
try {
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isOpened()
- || !rs_b.isOpened()) {
+ if (rs_a == null || !rs_a.isOpenedOnServer(serverName)
+ || rs_b == null || !rs_b.isOpenedOnServer(serverName)) {
return "Some daughter is not in a state to merge on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
@@ -2353,7 +2360,7 @@ public class AssignmentManager {
final ServerName serverName, final RegionStateTransition transition) {
// The region must be in merging_new state, and the daughters must be
// merging. To check RPC retry, we use server holding info.
- if (current == null || !current.isOnServer(serverName) || !current.isMergingNew()) {
+ if (current != null && !current.isMergingNewOnServer(serverName)) {
return hri.getShortNameToLog() + " is not merging on " + serverName;
}
@@ -2361,13 +2368,19 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isMerging()
- || !rs_b.isMerging()) {
+ if (rs_a == null || !rs_a.isMergingOnServer(serverName)
+ || rs_b == null || !rs_b.isMergingOnServer(serverName)) {
return "Some daughter is not known to be merging on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
+ // Master could have restarted and lost the new region state
+ if (current == null) {
+ regionStates.createRegionState(
+ hri, State.MERGING_NEW, serverName, null);
+ }
+
+ // Just return in case of retrying
if (regionStates.isRegionOnServer(hri, serverName)) {
return null;
}
@@ -2387,11 +2400,11 @@ public class AssignmentManager {
// merging on this server.
// If current state is already opened on the same server,
// it could be a reportRegionTransition RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isMergingNew() || current.isOpened())) {
+ if (current == null || !current.isMergingNewOrOpenedOnServer(serverName)) {
return hri.getShortNameToLog() + " is not merging on " + serverName;
}
+ // Just return in case of retrying
if (current.isOpened()) {
return null;
}
@@ -2400,9 +2413,8 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isMerging()
- || !rs_b.isMerging()) {
+ if (rs_a == null || !rs_a.isMergingOnServer(serverName)
+ || rs_b == null || !rs_b.isMergingOnServer(serverName)) {
return "Some daughter is not known to be merging on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
@@ -2433,11 +2445,11 @@ public class AssignmentManager {
// The region must be in merging_new state, and the daughters must be
// merging on this server.
// If the region is in offline state, it could be an RPC retry.
- if (current == null || !current.isOnServer(serverName)
- || !(current.isMergingNew() || current.isOffline())) {
+ if (current == null || !current.isMergingNewOrOfflineOnServer(serverName)) {
return hri.getShortNameToLog() + " is not merging on " + serverName;
}
+ // Just return in case of retrying
if (current.isOffline()) {
return null;
}
@@ -2446,9 +2458,8 @@ public class AssignmentManager {
final HRegionInfo b = HRegionInfo.convert(transition.getRegionInfo(2));
RegionState rs_a = regionStates.getRegionState(a);
RegionState rs_b = regionStates.getRegionState(b);
- if (rs_a == null || rs_b == null || !rs_a.isOnServer(serverName)
- || !rs_b.isOnServer(serverName) || !rs_a.isMerging()
- || !rs_b.isMerging()) {
+ if (rs_a == null || !rs_a.isMergingOnServer(serverName)
+ || rs_b == null || !rs_b.isMergingOnServer(serverName)) {
return "Some daughter is not known to be merging on " + serverName
+ ", a=" + rs_a + ", b=" + rs_b;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4c07e9a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 85c77f1..b4a9047 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -335,7 +335,7 @@ public class RegionStates {
}
lastAssignments.put(encodedName, lastHost);
regionAssignments.put(hri, lastHost);
- } else if (!regionState.isUnassignable()) {
+ } else if (!isOneOfStates(regionState, State.MERGED, State.SPLIT, State.OFFLINE)) {
regionsInTransition.put(encodedName, regionState);
}
if (lastHost != null && newState != State.SPLIT) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4c07e9a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index 07c1a58..1a14571 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -26,12 +26,15 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -51,8 +54,13 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.master.RegionStates;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -60,12 +68,15 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import com.google.common.base.Joiner;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
/**
* Like {@link TestRegionMergeTransaction} in that we're testing
@@ -97,7 +108,7 @@ public class TestRegionMergeTransactionOnCluster {
@BeforeClass
public static void beforeAllTests() throws Exception {
// Start a cluster
- TEST_UTIL.startMiniCluster(NB_SERVERS);
+ TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
master = cluster.getMaster();
master.balanceSwitch(false);
@@ -156,6 +167,31 @@ public class TestRegionMergeTransactionOnCluster {
table.close();
}
+ /**
+ * Not really restarting the master. Simulate it by clear of new region
+ * state since it is not persisted, will be lost after master restarts.
+ */
+ @Test
+ public void testMergeAndRestartingMaster() throws Exception {
+ LOG.info("Starting testMergeAndRestartingMaster");
+ final TableName tableName = TableName.valueOf("testMergeAndRestartingMaster");
+
+ // Create table and load data.
+ Table table = createTableAndLoadData(master, tableName);
+
+ try {
+ MyMasterRpcServices.enabled.set(true);
+
+ // Merge 1st and 2nd region
+ mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
+ INITIAL_REGION_NUM - 1);
+ } finally {
+ MyMasterRpcServices.enabled.set(false);
+ }
+
+ table.close();
+ }
+
@SuppressWarnings("deprecation")
@Test
public void testCleanMergeReference() throws Exception {
@@ -433,4 +469,45 @@ public class TestRegionMergeTransactionOnCluster {
assertEquals(expectedRegionNum, rowCount);
scanner.close();
}
+
+ // Make it public so that JVMClusterUtil can access it.
+ public static class MyMaster extends HMaster {
+ public MyMaster(Configuration conf, CoordinatedStateManager cp)
+ throws IOException, KeeperException,
+ InterruptedException {
+ super(conf, cp);
+ }
+
+ @Override
+ protected RSRpcServices createRpcServices() throws IOException {
+ return new MyMasterRpcServices(this);
+ }
+ }
+
+ static class MyMasterRpcServices extends MasterRpcServices {
+ static AtomicBoolean enabled = new AtomicBoolean(false);
+
+ private HMaster myMaster;
+ public MyMasterRpcServices(HMaster master) throws IOException {
+ super(master);
+ myMaster = master;
+ }
+
+ @Override
+ public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
+ ReportRegionStateTransitionRequest req) throws ServiceException {
+ ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
+ if (enabled.get() && req.getTransition(0).getTransitionCode()
+ == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
+ RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
+ for (RegionState regionState: regionStates.getRegionsInTransition().values()) {
+ // Find the merging_new region and remove it
+ if (regionState.isMergingNew()) {
+ regionStates.deleteRegion(regionState.getRegion());
+ }
+ }
+ }
+ return resp;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4c07e9a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 7ae4041..45dd273 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -30,6 +30,7 @@ import java.io.InterruptedIOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -73,10 +74,14 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -98,6 +103,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
/**
@@ -106,6 +112,7 @@ import com.google.protobuf.ServiceException;
* is tests against a bare {@link HRegion}.
*/
@Category({RegionServerTests.class, LargeTests.class})
+@SuppressWarnings("deprecation")
public class TestSplitTransactionOnCluster {
private static final Log LOG =
LogFactory.getLog(TestSplitTransactionOnCluster.class);
@@ -120,7 +127,7 @@ public class TestSplitTransactionOnCluster {
@BeforeClass public static void before() throws Exception {
TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
- TESTING_UTIL.startMiniCluster(NB_SERVERS);
+ TESTING_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
}
@AfterClass public static void after() throws Exception {
@@ -211,7 +218,6 @@ public class TestSplitTransactionOnCluster {
}
}
@Test(timeout = 60000)
- @SuppressWarnings("deprecation")
public void testSplitFailedCompactionAndSplit() throws Exception {
final byte[] tableName = Bytes.toBytes("testSplitFailedCompactionAndSplit");
Configuration conf = TESTING_UTIL.getConfiguration();
@@ -573,7 +579,7 @@ public class TestSplitTransactionOnCluster {
this.admin.split(hri.getRegionNameAsString());
checkAndGetDaughters(tableName);
- MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
+ HMaster master = abortAndWaitForMaster();
this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
@@ -826,6 +832,52 @@ public class TestSplitTransactionOnCluster {
}
}
+ /**
+ * Not really restarting the master. Simulate it by clear of new region
+ * state since it is not persisted, will be lost after master restarts.
+ */
+ @Test(timeout = 180000)
+ public void testSplitAndRestartingMaster() throws Exception {
+ LOG.info("Starting testSplitAndRestartingMaster");
+ final TableName tableName = TableName.valueOf("testSplitAndRestartingMaster");
+ // Create table then get the single region for our new table.
+ createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
+ List<HRegion> regions = cluster.getRegions(tableName);
+ HRegionInfo hri = getAndCheckSingleTableRegion(regions);
+ ensureTableRegionNotOnSameServerAsMeta(admin, hri);
+ int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
+ HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
+ // Turn off balancer so it doesn't cut in and mess up our placements.
+ this.admin.setBalancerRunning(false, true);
+ // Turn off the meta scanner so it don't remove parent on us.
+ cluster.getMaster().setCatalogJanitorEnabled(false);
+ try {
+ MyMasterRpcServices.enabled.set(true);
+ // find a splittable region. Refresh the regions list
+ regions = cluster.getRegions(tableName);
+ final HRegion region = findSplittableRegion(regions);
+ assertTrue("not able to find a splittable region", region != null);
+
+ // Now split.
+ SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
+ try {
+ st.prepare();
+ st.execute(regionServer, regionServer);
+ } catch (IOException e) {
+ fail("Split execution should have succeeded with no exceptions thrown");
+ }
+
+ // Postcondition
+ List<HRegion> daughters = cluster.getRegions(tableName);
+ LOG.info("xxx " + regions.size() + AssignmentManager.TEST_SKIP_SPLIT_HANDLING);
+ assertTrue(daughters.size() == 2);
+ } finally {
+ MyMasterRpcServices.enabled.set(false);
+ admin.setBalancerRunning(true, false);
+ cluster.getMaster().setCatalogJanitorEnabled(true);
+ }
+ }
+
@Test(timeout = 180000)
public void testSplitHooksBeforeAndAfterPONR() throws Exception {
String firstTable = "testSplitHooksBeforeAndAfterPONR_1";
@@ -962,14 +1014,11 @@ public class TestSplitTransactionOnCluster {
return daughters;
}
- private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
+ private HMaster abortAndWaitForMaster()
throws IOException, InterruptedException {
cluster.abortMaster(0);
cluster.waitOnMaster(0);
- cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
- MockMasterWithoutCatalogJanitor.class, HMaster.class);
- MockMasterWithoutCatalogJanitor master = null;
- master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
+ HMaster master = cluster.startMaster().getMaster();
cluster.waitForActiveAndReadyMaster();
return master;
}
@@ -1107,20 +1156,52 @@ public class TestSplitTransactionOnCluster {
return t;
}
- public static class MockMasterWithoutCatalogJanitor extends HMaster {
+ private static class SplittingNodeCreationFailedException extends IOException {
+ private static final long serialVersionUID = 1652404976265623004L;
+
+ public SplittingNodeCreationFailedException () {
+ super();
+ }
+ }
- public MockMasterWithoutCatalogJanitor(Configuration conf, CoordinatedStateManager cp)
+ // Make it public so that JVMClusterUtil can access it.
+ public static class MyMaster extends HMaster {
+ public MyMaster(Configuration conf, CoordinatedStateManager cp)
throws IOException, KeeperException,
InterruptedException {
super(conf, cp);
}
+
+ @Override
+ protected RSRpcServices createRpcServices() throws IOException {
+ return new MyMasterRpcServices(this);
+ }
}
- private static class SplittingNodeCreationFailedException extends IOException {
- private static final long serialVersionUID = 1652404976265623004L;
+ static class MyMasterRpcServices extends MasterRpcServices {
+ static AtomicBoolean enabled = new AtomicBoolean(false);
- public SplittingNodeCreationFailedException () {
- super();
+ private HMaster myMaster;
+ public MyMasterRpcServices(HMaster master) throws IOException {
+ super(master);
+ myMaster = master;
+ }
+
+ @Override
+ public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
+ ReportRegionStateTransitionRequest req) throws ServiceException {
+ ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
+ if (enabled.get() && req.getTransition(0).getTransitionCode().equals(
+ TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) {
+ RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
+ for (RegionState regionState: regionStates.getRegionsInTransition().values()) {
+ // Find the merging_new region and remove it
+ if (regionState.isSplittingNew()) {
+ regionStates.deleteRegion(regionState.getRegion());
+ }
+ }
+ }
+ return resp;
}
}
@@ -1177,7 +1258,6 @@ public class TestSplitTransactionOnCluster {
HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
st.stepsAfterPONR(rs, rs, daughterRegions);
}
-
}
}