You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/12/14 00:14:52 UTC
[24/50] [abbrv] hbase git commit: HBASE-19553 Old replica regions
should be cleared from AM memory after primary region split or merge
HBASE-19553 Old replica regions should be cleared from AM memory after primary region split or merge
Signed-off-by: Ted Yu <te...@apache.org>
Signed-off-by: Huaxiang Sun <hu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5c799c18
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5c799c18
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5c799c18
Branch: refs/heads/branch-1.3
Commit: 5c799c18f684121352b1f071676a442a8efdc8a1
Parents: b3f911c
Author: Pankaj Kumar <pa...@huawei.com>
Authored: Thu Jan 25 13:25:22 2018 +0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Dec 12 18:08:18 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/master/AssignmentManager.java | 17 ++++
.../apache/hadoop/hbase/client/TestAdmin1.java | 102 +++++++++++++++++--
2 files changed, 111 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c799c18/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 63ef5ff..79fe596 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -4256,6 +4256,8 @@ public class AssignmentManager extends ZooKeeperListener {
LOG.warn("Couldn't assign all replica(s) of region " + mergedHri+ " because of " +
ie.getMessage());
}
+ // Remove merged region's replica from AM's memory
+ clearReplicaRegions(c);
}
private void doSplittingOfReplicas(final HRegionInfo parentHri, final HRegionInfo hri_a,
@@ -4299,6 +4301,21 @@ public class AssignmentManager extends ZooKeeperListener {
} catch (InterruptedException e) {
LOG.warn("Caught exception " + e + " while trying to assign replica(s) of daughter(s)");
}
+ // Remove parent region's replica from AM's memory
+ clearReplicaRegions(c);
+ }
+
+ /*
+ * Clear the replica regions after region split or merge.
+ */
+ private void clearReplicaRegions(Collection<List<HRegionInfo>> regionInfos) {
+ for (List<HRegionInfo> regionInfoList : regionInfos) {
+ for (HRegionInfo regionInfo : regionInfoList) {
+ if (!RegionReplicaUtil.isDefaultReplica(regionInfo)) {
+ regionStates.deleteRegion(regionInfo);
+ }
+ }
+ }
}
private void prepareDaughterReplicaForAssignment(HRegionInfo daughterHri, HRegionInfo parentHri,
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c799c18/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index d153a85..e9fa3e3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.util.Bytes;
@@ -56,7 +57,9 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
+import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -1168,15 +1171,8 @@ public class TestAdmin1 {
// regions.
// Set up a table with 3 regions and replication set to 3
TableName tableName = TableName.valueOf("testSplitAndMergeWithReplicaTable");
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.setRegionReplication(3);
byte[] cf = "f".getBytes();
- HColumnDescriptor hcd = new HColumnDescriptor(cf);
- desc.addFamily(hcd);
- byte[][] splitRows = new byte[2][];
- splitRows[0] = new byte[]{(byte)'4'};
- splitRows[1] = new byte[]{(byte)'7'};
- TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+ createReplicaTable(tableName, cf);
List<HRegion> oldRegions;
do {
oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
@@ -1284,6 +1280,96 @@ public class TestAdmin1 {
}
/**
+ * Test case to validate whether parent's replica regions are cleared from AM's memory after
+ * SPLIT/MERGE.
+ */
+ @Test
+ public void testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge");
+ createReplicaTable(tableName, "f".getBytes());
+ final int regionReplication = admin.getTableDescriptor(tableName).getRegionReplication();
+
+ List<Pair<HRegionInfo, ServerName>> regions = MetaTableAccessor.getTableRegionsAndLocations(
+ TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(), tableName);
+ assertEquals(9, regions.size());
+ final int primaryRegionCount = regions.size() / regionReplication;
+
+ final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
+ List<HRegionInfo> splitRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+ assertEquals(0, splitRegions.size());
+
+ // Validate region split
+ byte[] regionName = regions.get(0).getFirst().getRegionName();
+ try {
+ TEST_UTIL.getHBaseAdmin().split(regionName, Bytes.toBytes('2'));
+ } catch (IllegalArgumentException ex) {
+ fail("Exception occured during region split" + ex);
+ }
+
+ // Wait for replica region to become online
+ TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+ .size() == (primaryRegionCount + 1) * regionReplication;
+ }
+ });
+
+ regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+ TEST_UTIL.getConnection(), tableName);
+ assertEquals(12, regions.size());
+ final int primaryRegionCountAfterSplit = regions.size() / regionReplication;
+
+ // Split region after region split
+ splitRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+ // Parent's replica region should be removed from AM's memory.
+ assertEquals(1, splitRegions.size());
+
+ // Validate region merge
+ HRegionInfo regionA = regions.get(3).getFirst();
+ HRegionInfo regionB = regions.get(6).getFirst();
+ try {
+ TEST_UTIL.getHBaseAdmin().mergeRegions(regionA.getRegionName(), regionB.getRegionName(),
+ true);
+ } catch (IllegalArgumentException ex) {
+ fail("Exception occured during region merge" + ex);
+ }
+
+ // Wait for replica regions to become online
+ TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+ .size() == (primaryRegionCountAfterSplit - 1) * regionReplication;
+ }
+ });
+
+ regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+ TEST_UTIL.getConnection(), tableName);
+ assertEquals(9, regions.size());
+ // Offline region after region merge
+ List<HRegionInfo> offlineRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE);
+ // Parent's replica region should be removed from AM's memory.
+ assertEquals(0, offlineRegions.size());
+ }
+
+ private byte[] createReplicaTable(TableName tableName, byte[] cf) throws IOException {
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.setRegionReplication(3);
+ HColumnDescriptor hcd = new HColumnDescriptor(cf);
+ desc.addFamily(hcd);
+ byte[][] splitRows = new byte[2][];
+ splitRows[0] = new byte[] { (byte) '4' };
+ splitRows[1] = new byte[] { (byte) '7' };
+ TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+ return cf;
+ }
+
+ /**
* HADOOP-2156
* @throws IOException
*/