You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/10/13 03:42:19 UTC
[hbase] 06/09: HBASE-24606 Implement meta merge (#2311)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-24950
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 00a042806074e94312946ae8c5956458f6bda9b1
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Sep 9 12:38:32 2020 +0800
HBASE-24606 Implement meta merge (#2311)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../hbase/master/assignment/AssignmentManager.java | 2 +-
.../hbase/master/assignment/RegionStateStore.java | 8 ++-
...etaSplit.java => TestSimpleMetaSplitMerge.java} | 73 +++++++++++++++++-----
3 files changed, 65 insertions(+), 18 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index ab87716..15712e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -1870,7 +1870,7 @@ public class AssignmentManager {
* References removed).
*/
public void markRegionAsMerged(final RegionInfo child, final ServerName serverName,
- RegionInfo [] mergeParents)
+ RegionInfo[] mergeParents)
throws IOException {
final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child);
node.setState(State.MERGED);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
index 80eafa9..c8bbfa0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
@@ -314,8 +314,12 @@ public class RegionStateStore {
private Result getRegionCatalogResult(RegionInfo region) throws IOException {
Get get =
new Get(CatalogFamilyFormat.getMetaKeyForRegion(region)).addFamily(HConstants.CATALOG_FAMILY);
- try (Table table = getMetaTable()) {
- return table.get(get);
+ if (region.isMetaRegion()) {
+ return masterRegion.get(get);
+ } else {
+ try (Table table = getMetaTable()) {
+ return table.get(get);
+ }
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
similarity index 59%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
index 110309c..7cbd245 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
@@ -20,17 +20,23 @@ package org.apache.hadoop.hbase;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -38,11 +44,11 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ MiscTests.class, MediumTests.class })
-public class TestSimpleMetaSplit {
+public class TestSimpleMetaSplitMerge {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestSimpleMetaSplit.class);
+ HBaseClassTestRule.forClass(TestSimpleMetaSplitMerge.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
@@ -70,29 +76,66 @@ public class TestSimpleMetaSplit {
UTIL.shutdownMiniCluster();
}
+ private void assertMetaRegionCount(int count) {
+ // do not count it from client as it will reset the location cache for meta table
+ assertEquals(count, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
+ .getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size());
+ }
+
+ private void clearCache(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ locator.clearRegionLocationCache();
+ }
+ }
+
+ private void assertValue(TableName tableName, String row) throws IOException {
+ try (Table table = UTIL.getConnection().getTable(tableName)) {
+ Result result = table.get(new Get(Bytes.toBytes(row)));
+ assertEquals(row, Bytes.toString(result.getValue(CF, CQ)));
+ }
+ }
+
@Test
- public void test() throws IOException {
+ public void test() throws Exception {
try (Table table = UTIL.getConnection().getTable(TD1.getTableName())) {
table.put(new Put(Bytes.toBytes("row1")).addColumn(CF, CQ, Bytes.toBytes("row1")));
}
try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) {
table.put(new Put(Bytes.toBytes("row2")).addColumn(CF, CQ, Bytes.toBytes("row2")));
}
+ Admin admin = UTIL.getAdmin();
// split meta
- UTIL.getAdmin().split(TableName.META_TABLE_NAME, Bytes.toBytes("b"));
- // do not count it from client as it will reset the location cache for meta table
- assertEquals(2, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
- .getRegionsOfTable(TableName.META_TABLE_NAME).size());
+ admin.split(TableName.META_TABLE_NAME, Bytes.toBytes("b"));
+ assertMetaRegionCount(2);
// clear the cache for table 'b'
- try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TD2.getTableName())) {
- locator.clearRegionLocationCache();
- }
+ clearCache(TD2.getTableName());
// make sure that we could get the location of the TD2 from the second meta region
- try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) {
- Result result = table.get(new Get(Bytes.toBytes("row2")));
- assertEquals("row2", Bytes.toString(result.getValue(CF, CQ)));
- }
+ assertValue(TD2.getTableName(), "row2");
// assert from client side
- assertEquals(2, UTIL.getAdmin().getRegions(TableName.META_TABLE_NAME).size());
+ List<RegionInfo> regions = admin.getRegions(TableName.META_TABLE_NAME);
+ assertEquals(2, regions.size());
+ // compact to make sure we can merge
+ for (JVMClusterUtil.RegionServerThread t : UTIL.getMiniHBaseCluster()
+ .getRegionServerThreads()) {
+ for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) {
+ if (TableName.isMetaTableName(r.getRegionInfo().getTable())) {
+ r.compact(true);
+ for (HStore store : r.getStores()) {
+ store.closeAndArchiveCompactedFiles();
+ }
+ }
+ }
+ }
+ // merge the 2 regions back to 1
+ admin.mergeRegionsAsync(regions.stream().map(RegionInfo::getRegionName).toArray(byte[][]::new),
+ false).get();
+ assertMetaRegionCount(1);
+ // clear the cache for table 'a' and 'b'
+ clearCache(TD1.getTableName());
+ clearCache(TD2.getTableName());
+
+ // make sure that we could still get the locations from the new meta region
+ assertValue(TD2.getTableName(), "row2");
+ assertValue(TD1.getTableName(), "row1");
}
}