[hbase] branch HBASE-24950 updated: HBASE-24606 Implement meta merge (#2311)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch HBASE-24950
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/HBASE-24950 by this push:
     new 9246209  HBASE-24606 Implement meta merge (#2311)
9246209 is described below

commit 9246209f5afa8bf102681498c458b06eaa881860
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Sep 9 12:38:32 2020 +0800

    HBASE-24606 Implement meta merge (#2311)
    
    Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
 .../org/apache/hadoop/hbase/MetaTableAccessor.java |  1 -
 .../hbase/master/assignment/AssignmentManager.java |  2 +-
 .../hbase/master/assignment/RegionStateStore.java  | 11 ++--
 ...etaSplit.java => TestSimpleMetaSplitMerge.java} | 73 +++++++++++++++++-----
 4 files changed, 66 insertions(+), 21 deletions(-)

diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 46602d8..d2fd5e8 100644
--- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -994,7 +994,6 @@ public final class MetaTableAccessor {
     LOG.debug("Overwritten regions: {} ", regionInfos);
   }
 
-
   public static Put addRegionInfo(final Put p, final RegionInfo hri) throws IOException {
     p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow())
       .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.REGIONINFO_QUALIFIER)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 64bcafd..fc621ff 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -1857,7 +1857,7 @@ public class AssignmentManager {
    * References removed).
    */
   public void markRegionAsMerged(final RegionInfo child, final ServerName serverName,
-        RegionInfo [] mergeParents)
+    RegionInfo[] mergeParents)
       throws IOException {
     final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child);
     node.setState(State.MERGED);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
index b2aa339..65cc23d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
@@ -313,12 +313,15 @@ public class RegionStateStore {
   private Result getRegionCatalogResult(RegionInfo region) throws IOException {
     Get get =
       new Get(CatalogFamilyFormat.getMetaKeyForRegion(region)).addFamily(HConstants.CATALOG_FAMILY);
-    try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
-      return table.get(get);
+    if (region.isMetaRegion()) {
+      return masterRegion.get(get);
+    } else {
+      try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
+        return table.get(get);
+      }
     }
   }
 
-
   private static Put addSequenceNum(Put p, long openSeqNum, int replicaId) throws IOException {
     return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow())
       .setFamily(HConstants.CATALOG_FAMILY)
@@ -501,7 +504,7 @@ public class RegionStateStore {
   }
 
   // ============================================================================================
-  //  Delete Region State helpers
+  // Delete Region State helpers
   // ============================================================================================
   public void deleteRegion(final RegionInfo regionInfo) throws IOException {
     deleteRegions(Collections.singletonList(regionInfo));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
similarity index 59%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
index 110309c..7cbd245 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java
@@ -20,17 +20,23 @@ package org.apache.hadoop.hbase;
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -38,11 +44,11 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 @Category({ MiscTests.class, MediumTests.class })
-public class TestSimpleMetaSplit {
+public class TestSimpleMetaSplitMerge {
 
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =
-    HBaseClassTestRule.forClass(TestSimpleMetaSplit.class);
+    HBaseClassTestRule.forClass(TestSimpleMetaSplitMerge.class);
 
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
 
@@ -70,29 +76,66 @@ public class TestSimpleMetaSplit {
     UTIL.shutdownMiniCluster();
   }
 
+  private void assertMetaRegionCount(int count) {
+    // do not count it from client as it will reset the location cache for meta table
+    assertEquals(count, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
+      .getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size());
+  }
+
+  private void clearCache(TableName tableName) throws IOException {
+    try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+      locator.clearRegionLocationCache();
+    }
+  }
+
+  private void assertValue(TableName tableName, String row) throws IOException {
+    try (Table table = UTIL.getConnection().getTable(tableName)) {
+      Result result = table.get(new Get(Bytes.toBytes(row)));
+      assertEquals(row, Bytes.toString(result.getValue(CF, CQ)));
+    }
+  }
+
   @Test
-  public void test() throws IOException {
+  public void test() throws Exception {
     try (Table table = UTIL.getConnection().getTable(TD1.getTableName())) {
       table.put(new Put(Bytes.toBytes("row1")).addColumn(CF, CQ, Bytes.toBytes("row1")));
     }
     try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) {
       table.put(new Put(Bytes.toBytes("row2")).addColumn(CF, CQ, Bytes.toBytes("row2")));
     }
+    Admin admin = UTIL.getAdmin();
     // split meta
-    UTIL.getAdmin().split(TableName.META_TABLE_NAME, Bytes.toBytes("b"));
-    // do not count it from client as it will reset the location cache for meta table
-    assertEquals(2, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
-      .getRegionsOfTable(TableName.META_TABLE_NAME).size());
+    admin.split(TableName.META_TABLE_NAME, Bytes.toBytes("b"));
+    assertMetaRegionCount(2);
     // clear the cache for table 'b'
-    try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TD2.getTableName())) {
-      locator.clearRegionLocationCache();
-    }
+    clearCache(TD2.getTableName());
     // make sure that we could get the location of the TD2 from the second meta region
-    try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) {
-      Result result = table.get(new Get(Bytes.toBytes("row2")));
-      assertEquals("row2", Bytes.toString(result.getValue(CF, CQ)));
-    }
+    assertValue(TD2.getTableName(), "row2");
     // assert from client side
-    assertEquals(2, UTIL.getAdmin().getRegions(TableName.META_TABLE_NAME).size());
+    List<RegionInfo> regions = admin.getRegions(TableName.META_TABLE_NAME);
+    assertEquals(2, regions.size());
+    // compact to make sure we can merge
+    for (JVMClusterUtil.RegionServerThread t : UTIL.getMiniHBaseCluster()
+      .getRegionServerThreads()) {
+      for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) {
+        if (TableName.isMetaTableName(r.getRegionInfo().getTable())) {
+          r.compact(true);
+          for (HStore store : r.getStores()) {
+            store.closeAndArchiveCompactedFiles();
+          }
+        }
+      }
+    }
+    // merge the 2 regions back to 1
+    admin.mergeRegionsAsync(regions.stream().map(RegionInfo::getRegionName).toArray(byte[][]::new),
+      false).get();
+    assertMetaRegionCount(1);
+    // clear the cache for table 'a' and 'b'
+    clearCache(TD1.getTableName());
+    clearCache(TD2.getTableName());
+
+    // make sure that we could still get the locations from the new meta region
+    assertValue(TD2.getTableName(), "row2");
+    assertValue(TD1.getTableName(), "row1");
   }
 }