You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zg...@apache.org on 2020/05/26 09:03:28 UTC
[hbase] branch master updated: HBASE-24416 RegionNormalizer
spliting region should not be limited by hbase.normalizer.min.region.count
(#1770)
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new e06e1db HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770)
e06e1db is described below
commit e06e1dba100802a710de32fe1532779f740ddfa8
Author: XinSun <dd...@gmail.com>
AuthorDate: Tue May 26 17:03:14 2020 +0800
HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../normalizer/AbstractRegionNormalizer.java | 10 +++++
.../hbase/master/normalizer/MergeNormalizer.java | 10 ++---
.../master/normalizer/SimpleRegionNormalizer.java | 25 ++++++-------
.../normalizer/TestSimpleRegionNormalizer.java | 43 ++++++++++++++++++++++
4 files changed, 69 insertions(+), 19 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java
index 8b17f77..c79c1d4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Size;
@@ -42,8 +43,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
@InterfaceAudience.Private
public abstract class AbstractRegionNormalizer implements RegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class);
+
+ public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY =
+ "hbase.normalizer.min.region.count";
+ public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3;
+
protected MasterServices masterServices;
protected MasterRpcServices masterRpcServices;
+ protected int minRegionCount;
/**
* Set the master service.
@@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer {
@Override
public void setMasterServices(MasterServices masterServices) {
this.masterServices = masterServices;
+ minRegionCount = masterServices.getConfiguration().getInt(
+ HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY,
+ HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT);
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java
index 444c27c..f006103 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java
@@ -23,10 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,13 +63,13 @@ import org.slf4j.LoggerFactory;
public class MergeNormalizer extends AbstractRegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class);
- private int minRegionCount;
private int minRegionAge;
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
- public MergeNormalizer() {
- Configuration conf = HBaseConfiguration.create();
- minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3);
+ @Override
+ public void setMasterServices(MasterServices masterServices) {
+ super.setMasterServices(masterServices);
+ Configuration conf = masterServices.getConfiguration();
minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index bd90f5b..b68c068 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import org.apache.hadoop.hbase.HBaseConfiguration;
+
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -51,13 +51,8 @@ import org.slf4j.LoggerFactory;
public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class);
- private int minRegionCount;
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
- public SimpleRegionNormalizer() {
- minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3);
- }
-
@Override
public void planSkipped(RegionInfo hri, PlanType type) {
skippedCount[type.ordinal()]++;
@@ -112,11 +107,7 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
List<RegionInfo> tableRegions =
masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table);
- if (tableRegions == null || tableRegions.size() < minRegionCount) {
- int nrRegions = tableRegions == null ? 0 : tableRegions.size();
- LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is "
- + "{}, not running normalizer",
- table, nrRegions, minRegionCount);
+ if (tableRegions == null) {
return null;
}
@@ -131,9 +122,15 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
}
if (mergeEnabled) {
- List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
- if (mergePlans != null) {
- plans.addAll(mergePlans);
+ if (tableRegions.size() < minRegionCount) {
+ LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" +
+ " is {}, not running normalizer",
+ table, tableRegions.size(), minRegionCount);
+ } else {
+ List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
+ if (mergePlans != null) {
+ plans.addAll(mergePlans);
+ }
}
}
if (plans.isEmpty()) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index a05b3a8..5c1c4e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.normalizer;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -30,7 +31,10 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
@@ -526,6 +530,45 @@ public class TestSimpleRegionNormalizer {
assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
}
+ @Test
+ public void testSplitIfTooFewRegions() throws HBaseIOException {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ List<RegionInfo> RegionInfo = new ArrayList<>();
+ Map<byte[], Integer> regionSizes = new HashMap<>();
+
+ RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("aaa"))
+ .setEndKey(Bytes.toBytes("bbb"))
+ .build();
+ RegionInfo.add(hri1);
+ regionSizes.put(hri1.getRegionName(), 1);
+
+ RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("bbb"))
+ .setEndKey(Bytes.toBytes("ccc"))
+ .build();
+ RegionInfo.add(hri2);
+ regionSizes.put(hri2.getRegionName(), 1);
+ // the third region is huge one
+ RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("ccc"))
+ .setEndKey(Bytes.toBytes("ddd"))
+ .build();
+ RegionInfo.add(hri3);
+ regionSizes.put(hri3.getRegionName(), 10);
+
+ setupMocksForNormalizer(regionSizes, RegionInfo);
+
+ Configuration configuration = HBaseConfiguration.create();
+ configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4);
+ when(masterServices.getConfiguration()).thenReturn(configuration);
+
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
+ assertNotNull(plans);
+ NormalizationPlan plan = plans.get(0);
+ assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo());
+ }
+
@SuppressWarnings("MockitoCast")
private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
List<RegionInfo> RegionInfo) {