You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2023/09/21 08:38:18 UTC
[hbase] branch branch-3 updated: HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions property … (#5403)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 302d78eb4ab HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions property … (#5403)
302d78eb4ab is described below
commit 302d78eb4ab352175674d6f5f977985b12ea443b
Author: Rahul Kumar <rk...@gmail.com>
AuthorDate: Wed Sep 20 20:47:37 2023 +0530
HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions property … (#5403)
Authored-by: Rahul Kumar <ra...@rahulku-ltmdgmc.internal.salesforce.com>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
hbase-common/src/main/resources/hbase-default.xml | 6 +++
.../master/normalizer/SimpleRegionNormalizer.java | 55 +++++++++++++++++-----
.../normalizer/TestSimpleRegionNormalizer.java | 36 ++++++++++++++
3 files changed, 84 insertions(+), 13 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index d2ecef6bda3..61eb4a0059f 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -656,6 +656,12 @@ possible configurations would overwhelm and obscure the important.
<description>The minimum size for a region to be considered for a merge, in whole
MBs.</description>
</property>
+ <property>
+ <name>hbase.normalizer.merge.merge_request_max_number_of_regions</name>
+ <value>50</value>
+ <description>The maximum number of region count in a merge request for merge
+ normalization.</description>
+ </property>
<property>
<name>hbase.table.normalization.enabled</name>
<value>false</value>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index dfae394b75a..934b8de1b94 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -81,6 +81,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3;
static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb";
static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 0;
+ static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY =
+ "hbase.normalizer.merge.merge_request_max_number_of_regions";
+ static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100;
private MasterServices masterServices;
private NormalizerConfiguration normalizerConfiguration;
@@ -138,6 +141,16 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
return settledValue;
}
+ private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) {
+ final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+ DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT);
+ final long settledValue = Math.max(2, parsedValue);
+ if (parsedValue != settledValue) {
+ warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue);
+ }
+ return settledValue;
+ }
+
private static <T> void warnInvalidValue(final String key, final T parsedValue,
final T settledValue) {
LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue,
@@ -186,6 +199,10 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
return normalizerConfiguration.getMergeMinRegionSizeMb();
}
+ public long getMergeRequestMaxNumberOfRegionsCount() {
+ return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount();
+ }
+
@Override
public void setMasterServices(final MasterServices masterServices) {
this.masterServices = masterServices;
@@ -382,19 +399,21 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
break;
}
if (
- rangeMembers.isEmpty() // when there are no range members, seed the range with whatever
- // we have. this way we're prepared in case the next region is
- // 0-size.
- || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one
- // region and the size is 0,
- // seed the range with
- // whatever we have.
- || regionSizeMb == 0 // always add an empty region to the current range.
- || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
- ) { // add the current region
- // to the range when
- // there's capacity
- // remaining.
+ // when there are no range members, seed the range with whatever we have. this way we're
+ // prepared in case the next region is 0-size.
+ rangeMembers.isEmpty()
+ // when there is only one region and the size is 0, seed the range with whatever we
+ // have.
+ || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0)
+ // add an empty region to the current range only if it doesn't exceed max merge request
+ // region count
+ || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())
+ // add region if current range region size is less than avg region size of table
+ // and current range doesn't exceed max merge request region count
+ || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
+ && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()))
+ ) {
+ // add the current region to the range when there's capacity remaining.
rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb));
sumRangeMembersSizeMb += regionSizeMb;
continue;
@@ -502,6 +521,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
private final int mergeMinRegionCount;
private final Period mergeMinRegionAge;
private final long mergeMinRegionSizeMb;
+ private final long mergeRequestMaxNumberOfRegionsCount;
private final long cumulativePlansSizeLimitMb;
private NormalizerConfiguration() {
@@ -511,6 +531,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT;
mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
+ mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT;
cumulativePlansSizeLimitMb = DEFAULT_CUMULATIVE_SIZE_LIMIT_MB;
}
@@ -522,6 +543,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
mergeMinRegionCount = parseMergeMinRegionCount(conf);
mergeMinRegionAge = parseMergeMinRegionAge(conf);
mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
+ mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf);
cumulativePlansSizeLimitMb =
conf.getLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, DEFAULT_CUMULATIVE_SIZE_LIMIT_MB);
logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(),
@@ -534,6 +556,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge);
logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY,
currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb);
+ logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+ currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(),
+ mergeRequestMaxNumberOfRegionsCount);
}
public Configuration getConf() {
@@ -597,6 +622,10 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
return mergeMinRegionSizeMb;
}
+ public long getMergeRequestMaxNumberOfRegionsCount() {
+ return mergeRequestMaxNumberOfRegionsCount;
+ }
+
private long getCumulativePlansSizeLimitMb() {
return cumulativePlansSizeLimitMb;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index 5dba036bb70..902205c7463 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.M
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
+import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -503,6 +504,41 @@ public class TestSimpleRegionNormalizer {
assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
}
+ @Test
+ public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
+ conf.setBoolean(SPLIT_ENABLED_KEY, false);
+ conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+ conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+ conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
+ final TableName tableName = name.getTableName();
+ final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
+ final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+ assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+ List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+ assertThat(plans,
+ contains(
+ new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+ .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
+ new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
+ .addTarget(regionInfos.get(4), 0).build()));
+ }
+
+ @Test
+ public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
+ conf.setBoolean(SPLIT_ENABLED_KEY, false);
+ conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+ conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+ final TableName tableName = name.getTableName();
+ final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
+ final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+ assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+ List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+ assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+ .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
+ }
+
@Test
public void testMergeEmptyRegions0() {
conf.setBoolean(SPLIT_ENABLED_KEY, false);