You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2021/01/28 17:34:06 UTC
[hbase] branch master updated: HBASE-25528: Dedicated merge
dispatch threadpool on master (#2904)
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new de51a40 HBASE-25528: Dedicated merge dispatch threadpool on master (#2904)
de51a40 is described below
commit de51a40b53ec16d1847537211f4e9b1e2808ac66
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Thu Jan 28 09:33:35 2021 -0800
HBASE-25528: Dedicated merge dispatch threadpool on master (#2904)
Adds "hbase.master.executor.merge.dispatch.threads" and defaults to 2.
Also adds additional logging that includes the number of split plans
and merge plans computed for each normalizer run.
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../src/main/java/org/apache/hadoop/hbase/HConstants.java | 7 +++++++
.../java/org/apache/hadoop/hbase/executor/EventType.java | 2 +-
.../java/org/apache/hadoop/hbase/executor/ExecutorType.java | 1 +
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 3 +++
.../hbase/master/normalizer/SimpleRegionNormalizer.java | 13 ++++++++++---
5 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index d31cadd..48fa00c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1605,6 +1605,13 @@ public final class HConstants {
"hbase.master.executor.serverops.threads";
public static final int MASTER_SERVER_OPERATIONS_THREADS_DEFAULT = 5;
+ /**
+ * Number of threads used to dispatch merge operations to the regionservers.
+ */
+ public static final String MASTER_MERGE_DISPATCH_THREADS =
+ "hbase.master.executor.merge.dispatch.threads";
+ public static final int MASTER_MERGE_DISPATCH_THREADS_DEFAULT = 2;
+
public static final String MASTER_META_SERVER_OPERATIONS_THREADS =
"hbase.master.executor.meta.serverops.threads";
public static final int MASTER_META_SERVER_OPERATIONS_THREADS_DEFAULT = 5;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index a674479..600c96c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -152,7 +152,7 @@ public enum EventType {
* C_M_MERGE_REGION<br>
* Client asking Master to merge regions.
*/
- C_M_MERGE_REGION (30, ExecutorType.MASTER_TABLE_OPERATIONS),
+ C_M_MERGE_REGION (30, ExecutorType.MASTER_MERGE_OPERATIONS),
/**
* Messages originating from Client to Master.<br>
* C_M_DELETE_TABLE<br>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index d06bd54..36958c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -35,6 +35,7 @@ public enum ExecutorType {
MASTER_META_SERVER_OPERATIONS (6),
M_LOG_REPLAY_OPS (7),
MASTER_SNAPSHOT_OPERATIONS (8),
+ MASTER_MERGE_OPERATIONS (9),
// RegionServer executor services
RS_OPEN_REGION (20),
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 94f3bf2..9911f01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1324,6 +1324,9 @@ public class HMaster extends HRegionServer implements MasterServices {
HConstants.MASTER_LOG_REPLAY_OPS_THREADS, HConstants.MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT));
this.executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, conf.getInt(
SnapshotManager.SNAPSHOT_POOL_THREADS_KEY, SnapshotManager.SNAPSHOT_POOL_THREADS_DEFAULT));
+ this.executorService.startExecutorService(ExecutorType.MASTER_MERGE_OPERATIONS, conf.getInt(
+ HConstants.MASTER_MERGE_DISPATCH_THREADS,
+ HConstants.MASTER_MERGE_DISPATCH_THREADS_DEFAULT));
// We depend on there being only one instance of this executor running
// at a time. To do concurrency, would need fencing of enable/disable of
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 61bc922..5245568 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -209,14 +209,21 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
ctx.getTableRegions().size());
final List<NormalizationPlan> plans = new ArrayList<>();
+ int splitPlansCount = 0;
if (proceedWithSplitPlanning) {
- plans.addAll(computeSplitNormalizationPlans(ctx));
+ List<NormalizationPlan> splitPlans = computeSplitNormalizationPlans(ctx);
+ splitPlansCount = splitPlans.size();
+ plans.addAll(splitPlans);
}
+ int mergePlansCount = 0;
if (proceedWithMergePlanning) {
- plans.addAll(computeMergeNormalizationPlans(ctx));
+ List<NormalizationPlan> mergePlans = computeMergeNormalizationPlans(ctx);
+ mergePlansCount = mergePlans.size();
+ plans.addAll(mergePlans);
}
- LOG.debug("Computed {} normalization plans for table {}", plans.size(), table);
+ LOG.debug("Computed normalization plans for table {}. Total plans: {}, split plans: {}, " +
+ "merge plans: {}", table, plans.size(), splitPlansCount, mergePlansCount);
return plans;
}