You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2021/01/29 00:09:56 UTC
[hbase] branch branch-2 updated: HBASE-25528: Dedicated merge
dispatch threadpool on master (#2910)
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new b930ea1 HBASE-25528: Dedicated merge dispatch threadpool on master (#2910)
b930ea1 is described below
commit b930ea1ebbfc9a4e741c86502548bb10d8002652
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Thu Jan 28 16:09:18 2021 -0800
HBASE-25528: Dedicated merge dispatch threadpool on master (#2910)
Adds "hbase.master.executor.merge.dispatch.threads" and defaults to 2.
Also adds additional logging that includes the number of split plans
and merge plans computed for each normalizer run.
(cherry picked from commit 36b4698cea9485f4d491612342b8c841e1c3db86)
Signed-off-by: Viraj Jasani <vj...@apache.org>
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
Signed-off-by: Andrew Purtell <ap...@apache.org>
---
.../src/main/java/org/apache/hadoop/hbase/HConstants.java | 7 +++++++
.../java/org/apache/hadoop/hbase/executor/EventType.java | 2 +-
.../java/org/apache/hadoop/hbase/executor/ExecutorType.java | 1 +
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 3 +++
.../hbase/master/normalizer/SimpleRegionNormalizer.java | 13 ++++++++++---
5 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 1c5363b..0165778 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1628,6 +1628,13 @@ public final class HConstants {
"hbase.master.executor.serverops.threads";
public static final int MASTER_SERVER_OPERATIONS_THREADS_DEFAULT = 5;
+ /**
+ * Number of threads used to dispatch merge operations to the regionservers.
+ */
+ public static final String MASTER_MERGE_DISPATCH_THREADS =
+ "hbase.master.executor.merge.dispatch.threads";
+ public static final int MASTER_MERGE_DISPATCH_THREADS_DEFAULT = 2;
+
public static final String MASTER_META_SERVER_OPERATIONS_THREADS =
"hbase.master.executor.meta.serverops.threads";
public static final int MASTER_META_SERVER_OPERATIONS_THREADS_DEFAULT = 5;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index 80c2717..32993f5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -152,7 +152,7 @@ public enum EventType {
* C_M_MERGE_REGION<br>
* Client asking Master to merge regions.
*/
- C_M_MERGE_REGION (30, ExecutorType.MASTER_TABLE_OPERATIONS),
+ C_M_MERGE_REGION (30, ExecutorType.MASTER_MERGE_OPERATIONS),
/**
* Messages originating from Client to Master.<br>
* C_M_DELETE_TABLE<br>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index 66baccb..ca82d71 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -35,6 +35,7 @@ public enum ExecutorType {
MASTER_META_SERVER_OPERATIONS (6),
M_LOG_REPLAY_OPS (7),
MASTER_SNAPSHOT_OPERATIONS (8),
+ MASTER_MERGE_OPERATIONS (9),
// RegionServer executor services
RS_OPEN_REGION (20),
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 5538aed..9fa6653 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1307,6 +1307,9 @@ public class HMaster extends HRegionServer implements MasterServices {
HConstants.MASTER_LOG_REPLAY_OPS_THREADS, HConstants.MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT));
this.executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, conf.getInt(
SnapshotManager.SNAPSHOT_POOL_THREADS_KEY, SnapshotManager.SNAPSHOT_POOL_THREADS_DEFAULT));
+ this.executorService.startExecutorService(ExecutorType.MASTER_MERGE_OPERATIONS, conf.getInt(
+ HConstants.MASTER_MERGE_DISPATCH_THREADS,
+ HConstants.MASTER_MERGE_DISPATCH_THREADS_DEFAULT));
// We depend on there being only one instance of this executor running
// at a time. To do concurrency, would need fencing of enable/disable of
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index cea4b32..9457b02 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -209,14 +209,21 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
ctx.getTableRegions().size());
final List<NormalizationPlan> plans = new ArrayList<>();
+ int splitPlansCount = 0;
if (proceedWithSplitPlanning) {
- plans.addAll(computeSplitNormalizationPlans(ctx));
+ List<NormalizationPlan> splitPlans = computeSplitNormalizationPlans(ctx);
+ splitPlansCount = splitPlans.size();
+ plans.addAll(splitPlans);
}
+ int mergePlansCount = 0;
if (proceedWithMergePlanning) {
- plans.addAll(computeMergeNormalizationPlans(ctx));
+ List<NormalizationPlan> mergePlans = computeMergeNormalizationPlans(ctx);
+ mergePlansCount = mergePlans.size();
+ plans.addAll(mergePlans);
}
- LOG.debug("Computed {} normalization plans for table {}", plans.size(), table);
+ LOG.debug("Computed normalization plans for table {}. Total plans: {}, split plans: {}, " +
+ "merge plans: {}", table, plans.size(), splitPlansCount, mergePlansCount);
return plans;
}