You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zg...@apache.org on 2019/07/04 01:20:23 UTC
[hbase] branch branch-2 updated: HBASE-22403 Balance in RSGroup
should consider throttling and a failure affects the whole
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 5087fdd HBASE-22403 Balance in RSGroup should consider throttling and a failure affects the whole
5087fdd is described below
commit 5087fddedf3f2a9d772bad5951861a737d95a852
Author: haxiaolin <li...@gmail.com>
AuthorDate: Thu Jun 27 10:52:01 2019 +0800
HBASE-22403 Balance in RSGroup should consider throttling and a failure affects the whole
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../hadoop/hbase/rsgroup/RSGroupAdminServer.java | 10 +--
.../org/apache/hadoop/hbase/master/HMaster.java | 80 ++++++++++++----------
.../apache/hadoop/hbase/master/MasterServices.java | 8 +++
.../hbase/master/MockNoopMasterServices.java | 7 +-
4 files changed, 59 insertions(+), 46 deletions(-)
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
index 2dae853..8cc7ab7 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
@@ -409,7 +409,6 @@ public class RSGroupAdminServer implements RSGroupAdmin {
@Override
public boolean balanceRSGroup(String groupName) throws IOException {
ServerManager serverManager = master.getServerManager();
- AssignmentManager assignmentManager = master.getAssignmentManager();
LoadBalancer balancer = master.getLoadBalancer();
synchronized (balancer) {
@@ -447,16 +446,11 @@ public class RSGroupAdminServer implements RSGroupAdmin {
plans.addAll(partialPlans);
}
}
- long startTime = System.currentTimeMillis();
boolean balancerRan = !plans.isEmpty();
if (balancerRan) {
LOG.info("RSGroup balance {} starting with plan count: {}", groupName, plans.size());
- for (RegionPlan plan: plans) {
- LOG.info("balance {}", plan);
- assignmentManager.moveAsync(plan);
- }
- LOG.info("RSGroup balance {} completed after {} seconds", groupName,
- (System.currentTimeMillis() - startTime));
+ master.executeRegionPlansWithThrottling(plans);
+ LOG.info("RSGroup balance " + groupName + " completed");
}
return balancerRan;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index be01116..d450aba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1649,7 +1649,6 @@ public class HMaster extends HRegionServer implements MasterServices {
return false;
}
- int maxRegionsInTransition = getMaxRegionsInTransition();
synchronized (this.balancer) {
// If balance not true, don't run balancer.
if (!this.loadBalancerTracker.isBalancerOn()) return false;
@@ -1708,45 +1707,11 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
- long balanceStartTime = System.currentTimeMillis();
- long cutoffTime = balanceStartTime + this.maxBlancingTime;
- int rpCount = 0; // number of RegionPlans balanced so far
- if (plans != null && !plans.isEmpty()) {
- int balanceInterval = this.maxBlancingTime / plans.size();
- LOG.info("Balancer plans size is " + plans.size() + ", the balance interval is "
- + balanceInterval + " ms, and the max number regions in transition is "
- + maxRegionsInTransition);
-
- for (RegionPlan plan: plans) {
- LOG.info("balance " + plan);
- //TODO: bulk assign
- try {
- this.assignmentManager.moveAsync(plan);
- } catch (HBaseIOException hioe) {
- //should ignore failed plans here, avoiding the whole balance plans be aborted
- //later calls of balance() can fetch up the failed and skipped plans
- LOG.warn("Failed balance plan: {}, just skip it", plan, hioe);
- }
- //rpCount records balance plans processed, does not care if a plan succeeds
- rpCount++;
-
- balanceThrottling(balanceStartTime + rpCount * balanceInterval, maxRegionsInTransition,
- cutoffTime);
-
- // if performing next balance exceeds cutoff time, exit the loop
- if (rpCount < plans.size() && System.currentTimeMillis() > cutoffTime) {
- // TODO: After balance, there should not be a cutoff time (keeping it as
- // a security net for now)
- LOG.debug("No more balancing till next balance run; maxBalanceTime="
- + this.maxBlancingTime);
- break;
- }
- }
- }
+ List<RegionPlan> sucRPs = executeRegionPlansWithThrottling(plans);
if (this.cpHost != null) {
try {
- this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
+ this.cpHost.postBalance(sucRPs);
} catch (IOException ioe) {
// balancing already succeeded so don't change the result
LOG.error("Error invoking master coprocessor postBalance()", ioe);
@@ -1758,6 +1723,47 @@ public class HMaster extends HRegionServer implements MasterServices {
return true;
}
+ public List<RegionPlan> executeRegionPlansWithThrottling(List<RegionPlan> plans) {
+ List<RegionPlan> sucRPs = new ArrayList<>();
+ int maxRegionsInTransition = getMaxRegionsInTransition();
+ long balanceStartTime = System.currentTimeMillis();
+ long cutoffTime = balanceStartTime + this.maxBlancingTime;
+ int rpCount = 0; // number of RegionPlans balanced so far
+ if (plans != null && !plans.isEmpty()) {
+ int balanceInterval = this.maxBlancingTime / plans.size();
+ LOG.info("Balancer plans size is " + plans.size() + ", the balance interval is "
+ + balanceInterval + " ms, and the max number regions in transition is "
+ + maxRegionsInTransition);
+
+ for (RegionPlan plan: plans) {
+ LOG.info("balance " + plan);
+ //TODO: bulk assign
+ try {
+ this.assignmentManager.moveAsync(plan);
+ } catch (HBaseIOException hioe) {
+ //should ignore failed plans here, avoiding the whole balance plans be aborted
+ //later calls of balance() can fetch up the failed and skipped plans
+ LOG.warn("Failed balance plan: {}, just skip it", plan, hioe);
+ }
+ //rpCount records balance plans processed, does not care if a plan succeeds
+ rpCount++;
+
+ balanceThrottling(balanceStartTime + rpCount * balanceInterval, maxRegionsInTransition,
+ cutoffTime);
+
+ // if performing next balance exceeds cutoff time, exit the loop
+ if (rpCount < plans.size() && System.currentTimeMillis() > cutoffTime) {
+ // TODO: After balance, there should not be a cutoff time (keeping it as
+ // a security net for now)
+ LOG.debug("No more balancing till next balance run; maxBalanceTime="
+ + this.maxBlancingTime);
+ break;
+ }
+ }
+ }
+ return sucRPs;
+ }
+
@Override
@VisibleForTesting
public RegionNormalizer getRegionNormalizer() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index f9d437a..3524f63 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -514,4 +514,12 @@ public interface MasterServices extends Server {
* @return the {@link ZKPermissionWatcher}
*/
ZKPermissionWatcher getZKPermissionWatcher();
+
+ /**
+ * Execute region plans with throttling
+ * @param plans to execute
+ * @return succeeded plans
+ */
+ List<RegionPlan> executeRegionPlansWithThrottling(List<RegionPlan> plans);
+
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 951eae6..6b22201 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -474,4 +474,9 @@ public class MockNoopMasterServices implements MasterServices {
public ZKPermissionWatcher getZKPermissionWatcher() {
return null;
}
-}
\ No newline at end of file
+
+ @Override
+ public List<RegionPlan> executeRegionPlansWithThrottling(List<RegionPlan> plans) {
+ return null;
+ }
+}