You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ya...@apache.org on 2016/11/30 10:04:01 UTC
hbase git commit: HBASE-17178 Add region balance throttling
Repository: hbase
Updated Branches:
refs/heads/master 890fcbd0e -> ea912478e
HBASE-17178 Add region balance throttling
Signed-off-by: Phil Yang <ya...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ea912478
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ea912478
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ea912478
Branch: refs/heads/master
Commit: ea912478e28f1f041a6058c09a9d9eab64e01352
Parents: 890fcbd
Author: Guanghao Zhang <zg...@gmail.com>
Authored: Mon Nov 28 19:12:39 2016 +0800
Committer: Phil Yang <ya...@apache.org>
Committed: Wed Nov 30 18:03:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 16 ++
.../src/main/resources/hbase-default.xml | 8 +
.../org/apache/hadoop/hbase/master/HMaster.java | 99 +++++++++---
.../hadoop/hbase/master/RegionStates.java | 11 +-
.../hbase/master/balancer/BalancerChore.java | 3 +-
.../master/TestMasterBalanceThrottling.java | 151 +++++++++++++++++++
.../TestSplitTransactionOnCluster.java | 2 +-
7 files changed, 265 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3f9b430..08be194 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -129,6 +129,22 @@ public final class HConstants {
/** Config for balancing the cluster by table */
public static final String HBASE_MASTER_LOADBALANCE_BYTABLE = "hbase.master.loadbalance.bytable";
+ /** Config for the max percent of regions in transition */
+ public static final String HBASE_MASTER_BALANCER_MAX_RIT_PERCENT =
+ "hbase.master.balancer.maxRitPercent";
+
+ /** Default value for the max percent of regions in transition */
+ public static final double DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT = 1.0;
+
+ /** Config for the max balancing time */
+ public static final String HBASE_BALANCER_MAX_BALANCING = "hbase.balancer.max.balancing";
+
+ /** Config for the balancer period */
+ public static final String HBASE_BALANCER_PERIOD = "hbase.balancer.period";
+
+ /** Default value for the balancer period */
+ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 300000;
+
/** The name of the ensemble table */
public static final String ENSEMBLE_TABLE_NAME = "hbase:ensemble";
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 7b33f92..d8baa00 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -542,6 +542,14 @@ possible configurations would overwhelm and obscure the important.
0 means never give up.</description>
</property>
<property>
+ <name>hbase.master.balancer.maxRitPercent</name>
+ <value>1.0</value>
+ <description>The max percent of regions in transition when balancing.
+ The default value is 1.0. So there are no balancer throttling. If set this config to 0.01,
+ It means that there are at most 1% regions in transition when balancing.
+ Then the cluster's availability is at least 99% when balancing.</description>
+ </property>
+ <property>
<name>hbase.balancer.period
</name>
<value>300000</value>
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0136ff5..7be1282 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -302,6 +302,11 @@ public class HMaster extends HRegionServer implements MasterServices {
private final ProcedureEvent serverCrashProcessingEnabled =
new ProcedureEvent("server crash processing");
+ // Maximum time we should run balancer for
+ private final int maxBlancingTime;
+ // Maximum percent of regions in transition when balancing
+ private final double maxRitPercent;
+
private LoadBalancer balancer;
private RegionNormalizer normalizer;
private BalancerChore balancerChore;
@@ -431,6 +436,10 @@ public class HMaster extends HRegionServer implements MasterServices {
// preload table descriptor at startup
this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
+ this.maxBlancingTime = getMaxBalancingTime();
+ this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
+ HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
+
// Do we publish the status?
boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
@@ -1114,14 +1123,60 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* @return Maximum time we should run balancer for
*/
- private int getBalancerCutoffTime() {
- int balancerCutoffTime = getConfiguration().getInt("hbase.balancer.max.balancing", -1);
- if (balancerCutoffTime == -1) {
- // if cutoff time isn't set, defaulting it to period time
- int balancerPeriod = getConfiguration().getInt("hbase.balancer.period", 300000);
- balancerCutoffTime = balancerPeriod;
+ private int getMaxBalancingTime() {
+ int maxBalancingTime = getConfiguration().getInt(HConstants.HBASE_BALANCER_MAX_BALANCING, -1);
+ if (maxBalancingTime == -1) {
+ // if max balancing time isn't set, defaulting it to period time
+ maxBalancingTime = getConfiguration().getInt(HConstants.HBASE_BALANCER_PERIOD,
+ HConstants.DEFAULT_HBASE_BALANCER_PERIOD);
+ }
+ return maxBalancingTime;
+ }
+
+ /**
+ * @return Maximum number of regions in transition
+ */
+ private int getMaxRegionsInTransition() {
+ int numRegions = this.assignmentManager.getRegionStates().getRegionAssignments().size();
+ return Math.max((int) Math.floor(numRegions * this.maxRitPercent), 1);
+ }
+
+ /**
+ * It first sleep to the next balance plan start time. Meanwhile, throttling by the max
+ * number regions in transition to protect availability.
+ * @param nextBalanceStartTime The next balance plan start time
+ * @param maxRegionsInTransition max number of regions in transition
+ * @param cutoffTime when to exit balancer
+ */
+ private void balanceThrottling(long nextBalanceStartTime, int maxRegionsInTransition,
+ long cutoffTime) {
+ boolean interrupted = false;
+
+ // Sleep to next balance plan start time
+ // But if there are zero regions in transition, it can skip sleep to speed up.
+ while (!interrupted && System.currentTimeMillis() < nextBalanceStartTime
+ && this.assignmentManager.getRegionStates().getRegionsInTransitionCount() != 0) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ie) {
+ interrupted = true;
+ }
+ }
+
+ // Throttling by max number regions in transition
+ while (!interrupted
+ && maxRegionsInTransition > 0
+ && this.assignmentManager.getRegionStates().getRegionsInTransitionCount()
+ >= maxRegionsInTransition && System.currentTimeMillis() <= cutoffTime) {
+ try {
+ // sleep if the number of regions in transition exceeds the limit
+ Thread.sleep(100);
+ } catch (InterruptedException ie) {
+ interrupted = true;
+ }
}
- return balancerCutoffTime;
+
+ if (interrupted) Thread.currentThread().interrupt();
}
public boolean balance() throws IOException {
@@ -1140,8 +1195,7 @@ public class HMaster extends HRegionServer implements MasterServices {
return false;
}
- // Do this call outside of synchronized block.
- int maximumBalanceTime = getBalancerCutoffTime();
+ int maxRegionsInTransition = getMaxRegionsInTransition();
synchronized (this.balancer) {
// If balance not true, don't run balancer.
if (!this.loadBalancerTracker.isBalancerOn()) return false;
@@ -1188,28 +1242,35 @@ public class HMaster extends HRegionServer implements MasterServices {
if (partialPlans != null) plans.addAll(partialPlans);
}
- long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
+ long balanceStartTime = System.currentTimeMillis();
+ long cutoffTime = balanceStartTime + this.maxBlancingTime;
int rpCount = 0; // number of RegionPlans balanced so far
- long totalRegPlanExecTime = 0;
if (plans != null && !plans.isEmpty()) {
+ int balanceInterval = this.maxBlancingTime / plans.size();
+ LOG.info("Balancer plans size is " + plans.size() + ", the balance interval is "
+ + balanceInterval + " ms, and the max number regions in transition is "
+ + maxRegionsInTransition);
+
for (RegionPlan plan: plans) {
LOG.info("balance " + plan);
- long balStartTime = System.currentTimeMillis();
//TODO: bulk assign
this.assignmentManager.balance(plan);
- totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
rpCount++;
- if (rpCount < plans.size() &&
- // if performing next balance exceeds cutoff time, exit the loop
- (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
- //TODO: After balance, there should not be a cutoff time (keeping it as
+
+ balanceThrottling(balanceStartTime + rpCount * balanceInterval, maxRegionsInTransition,
+ cutoffTime);
+
+ // if performing next balance exceeds cutoff time, exit the loop
+ if (rpCount < plans.size() && System.currentTimeMillis() > cutoffTime) {
+ // TODO: After balance, there should not be a cutoff time (keeping it as
// a security net for now)
- LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
- maximumBalanceTime);
+ LOG.debug("No more balancing till next balance run; maxBalanceTime="
+ + this.maxBlancingTime);
break;
}
}
}
+
if (this.cpHost != null) {
try {
this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index f69925a..fbc5c68 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -20,13 +20,10 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -51,7 +48,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.constraint.ConstraintException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -236,6 +232,13 @@ public class RegionStates {
}
/**
+ * Get the number of regions in transition.
+ */
+ public synchronized int getRegionsInTransitionCount() {
+ return regionsInTransition.size();
+ }
+
+ /**
* @return True if specified region in transition.
*/
public synchronized boolean isRegionInTransition(final HRegionInfo hri) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java
index bbbfdf2..37004ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.HMaster;
@@ -38,7 +39,7 @@ public class BalancerChore extends ScheduledChore {
public BalancerChore(HMaster master) {
super(master.getServerName() + "-BalancerChore", master, master.getConfiguration().getInt(
- "hbase.balancer.period", 300000));
+ HConstants.HBASE_BALANCER_PERIOD, HConstants.DEFAULT_HBASE_BALANCER_PERIOD));
this.master = master;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
new file mode 100644
index 0000000..74f2c91
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
@@ -0,0 +1,151 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestMasterBalanceThrottling {
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
+
+ @Before
+ public void setupConfiguration() {
+ TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+ "org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer");
+ }
+
+ @After
+ public void shutdown() throws Exception {
+ TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_MAX_BALANCING,
+ HConstants.DEFAULT_HBASE_BALANCER_PERIOD);
+ TEST_UTIL.getConfiguration().setDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
+ HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test(timeout = 60000)
+ public void testThrottlingByBalanceInterval() throws Exception {
+ // Use default config and start a cluster of two regionservers.
+ TEST_UTIL.startMiniCluster(2);
+
+ TableName tableName = createTable("testNoThrottling");
+ final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
+
+ // Default max balancing time is 300000 ms and there are 50 regions to balance
+ // The balance interval is 6000 ms, much longger than the normal region in transition duration
+ // So the master can balance the region one by one
+ unbalance(master, tableName);
+ AtomicInteger maxCount = new AtomicInteger(0);
+ AtomicBoolean stop = new AtomicBoolean(false);
+ Thread checker = startBalancerChecker(master, maxCount, stop);
+ master.balance();
+ stop.set(true);
+ checker.interrupt();
+ checker.join();
+ assertTrue("max regions in transition: " + maxCount.get(), maxCount.get() == 1);
+
+ TEST_UTIL.deleteTable(tableName);
+ }
+
+ @Test(timeout = 60000)
+ public void testThrottlingByMaxRitPercent() throws Exception {
+ // Set max balancing time to 500 ms and max percent of regions in transition to 0.05
+ TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_MAX_BALANCING, 500);
+ TEST_UTIL.getConfiguration().setDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT, 0.05);
+ TEST_UTIL.startMiniCluster(2);
+
+ TableName tableName = createTable("testThrottlingByMaxRitPercent");
+ final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
+
+ unbalance(master, tableName);
+ AtomicInteger maxCount = new AtomicInteger(0);
+ AtomicBoolean stop = new AtomicBoolean(false);
+ Thread checker = startBalancerChecker(master, maxCount, stop);
+ master.balance();
+ stop.set(true);
+ checker.interrupt();
+ checker.join();
+ // The max number of regions in transition is 100 * 0.05 = 5
+ assertTrue("max regions in transition: " + maxCount.get(), maxCount.get() == 5);
+
+ TEST_UTIL.deleteTable(tableName);
+ }
+
+ private TableName createTable(String table) throws IOException {
+ TableName tableName = TableName.valueOf(table);
+ byte[] startKey = new byte[] { 0x00 };
+ byte[] stopKey = new byte[] { 0x7f };
+ TEST_UTIL.createTable(tableName, new byte[][] { FAMILYNAME }, 1, startKey, stopKey,
+ 100);
+ return tableName;
+ }
+
+ private Thread startBalancerChecker(final HMaster master, final AtomicInteger maxCount,
+ final AtomicBoolean stop) {
+ Runnable checker = new Runnable() {
+ @Override
+ public void run() {
+ while (!stop.get()) {
+ maxCount.set(Math.max(maxCount.get(), master.getAssignmentManager().getRegionStates()
+ .getRegionsInTransitionCount()));
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ };
+ Thread thread = new Thread(checker);
+ thread.start();
+ return thread;
+ }
+
+ private void unbalance(HMaster master, TableName tableName) throws Exception {
+ while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) {
+ Thread.sleep(100);
+ }
+ HRegionServer biasedServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+ for (HRegionInfo regionInfo : TEST_UTIL.getAdmin().getTableRegions(tableName)) {
+ master.move(regionInfo.getEncodedNameAsBytes(),
+ Bytes.toBytes(biasedServer.getServerName().getServerName()));
+ }
+ while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) {
+ Thread.sleep(100);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea912478/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 495da8f..364f355 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -118,7 +118,7 @@ public class TestSplitTransactionOnCluster {
new HBaseTestingUtility();
@BeforeClass public static void before() throws Exception {
- TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
+ TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000);
TESTING_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
}