You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2011/03/25 01:40:56 UTC
svn commit: r1085223 - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/master/ src/main/resources/
src/test/java/org/apache/hadoop/hbase/master/
Author: jdcryans
Date: Fri Mar 25 00:40:55 2011
New Revision: 1085223
URL: http://svn.apache.org/viewvc?rev=1085223&view=rev
Log:
HBASE-3681 Check the sloppiness of the region load before balancing (Ted Yu via JD)
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
hbase/trunk/src/main/resources/hbase-default.xml
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1085223&r1=1085222&r2=1085223&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Mar 25 00:40:55 2011
@@ -206,6 +206,7 @@ Release 0.90.2 - Unreleased
HBASE-3600 Update our jruby to 1.6.0
HBASE-3640 [replication] Transferring queues shouldn't be done inline with RS startup
HBASE-3658 Alert when heap is over committed (Subbu M Iyer via Stack)
+ HBASE-3681 Check the sloppiness of the region load before balancing (Ted Yu via JD)
Release 0.90.1 - February 9th, 2011
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1085223&r1=1085222&r2=1085223&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri Mar 25 00:40:55 2011
@@ -163,7 +163,7 @@ implements HMasterInterface, HMasterRegi
// Instance of the hbase executor service.
ExecutorService executorService;
- private LoadBalancer balancer = new LoadBalancer();
+ private LoadBalancer balancer;
private Thread balancerChore;
// If 'true', the balancer is 'on'. If 'false', the balancer will not run.
private volatile boolean balanceSwitch = true;
@@ -358,6 +358,7 @@ implements HMasterInterface, HMasterRegi
this.assignmentManager = new AssignmentManager(this, serverManager,
this.catalogTracker, this.executorService);
+ this.balancer = new LoadBalancer(conf);
zooKeeper.registerListenerFirst(assignmentManager);
this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java?rev=1085223&r1=1085222&r2=1085223&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java Fri Mar 25 00:40:55 2011
@@ -32,6 +32,7 @@ import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -59,7 +60,15 @@ import org.apache.hadoop.hbase.HServerIn
public class LoadBalancer {
private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
private static final Random RANDOM = new Random(System.currentTimeMillis());
+ // slop for regions
+ private float slop;
+ LoadBalancer(Configuration conf) {
+ this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2);
+ if (slop < 0) slop = 0;
+ else if (slop > 1) slop = 1;
+ }
+
static class RegionPlanComparator implements Comparator<RegionPlan> {
@Override
public int compare(RegionPlan l, RegionPlan r) {
@@ -165,10 +174,11 @@ public class LoadBalancer {
// Check if we even need to do any load balancing
float average = (float)numRegions / numServers; // for logging
- int min = numRegions / numServers;
- int max = numRegions % numServers == 0 ? min : min + 1;
- if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= max &&
- serversByLoad.firstKey().getLoad().getNumberOfRegions() >= min) {
+ // HBASE-3681 check sloppiness first
+ int floor = (int) Math.floor(average * (1 - slop));
+ int ceiling = (int) Math.ceil(average * (1 + slop));
+ if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= ceiling &&
+ serversByLoad.firstKey().getLoad().getNumberOfRegions() >= floor) {
// Skipped because no server outside (min,max) range
LOG.info("Skipping load balancing. servers=" + numServers + " " +
"regions=" + numRegions + " average=" + average + " " +
@@ -176,6 +186,8 @@ public class LoadBalancer {
" leastloaded=" + serversByLoad.firstKey().getLoad().getNumberOfRegions());
return null;
}
+ int min = numRegions / numServers;
+ int max = numRegions % numServers == 0 ? min : min + 1;
// Balance the cluster
// TODO: Look at data block locality or a more complex load to do this
Modified: hbase/trunk/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/resources/hbase-default.xml?rev=1085223&r1=1085222&r2=1085223&view=diff
==============================================================================
--- hbase/trunk/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/src/main/resources/hbase-default.xml Fri Mar 25 00:40:55 2011
@@ -282,6 +282,13 @@
</description>
</property>
<property>
+ <name>hbase.regions.slop</name>
+ <value>0.2</value>
+ <description>Rebalance if any regionserver has average + (average * slop) regions.
+ Default is 20% slop.
+ </description>
+ </property>
+ <property>
<name>hbase.master.logcleaner.ttl</name>
<value>600000</value>
<description>Maximum time a HLog can stay in the .oldlogdir directory,
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java?rev=1085223&r1=1085222&r2=1085223&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java Fri Mar 25 00:40:55 2011
@@ -39,6 +39,8 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
@@ -57,7 +59,9 @@ public class TestLoadBalancer {
@BeforeClass
public static void beforeAllTests() throws Exception {
- loadBalancer = new LoadBalancer();
+ Configuration conf = HBaseConfiguration.create();
+ conf.set("hbase.regions.slop", "0");
+ loadBalancer = new LoadBalancer(conf);
rand = new Random();
}