You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/03/08 00:38:08 UTC
svn commit: r1079026 - in /hbase/branches/0.90: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
Author: stack
Date: Mon Mar 7 23:38:08 2011
New Revision: 1079026
URL: http://svn.apache.org/viewvc?rev=1079026&view=rev
Log:
HBASE-3586 Improve the selection of regions to balance
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1079026&r1=1079025&r2=1079026&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Mon Mar 7 23:38:08 2011
@@ -27,6 +27,7 @@ Release 0.90.2 - February 9th, 2011
HBASE-3603 Remove -XX:+HeapDumpOnOutOfMemoryError autodump of heap option
on OOME
HBASE-3285 Hlog recovery takes too much time
+ HBASE-3586 Improve the selection of regions to balance
Release 0.90.1 - February 9th, 2011
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java?rev=1079026&r1=1079025&r2=1079026&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java Mon Mar 7 23:38:08 2011
@@ -58,7 +58,7 @@ import org.apache.hadoop.hbase.HServerIn
*/
public class LoadBalancer {
private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
- private static final Random rand = new Random();
+ private static final Random RANDOM = new Random(System.currentTimeMillis());
static class RegionPlanComparator implements Comparator<RegionPlan> {
@Override
@@ -195,7 +195,7 @@ public class LoadBalancer {
break;
}
serversOverloaded++;
- List<HRegionInfo> regions = server.getValue();
+ List<HRegionInfo> regions = randomize(server.getValue());
int numToOffload = Math.min(regionCount - max, regions.size());
int numTaken = 0;
for (int i = regions.size() - 1; i >= 0; i--) {
@@ -209,8 +209,6 @@ public class LoadBalancer {
serverBalanceInfo.put(serverInfo,
new BalanceInfo(numToOffload, (-1)*numTaken));
}
- // put young regions at the beginning of regionsToMove
- Collections.sort(regionsToMove, rpComparator);
// Walk down least loaded, filling each to the min
int serversUnderloaded = 0; // number of servers that get new regions
@@ -336,6 +334,15 @@ public class LoadBalancer {
}
/**
+ * @param regions
+ * @return Randomization of passed <code>regions</code>
+ */
+ static List<HRegionInfo> randomize(final List<HRegionInfo> regions) {
+ Collections.shuffle(regions, RANDOM);
+ return regions;
+ }
+
+ /**
* Stores additional per-server information about the regions added/removed
* during the run of the balancing algorithm.
*
@@ -396,7 +403,7 @@ public class LoadBalancer {
int max = (int)Math.ceil((float)numRegions/numServers);
int serverIdx = 0;
if (numServers > 1) {
- serverIdx = rand.nextInt(numServers);
+ serverIdx = RANDOM.nextInt(numServers);
}
int regionIdx = 0;
for (int j = 0; j < numServers; j++) {
@@ -444,7 +451,7 @@ public class LoadBalancer {
if (server != null) {
assignments.get(server).add(region.getKey());
} else {
- assignments.get(servers.get(rand.nextInt(assignments.size()))).add(
+ assignments.get(servers.get(RANDOM.nextInt(assignments.size()))).add(
region.getKey());
}
}
@@ -575,7 +582,7 @@ public class LoadBalancer {
Map<HRegionInfo,HServerInfo> assignments =
new TreeMap<HRegionInfo,HServerInfo>();
for(HRegionInfo region : regions) {
- assignments.put(region, servers.get(rand.nextInt(servers.size())));
+ assignments.put(region, servers.get(RANDOM.nextInt(servers.size())));
}
return assignments;
}
@@ -585,7 +592,7 @@ public class LoadBalancer {
LOG.warn("Wanted to do random assignment but no servers to assign to");
return null;
}
- return servers.get(rand.nextInt(servers.size()));
+ return servers.get(RANDOM.nextInt(servers.size()));
}
/**
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java?rev=1079026&r1=1079025&r2=1079026&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java Mon Mar 7 23:38:08 2011
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -34,6 +35,8 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -42,6 +45,7 @@ import org.apache.hadoop.hbase.HServerIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tools.ant.taskdefs.PathConvert.MapEntry;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -134,6 +138,38 @@ public class TestLoadBalancer {
new int [] { 12, 100 },
};
+ @Test
+ public void testRandomizer() {
+ for(int [] mockCluster : clusterStateMocks) {
+ if (mockCluster.length < 5) continue;
+ Map<HServerInfo, List<HRegionInfo>> servers =
+ mockClusterServers(mockCluster);
+ for (Map.Entry<HServerInfo, List<HRegionInfo>> e: servers.entrySet()) {
+ List<HRegionInfo> original = e.getValue();
+ if (original.size() < 5) continue;
+ // Try ten times in case random chances upon original order more than
+ // one or two times in a row.
+ boolean same = true;
+ for (int i = 0; i < 10 && same; i++) {
+ List<HRegionInfo> copy = new ArrayList<HRegionInfo>(original);
+ System.out.println("Randomizing before " + copy.size());
+ for (HRegionInfo hri: copy) {
+ System.out.println(hri.getEncodedName());
+ }
+ List<HRegionInfo> randomized = LoadBalancer.randomize(copy);
+ System.out.println("Randomizing after " + randomized.size());
+ for (HRegionInfo hri: randomized) {
+ System.out.println(hri.getEncodedName());
+ }
+ if (original.equals(randomized)) continue;
+ same = false;
+ break;
+ }
+ assertFalse(same);
+ }
+ }
+ }
+
/**
* Test the load balancing algorithm.
*
@@ -410,7 +446,7 @@ public class TestLoadBalancer {
Bytes.putInt(start, 0, numRegions << 1);
Bytes.putInt(end, 0, (numRegions << 1) + 1);
HRegionInfo hri = new HRegionInfo(
- new HTableDescriptor(Bytes.toBytes("table")), start, end);
+ new HTableDescriptor(Bytes.toBytes("table" + i)), start, end);
regions.add(hri);
}
return regions;