You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/01/05 23:22:19 UTC
svn commit: r1227873 - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
main/java/org/apache/hadoop/hbase/master/HMaster.java
test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
Author: tedyu
Date: Thu Jan 5 22:22:19 2012
New Revision: 1227873
URL: http://svn.apache.org/viewvc?rev=1227873&view=rev
Log:
HBASE-3373 Allow regions to be load-balanced by table
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1227873&r1=1227872&r2=1227873&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Thu Jan 5 22:22:19 2012
@@ -49,6 +49,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.Chore;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
@@ -2782,6 +2783,55 @@ public class AssignmentManager extends Z
}
/**
+ * This is an EXPENSIVE clone. Cloning though is the safest thing to do.
+ * Can't let out original since it can change and at least the loadbalancer
+ * wants to iterate this exported list. We need to synchronize on regions
+ * since all access to this.servers is under a lock on this.regions.
+ *
+ * @return A clone of current assignments by table.
+ */
+ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
+ Map<String, Map<ServerName, List<HRegionInfo>>> result = null;
+ synchronized (this.regions) {
+ result = new HashMap<String, Map<ServerName,List<HRegionInfo>>>();
+ if (!this.master.getConfiguration().
+ getBoolean("hbase.master.loadbalance.bytable", true)) {
+ result.put("ensemble", getAssignments());
+ } else {
+ for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
+ for (HRegionInfo hri : e.getValue()) {
+ if (hri.isMetaRegion() || hri.isRootRegion()) continue;
+ String tablename = hri.getTableNameAsString();
+ Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
+ if (svrToRegions == null) {
+ svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(this.servers.size());
+ result.put(tablename, svrToRegions);
+ }
+ List<HRegionInfo> regions = null;
+ if (!svrToRegions.containsKey(e.getKey())) {
+ regions = new ArrayList<HRegionInfo>();
+ svrToRegions.put(e.getKey(), regions);
+ } else {
+ regions = svrToRegions.get(e.getKey());
+ }
+ regions.add(hri);
+ }
+ }
+ }
+ }
+ Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
+ // Take care of servers w/o assignments.
+ for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
+ for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
+ if (!map.containsKey(svrEntry.getKey())) {
+ map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
* @return A clone of current assignments. Note, this is assignments only.
* If a new server has come in and it has no regions, it will not be included
* in the returned Map.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1227873&r1=1227872&r2=1227873&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Jan 5 22:22:19 2012
@@ -945,16 +945,14 @@ implements HMasterInterface, HMasterRegi
}
}
- Map<ServerName, List<HRegionInfo>> assignments =
- this.assignmentManager.getAssignments();
- // Returned Map from AM does not include mention of servers w/o assignments.
- for (Map.Entry<ServerName, HServerLoad> e:
- this.serverManager.getOnlineServers().entrySet()) {
- if (!assignments.containsKey(e.getKey())) {
- assignments.put(e.getKey(), new ArrayList<HRegionInfo>());
- }
+ Map<String, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
+ this.assignmentManager.getAssignmentsByTable();
+
+ List<RegionPlan> plans = new ArrayList<RegionPlan>();
+ for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
+ List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
+ if (partialPlans != null) plans.addAll(partialPlans);
}
- List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
int rpCount = 0; // number of RegionPlans balanced so far
long totalRegPlanExecTime = 0;
balancerRan = plans != null;
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=1227873&r1=1227872&r2=1227873&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Thu Jan 5 22:22:19 2012
@@ -169,9 +169,16 @@ public class TestRegionRebalancing {
LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
&& serverLoad >= avgLoadMinusSlop)) {
- LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
- " actual: " + serverLoad + " slop: " + slop);
- success = false;
+ for (HRegionInfo hri : server.getOnlineRegions()) {
+ if (hri.isMetaRegion() || hri.isRootRegion()) serverLoad--;
+ // LOG.debug(hri.getRegionNameAsString());
+ }
+ if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) {
+ LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
+ " actual: " + serverLoad + " slop: " + slop);
+ success = false;
+ break;
+ }
}
}