You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/01/23 18:54:23 UTC
svn commit: r1234922 - in /hbase/branches/0.92: ./
src/main/java/org/apache/hadoop/hbase/master/
src/test/java/org/apache/hadoop/hbase/
Author: tedyu
Date: Mon Jan 23 17:54:23 2012
New Revision: 1234922
URL: http://svn.apache.org/viewvc?rev=1234922&view=rev
Log:
HBASE-5231 Backport HBASE-3373 (per-table load balancing) to 0.92
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1234922&r1=1234921&r2=1234922&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Mon Jan 23 17:54:23 2012
@@ -3,6 +3,7 @@ HBase Change Log
Release 0.92.1 - Unreleased
INCOMPATIBLE CHANGES
HBASE-5228 [REST] Rip out "transform" feature
+ HBASE-5231 Backport HBASE-3373 (per-table load balancing) to 0.92
BUG FIXES
HBASE-5176 AssignmentManager#getRegion: logging nit adds a redundant '+' (Karthik K)
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1234922&r1=1234921&r2=1234922&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Mon Jan 23 17:54:23 2012
@@ -49,6 +49,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.Chore;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
@@ -2775,6 +2776,55 @@ public class AssignmentManager extends Z
}
/**
+ * This is an EXPENSIVE clone. Cloning though is the safest thing to do.
+ * Can't let out original since it can change and at least the loadbalancer
+ * wants to iterate this exported list. We need to synchronize on regions
+ * since all access to this.servers is under a lock on this.regions.
+ *
+ * @return A clone of current assignments by table.
+ */
+ Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
+ Map<String, Map<ServerName, List<HRegionInfo>>> result = null;
+ synchronized (this.regions) {
+ result = new HashMap<String, Map<ServerName,List<HRegionInfo>>>();
+ if (!this.master.getConfiguration().
+ getBoolean("hbase.master.loadbalance.bytable", true)) {
+ result.put("ensemble", getAssignments());
+ } else {
+ for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
+ for (HRegionInfo hri : e.getValue()) {
+ if (hri.isMetaRegion() || hri.isRootRegion()) continue;
+ String tablename = hri.getTableNameAsString();
+ Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
+ if (svrToRegions == null) {
+ svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(this.servers.size());
+ result.put(tablename, svrToRegions);
+ }
+ List<HRegionInfo> regions = null;
+ if (!svrToRegions.containsKey(e.getKey())) {
+ regions = new ArrayList<HRegionInfo>();
+ svrToRegions.put(e.getKey(), regions);
+ } else {
+ regions = svrToRegions.get(e.getKey());
+ }
+ regions.add(hri);
+ }
+ }
+ }
+ }
+ Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
+ // Take care of servers w/o assignments.
+ for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
+ for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
+ if (!map.containsKey(svrEntry.getKey())) {
+ map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
* @return A clone of current assignments. Note, this is assignments only.
* If a new server has come in and it has no regions, it will not be included
* in the returned Map.
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java?rev=1234922&r1=1234921&r2=1234922&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java Mon Jan 23 17:54:23 2012
@@ -245,7 +245,7 @@ public class DefaultLoadBalancer impleme
if (serversByLoad.lastKey().getLoad() <= ceiling &&
serversByLoad.firstKey().getLoad() >= floor) {
// Skipped because no server outside (min,max) range
- LOG.info("Skipping load balancing because balanced cluster; " +
+ LOG.debug("Skipping load balancing because balanced cluster; " +
"servers=" + numServers + " " +
"regions=" + numRegions + " average=" + average + " " +
"mostloaded=" + serversByLoad.lastKey().getLoad() +
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1234922&r1=1234921&r2=1234922&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mon Jan 23 17:54:23 2012
@@ -901,16 +901,14 @@ implements HMasterInterface, HMasterRegi
}
}
- Map<ServerName, List<HRegionInfo>> assignments =
- this.assignmentManager.getAssignments();
- // Returned Map from AM does not include mention of servers w/o assignments.
- for (Map.Entry<ServerName, HServerLoad> e:
- this.serverManager.getOnlineServers().entrySet()) {
- if (!assignments.containsKey(e.getKey())) {
- assignments.put(e.getKey(), new ArrayList<HRegionInfo>());
- }
+ Map<String, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
+ this.assignmentManager.getAssignmentsByTable();
+
+ List<RegionPlan> plans = new ArrayList<RegionPlan>();
+ for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
+ List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
+ if (partialPlans != null) plans.addAll(partialPlans);
}
- List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
int rpCount = 0; // number of RegionPlans balanced so far
long totalRegPlanExecTime = 0;
balancerRan = plans != null;
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=1234922&r1=1234921&r2=1234922&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Mon Jan 23 17:54:23 2012
@@ -166,9 +166,16 @@ public class TestRegionRebalancing {
LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
&& serverLoad >= avgLoadMinusSlop)) {
- LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
- " actual: " + serverLoad + " slop: " + slop);
- success = false;
+ for (HRegionInfo hri : server.getOnlineRegions()) {
+ if (hri.isMetaRegion() || hri.isRootRegion()) serverLoad--;
+ // LOG.debug(hri.getRegionNameAsString());
+ }
+ if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) {
+ LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
+ " actual: " + serverLoad + " slop: " + slop);
+ success = false;
+ break;
+ }
}
}
@@ -214,4 +221,4 @@ public class TestRegionRebalancing {
} catch (InterruptedException e) {}
}
}
-}
\ No newline at end of file
+}