You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/03/30 18:03:32 UTC
[30/50] [abbrv] hbase git commit: HBASE-15515 Improve
LocalityBasedCandidateGenerator in Balancer (Guanghao Zhang)
HBASE-15515 Improve LocalityBasedCandidateGenerator in Balancer (Guanghao Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ad3feaa4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ad3feaa4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ad3feaa4
Branch: refs/heads/hbase-12439
Commit: ad3feaa44800f10d102255a240c38ccf23a82d49
Parents: 7c93098
Author: tedyu <yu...@gmail.com>
Authored: Thu Mar 24 11:10:58 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Mar 24 11:10:58 2016 -0700
----------------------------------------------------------------------
.../hbase/master/balancer/BaseLoadBalancer.java | 30 ++++++++++++++++----
.../master/balancer/RegionLocationFinder.java | 16 +++++++++++
.../master/balancer/StochasticLoadBalancer.java | 6 ++--
3 files changed, 42 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ad3feaa4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index bde5c61..6a28006 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -828,7 +828,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
int getLowestLocalityRegionOnServer(int serverIndex) {
if (regionFinder != null) {
float lowestLocality = 1.0f;
- int lowestLocalityRegionIndex = 0;
+ int lowestLocalityRegionIndex = -1;
if (regionsPerServer[serverIndex].length == 0) {
// No regions on that region server
return -1;
@@ -838,15 +838,24 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
HDFSBlocksDistribution distribution = regionFinder
.getBlockDistribution(regions[regionIndex]);
float locality = distribution.getBlockLocalityIndex(servers[serverIndex].getHostname());
+ // skip empty region
+ if (distribution.getUniqueBlocksTotalWeight() == 0) {
+ continue;
+ }
if (locality < lowestLocality) {
lowestLocality = locality;
lowestLocalityRegionIndex = j;
}
}
+ if (lowestLocalityRegionIndex == -1) {
+ return -1;
+ }
if (LOG.isTraceEnabled()) {
- LOG.trace(" Lowest locality region index is " + lowestLocalityRegionIndex
- + " and its region server contains " + regionsPerServer[serverIndex].length
- + " regions");
+ LOG.trace("Lowest locality region is "
+ + regions[regionsPerServer[serverIndex][lowestLocalityRegionIndex]]
+ .getRegionNameAsString() + " with locality " + lowestLocality
+ + " and its region server contains " + regionsPerServer[serverIndex].length
+ + " regions");
}
return regionsPerServer[serverIndex][lowestLocalityRegionIndex];
} else {
@@ -863,9 +872,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
}
}
- int getLeastLoadedTopServerForRegion(int region) {
+ /**
+ * Returns a least loaded server which has better locality for this region
+ * than the current server.
+ */
+ int getLeastLoadedTopServerForRegion(int region, int currentServer) {
if (regionFinder != null) {
- List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region]);
+ List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region],
+ servers[currentServer].getHostname());
int leastLoadedServerIndex = -1;
int load = Integer.MAX_VALUE;
for (ServerName sn : topLocalServers) {
@@ -882,6 +896,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
load = tempLoad;
}
}
+ if (leastLoadedServerIndex != -1) {
+ LOG.debug("Pick the least loaded server " + servers[leastLoadedServerIndex].getHostname()
+ + " with better locality for region " + regions[region]);
+ }
return leastLoadedServerIndex;
} else {
return -1;
http://git-wip-us.apache.org/repos/asf/hbase/blob/ad3feaa4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index 9657e53..a6724ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -171,6 +171,22 @@ class RegionLocationFinder {
}
/**
+ * Returns an ordered list of hosts which have better locality for this region
+ * than the current host.
+ */
+ protected List<ServerName> getTopBlockLocations(HRegionInfo region, String currentHost) {
+ HDFSBlocksDistribution blocksDistribution = getBlockDistribution(region);
+ List<String> topHosts = new ArrayList<String>();
+ for (String host : blocksDistribution.getTopHosts()) {
+ if (host.equals(currentHost)) {
+ break;
+ }
+ topHosts.add(host);
+ }
+ return mapHostNameToServerName(topHosts);
+ }
+
+ /**
* Returns an ordered list of hosts that are hosting the blocks for this
* region. The weight of each host is the sum of the block lengths of all
* files on that host, so the first host in the list is the server which holds
http://git-wip-us.apache.org/repos/asf/hbase/blob/ad3feaa4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index add3242..d01f510 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -705,9 +705,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
return pickRandomRegions(cluster, thisServer, otherServer);
}
- cluster.calculateRegionServerLocalities();
- // Pick server with lowest locality
- int thisServer = pickLowestLocalityServer(cluster);
+ int thisServer = pickRandomServer(cluster);
int thisRegion;
if (thisServer == -1) {
LOG.warn("Could not pick lowest locality region server");
@@ -722,7 +720,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
// Pick the least loaded server with good locality for the region
- int otherServer = cluster.getLeastLoadedTopServerForRegion(thisRegion);
+ int otherServer = cluster.getLeastLoadedTopServerForRegion(thisRegion, thisServer);
if (otherServer == -1) {
return Cluster.NullAction;