You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2015/08/26 07:09:46 UTC
[2/2] hbase git commit: Revert "HBASE-13376 Improvements to
Stochastic load balancer (Vandana Ayyalasomayajula)"
Revert "HBASE-13376 Improvements to Stochastic load balancer (Vandana Ayyalasomayajula)"
This reverts commit 2fd9c8df3b261dd596f68fff1c09c803da1ccdc3.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4aa14b6c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4aa14b6c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4aa14b6c
Branch: refs/heads/0.98
Commit: 4aa14b6c9074002abb92181d99bf65e7f0e46a22
Parents: 10388f6
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue Aug 25 22:07:36 2015 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Aug 25 22:07:36 2015 -0700
----------------------------------------------------------------------
.../hbase/master/balancer/BaseLoadBalancer.java | 127 +------------------
.../master/balancer/RegionLocationFinder.java | 75 +++++------
.../master/balancer/StochasticLoadBalancer.java | 91 ++++++-------
.../balancer/TestStochasticLoadBalancer.java | 3 -
4 files changed, 72 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4aa14b6c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 0d10531..3081811 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -35,7 +35,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
@@ -67,20 +66,16 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
ArrayList<String> tables;
HRegionInfo[] regions;
Deque<RegionLoad>[] regionLoads;
- RegionLocationFinder regionFinder;
int[][] regionLocations; //regionIndex -> list of serverIndex sorted by locality
int[][] regionsPerServer; //serverIndex -> region list
- float[] localityPerServer;
int[] regionIndexToServerIndex; //regionIndex -> serverIndex
int[] initialRegionIndexToServerIndex; //regionIndex -> serverIndex (initial cluster state)
int[] regionIndexToTableIndex; //regionIndex -> tableIndex
int[][] numRegionsPerServerPerTable; //serverIndex -> tableIndex -> # regions
int[] numMaxRegionsPerTable; //tableIndex -> max number of regions in a single RS
-
Integer[] serverIndicesSortedByRegionCount;
- Integer[] serverIndicesSortedByLocality;
Map<String, Integer> serversToIndex;
Map<String, Integer> tablesToIndex;
@@ -101,7 +96,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
//TODO: We should get the list of tables from master
tables = new ArrayList<String>();
- this.regionFinder = regionFinder;
numRegions = 0;
@@ -132,8 +126,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
regionLoads = new Deque[numRegions];
regionLocations = new int[numRegions][];
serverIndicesSortedByRegionCount = new Integer[numServers];
- serverIndicesSortedByLocality = new Integer[numServers];
- localityPerServer = new float[numServers];
int tableIndex = 0, regionIndex = 0, regionPerServerIndex = 0;
@@ -155,7 +147,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
regionsPerServer[serverIndex] = new int[entry.getValue().size()];
}
serverIndicesSortedByRegionCount[serverIndex] = serverIndex;
- serverIndicesSortedByLocality[serverIndex] = serverIndex;
}
for (Entry<ServerName, List<HRegionInfo>> entry : clusterState.entrySet()) {
@@ -198,6 +189,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
(serversToIndex.get(loc.get(i).getHostAndPort()) == null ? -1 : serversToIndex.get(loc.get(i).getHostAndPort()));
}
}
+
regionIndex++;
}
}
@@ -311,18 +303,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
Arrays.sort(serverIndicesSortedByRegionCount, numRegionsComparator);
}
- void sortServersByLocality() {
- Arrays.sort(serverIndicesSortedByLocality, localityComparator);
- }
-
int getNumRegions(int server) {
return regionsPerServer[server].length;
}
- float getLocality(int server) {
- return localityPerServer[server];
- }
-
private Comparator<Integer> numRegionsComparator = new Comparator<Integer>() {
@Override
public int compare(Integer integer, Integer integer2) {
@@ -330,111 +314,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
}
};
- private Comparator<Integer> localityComparator = new Comparator<Integer>() {
- @Override
- public int compare(Integer integer, Integer integer2) {
- float locality1 = getLocality(integer);
- float locality2 = getLocality(integer2);
- if (locality1 < locality2) {
- return -1;
- } else if (locality1 > locality2) {
- return 1;
- } else {
- return 0;
- }
- }
- };
-
- int getLowestLocalityRegionServer() {
- if (regionFinder == null) {
- return -1;
- } else {
- sortServersByLocality();
- // We want to find server with non zero regions having lowest locality.
- int i = 0;
- int lowestLocalityServerIndex = serverIndicesSortedByLocality[i];
- while (localityPerServer[lowestLocalityServerIndex] == 0
- && (regionsPerServer[lowestLocalityServerIndex].length == 0)) {
- i++;
- lowestLocalityServerIndex = serverIndicesSortedByLocality[i];
- }
- LOG.debug("Lowest locality region server with non zero regions is "
- + servers[lowestLocalityServerIndex].getHostname() + " with locality "
- + localityPerServer[lowestLocalityServerIndex]);
- return lowestLocalityServerIndex;
- }
- }
-
- int getLowestLocalityRegionOnServer(int serverIndex) {
- if (regionFinder != null) {
- float lowestLocality = 1.0f;
- int lowestLocalityRegionIndex = 0;
- if (regionsPerServer[serverIndex].length == 0) {
- // No regions on that region server
- return -1;
- }
- for (int j = 0; j < regionsPerServer[serverIndex].length; j++) {
- int regionIndex = regionsPerServer[serverIndex][j];
- HDFSBlocksDistribution distribution = regionFinder
- .getBlockDistribution(regions[regionIndex]);
- float locality = distribution.getBlockLocalityIndex(servers[serverIndex].getHostname());
- if (locality < lowestLocality) {
- lowestLocality = locality;
- lowestLocalityRegionIndex = j;
- }
- }
- LOG.debug(" Lowest locality region index is " + lowestLocalityRegionIndex
- + " and its region server contains " + regionsPerServer[serverIndex].length
- + " regions");
- return regionsPerServer[serverIndex][lowestLocalityRegionIndex];
- } else {
- return -1;
- }
- }
-
- float getLocalityOfRegion(int region, int server) {
- HDFSBlocksDistribution distribution = regionFinder.getBlockDistribution(regions[region]);
- return distribution.getBlockLocalityIndex(servers[server].getHostname());
- }
-
- int getLeastLoadedTopServerForRegion(int region) {
- if (regionFinder != null) {
- List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region]);
- int leastLoadedServerIndex = -1;
- int load = Integer.MAX_VALUE;
- for (ServerName sn : topLocalServers) {
- int index = serversToIndex.get(sn);
- int tempLoad = regionsPerServer[index].length;
- if (tempLoad <= load) {
- leastLoadedServerIndex = index;
- load = tempLoad;
- }
- }
- return leastLoadedServerIndex;
- } else {
- return -1;
- }
- }
-
- void calculateRegionServerLocalities() {
- if (regionFinder == null) {
- LOG.warn("Region location finder found null, skipping locality calculations.");
- return;
- }
- for (int i = 0; i < regionsPerServer.length; i++) {
- HDFSBlocksDistribution distribution = new HDFSBlocksDistribution();
- if (regionsPerServer[i].length > 0) {
- for (int j = 0; j < regionsPerServer[i].length; j++) {
- int regionIndex = regionsPerServer[i][j];
- distribution.add(regionFinder.getBlockDistribution(regions[regionIndex]));
- }
- } else {
- LOG.debug("Server " + servers[i].getHostname() + " had 0 regions.");
- }
- localityPerServer[i] = distribution.getBlockLocalityIndex(servers[i].getHostname());
- }
- }
-
@Override
public String toString() {
String desc = "Cluster{" +
@@ -471,7 +350,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
// slop for regions
protected float slop;
private Configuration config;
- static final Random RANDOM = new Random(System.currentTimeMillis());
+ private static final Random RANDOM = new Random(System.currentTimeMillis());
private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
protected final MetricsBalancer metricsBalancer = new MetricsBalancer();
@@ -598,7 +477,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
*/
@Override
public Map<HRegionInfo, ServerName> immediateAssignment(List<HRegionInfo> regions,
- List<ServerName> servers) throws HBaseIOException {
+ List<ServerName> servers) {
metricsBalancer.incrMiscInvocations();
Map<HRegionInfo, ServerName> assignments = new TreeMap<HRegionInfo, ServerName>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/4aa14b6c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index 4475d02..690d8c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -30,18 +31,17 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.regionserver.HRegion;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Lists;
/**
* This will find where data for a region is located in HDFS. It ranks
@@ -54,27 +54,31 @@ class RegionLocationFinder {
private static Log LOG = LogFactory.getLog(RegionLocationFinder.class);
private Configuration conf;
- private volatile ClusterStatus status;
+ private ClusterStatus status;
private MasterServices services;
- private CacheLoader<HRegionInfo, HDFSBlocksDistribution> loader =
- new CacheLoader<HRegionInfo, HDFSBlocksDistribution>() {
+ private CacheLoader<HRegionInfo, List<ServerName>> loader =
+ new CacheLoader<HRegionInfo, List<ServerName>>() {
- @Override
- public HDFSBlocksDistribution load(HRegionInfo key) throws Exception {
- return internalGetTopBlockLocation(key);
- }
- };
+ @Override
+ public List<ServerName> load(HRegionInfo key) throws Exception {
+ List<ServerName> servers = internalGetTopBlockLocation(key);
+ if (servers == null) {
+ return new LinkedList<ServerName>();
+ }
+ return servers;
+ }
+ };
// The cache for where regions are located.
- private LoadingCache<HRegionInfo, HDFSBlocksDistribution> cache = null;
+ private LoadingCache<HRegionInfo, List<ServerName>> cache = null;
/**
* Create a cache for region to list of servers
* @param mins Number of mins to cache
* @return A new Cache.
*/
- private LoadingCache<HRegionInfo, HDFSBlocksDistribution> createCache(int mins) {
+ private LoadingCache<HRegionInfo, List<ServerName>> createCache(int mins) {
return CacheBuilder.newBuilder().expireAfterAccess(mins, TimeUnit.MINUTES).build(loader);
}
@@ -96,9 +100,14 @@ class RegionLocationFinder {
}
protected List<ServerName> getTopBlockLocations(HRegionInfo region) {
- HDFSBlocksDistribution blocksDistribution = getBlockDistribution(region);
- List<String> topHosts = blocksDistribution.getTopHosts();
- return mapHostNameToServerName(topHosts);
+ List<ServerName> servers = null;
+ try {
+ servers = cache.get(region);
+ } catch (ExecutionException ex) {
+ servers = new LinkedList<ServerName>();
+ }
+ return servers;
+
}
/**
@@ -110,20 +119,22 @@ class RegionLocationFinder {
* @param region region
* @return ordered list of hosts holding blocks of the specified region
*/
- protected HDFSBlocksDistribution internalGetTopBlockLocation(HRegionInfo region) {
+ protected List<ServerName> internalGetTopBlockLocation(HRegionInfo region) {
+ List<ServerName> topServerNames = null;
try {
HTableDescriptor tableDescriptor = getTableDescriptor(region.getTable());
if (tableDescriptor != null) {
HDFSBlocksDistribution blocksDistribution =
HRegion.computeHDFSBlocksDistribution(getConf(), tableDescriptor, region);
- return blocksDistribution;
+ List<String> topHosts = blocksDistribution.getTopHosts();
+ topServerNames = mapHostNameToServerName(topHosts);
}
} catch (IOException ioe) {
- LOG.warn("IOException during HDFSBlocksDistribution computation. for " + "region = "
+ LOG.debug("IOException during HDFSBlocksDistribution computation. for " + "region = "
+ region.getEncodedName(), ioe);
}
- return new HDFSBlocksDistribution();
+ return topServerNames;
}
/**
@@ -156,10 +167,7 @@ class RegionLocationFinder {
*/
protected List<ServerName> mapHostNameToServerName(List<String> hosts) {
if (hosts == null || status == null) {
- if (hosts == null) {
- LOG.warn("RegionLocationFinder top hosts is null");
- }
- return Lists.newArrayList();
+ return null;
}
List<ServerName> topServerNames = new ArrayList<ServerName>();
@@ -181,25 +189,4 @@ class RegionLocationFinder {
}
return topServerNames;
}
-
- public HDFSBlocksDistribution getBlockDistribution(HRegionInfo hri) {
- HDFSBlocksDistribution blockDistbn = null;
- try {
- if (cache.asMap().containsKey(hri)) {
- blockDistbn = cache.get(hri);
- return blockDistbn;
- } else {
- LOG.debug("HDFSBlocksDistribution not found in cache for region "
- + hri.getRegionNameAsString());
- blockDistbn = internalGetTopBlockLocation(hri);
- cache.put(hri, blockDistbn);
- return blockDistbn;
- }
- } catch (ExecutionException e) {
- LOG.warn("Error while fetching cache entry ", e);
- blockDistbn = internalGetTopBlockLocation(hri);
- cache.put(hri, blockDistbn);
- return blockDistbn;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4aa14b6c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 25c4f18..9e1c9f5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.master.balancer;
-
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Deque;
@@ -31,6 +30,7 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
@@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.util.Bytes;
@@ -137,7 +136,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
localityPicker = new LocalityBasedPicker(services);
localityCost = new LocalityCostFunction(conf, services);
- pickers = new RegionPicker[] { new RandomRegionPicker(), new LoadPicker(), localityPicker };
+ pickers = new RegionPicker[] {
+ new RandomRegionPicker(),
+ new LoadPicker(),
+ localityPicker
+ };
regionLoadFunctions = new CostFromRegionLoadFunction[] {
new ReadRequestCostFunction(conf),
@@ -189,10 +192,10 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
*/
@Override
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
-
if (!needsBalance(new ClusterLoadState(clusterState))) {
return null;
}
+
long startTime = EnvironmentEdgeManager.currentTimeMillis();
// On clusters with lots of HFileLinks or lots of reference files,
@@ -203,13 +206,14 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
if (this.localityCost != null && this.localityCost.getMultiplier() > 0) {
finder = this.regionFinder;
}
-
+
// Keep track of servers to iterate through them.
Cluster cluster = new Cluster(clusterState, loads, finder);
double currentCost = computeCost(cluster, Double.MAX_VALUE);
double initCost = currentCost;
double newCost = currentCost;
+
long computedMaxSteps = Math.min(this.maxSteps,
((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));
// Perform a stochastic walk to see if we can get a good fit.
@@ -463,7 +467,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
@Override
Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> pick(Cluster cluster) {
cluster.sortServersByRegionCount();
- int thisServer = pickMostLoadedServer(cluster);
+ int thisServer = pickMostLoadedServer(cluster, -1);
int otherServer = pickLeastLoadedServer(cluster, thisServer);
Pair<Integer, Integer> regions = pickRandomRegions(cluster, thisServer, otherServer);
@@ -487,24 +491,17 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
return servers[index];
}
- /**
- * Pick a random server which is loaded above average.
- *
- * @param cluster
- * @return index of the region server picked.
- */
- private int pickMostLoadedServer(final Cluster cluster) {
+ private int pickMostLoadedServer(final Cluster cluster, int thisServer) {
Integer[] servers = cluster.serverIndicesSortedByRegionCount;
- float averageLoad = (float) cluster.regions.length / cluster.servers.length;
- int startIndex = 0;
- for ( int i = 0 ; i < servers.length; i++) {
- if (cluster.getNumRegions(servers[i]) >= averageLoad) {
- startIndex = i;
- break;
+
+ int index = servers.length - 1;
+ while (servers[index] == null || servers[index] == thisServer) {
+ index--;
+ if (index < 0) {
+ return -1;
}
}
- int randomServerIndex = RANDOM.nextInt(servers.length - startIndex) + startIndex;
- return servers[randomServerIndex];
+ return servers[index];
}
}
@@ -518,30 +515,23 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
@Override
Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> pick(Cluster cluster) {
- Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> nothingPicked =
- new Pair<Pair<Integer, Integer>, Pair<Integer, Integer>>(
- new Pair<Integer, Integer>(-1, -1), new Pair<Integer, Integer>(-1, -1));
if (this.masterServices == null) {
- return nothingPicked;
- }
- cluster.calculateRegionServerLocalities();
- // Pick lowest locality region server
- int thisServer = pickLowestLocalityServer(cluster);
- int thisRegion;
- if ( thisServer == -1) {
- LOG.warn("Could not pick lowest locality region server");
- return nothingPicked;
- } else {
- // Pick lowest locality region on this server
- thisRegion = pickLowestLocalityRegionOnServer(cluster, thisServer);
+ return new Pair<Pair<Integer, Integer>, Pair<Integer, Integer>>(
+ new Pair<Integer, Integer>(-1,-1),
+ new Pair<Integer, Integer>(-1,-1)
+ );
}
+ // Pick a random region server
+ int thisServer = pickRandomServer(cluster);
+
+ // Pick a random region on this server
+ int thisRegion = pickRandomRegion(cluster, thisServer, 0.0f);
if (thisRegion == -1) {
- if (cluster.regionsPerServer[thisServer].length > 0) {
- LOG.warn("Could not pick lowest locality region even when region server held "
- + cluster.regionsPerServer[thisServer].length + " regions");
- }
- return nothingPicked;
+ return new Pair<Pair<Integer, Integer>, Pair<Integer, Integer>>(
+ new Pair<Integer, Integer>(-1,-1),
+ new Pair<Integer, Integer>(-1,-1)
+ );
}
// Pick the server with the highest locality
@@ -556,19 +546,10 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
);
}
- private int pickLowestLocalityServer(Cluster cluster) {
- return cluster.getLowestLocalityRegionServer();
- }
-
- private int pickLowestLocalityRegionOnServer(Cluster cluster, int server) {
- return cluster.getLowestLocalityRegionOnServer(server);
- }
-
private int pickHighestLocalityServer(Cluster cluster, int thisServer, int thisRegion) {
int[] regionLocations = cluster.regionLocations[thisRegion];
if (regionLocations == null || regionLocations.length <= 1) {
- LOG.warn("Picking random destination as pickHighestLocalityServer did not give good result");
return pickOtherRandomServer(cluster, thisServer);
}
@@ -579,7 +560,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
// no location found
- LOG.warn("Picking random destination as pickHighestLocalityServer did not give good result");
return pickOtherRandomServer(cluster, thisServer);
}
@@ -651,6 +631,8 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
return scaled;
}
+
+
private double getSum(double[] stats) {
double total = 0;
for(double s:stats) {
@@ -752,6 +734,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
for (int i =0; i < cluster.numServers; i++) {
stats[i] = cluster.regionsPerServer[i].length;
}
+
return costFromArray(stats);
}
}
@@ -837,9 +820,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
if (index < 0) {
- cost += 1;
+ if (regionLocations.length > 0) {
+ cost += 1;
+ }
} else {
- cost += (1 - cluster.getLocalityOfRegion(i, index));
+ cost += (double) index / (double) regionLocations.length;
}
}
return scale(0, max, cost);
http://git-wip-us.apache.org/repos/asf/hbase/blob/4aa14b6c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index aac9fd6..9f8ea42 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -58,8 +58,6 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
Configuration conf = HBaseConfiguration.create();
conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
conf.setFloat("hbase.regions.slop", 0.0f);
- conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
- conf.setInt("hbase.master.balancer.stochastic.maxSteps", 10000000);
loadBalancer = new StochasticLoadBalancer();
loadBalancer.setConf(conf);
}
@@ -134,7 +132,6 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
new int[]{13, 14, 6, 10, 10, 10, 8, 10},
new int[]{130, 14, 60, 10, 100, 10, 80, 10},
new int[]{130, 140, 60, 100, 100, 100, 80, 100},
- new int[]{0, 5 , 5, 5, 5},
largeCluster,
};