You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2017/01/23 02:35:46 UTC
hbase git commit: HBASE-17462 Use sliding window for read/write
request costs in StochasticLoadBalancer (Tim Brown)
Repository: hbase
Updated Branches:
refs/heads/master f254e278e -> 7754a9620
HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim Brown)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7754a962
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7754a962
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7754a962
Branch: refs/heads/master
Commit: 7754a9620eff44d1d570fda534f9159a756310cd
Parents: f254e27
Author: tedyu <yu...@gmail.com>
Authored: Sun Jan 22 18:35:38 2017 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Sun Jan 22 18:35:38 2017 -0800
----------------------------------------------------------------------
.../master/balancer/StochasticLoadBalancer.java | 47 ++++++++++++++------
.../balancer/TestStochasticLoadBalancer.java | 24 ++++++++++
2 files changed, 57 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7754a962/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index b02aac1..4fbae6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.Type;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRegionAction;
@@ -1300,21 +1299,42 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
double cost = 0;
-
for (RegionLoad rl : regionLoadList) {
- double toAdd = getCostFromRl(rl);
+ cost += getCostFromRl(rl);
+ }
+ return cost / regionLoadList.size();
+ }
+
+ protected abstract double getCostFromRl(RegionLoad rl);
+ }
- if (cost == 0) {
- cost = toAdd;
+ /**
+ * Class to be used for the subset of RegionLoad costs that should be treated as rates.
+ * We do not compare about the actual rate in requests per second but rather the rate relative
+ * to the rest of the regions.
+ */
+ abstract static class CostFromRegionLoadAsRateFunction extends CostFromRegionLoadFunction {
+
+ CostFromRegionLoadAsRateFunction(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
+ double cost = 0;
+ double previous = 0;
+ boolean isFirst = true;
+ for (RegionLoad rl : regionLoadList) {
+ double current = getCostFromRl(rl);
+ if (isFirst) {
+ isFirst = false;
} else {
- cost = (.5 * cost) + (.5 * toAdd);
+ cost += current - previous;
}
+ previous = current;
}
-
- return cost;
+ return Math.max(0, cost / (regionLoadList.size() - 1));
}
-
- protected abstract double getCostFromRl(RegionLoad rl);
}
/**
@@ -1322,7 +1342,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* computed cost will be. This uses a rolling average of regionload.
*/
- static class ReadRequestCostFunction extends CostFromRegionLoadFunction {
+ static class ReadRequestCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String READ_REQUEST_COST_KEY =
"hbase.master.balancer.stochastic.readRequestCost";
@@ -1333,7 +1353,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
this.setMultiplier(conf.getFloat(READ_REQUEST_COST_KEY, DEFAULT_READ_REQUEST_COST));
}
-
@Override
protected double getCostFromRl(RegionLoad rl) {
return rl.getReadRequestsCount();
@@ -1344,7 +1363,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* Compute the cost of total number of write requests. The more unbalanced the higher the
* computed cost will be. This uses a rolling average of regionload.
*/
- static class WriteRequestCostFunction extends CostFromRegionLoadFunction {
+ static class WriteRequestCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String WRITE_REQUEST_COST_KEY =
"hbase.master.balancer.stochastic.writeRequestCost";
@@ -1522,7 +1541,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* Compute the cost of total memstore size. The more unbalanced the higher the
* computed cost will be. This uses a rolling average of regionload.
*/
- static class MemstoreSizeCostFunction extends CostFromRegionLoadFunction {
+ static class MemstoreSizeCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String MEMSTORE_SIZE_COST_KEY =
"hbase.master.balancer.stochastic.memstoreSizeCost";
http://git-wip-us.apache.org/repos/asf/hbase/blob/7754a962/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index 094687b..3d975b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -231,6 +231,30 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
}
@Test
+ public void testRegionLoadCost() {
+ List<RegionLoad> regionLoads = new ArrayList<>();
+ for (int i = 1; i < 5; i++) {
+ RegionLoad regionLoad = mock(RegionLoad.class);
+ when(regionLoad.getReadRequestsCount()).thenReturn(new Long(i));
+ when(regionLoad.getStorefileSizeMB()).thenReturn(i);
+ regionLoads.add(regionLoad);
+ }
+
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.ReadRequestCostFunction readCostFunction =
+ new StochasticLoadBalancer.ReadRequestCostFunction(conf);
+ double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
+ // read requests are treated as a rate so the average rate here is simply 1
+ assertEquals(1, rateResult, 0.01);
+
+ StochasticLoadBalancer.StoreFileCostFunction storeFileCostFunction =
+ new StochasticLoadBalancer.StoreFileCostFunction(conf);
+ double result = storeFileCostFunction.getRegionLoadCost(regionLoads);
+ // storefile size cost is simply an average of it's value over time
+ assertEquals(2.5, result, 0.01);
+ }
+
+ @Test
public void testCostFromArray() {
Configuration conf = HBaseConfiguration.create();
StochasticLoadBalancer.CostFromRegionLoadFunction