You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/02/02 08:53:33 UTC
[26/41] hbase git commit: HBASE-19913 Split
TestStochasticLoadBalancer2
HBASE-19913 Split TestStochasticLoadBalancer2
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94dad844
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94dad844
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94dad844
Branch: refs/heads/HBASE-19064
Commit: 94dad844af239eafd09dac951bbc9ac2243fac82
Parents: 3622bb0
Author: zhangduo <zh...@apache.org>
Authored: Thu Feb 1 15:56:25 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri Feb 2 08:29:28 2018 +0800
----------------------------------------------------------------------
.../master/balancer/BalancerTestBase2.java | 40 ++++++++
.../balancer/TestStochasticLoadBalancer2.java | 96 --------------------
...TestStochasticLoadBalancerRegionReplica.java | 4 +-
...oadBalancerRegionReplicaHighReplication.java | 46 ++++++++++
...icLoadBalancerRegionReplicaLargeCluster.java | 43 +++++++++
...sticLoadBalancerRegionReplicaMidCluster.java | 43 +++++++++
...onReplicaReplicationGreaterThanNumNodes.java | 46 ++++++++++
7 files changed, 220 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java
new file mode 100644
index 0000000..dc952da
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.junit.After;
+import org.junit.Before;
+
+public class BalancerTestBase2 extends BalancerTestBase {
+
+ @Before
+ public void before() {
+ conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
+ conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
+ conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
+ conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
+ conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f);
+ loadBalancer.setConf(conf);
+ }
+
+ @After
+ public void after() {
+ // reset config to make sure balancer run
+ loadBalancer.setConf(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
deleted file mode 100644
index da573da..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.balancer;
-
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.testclassification.FlakeyTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({FlakeyTests.class, LargeTests.class})
-public class TestStochasticLoadBalancer2 extends BalancerTestBase {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestStochasticLoadBalancer2.class);
-
- @Before
- public void before() {
- conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
- conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
- conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
- conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
- conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f);
- loadBalancer.setConf(conf);
- }
-
- @After
- public void after() {
- // reset config to make sure balancer run
- loadBalancer.setConf(conf);
- }
-
- @Test
- public void testRegionReplicasOnMidCluster() {
- int numNodes = 200;
- int numRegions = 40 * 200;
- int replication = 3; // 3 replicas per region
- int numRegionsPerServer = 30; //all regions are mostly balanced
- int numTables = 10;
- testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
- }
-
- @Test
- public void testRegionReplicasOnLargeCluster() {
- int numNodes = 1000;
- int numRegions = 20 * numNodes; // 20 * replication regions per RS
- int numRegionsPerServer = 19; // all servers except one
- int numTables = 100;
- int replication = 3;
- testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
- }
-
- @Test
- public void testRegionReplicasOnMidClusterHighReplication() {
- conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L);
- conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
- loadBalancer.setConf(conf);
- int numNodes = 80;
- int numRegions = 6 * numNodes;
- int replication = 80; // 80 replicas per region, one for each server
- int numRegionsPerServer = 5;
- int numTables = 10;
- testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true);
- }
-
- @Test
- public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() {
- conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
- loadBalancer.setConf(conf);
- int numNodes = 40;
- int numRegions = 6 * 50;
- int replication = 50; // 50 replicas per region, more than numNodes
- int numRegionsPerServer = 6;
- int numTables = 10;
- testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java
index cb48e1d..5b06f86 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java
@@ -35,13 +35,13 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RackManager;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-@Category({ MasterTests.class, LargeTests.class })
+@Category({ MasterTests.class, MediumTests.class })
public class TestStochasticLoadBalancerRegionReplica extends BalancerTestBase {
@ClassRule
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java
new file mode 100644
index 0000000..1253304
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestStochasticLoadBalancerRegionReplicaHighReplication extends BalancerTestBase2 {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaHighReplication.class);
+
+ @Test
+ public void testRegionReplicasOnMidClusterHighReplication() {
+ conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L);
+ conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
+ loadBalancer.setConf(conf);
+ int numNodes = 80;
+ int numRegions = 6 * numNodes;
+ int replication = 80; // 80 replicas per region, one for each server
+ int numRegionsPerServer = 5;
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java
new file mode 100644
index 0000000..f43af33
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestStochasticLoadBalancerRegionReplicaLargeCluster extends BalancerTestBase2 {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaLargeCluster.class);
+
+ @Test
+ public void testRegionReplicasOnLargeCluster() {
+ int numNodes = 1000;
+ int numRegions = 20 * numNodes; // 20 * replication regions per RS
+ int numRegionsPerServer = 19; // all servers except one
+ int numTables = 100;
+ int replication = 3;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java
new file mode 100644
index 0000000..99c46a5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestStochasticLoadBalancerRegionReplicaMidCluster extends BalancerTestBase2 {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaMidCluster.class);
+
+ @Test
+ public void testRegionReplicasOnMidCluster() {
+ int numNodes = 200;
+ int numRegions = 40 * 200;
+ int replication = 3; // 3 replicas per region
+ int numRegionsPerServer = 30; // all regions are mostly balanced
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java
new file mode 100644
index 0000000..fd0cc98
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes
+ extends BalancerTestBase2 {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule
+ .forClass(TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.class);
+
+ @Test
+ public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() {
+ conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
+ loadBalancer.setConf(conf);
+ int numNodes = 40;
+ int numRegions = 6 * 50;
+ int replication = 50; // 50 replicas per region, more than numNodes
+ int numRegionsPerServer = 6;
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false);
+ }
+}