You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/11/04 23:20:44 UTC
hbase git commit: HBASE-14706 RegionLocationFinder should return
multiple servernames by top host (Guanghao Zhang)
Repository: hbase
Updated Branches:
refs/heads/master 3e4575541 -> 9fc7858c6
HBASE-14706 RegionLocationFinder should return multiple servernames by top host (Guanghao Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9fc7858c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9fc7858c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9fc7858c
Branch: refs/heads/master
Commit: 9fc7858c6bb2d0356267cd7bc4aac9e38294c40c
Parents: 3e45755
Author: tedyu <yu...@gmail.com>
Authored: Wed Nov 4 14:20:34 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Wed Nov 4 14:20:34 2015 -0800
----------------------------------------------------------------------
.../master/balancer/RegionLocationFinder.java | 22 ++-
.../balancer/TestRegionLocationFinder.java | 145 +++++++++++++++++++
2 files changed, 160 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fc7858c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index 3774048..9657e53 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -235,17 +235,25 @@ class RegionLocationFinder {
Collection<ServerName> regionServers = status.getServers();
// create a mapping from hostname to ServerName for fast lookup
- HashMap<String, ServerName> hostToServerName = new HashMap<String, ServerName>();
+ HashMap<String, List<ServerName>> hostToServerName = new HashMap<String, List<ServerName>>();
for (ServerName sn : regionServers) {
- hostToServerName.put(sn.getHostname(), sn);
+ String host = sn.getHostname();
+ if (!hostToServerName.containsKey(host)) {
+ hostToServerName.put(host, new ArrayList<ServerName>());
+ }
+ hostToServerName.get(host).add(sn);
}
for (String host : hosts) {
- ServerName sn = hostToServerName.get(host);
- // it is possible that HDFS is up ( thus host is valid ),
- // but RS is down ( thus sn is null )
- if (sn != null) {
- topServerNames.add(sn);
+ if (!hostToServerName.containsKey(host)) {
+ continue;
+ }
+ for (ServerName sn : hostToServerName.get(host)) {
+ // it is possible that HDFS is up ( thus host is valid ),
+ // but RS is down ( thus sn is null )
+ if (sn != null) {
+ topServerNames.add(sn);
+ }
}
}
return topServerNames;
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fc7858c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionLocationFinder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionLocationFinder.java
new file mode 100644
index 0000000..177adfd
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionLocationFinder.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestRegionLocationFinder {
+ private static final Log LOG = LogFactory.getLog(TestRegionLocationFinder.class);
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static MiniHBaseCluster cluster;
+
+ private final static TableName tableName = TableName.valueOf("table");
+ private final static byte[] FAMILY = Bytes.toBytes("cf");
+ private static Table table;
+ private final static int ServerNum = 5;
+
+ private static RegionLocationFinder finder = new RegionLocationFinder();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ cluster = TEST_UTIL.startMiniCluster(1, ServerNum);
+ table = TEST_UTIL.createTable(tableName, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ TEST_UTIL.waitTableAvailable(tableName, 1000);
+ TEST_UTIL.loadTable(table, FAMILY);
+
+ for (int i = 0; i < ServerNum; i++) {
+ HRegionServer server = cluster.getRegionServer(i);
+ for (Region region : server.getOnlineRegions(tableName)) {
+ region.flush(true);
+ }
+ }
+
+ finder.setConf(TEST_UTIL.getConfiguration());
+ finder.setServices(cluster.getMaster());
+ finder.setClusterStatus(cluster.getMaster().getClusterStatus());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ table.close();
+ TEST_UTIL.deleteTable(tableName);
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testInternalGetTopBlockLocation() throws Exception {
+ for (int i = 0; i < ServerNum; i++) {
+ HRegionServer server = cluster.getRegionServer(i);
+ for (Region region : server.getOnlineRegions(tableName)) {
+ // get region's hdfs block distribution by region and RegionLocationFinder,
+ // they should have same result
+ HDFSBlocksDistribution blocksDistribution1 = region.getHDFSBlocksDistribution();
+ HDFSBlocksDistribution blocksDistribution2 = finder.getBlockDistribution(region
+ .getRegionInfo());
+ assertEquals(blocksDistribution1.getUniqueBlocksTotalWeight(),
+ blocksDistribution2.getUniqueBlocksTotalWeight());
+ if (blocksDistribution1.getUniqueBlocksTotalWeight() != 0) {
+ assertEquals(blocksDistribution1.getTopHosts().get(0), blocksDistribution2.getTopHosts()
+ .get(0));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMapHostNameToServerName() throws Exception {
+ List<String> topHosts = new ArrayList<String>();
+ for (int i = 0; i < ServerNum; i++) {
+ HRegionServer server = cluster.getRegionServer(i);
+ String serverHost = server.getServerName().getHostname();
+ if (!topHosts.contains(serverHost)) {
+ topHosts.add(serverHost);
+ }
+ }
+ List<ServerName> servers = finder.mapHostNameToServerName(topHosts);
+ // mini cluster, all rs in one host
+ assertEquals(1, topHosts.size());
+ for (int i = 0; i < ServerNum; i++) {
+ ServerName server = cluster.getRegionServer(i).getServerName();
+ assertTrue(servers.contains(server));
+ }
+ }
+
+ @Test
+ public void testGetTopBlockLocations() throws Exception {
+ for (int i = 0; i < ServerNum; i++) {
+ HRegionServer server = cluster.getRegionServer(i);
+ for (Region region : server.getOnlineRegions(tableName)) {
+ List<ServerName> servers = finder.getTopBlockLocations(region.getRegionInfo());
+ // test table may have empty region
+ if (region.getHDFSBlocksDistribution().getUniqueBlocksTotalWeight() == 0) {
+ continue;
+ }
+ List<String> topHosts = region.getHDFSBlocksDistribution().getTopHosts();
+ // rs and datanode may have different host in local machine test
+ if (!topHosts.contains(server.getServerName().getHostname())) {
+ continue;
+ }
+ for (int j = 0; j < ServerNum; j++) {
+ ServerName serverName = cluster.getRegionServer(j).getServerName();
+ assertTrue(servers.contains(serverName));
+ }
+ }
+ }
+ }
+}