You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:07:50 UTC
[14/50] [abbrv] git commit: [HBASE-11336] Show region sizes in table
page of master info server.
[HBASE-11336] Show region sizes in table page of master info server.
Summary: Extract sizes of regions on HDFS, and show them on info server table page.
Test Plan:
`TestFSUtils`
The web page on testcase.
Reviewers: manukranthk, adela, gauravm, elliott, fan
Reviewed By: fan
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1377657
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42820 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7403a0bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7403a0bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7403a0bf
Branch: refs/heads/0.89-fb
Commit: 7403a0bf209345e1cf3a2f666f39b1727c1bafe2
Parents: eae4c26
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Thu Jun 12 01:34:51 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +
.../org/apache/hadoop/hbase/util/FSUtils.java | 37 +++++++
.../resources/hbase-webapps/master/table.jsp | 28 ++++--
.../apache/hadoop/hbase/util/TestFSUtils.java | 100 +++++++++++++++++++
4 files changed, 159 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 07f559e..0d008ff 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1255,6 +1255,8 @@ public class HMaster extends HasThread implements HMasterInterface,
this.infoServer = new InfoServer(MASTER, a, port, false, conf);
this.infoServer.setAttribute(MASTER, this);
this.infoServer.start();
+ LOG.info("Master info server started at: " + a + ":"
+ + this.infoServer.getPort());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Started service threads");
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 7589ecf..6bfb630 100755
--- a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -27,6 +27,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
@@ -480,6 +481,42 @@ public class FSUtils {
}
/**
+ * Returns the sizes of some regions on DFS.
+ */
+ public static Map<HRegionInfo, Long> getRegionSizesInBytes(HMaster master,
+ Set<HRegionInfo> regions) throws IOException {
+ Configuration conf = master.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ return getRegionSizesInBytes(regions, fs, master.getRootDir());
+ }
+
+ /**
+ * Returns the sizes of some regions on DFS.
+ */
+ public static Map<HRegionInfo, Long> getRegionSizesInBytes(
+ Set<HRegionInfo> regions, FileSystem fs, Path hbaseRootDir)
+ throws IOException {
+ Map<HRegionInfo, Long> sizes = new ConcurrentHashMap<>();
+
+ DirFilter df = new DirFilter(fs);
+ for (HRegionInfo region : regions) {
+ long size = 0L;
+ Path regionDir = HRegion.getRegionDir(hbaseRootDir, region);
+ for (FileStatus familyDir : fs.listStatus(regionDir, df)) {
+ for (FileStatus storeFile : fs.listStatus(familyDir.getPath())) {
+ if (storeFile.isDir()) {
+ continue;
+ }
+ size += storeFile.getLen();
+ }
+ }
+ sizes.put(region, size);
+ }
+
+ return sizes;
+ }
+
+ /**
* Expects to find -ROOT- directory.
* @param fs filesystem
* @param hbaseRootDir hbase root directory
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp
index e690ae7..65bfa49 100644
--- a/src/main/resources/hbase-webapps/master/table.jsp
+++ b/src/main/resources/hbase-webapps/master/table.jsp
@@ -11,6 +11,8 @@
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.master.MetaRegion"
import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.util.StringUtils"
import="java.util.Map"
import="org.apache.hadoop.hbase.HConstants"%><%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
@@ -135,20 +137,30 @@
<%
Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
if(regions != null && regions.size() > 0) { %>
-<%= tableHeader %>
+ <h2>Table Regions</h2>
+ <table>
+ <tr>
+ <th>Name</th>
+ <th>Region Server</th>
+ <th>Start Key</th>
+ <th>End Key</th>
+ <th>Bytes</th>
+ </tr>
<%
+ Map<HRegionInfo, Long> regionSizes = FSUtils.getRegionSizesInBytes(master, regions.keySet());
for(Map.Entry<HRegionInfo, HServerAddress> hriEntry : regions.entrySet()) {
String urlRegionServer =
"http://" + hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort + "/";
%>
-<tr>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getRegionName())%></td>
- <td><a href="<%= urlRegionServer %>"><%= hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort %></a></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getStartKey())%></td>
- <td><%= Bytes.toStringBinary(hriEntry.getKey().getEndKey())%></td>
-</tr>
+ <tr>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getRegionName())%></td>
+ <td><a href="<%= urlRegionServer %>"><%= hriEntry.getValue().getHostname().toString() + ":" + rsInfoPort %></a></td>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getStartKey())%></td>
+ <td><%= Bytes.toStringBinary(hriEntry.getKey().getEndKey())%></td>
+ <td><%= StringUtils.byteDesc(regionSizes.get(hriEntry.getKey())) %></td>
+ </tr>
<% } %>
-</table>
+ </table>
<% }
} catch(Exception ex) {
ex.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hbase/blob/7403a0bf/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
new file mode 100644
index 0000000..e118829
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -0,0 +1,100 @@
+/**
+ * Copyright 2014 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestFSUtils {
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ // Setting this make master start the info server.
+ conf.setInt(HConstants.MASTER_INFO_PORT, 0);
+ TEST_UTIL.startMiniCluster();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Test method for {@link org.apache.hadoop.hbase.util.FSUtils#getRegionSizesInBytes(java.util.Set, org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path)}.
+ */
+ @Test
+ public void testGetRegionSizesInBytes() throws Exception {
+ final StringBytes TABLE = new StringBytes("testGetRegionSizesInBytes");
+ final byte[] FAMILY = Bytes.toBytes("f");
+ Configuration conf = TEST_UTIL.getConfiguration();
+ Path rootDir = FSUtils.getRootDir(conf);
+ try (HTable table = TEST_UTIL.createTable(TABLE, FAMILY)) {
+ TEST_UTIL.loadTable(table, FAMILY);
+
+ // Flush to disk
+ for (HRegionServer rs : TEST_UTIL.getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ rs.flushRegion(region.getRegionName());
+ }
+ }
+
+ Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
+ FileSystem fs = FileSystem.get(conf);
+ Map<HRegionInfo, Long> regionSizes =
+ FSUtils.getRegionSizesInBytes(regions.keySet(), fs, rootDir);
+ System.out.println(regionSizes);
+
+ Assert.assertEquals("Number of regions", regions.size(),
+ regionSizes.size());
+
+ for (HRegionInfo region : regions.keySet()) {
+ Assert.assertNotEquals(
+ "Size of region " + region.getRegionNameAsString(), 0L,
+ regionSizes.get(region).longValue());
+ }
+ }
+ }
+}