You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/01/05 07:03:06 UTC
svn commit: r895912 - in /hadoop/hbase/trunk/src:
java/org/apache/hadoop/hbase/master/
java/org/apache/hadoop/hbase/regionserver/
java/org/apache/hadoop/hbase/regionserver/metrics/
java/org/apache/hadoop/hbase/util/ webapps/master/
Author: stack
Date: Tue Jan 5 06:02:52 2010
New Revision: 895912
URL: http://svn.apache.org/viewvc?rev=895912&view=rev
Log:
HBASE-2021 Add compaction details to master UI
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
hadoop/hbase/trunk/src/webapps/master/master.jsp
hadoop/hbase/trunk/src/webapps/master/table.jsp
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java Tue Jan 5 06:02:52 2010
@@ -151,6 +151,9 @@
private final ServerManager serverManager;
private final RegionManager regionManager;
+ private long lastFragmentationQuery = -1L;
+ private Map<String, Integer> fragmentation = null;
+
/**
* Constructor
* @param conf configuration
@@ -1233,6 +1236,17 @@
}
}
+ public Map<String, Integer> getTableFragmentation() throws IOException {
+ long now = System.currentTimeMillis();
+ // only check every two minutes by default
+ int check = this.conf.getInt("hbase.master.fragmentation.check.frequency", 2 * 60 * 1000);
+ if (lastFragmentationQuery == -1 || now - lastFragmentationQuery > check) {
+ fragmentation = FSUtils.getTableFragmentation(this);
+ lastFragmentationQuery = now;
+ }
+ return fragmentation;
+ }
+
/**
* Main program
* @param args
@@ -1240,4 +1254,4 @@
public static void main(String [] args) {
doMain(args, HMaster.class);
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Tue Jan 5 06:02:52 2010
@@ -217,4 +217,14 @@
this.interrupt();
}
}
+
+ /**
+ * Returns the current size of the queue containing regions that are
+ * processed.
+ *
+ * @return The current size of the regions queue.
+ */
+ public int getCompactionQueueSize() {
+ return compactionQueue.size();
+ }
}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Jan 5 06:02:52 2010
@@ -1010,6 +1010,8 @@
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int)(memstoreSize/(1024*1024)));
this.metrics.storefileIndexSizeMB.set((int)(storefileIndexSize/(1024*1024)));
+ this.metrics.compactionQueueSize.set(compactSplitThread.
+ getCompactionQueueSize());
LruBlockCache lruBlockCache = (LruBlockCache)StoreFile.getBlockCache(conf);
if (lruBlockCache != null) {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Tue Jan 5 06:02:52 2010
@@ -108,6 +108,12 @@
new MetricsIntValue("memstoreSizeMB", registry);
/**
+ * Size of the compaction queue.
+ */
+ public final MetricsIntValue compactionQueueSize =
+ new MetricsIntValue("compactionQueueSize", registry);
+
+ /**
* filesystem read latency
*/
public final MetricsTimeVaryingRate fsReadLatency =
@@ -158,7 +164,7 @@
this.memstoreSizeMB.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
-
+ this.compactionQueueSize.pushMetric(this.metricsRecord);
this.blockCacheSize.pushMetric(this.metricsRecord);
this.blockCacheFree.pushMetric(this.metricsRecord);
this.blockCacheCount.pushMetric(this.metricsRecord);
@@ -218,6 +224,8 @@
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
+ sb = Strings.appendKeyValue(sb, "compactionQueueSize",
+ Integer.valueOf(this.compactionQueueSize.get()));
// Duplicate from jvmmetrics because metrics are private there so
// inaccessible.
MemoryUsage memory =
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java Tue Jan 5 06:02:52 2010
@@ -23,6 +23,8 @@
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,6 +38,7 @@
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -361,6 +364,98 @@
}
/**
+ * Returns the total overall fragmentation percentage. Includes .META. and
+ * -ROOT- as well.
+ *
+ * @param master The master defining the HBase root and file system.
+ * @return A map for each table and its percentage.
+ * @throws IOException When scanning the directory fails.
+ */
+ public static int getTotalTableFragmentation(final HMaster master)
+ throws IOException {
+ Map<String, Integer> map = getTableFragmentation(master);
+ return map != null && map.size() > 0 ? map.get("-TOTAL-").intValue() : -1;
+ }
+
+ /**
+ * Runs through the HBase rootdir and checks how many stores for each table
+ * have more than one file in them. Checks -ROOT- and .META. too. The total
+ * percentage across all tables is stored under the special key "-TOTAL-".
+ *
+ * @param master The master defining the HBase root and file system.
+ * @return A map for each table and its percentage.
+ * @throws IOException When scanning the directory fails.
+ */
+ public static Map<String, Integer> getTableFragmentation(
+ final HMaster master)
+ throws IOException {
+ Path path = master.getRootDir();
+ // since HMaster.getFileSystem() is package private
+ FileSystem fs = path.getFileSystem(master.getConfiguration());
+ return getTableFragmentation(fs, path);
+ }
+
+ /**
+ * Runs through the HBase rootdir and checks how many stores for each table
+ * have more than one file in them. Checks -ROOT- and .META. too. The total
+ * percentage across all tables is stored under the special key "-TOTAL-".
+ *
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @return A map for each table and its percentage.
+ * @throws IOException When scanning the directory fails.
+ */
+ public static Map<String, Integer> getTableFragmentation(
+ final FileSystem fs, final Path hbaseRootDir)
+ throws IOException {
+ Map<String, Integer> frags = new HashMap<String, Integer>();
+ int cfCountTotal = 0;
+ int cfFragTotal = 0;
+ DirFilter df = new DirFilter(fs);
+ // presumes any directory under hbase.rootdir is a table
+ FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
+ for (int i = 0; i < tableDirs.length; i++) {
+ // Skip the .log directory. All others should be tables. Inside a table,
+ // there are compaction.dir directories to skip. Otherwise, all else
+ // should be regions. Then in each region, should only be family
+ // directories. Under each of these, should be one file only.
+ Path d = tableDirs[i].getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
+ continue;
+ }
+ int cfCount = 0;
+ int cfFrag = 0;
+ FileStatus [] regionDirs = fs.listStatus(d, df);
+ for (int j = 0; j < regionDirs.length; j++) {
+ Path dd = regionDirs[j].getPath();
+ if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
+ continue;
+ }
+ // else its a region name, now look in region for families
+ FileStatus [] familyDirs = fs.listStatus(dd, df);
+ for (int k = 0; k < familyDirs.length; k++) {
+ cfCount++;
+ cfCountTotal++;
+ Path family = familyDirs[k].getPath();
+ // now in family make sure only one file
+ FileStatus [] familyStatus = fs.listStatus(family);
+ if (familyStatus.length > 1) {
+ cfFrag++;
+ cfFragTotal++;
+ }
+ }
+ }
+ // compute percentage per table and store in result list
+ frags.put(d.getName(), new Integer(
+ Math.round((float) cfFrag / cfCount * 100)));
+ }
+ // set overall percentage for all tables
+ frags.put("-TOTAL-", new Integer(
+ Math.round((float) cfFragTotal / cfCountTotal * 100)));
+ return frags;
+ }
+
+ /**
* Expects to find -ROOT- directory.
* @param fs
* @param hbaseRootDir
Modified: hadoop/hbase/trunk/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/webapps/master/master.jsp?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/trunk/src/webapps/master/master.jsp Tue Jan 5 06:02:52 2010
@@ -4,6 +4,7 @@
import="org.apache.hadoop.conf.Configuration"
import="org.apache.hadoop.io.Text"
import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.master.MetaRegion"
@@ -24,6 +25,7 @@
if (interval == 0) {
interval = 1;
}
+ Map<String, Integer> frags = master.getTableFragmentation();
%><?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -49,6 +51,7 @@
<tr><td>HBase Root Directory</td><td><%= master.getRootDir().toString() %></td><td>Location of HBase home directory</td></tr>
<tr><td>Load average</td><td><%= master.getServerManager().getAverageLoad() %></td><td>Average number of regions per regionserver. Naive computation.</td></tr>
<tr><td>Regions On FS</td><td><%= master.getRegionManager().countRegionsOnFS() %></td><td>Number of regions on FileSystem. Rough count.</td></tr>
+<tr><td>Fragmentation</td><td><%= frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %></td><td>Overall fragmentation of all tables, including .META. and -ROOT-.</td></tr>
<tr><td>Zookeeper Quorum</td><td><%= master.getZooKeeperWrapper().getQuorumServers() %></td><td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td></tr>
</table>
@@ -56,11 +59,17 @@
<%
if (rootLocation != null) { %>
<table>
-<tr><th>Table</th><th>Description</th></tr>
-<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %>"><%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %></a></td><td>The -ROOT- table holds references to all .META. regions.</td></tr>
+<tr><th>Table</th><th title="Fragmentation - Will be 0% after a major compaction and fluctuate during normal usage.">Frag.</th><th>Description</th></tr>
+<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %>"><%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %></a></td>
+<td align="center"><%= frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %></td>
+<td>The -ROOT- table holds references to all .META. regions.</td>
+</tr>
<%
if (onlineRegions != null && onlineRegions.size() > 0) { %>
-<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.META_TABLE_NAME) %>"><%= Bytes.toString(HConstants.META_TABLE_NAME) %></a></td><td>The .META. table holds references to all User Table regions</td></tr>
+<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.META_TABLE_NAME) %>"><%= Bytes.toString(HConstants.META_TABLE_NAME) %></a></td>
+<td align="center"><%= frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %></td>
+<td>The .META. table holds references to all User Table regions</td>
+</tr>
<% } %>
</table>
@@ -70,9 +79,12 @@
<% HTableDescriptor[] tables = new HBaseAdmin(conf).listTables();
if(tables != null && tables.length > 0) { %>
<table>
-<tr><th>Table</th><th>Description</th></tr>
+<tr><th>Table</th><th title="Fragmentation - Will be 0% after a major compaction and fluctuate during normal usage.">Frag.</th><th>Description</th></tr>
<% for(HTableDescriptor htDesc : tables ) { %>
-<tr><td><a href=/table.jsp?name=<%= htDesc.getNameAsString() %>><%= htDesc.getNameAsString() %></a> </td><td><%= htDesc.toString() %></td></tr>
+<tr><td><a href=/table.jsp?name=<%= htDesc.getNameAsString() %>><%= htDesc.getNameAsString() %></a> </td>
+<td align="center"><%= frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %></td>
+<td><%= htDesc.toString() %></td>
+</tr>
<% } %>
<p> <%= tables.length %> table(s) in set.</p>
Modified: hadoop/hbase/trunk/src/webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/webapps/master/table.jsp?rev=895912&r1=895911&r2=895912&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/webapps/master/table.jsp (original)
+++ hadoop/hbase/trunk/src/webapps/master/table.jsp Tue Jan 5 06:02:52 2010
@@ -28,6 +28,7 @@
master.getServerManager().getServerAddressToServerInfo();
String tableHeader = "<h2>Table Regions</h2><table><tr><th>Name</th><th>Region Server</th><th>Encoded Name</th><th>Start Key</th><th>End Key</th></tr>";
HServerAddress rootLocation = master.getRegionManager().getRootRegionLocation();
+ Map<String, Integer> frags = master.getTableFragmentation();
%>
<?xml version="1.0" encoding="UTF-8" ?>
@@ -121,6 +122,7 @@
<table>
<tr><th>Attribute Name</th><th>Value</th><th>Description</th></tr>
<tr><td>Enabled</td><td><%= hbadmin.isTableEnabled(table.getTableName()) %></td><td>Is the table enabled</td></tr>
+ <tr><td>Fragmentation</td><td><%= frags.get(tableName) != null ? frags.get(tableName).intValue() + "%" : "n/a" %></td><td>How fragmented is the table. After a major compaction it is 0%.</td></tr>
</table>
<%
Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();