You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2010/04/08 23:52:18 UTC
svn commit: r932137 - in /hadoop/hbase/branches/0.20: ./
src/java/org/apache/hadoop/hbase/master/
src/java/org/apache/hadoop/hbase/regionserver/
src/java/org/apache/hadoop/hbase/regionserver/metrics/
src/java/org/apache/hadoop/hbase/util/ src/webapps/m...
Author: jdcryans
Date: Thu Apr 8 21:52:18 2010
New Revision: 932137
URL: http://svn.apache.org/viewvc?rev=932137&view=rev
Log:
HBASE-2165 Improve fragmentation display and implementation
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/util/FSUtils.java
hadoop/hbase/branches/0.20/src/webapps/master/master.jsp
hadoop/hbase/branches/0.20/src/webapps/master/table.jsp
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Thu Apr 8 21:52:18 2010
@@ -8,6 +8,7 @@ Release 0.20.4 - Unreleased
HBASE-2378 Bulk insert with multiple reducers broken due to improper
ImmutableBytesWritable comparator (Todd Lipcon via Stack)
HBASE-2392 Upgrade to ZooKeeper 3.3.0
+ HBASE-2165 Improve fragmentation display and implementation
BUG FIXES
HBASE-2173 New idx javadoc not included with the rest
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/master/HMaster.java Thu Apr 8 21:52:18 2010
@@ -161,9 +161,6 @@ public class HMaster extends Thread impl
private MasterMetrics metrics;
- private long lastFragmentationQuery = -1L;
- private Map<String, Integer> fragmentation = null;
-
/**
* Build the HMaster out of a raw configuration item.
* @param conf configuration
@@ -1183,17 +1180,6 @@ public class HMaster extends Thread impl
}
}
- public Map<String, Integer> getTableFragmentation() throws IOException {
- long now = System.currentTimeMillis();
- // only check every two minutes by default
- int check = this.conf.getInt("hbase.master.fragmentation.check.frequency", 2 * 60 * 1000);
- if (lastFragmentationQuery == -1 || now - lastFragmentationQuery > check) {
- fragmentation = FSUtils.getTableFragmentation(this);
- lastFragmentationQuery = now;
- }
- return fragmentation;
- }
-
/*
* Main program
*/
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu Apr 8 21:52:18 2010
@@ -234,14 +234,4 @@ class CompactSplitThread extends Thread
this.interrupt();
}
}
-
- /**
- * Returns the current size of the queue containing regions that are
- * processed.
- *
- * @return The current size of the regions queue.
- */
- public int getCompactionQueueSize() {
- return compactionQueue.size();
- }
}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Apr 8 21:52:18 2010
@@ -1100,8 +1100,6 @@ public class HRegionServer implements HC
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int)(memstoreSize/(1024*1024)));
this.metrics.storefileIndexSizeMB.set((int)(storefileIndexSize/(1024*1024)));
- this.metrics.compactionQueueSize.set(compactSplitThread.
- getCompactionQueueSize());
LruBlockCache lruBlockCache = (LruBlockCache)StoreFile.getBlockCache(conf);
if (lruBlockCache != null) {
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Thu Apr 8 21:52:18 2010
@@ -108,12 +108,6 @@ public class RegionServerMetrics impleme
new MetricsIntValue("memstoreSizeMB", registry);
/**
- * Size of the compaction queue.
- */
- public final MetricsIntValue compactionQueueSize =
- new MetricsIntValue("compactionQueueSize", registry);
-
- /**
* filesystem read latency
*/
public final MetricsTimeVaryingRate fsReadLatency =
@@ -164,7 +158,7 @@ public class RegionServerMetrics impleme
this.memstoreSizeMB.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
- this.compactionQueueSize.pushMetric(this.metricsRecord);
+
this.blockCacheSize.pushMetric(this.metricsRecord);
this.blockCacheFree.pushMetric(this.metricsRecord);
this.blockCacheCount.pushMetric(this.metricsRecord);
@@ -237,8 +231,6 @@ public class RegionServerMetrics impleme
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
- sb = Strings.appendKeyValue(sb, "compactionQueueSize",
- Integer.valueOf(this.compactionQueueSize.get()));
// Duplicate from jvmmetrics because metrics are private there so
// inaccessible.
MemoryUsage memory =
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/util/FSUtils.java Thu Apr 8 21:52:18 2010
@@ -24,8 +24,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,7 +37,6 @@ import org.apache.hadoop.hbase.HBaseConf
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -371,98 +368,6 @@ public class FSUtils {
}
/**
- * Returns the total overall fragmentation percentage. Includes .META. and
- * -ROOT- as well.
- *
- * @param master The master defining the HBase root and file system.
- * @return A map for each table and its percentage.
- * @throws IOException When scanning the directory fails.
- */
- public static int getTotalTableFragmentation(final HMaster master)
- throws IOException {
- Map<String, Integer> map = getTableFragmentation(master);
- return map != null && map.size() > 0 ? map.get("-TOTAL-").intValue() : -1;
- }
-
- /**
- * Runs through the HBase rootdir and checks how many stores for each table
- * have more than one file in them. Checks -ROOT- and .META. too. The total
- * percentage across all tables is stored under the special key "-TOTAL-".
- *
- * @param master The master defining the HBase root and file system.
- * @return A map for each table and its percentage.
- * @throws IOException When scanning the directory fails.
- */
- public static Map<String, Integer> getTableFragmentation(
- final HMaster master)
- throws IOException {
- Path path = master.getRootDir();
- // since HMaster.getFileSystem() is package private
- FileSystem fs = path.getFileSystem(master.getConfiguration());
- return getTableFragmentation(fs, path);
- }
-
- /**
- * Runs through the HBase rootdir and checks how many stores for each table
- * have more than one file in them. Checks -ROOT- and .META. too. The total
- * percentage across all tables is stored under the special key "-TOTAL-".
- *
- * @param fs The file system to use.
- * @param hbaseRootDir The root directory to scan.
- * @return A map for each table and its percentage.
- * @throws IOException When scanning the directory fails.
- */
- public static Map<String, Integer> getTableFragmentation(
- final FileSystem fs, final Path hbaseRootDir)
- throws IOException {
- Map<String, Integer> frags = new HashMap<String, Integer>();
- int cfCountTotal = 0;
- int cfFragTotal = 0;
- DirFilter df = new DirFilter(fs);
- // presumes any directory under hbase.rootdir is a table
- FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
- for (int i = 0; i < tableDirs.length; i++) {
- // Skip the .log directory. All others should be tables. Inside a table,
- // there are compaction.dir directories to skip. Otherwise, all else
- // should be regions. Then in each region, should only be family
- // directories. Under each of these, should be one file only.
- Path d = tableDirs[i].getPath();
- if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
- continue;
- }
- int cfCount = 0;
- int cfFrag = 0;
- FileStatus [] regionDirs = fs.listStatus(d, df);
- for (int j = 0; j < regionDirs.length; j++) {
- Path dd = regionDirs[j].getPath();
- if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
- continue;
- }
- // else its a region name, now look in region for families
- FileStatus [] familyDirs = fs.listStatus(dd, df);
- for (int k = 0; k < familyDirs.length; k++) {
- cfCount++;
- cfCountTotal++;
- Path family = familyDirs[k].getPath();
- // now in family make sure only one file
- FileStatus [] familyStatus = fs.listStatus(family);
- if (familyStatus.length > 1) {
- cfFrag++;
- cfFragTotal++;
- }
- }
- }
- // compute percentage per table and store in result list
- frags.put(d.getName(), new Integer(
- Math.round((float) cfFrag / cfCount * 100)));
- }
- // set overall percentage for all tables
- frags.put("-TOTAL-", new Integer(
- Math.round((float) cfFragTotal / cfCountTotal * 100)));
- return frags;
- }
-
- /**
* Expects to find -ROOT- directory.
* @param fs
* @param hbaseRootDir
Modified: hadoop/hbase/branches/0.20/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/webapps/master/master.jsp?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/branches/0.20/src/webapps/master/master.jsp Thu Apr 8 21:52:18 2010
@@ -3,7 +3,6 @@
import="java.net.URLEncoder"
import="org.apache.hadoop.io.Text"
import="org.apache.hadoop.hbase.util.Bytes"
- import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.master.MetaRegion"
@@ -24,7 +23,6 @@
if (interval == 0) {
interval = 1;
}
- Map<String, Integer> frags = master.getTableFragmentation();
%><?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -50,7 +48,6 @@
<tr><td>HBase Root Directory</td><td><%= master.getRootDir().toString() %></td><td>Location of HBase home directory</td></tr>
<tr><td>Load average</td><td><%= master.getAverageLoad() %></td><td>Average number of regions per regionserver. Naive computation.</td></tr>
<tr><td>Regions On FS</td><td><%= master.countRegionsOnFS() %></td><td>Number of regions on FileSystem. Rough count.</td></tr>
-<tr><td>Fragmentation</td><td><%= frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %></td><td>Overall fragmentation of all tables, including .META. and -ROOT-.</td></tr>
<tr><td>Zookeeper Quorum</td><td><%= master.getZooKeeperWrapper().getQuorumServers() %></td><td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td></tr>
</table>
@@ -58,18 +55,11 @@
<%
if (rootLocation != null) { %>
<table>
-<tr><th>Table</th><th title="Fragmentation - Will be 0% after a major compaction and fluctuate during normal usage.">Frag.</th><th>Description</th></tr>
-<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %>"><%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %></a></td>
-<td align="center"><%= frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %></td>
-<td>The -ROOT- table holds references to all .META. regions.</td>
-</tr>
+<tr><th>Table</th><th>Description</th></tr>
+<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %>"><%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %></a></td><td>The -ROOT- table holds references to all .META. regions.</td></tr>
<%
if (onlineRegions != null && onlineRegions.size() > 0) { %>
-<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.META_TABLE_NAME) %>"><%= Bytes.toString(HConstants.META_TABLE_NAME) %></a></td>
-<td align="center"><%= frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %></td>
-<td>The .META. table holds references to all User Table regions</td>
-</tr>
-
+<tr><td><a href="/table.jsp?name=<%= Bytes.toString(HConstants.META_TABLE_NAME) %>"><%= Bytes.toString(HConstants.META_TABLE_NAME) %></a></td><td>The .META. table holds references to all User Table regions</td></tr>
<% } %>
</table>
<%} %>
@@ -78,12 +68,9 @@
<% HTableDescriptor[] tables = new HBaseAdmin(conf).listTables();
if(tables != null && tables.length > 0) { %>
<table>
-<tr><th>Table</th><th title="Fragmentation - Will be 0% after a major compaction and fluctuate during normal usage.">Frag.</th><th>Description</th></tr>
+<tr><th>Table</th><th>Description</th></tr>
<% for(HTableDescriptor htDesc : tables ) { %>
-<tr><td><a href=/table.jsp?name=<%= htDesc.getNameAsString() %>><%= htDesc.getNameAsString() %></a> </td>
-<td align="center"><%= frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %></td>
-<td><%= htDesc.toString() %></td>
-</tr>
+<tr><td><a href=/table.jsp?name=<%= htDesc.getNameAsString() %>><%= htDesc.getNameAsString() %></a> </td><td><%= htDesc.toString() %></td></tr>
<% } %>
<p> <%= tables.length %> table(s) in set.</p>
Modified: hadoop/hbase/branches/0.20/src/webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/webapps/master/table.jsp?rev=932137&r1=932136&r2=932137&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/webapps/master/table.jsp (original)
+++ hadoop/hbase/branches/0.20/src/webapps/master/table.jsp Thu Apr 8 21:52:18 2010
@@ -27,7 +27,6 @@
master.getServerAddressToServerInfo();
String tableHeader = "<h2>Table Regions</h2><table><tr><th>Name</th><th>Region Server</th><th>Encoded Name</th><th>Start Key</th><th>End Key</th></tr>";
HServerAddress rootLocation = master.getRootRegionLocation();
- Map<String, Integer> frags = master.getTableFragmentation();
%>
<?xml version="1.0" encoding="UTF-8" ?>
@@ -121,7 +120,6 @@
<table>
<tr><th>Attribute Name</th><th>Value</th><th>Description</th></tr>
<tr><td>Enabled</td><td><%= hbadmin.isTableEnabled(table.getTableName()) %></td><td>Is the table enabled</td></tr>
- <tr><td>Fragmentation</td><td><%= frags.get(tableName) != null ? frags.get(tableName).intValue() + "%" : "n/a" %></td><td>How fragmented is the table. After a major compaction it is 0%.</td></tr>
</table>
<%
Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();