You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/06/12 14:50:42 UTC
svn commit: r1349308 - in /hbase/branches/0.89-fb/src/main:
java/org/apache/hadoop/hbase/ java/org/apache/hadoop/hbase/regionserver/
resources/hbase-webapps/regionserver/
Author: mbautin
Date: Tue Jun 12 12:50:42 2012
New Revision: 1349308
URL: http://svn.apache.org/viewvc?rev=1349308&view=rev
Log:
HBASE-6194
Author: feifei
Summary: The region server currently lists all the region servers that it is hosting. It will be useful to report when those regions were opened on this server. It will also be useful to report what and when were the recent regions closed.
Test Plan:
create table test1, test2, test3, test4. disable these tables, then enable them. then you will see a list of recently closed regions info and the open date of those regions.
unit test :
(1) mvn -Dtest=org.apache.hadoop.hbase.regionserver.* test
there will be an error for org.apache.hadoop.hbase.regionserver.TestFSErrorsExposed
(2) mvn -Dtest=org.apache.hadoop.hbase.io.TestHeapSize test
Reviewers: pkhemani
Reviewed By: pkhemani
CC: mbautin
Differential Revision: https://phabricator.fb.com/D486302
Task ID: 1068217
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Tue Jun 12 12:50:42 2012
@@ -627,4 +627,4 @@ public class HRegionInfo extends Version
return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
}
-}
\ No newline at end of file
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Jun 12 12:50:42 2012
@@ -53,6 +53,9 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.Date;
+import java.util.Calendar;
+import java.text.SimpleDateFormat;
import org.apache.avro.ipc.Server;
import org.apache.commons.logging.Log;
@@ -153,6 +156,7 @@ public class HRegion implements HeapSize
static final String SPLITDIR = "splits";
static final String MERGEDIR = "merges";
+ static SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
final AtomicBoolean closed = new AtomicBoolean(false);
/* Closing can take some time; use the closing flag if there is stuff we don't
* want to do while in closing state; e.g. like offer this region up to the
@@ -186,6 +190,10 @@ public class HRegion implements HeapSize
private RequestMetrics writeRequests = null;
private HRegionServer regionServer = null;
+ /**
+ * The open date of the region is stored as a long obtained by currentTimeMillis()
+ */
+ private long openDate = 0;
/**
* The directory for the table this region is part of.
@@ -376,6 +384,19 @@ public class HRegion implements HeapSize
return m.get();
}
+ public String getOpenDateAsString() {
+ Date date = new Date(openDate);
+ return formatter.format(date);
+ }
+
+ public long getopenDate() {
+ return openDate;
+ }
+
+ public void setOpenDate(long dt) {
+ openDate = dt;
+ }
+
public static final long getWriteOps() {
return writeOps.getAndSet(0);
}
@@ -416,6 +437,7 @@ public class HRegion implements HeapSize
this.regionInfo = null;
this.threadWakeFrequency = 0L;
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+ this.openDate = 0;
}
/**
@@ -2343,16 +2365,16 @@ public class HRegion implements HeapSize
* new entries.
*/
private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap) {
- return applyFamilyMapToMemstore(familyMap, null);
+ return applyFamilyMapToMemstore(familyMap, null);
}
private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
- MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
+ MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
long start = EnvironmentEdgeManager.currentTimeMillis();
MultiVersionConsistencyControl.WriteEntry w = null;
long size = 0;
try {
- w = (writeEntryToUse == null)? mvcc.beginMemstoreInsert(): writeEntryToUse;
+ w = (writeEntryToUse == null)? mvcc.beginMemstoreInsert(): writeEntryToUse;
for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
byte[] family = e.getKey();
@@ -2366,12 +2388,12 @@ public class HRegion implements HeapSize
}
} finally {
if (writeEntryToUse == null) {
- long now = EnvironmentEdgeManager.currentTimeMillis();
- HRegion.memstoreInsertTime.addAndGet(now - start);
- start = now;
- mvcc.completeMemstoreInsert(w);
- now = EnvironmentEdgeManager.currentTimeMillis();
- HRegion.mvccWaitTime.addAndGet(now - start);
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ HRegion.memstoreInsertTime.addAndGet(now - start);
+ start = now;
+ mvcc.completeMemstoreInsert(w);
+ now = EnvironmentEdgeManager.currentTimeMillis();
+ HRegion.mvccWaitTime.addAndGet(now - start);
}
// else the calling function will take care of the mvcc completion and metrics.
}
@@ -3755,13 +3777,13 @@ public class HRegion implements HeapSize
flush = isFlushSize(this.incMemoryUsage(addedSize));
} finally {
// 8. roll mvcc forward
- long start = now;
- now = EnvironmentEdgeManager.currentTimeMillis();
- HRegion.memstoreInsertTime.addAndGet(now - start);
-
- mvcc.completeMemstoreInsert(w);
- now = EnvironmentEdgeManager.currentTimeMillis();
- HRegion.mvccWaitTime.addAndGet(now - start);
+ long start = now;
+ now = EnvironmentEdgeManager.currentTimeMillis();
+ HRegion.memstoreInsertTime.addAndGet(now - start);
+
+ mvcc.completeMemstoreInsert(w);
+ now = EnvironmentEdgeManager.currentTimeMillis();
+ HRegion.mvccWaitTime.addAndGet(now - start);
// 9. release region lock
this.updatesLock.readLock().unlock();
@@ -3851,7 +3873,7 @@ public class HRegion implements HeapSize
long after = EnvironmentEdgeManager.currentTimeMillis();
String signature = SchemaMetrics.generateSchemaMetricsPrefix(
this.getTableDesc().getNameAsString(), Bytes.toString(family));
- HRegion.incrTimeVaryingMetric(signature + "increment_", after - before);
+ HRegion.incrTimeVaryingMetric(signature + "increment_", after - before);
if (flush) {
// Request a cache flush. Do it outside update lock.
@@ -3876,7 +3898,7 @@ public class HRegion implements HeapSize
}
public static final long FIXED_OVERHEAD = ClassSize.align(
- (5 * Bytes.SIZEOF_LONG) + 2 * ClassSize.ARRAY +
+ (6 * Bytes.SIZEOF_LONG) + 2 * ClassSize.ARRAY +
(28 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Jun 12 12:50:42 2012
@@ -45,6 +45,9 @@ import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.Date;
+import java.util.Calendar;
+import java.text.SimpleDateFormat;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
@@ -131,6 +134,7 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
@@ -147,6 +151,8 @@ public class HRegionServer implements HR
private static final String UNABLE_TO_READ_MASTER_ADDRESS_ERR_MSG =
"Unable to read master address from ZooKeeper";
private static final ArrayList<Put> emptyPutArray = new ArrayList<Put>();
+ private static final int DEFAULT_NUM_TRACKED_CLOSED_REGION = 3;
+ private static SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// Set when a report to the master comes back with a message asking us to
// shutdown. Also set by call to stop when debugging or running unit tests
@@ -183,6 +189,10 @@ public class HRegionServer implements HR
protected final Map<Integer, HRegion> onlineRegions =
new ConcurrentHashMap<Integer, HRegion>();
+ // this is a list of region info that we recently closed
+ protected final List<ClosedRegionInfo> recentlyClosedRegions =
+ new ArrayList<ClosedRegionInfo>(DEFAULT_NUM_TRACKED_CLOSED_REGION + 1);
+
// This a list of regions that we need to re-try closing.
protected final Map<Integer, HRegion> retryCloseRegions = Collections
.synchronizedMap(new HashMap<Integer, HRegion>());
@@ -1045,7 +1055,49 @@ public class HRegionServer implements HR
return;
}
- /*
+ /**
+ * Inner class that stores information for recently closed regions.
+ * including region name, closeDate, startKey, endKey.
+ */
+ public static class ClosedRegionInfo {
+ private final String regionName;
+ private final long closeDate;
+ private final String startKey;
+ private final String endKey;
+
+ ClosedRegionInfo(String name, long date, byte[] startKey, byte[] endKey) {
+ this.regionName = name;
+ this.closeDate = date;
+ this.startKey = (startKey == null) ? "" : Bytes.toStringBinary(startKey);
+ this.endKey = (endKey == null) ? "" : Bytes.toStringBinary(endKey);
+ }
+
+ public String getRegionName() {
+ return regionName;
+ }
+
+ public long getCloseDate() {
+ return closeDate;
+ }
+
+ /**
+ * the date is represented as a string in format "yyyy-MM-dd HH:mm:ss"
+ */
+ public String getCloseDateAsString() {
+ Date date = new Date(closeDate);
+ return formatter.format(date);
+ }
+
+ public String getStartKey() {
+ return startKey;
+ }
+
+ public String getEndKey() {
+ return endKey;
+ }
+ }
+
+ /**
* Inner class that runs on a long period checking if regions need major
* compaction.
*/
@@ -1655,6 +1707,7 @@ public class HRegionServer implements HR
case MSG_REGION_CLOSE_WITHOUT_REPORT:
// Close a region, don't reply
+
closeRegion(e.msg.getRegionInfo(), false);
break;
@@ -1783,6 +1836,7 @@ public class HRegionServer implements HR
try {
this.onlineRegions.put(mapKey, region);
region.setRegionServer(this);
+ region.setOpenDate(EnvironmentEdgeManager.currentTimeMillis());
} finally {
this.lock.writeLock().unlock();
}
@@ -1850,6 +1904,12 @@ public class HRegionServer implements HR
getOutboundMsgs().add(new HMsg(HMsg.Type.MSG_REPORT_PROCESS_OPEN, hri));
}
+ private void addToRecentlyClosedRegions(ClosedRegionInfo info) {
+ recentlyClosedRegions.add(0, info);
+ if (recentlyClosedRegions.size() > DEFAULT_NUM_TRACKED_CLOSED_REGION)
+ recentlyClosedRegions.remove(DEFAULT_NUM_TRACKED_CLOSED_REGION);
+ }
+
@Override
public void closeRegion(final HRegionInfo hri, final boolean reportWhenCompleted)
throws IOException {
@@ -1871,6 +1931,10 @@ public class HRegionServer implements HR
this.addToRetryCloseRegions(region);
throw e;
}
+ ClosedRegionInfo info =
+ new ClosedRegionInfo(hri.getRegionNameAsString(), EnvironmentEdgeManager.currentTimeMillis() ,
+ hri.getStartKey(), hri.getEndKey());
+ addToRecentlyClosedRegions(info);
if(reportWhenCompleted) {
if(zkUpdater != null) {
HMsg hmsg = new HMsg(HMsg.Type.MSG_REPORT_CLOSE, hri, null);
@@ -2071,7 +2135,7 @@ public class HRegionServer implements HR
@Override
public void mutateRow(byte[] regionName, RowMutations arm)
throws IOException {
- mutateRow(regionName, Collections.singletonList(arm));
+ mutateRow(regionName, Collections.singletonList(arm));
}
@Override
@@ -2546,6 +2610,11 @@ public class HRegionServer implements HR
return lock.writeLock();
}
+ /** @return recent closed regions info*/
+ public List<ClosedRegionInfo> getRecentlyClosedRegionInfo() {
+ return recentlyClosedRegions;
+ }
+
/**
* @return Immutable list of this servers regions.
*/
@@ -2627,19 +2696,15 @@ public class HRegionServer implements HR
return region.getLastFlushTime();
}
- /**
- * @return The HRegionInfos from online regions sorted
- */
- public SortedSet<HRegionInfo> getSortedOnlineRegionInfos() {
- SortedSet<HRegionInfo> result = new TreeSet<HRegionInfo>();
+ public Map<HRegionInfo, String> getSortedOnlineRegionInfosAndOpenDate() {
+ TreeMap<HRegionInfo, String> result = new TreeMap<HRegionInfo, String>();
synchronized(this.onlineRegions) {
for (HRegion r: this.onlineRegions.values()) {
- result.add(r.getRegionInfo());
+ result.put(r.getRegionInfo(), r.getOpenDateAsString());
}
}
return result;
}
-
/**
* This method removes HRegion corresponding to hri from the Map of onlineRegions.
*
Modified: hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp (original)
+++ hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp Tue Jun 12 12:50:42 2012
@@ -3,6 +3,7 @@
import="java.io.IOException"
import="org.apache.hadoop.io.Text"
import="org.apache.hadoop.hbase.regionserver.HRegionServer"
+ import="org.apache.hadoop.hbase.regionserver.HRegionServer.ClosedRegionInfo"
import="org.apache.hadoop.hbase.regionserver.HRegion"
import="org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics"
import="org.apache.hadoop.hbase.util.Bytes"
@@ -18,7 +19,10 @@
e.printStackTrace();
}
RegionServerMetrics metrics = regionServer.getMetrics();
- Collection<HRegionInfo> onlineRegions = regionServer.getSortedOnlineRegionInfos();
+ Map<HRegionInfo, String> onlineRegionInfoAndOpenDate =
+ regionServer.getSortedOnlineRegionInfosAndOpenDate();
+ List<HRegionServer.ClosedRegionInfo> recentlyClosedRegionInfo =
+ regionServer.getRecentlyClosedRegionInfo();
int interval = regionServer.getConfiguration().getInt("hbase.regionserver.msginterval", 3000)/1000;
%><?xml version="1.0" encoding="UTF-8" ?>
@@ -46,27 +50,56 @@
</table>
<h2>Online Regions</h2>
-<% if (onlineRegions != null && onlineRegions.size() > 0) { %>
+<% if (onlineRegionInfoAndOpenDate != null && onlineRegionInfoAndOpenDate.size() > 0) { %>
<table>
<tr><th>Region Name</th>
<th>Read Requests/sec</th>
<th>Write Requests/sec</th>
<th>Start Key</th>
- <th>End Key</th><th>Metrics</th></tr>
-<% for (HRegionInfo r: onlineRegions) {
+ <th>End Key</th>
+ <th>Metrics</th>
+ <th>Open Time</th>
+</tr>
+<% for (HRegionInfo r : onlineRegionInfoAndOpenDate.keySet()) {
HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
%>
+
<tr><td><%= r.getRegionNameAsString() %></td>
<td><%= load.getReadRequestPerSec() %></td>
<td><%= load.getWriteRequestPerSec() %></td>
<td><%= Bytes.toStringBinary(r.getStartKey()) %></td>
<td><%= Bytes.toStringBinary(r.getEndKey()) %></td>
<td><%= load.toString() %></td>
+ <td><%= onlineRegionInfoAndOpenDate.get(r) %></td>
</tr>
<%
}
%>
</table>
+<h2>Recently Closed Regions</h2>
+<% if (recentlyClosedRegionInfo != null && recentlyClosedRegionInfo.size() > 0)
+ { %>
+<table>
+<tr><th>Region Name</th>
+ <th>Start Key</th>
+ <th>End Key</th>
+ <th>Closed Time</th>
+</tr>
+<%
+ for (HRegionServer.ClosedRegionInfo info: recentlyClosedRegionInfo) {
+%>
+<tr><td><%= info.getRegionName() %></td>
+ <td><%= info.getStartKey() %></td>
+ <td><%= info.getEndKey() %></td>
+ <td><%= info.getCloseDateAsString() %></td>
+</tr>
+<%
+ }
+%>
+</table>
+<%
+ }
+%>
<p>Region names are made of the containing table's name, a comma,
the start key, a comma, and a randomly generated region id. To illustrate,
the region named