You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/04/28 06:20:01 UTC
svn commit: r1331684 - in /hbase/branches/0.92: ./
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/java/org/apache/hadoop/hbase/regionserver/handler/
Author: tedyu
Date: Sat Apr 28 04:20:00 2012
New Revision: 1331684
URL: http://svn.apache.org/viewvc?rev=1331684&view=rev
Log:
HBASE-5611 Replayed edits from regions that failed to open during recovery aren't removed from the global MemStore size (Jieshan)
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1331684&r1=1331683&r2=1331684&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Sat Apr 28 04:20:00 2012
@@ -54,6 +54,8 @@ Release 0.92.2 - Unreleased
HBASE-5871 Usability regression, we don't parse compression algos anymore (Lars H)
HBASE-5873 TimeOut Monitor thread should be started after atleast one region server registers. (rajeshbabu)
HBASE-5893 Allow spaces in coprocessor conf (aka trim() className) (Matteo Bertozzi)
+ HBASE-5611 Replayed edits from regions that failed to open during recovery aren't removed from
+ the global MemStore size (Jieshan)
IMPROVEMENTS
HBASE-5592 Make it easier to get a table from shell (Ben West)
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1331684&r1=1331683&r2=1331684&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Apr 28 04:20:00 2012
@@ -275,6 +275,7 @@ public class HRegion implements HeapSize
long memstoreFlushSize;
private volatile long lastFlushTime;
final RegionServerServices rsServices;
+ private RegionServerAccounting rsAccounting;
private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
private long blockingMemStoreSize;
final long threadWakeFrequency;
@@ -367,6 +368,9 @@ public class HRegion implements HeapSize
// don't initialize coprocessors if not running within a regionserver
// TODO: revisit if coprocessors should load in other cases
if (rsServices != null) {
+ this.rsAccounting = this.rsServices.getRegionServerAccounting();
+ // don't initialize coprocessors if not running within a regionserver
+ // TODO: revisit if coprocessors should load in other cases
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
}
if (LOG.isDebugEnabled()) {
@@ -593,13 +597,8 @@ public class HRegion implements HeapSize
* @return the size of memstore in this region
*/
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
- if (this.rsServices != null) {
- RegionServerAccounting rsAccounting =
- this.rsServices.getRegionServerAccounting();
-
- if (rsAccounting != null) {
- rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
- }
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
}
return this.memstoreSize.getAndAdd(memStoreSize);
}
@@ -2315,6 +2314,13 @@ public class HRegion implements HeapSize
throw e;
}
}
+ // The edits size added into rsAccounting during this replaying will not
+ // be required any more. So just clear it.
+ if (this.rsAccounting != null) {
+ this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo
+ .getRegionName());
+ }
+
}
if (seqid > minSeqId) {
// Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2496,7 +2502,11 @@ public class HRegion implements HeapSize
* @return True if we should flush.
*/
protected boolean restoreEdit(final Store s, final KeyValue kv) {
- return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+ long kvSize = s.add(kv);
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+ }
+ return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
}
/*
@@ -3946,7 +3956,7 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+ 31 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
(4 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java?rev=1331684&r1=1331683&r2=1331684&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java Sat Apr 28 04:20:00 2012
@@ -19,8 +19,12 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.util.Bytes;
+
/**
* RegionServerAccounting keeps record of some basic real time information about
* the Region Server. Currently, it only keeps record the global memstore size.
@@ -29,6 +33,11 @@ public class RegionServerAccounting {
private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
+ // Store the edits size during replaying HLog. Use this to roll back the
+ // global memstore size once a region opening failed.
+ private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion =
+ new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+
/**
* @return the global Memstore size in the RegionServer
*/
@@ -44,5 +53,47 @@ public class RegionServerAccounting {
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
}
-
+
+ /***
+ * Add memStoreSize to replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+ * @return the replay edits size for the region.
+ */
+ public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+ AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+ if (replayEdistsSize == null) {
+ replayEdistsSize = new AtomicLong(0);
+ replayEditsPerRegion.put(regionName, replayEdistsSize);
+ }
+ return replayEdistsSize.addAndGet(memStoreSize);
+ }
+
+ /**
+ * Roll back the global MemStore size for a specified region when this region
+ * can't be opened.
+ *
+ * @param regionName the region which could not open.
+ * @return the global Memstore size in the RegionServer
+ */
+ public long rollbackRegionReplayEditsSize(byte[] regionName) {
+ AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+ long editsSizeLong = 0L;
+ if (replayEditsSize != null) {
+ editsSizeLong = -replayEditsSize.get();
+ clearRegionReplayEditsSize(regionName);
+ }
+ return addAndGetGlobalMemstoreSize(editsSizeLong);
+ }
+
+ /**
+ * Clear a region from replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ */
+ public void clearRegionReplayEditsSize(byte[] regionName) {
+ replayEditsPerRegion.remove(regionName);
+ }
+
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1331684&r1=1331683&r2=1331684&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java Sat Apr 28 04:20:00 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -343,8 +344,18 @@ public class OpenRegionHandler extends E
// We failed open. Our caller will see the 'null' return value
// and transition the node back to FAILED_OPEN. If that fails,
// we rely on the Timeout Monitor in the master to reassign.
- LOG.error("Failed open of region=" +
- this.regionInfo.getRegionNameAsString(), t);
+ LOG.error(
+ "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+ + ", starting to roll back the global memstore size.", t);
+ // Decrease the global memstore size.
+ if (this.rsServices != null) {
+ RegionServerAccounting rsAccounting = this.rsServices
+ .getRegionServerAccounting();
+ if (rsAccounting != null) {
+ rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo
+ .getRegionName());
+ }
+ }
}
return region;
}