You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/04/30 21:15:46 UTC
svn commit: r1332344 - in
/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver:
HRegion.java RegionServerAccounting.java handler/OpenRegionHandler.java
Author: tedyu
Date: Mon Apr 30 19:15:46 2012
New Revision: 1332344
URL: http://svn.apache.org/viewvc?rev=1332344&view=rev
Log:
HBASE-5611 Replayed edits from regions that failed to open during recovery aren't removed from the global MemStore size - v2 (Jieshan)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1332344&r1=1332343&r2=1332344&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Mon Apr 30 19:15:46 2012
@@ -299,6 +299,7 @@ public class HRegion implements HeapSize
final long timestampSlop;
private volatile long lastFlushTime;
final RegionServerServices rsServices;
+ private RegionServerAccounting rsAccounting;
private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
private long blockingMemStoreSize;
final long threadWakeFrequency;
@@ -403,9 +404,10 @@ public class HRegion implements HeapSize
"hbase.hregion.keyvalue.timestamp.slop.millisecs",
HConstants.LATEST_TIMESTAMP);
- // don't initialize coprocessors if not running within a regionserver
- // TODO: revisit if coprocessors should load in other cases
if (rsServices != null) {
+ this.rsAccounting = this.rsServices.getRegionServerAccounting();
+ // don't initialize coprocessors if not running within a regionserver
+ // TODO: revisit if coprocessors should load in other cases
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
}
if (LOG.isDebugEnabled()) {
@@ -662,14 +664,9 @@ public class HRegion implements HeapSize
* @return the size of memstore in this region
*/
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
- if (this.rsServices != null) {
- RegionServerAccounting rsAccounting =
- this.rsServices.getRegionServerAccounting();
-
- if (rsAccounting != null) {
- rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
- }
- }
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+ }
return this.memstoreSize.getAndAdd(memStoreSize);
}
@@ -2653,6 +2650,11 @@ public class HRegion implements HeapSize
throw e;
}
}
+ // The edits size added into rsAccounting during this replaying will not
+ // be required any more. So just clear it.
+ if (this.rsAccounting != null) {
+ this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+ }
}
if (seqid > minSeqId) {
// Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2834,7 +2836,11 @@ public class HRegion implements HeapSize
* @return True if we should flush.
*/
protected boolean restoreEdit(final Store s, final KeyValue kv) {
- return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+ long kvSize = s.add(kv);
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+ }
+ return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
}
/*
@@ -4611,8 +4617,8 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
- (6 * Bytes.SIZEOF_LONG) +
+ 32 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+ (5 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java?rev=1332344&r1=1332343&r2=1332344&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java Mon Apr 30 19:15:46 2012
@@ -19,8 +19,12 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hbase.util.Bytes;
+
/**
* RegionServerAccounting keeps record of some basic real time information about
* the Region Server. Currently, it only keeps record the global memstore size.
@@ -29,6 +33,11 @@ public class RegionServerAccounting {
private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
+ // Store the edits size during replaying HLog. Use this to roll back the
+ // global memstore size once a region opening failed.
+ private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion =
+ new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+
/**
* @return the global Memstore size in the RegionServer
*/
@@ -44,5 +53,46 @@ public class RegionServerAccounting {
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
}
-
+
+ /***
+ * Add memStoreSize to replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+ * @return the replay edits size for region hri.
+ */
+ public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+ AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+ if (replayEdistsSize == null) {
+ replayEdistsSize = new AtomicLong(0);
+ replayEditsPerRegion.put(regionName, replayEdistsSize);
+ }
+ return replayEdistsSize.addAndGet(memStoreSize);
+ }
+
+ /**
+ * Roll back the global MemStore size for a specified region when this region
+ * can't be opened.
+ *
+ * @param regionName the region which could not open.
+ * @return the global Memstore size in the RegionServer
+ */
+ public long rollbackRegionReplayEditsSize(byte[] regionName) {
+ AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+ long editsSizeLong = 0L;
+ if (replayEditsSize != null) {
+ editsSizeLong = -replayEditsSize.get();
+ clearRegionReplayEditsSize(regionName);
+ }
+ return addAndGetGlobalMemstoreSize(editsSizeLong);
+ }
+
+ /**
+ * Clear a region from replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ */
+ public void clearRegionReplayEditsSize(byte[] regionName) {
+ replayEditsPerRegion.remove(regionName);
+ }
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1332344&r1=1332343&r2=1332344&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java Mon Apr 30 19:15:46 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -343,8 +344,17 @@ public class OpenRegionHandler extends E
// We failed open. Our caller will see the 'null' return value
// and transition the node back to FAILED_OPEN. If that fails,
// we rely on the Timeout Monitor in the master to reassign.
- LOG.error("Failed open of region=" +
- this.regionInfo.getRegionNameAsString(), t);
+ LOG.error(
+ "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+ + ", starting to roll back the global memstore size.", t);
+ // Decrease the global memstore size.
+ if (this.rsServices != null) {
+ RegionServerAccounting rsAccounting =
+ this.rsServices.getRegionServerAccounting();
+ if (rsAccounting != null) {
+ rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
+ }
+ }
}
return region;
}