You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/04/28 06:15:16 UTC
svn commit: r1331681 - in
/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver:
HRegion.java RegionServerAccounting.java handler/OpenRegionHandler.java
Author: tedyu
Date: Sat Apr 28 04:15:15 2012
New Revision: 1331681
URL: http://svn.apache.org/viewvc?rev=1331681&view=rev
Log:
HBASE-5611 Replayed edits from regions that failed to open during recovery aren't removed from the global MemStore size (Jieshan)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1331681&r1=1331680&r2=1331681&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Apr 28 04:15:15 2012
@@ -312,6 +312,7 @@ public class HRegion implements HeapSize
final long rowProcessorTimeout;
private volatile long lastFlushTime;
final RegionServerServices rsServices;
+ private RegionServerAccounting rsAccounting;
private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
private long blockingMemStoreSize;
final long threadWakeFrequency;
@@ -443,9 +444,10 @@ public class HRegion implements HeapSize
this.rowProcessorTimeout = conf.getLong(
"hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
- // don't initialize coprocessors if not running within a regionserver
- // TODO: revisit if coprocessors should load in other cases
if (rsServices != null) {
+ this.rsAccounting = this.rsServices.getRegionServerAccounting();
+ // don't initialize coprocessors if not running within a regionserver
+ // TODO: revisit if coprocessors should load in other cases
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
}
if (LOG.isDebugEnabled()) {
@@ -702,13 +704,8 @@ public class HRegion implements HeapSize
* @return the size of memstore in this region
*/
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
- if (this.rsServices != null) {
- RegionServerAccounting rsAccounting =
- this.rsServices.getRegionServerAccounting();
-
- if (rsAccounting != null) {
- rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
- }
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
}
return this.memstoreSize.getAndAdd(memStoreSize);
}
@@ -2708,6 +2705,11 @@ public class HRegion implements HeapSize
throw e;
}
}
+ // The edits size added into rsAccounting during this replaying will not
+ // be required any more. So just clear it.
+ if (this.rsAccounting != null) {
+ this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+ }
}
if (seqid > minSeqId) {
// Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2889,7 +2891,11 @@ public class HRegion implements HeapSize
* @return True if we should flush.
*/
protected boolean restoreEdit(final Store s, final KeyValue kv) {
- return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+ long kvSize = s.add(kv);
+ if (this.rsAccounting != null) {
+ rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+ }
+ return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
}
/*
@@ -4775,7 +4781,7 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 33 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+ 34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
(6 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java?rev=1331681&r1=1331680&r2=1331681&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java Sat Apr 28 04:15:15 2012
@@ -19,9 +19,12 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* RegionServerAccounting keeps record of some basic real time information about
@@ -32,6 +35,11 @@ public class RegionServerAccounting {
private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
+ // Store the edits size during replaying HLog. Use this to roll back the
+ // global memstore size once a region opening failed.
+ private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion =
+ new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+
/**
* @return the global Memstore size in the RegionServer
*/
@@ -47,5 +55,47 @@ public class RegionServerAccounting {
public long addAndGetGlobalMemstoreSize(long memStoreSize) {
return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
}
-
+
+ /***
+ * Add memStoreSize to replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+ * @return the replay edits size for the region.
+ */
+ public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+ AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+ if (replayEdistsSize == null) {
+ replayEdistsSize = new AtomicLong(0);
+ replayEditsPerRegion.put(regionName, replayEdistsSize);
+ }
+ return replayEdistsSize.addAndGet(memStoreSize);
+ }
+
+ /**
+ * Roll back the global MemStore size for a specified region when this region
+ * can't be opened.
+ *
+ * @param regionName the region which could not open.
+ * @return the global Memstore size in the RegionServer
+ */
+ public long rollbackRegionReplayEditsSize(byte[] regionName) {
+ AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+ long editsSizeLong = 0L;
+ if (replayEditsSize != null) {
+ editsSizeLong = -replayEditsSize.get();
+ clearRegionReplayEditsSize(regionName);
+ }
+ return addAndGetGlobalMemstoreSize(editsSizeLong);
+ }
+
+ /**
+ * Clear a region from replayEditsPerRegion.
+ *
+ * @param regionName region name.
+ */
+ public void clearRegionReplayEditsSize(byte[] regionName) {
+ replayEditsPerRegion.remove(regionName);
+ }
+
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1331681&r1=1331680&r2=1331681&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java Sat Apr 28 04:15:15 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -345,8 +346,17 @@ public class OpenRegionHandler extends E
// We failed open. Our caller will see the 'null' return value
// and transition the node back to FAILED_OPEN. If that fails,
// we rely on the Timeout Monitor in the master to reassign.
- LOG.error("Failed open of region=" +
- this.regionInfo.getRegionNameAsString(), t);
+ LOG.error(
+ "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+ + ", starting to roll back the global memstore size.", t);
+ // Decrease the global memstore size.
+ if (this.rsServices != null) {
+ RegionServerAccounting rsAccounting =
+ this.rsServices.getRegionServerAccounting();
+ if (rsAccounting != null) {
+ rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
+ }
+ }
}
return region;
}