You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/11/24 05:32:08 UTC
svn commit: r1205723 - in /hbase/branches/0.92/src:
main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
Author: stack
Date: Thu Nov 24 04:32:07 2011
New Revision: 1205723
URL: http://svn.apache.org/viewvc?rev=1205723&view=rev
Log:
HBASE-4853 HBASE-4789 does overzealous pruning of seqids
Modified:
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Nov 24 04:32:07 2011
@@ -1165,8 +1165,8 @@ public class HRegion implements HeapSize
// rows then)
status.setStatus("Obtaining lock to block concurrent updates");
this.updatesLock.writeLock().lock();
+ long flushsize = this.memstoreSize.get();
status.setStatus("Preparing to flush by snapshotting stores");
- long currentMemStoreSize = 0;
List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
try {
sequenceId = (wal == null)? myseqid:
@@ -1184,7 +1184,8 @@ public class HRegion implements HeapSize
} finally {
this.updatesLock.writeLock().unlock();
}
- String s = "Finished snapshotting " + this + ", commencing wait for rwcc";
+ String s = "Finished snapshotting " + this +
+ ", commencing wait for mvcc, flushsize=" + flushsize;
status.setStatus(s);
LOG.debug(s);
@@ -1212,8 +1213,7 @@ public class HRegion implements HeapSize
storeFlushers.clear();
// Set down the memstore size by amount of flush.
- currentMemStoreSize =
- this.addAndGetGlobalMemstoreSize(-this.memstoreSize.get());
+ this.addAndGetGlobalMemstoreSize(-flushsize);
} catch (Throwable t) {
// An exception here means that the snapshot was not persisted.
// The hlog needs to be replayed so its content is restored to memstore.
@@ -1251,14 +1251,17 @@ public class HRegion implements HeapSize
}
long time = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ long memstoresize = this.memstoreSize.get();
String msg = "Finished memstore flush of ~" +
- StringUtils.humanReadableInt(currentMemStoreSize) + " for region " +
- this + " in " + time + "ms, sequenceid=" + sequenceId +
- ", compaction requested=" + compactionRequested +
- ((wal == null)? "; wal=null": "");
+ StringUtils.humanReadableInt(flushsize) + "/" + flushsize +
+ ", currentsize=" +
+ StringUtils.humanReadableInt(memstoresize) + "/" + memstoresize +
+ " for region " + this + " in " + time + "ms, sequenceid=" + sequenceId +
+ ", compaction requested=" + compactionRequested +
+ ((wal == null)? "; wal=null": "");
LOG.info(msg);
status.setStatus(msg);
- this.recentFlushes.add(new Pair<Long,Long>(time/1000,currentMemStoreSize));
+ this.recentFlushes.add(new Pair<Long,Long>(time/1000, flushsize));
return compactionRequested;
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Thu Nov 24 04:32:07 2011
@@ -1359,11 +1359,6 @@ public class HLog implements Syncable {
// Cleaning up of lastSeqWritten is in the finally clause because we
// don't want to confuse getOldestOutstandingSeqNum()
this.lastSeqWritten.remove(getSnapshotName(encodedRegionName));
- Long l = this.lastSeqWritten.remove(encodedRegionName);
- if (l != null) {
- LOG.warn("Why is there a raw encodedRegionName in lastSeqWritten? name=" +
- Bytes.toString(encodedRegionName) + ", seqid=" + l);
- }
this.cacheFlushLock.unlock();
}
}
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java Thu Nov 24 04:32:07 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.junit.Test;
@@ -67,7 +68,7 @@ public class TestGlobalMemStoreSize {
cluster = TEST_UTIL.getHBaseCluster();
LOG.info("Waiting for active/ready master");
cluster.waitForActiveAndReadyMaster();
-
+
// Create a table with regions
byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
byte [] family = Bytes.toBytes("family");
@@ -92,20 +93,56 @@ public class TestGlobalMemStoreSize {
// check the global memstore size after flush
int i = 0;
for (HRegionServer server : getOnlineRegionServers()) {
- LOG.info("Starting flushes on " + server.getServerName() + ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
- for(HRegionInfo regionInfo : server.getOnlineRegions()) {
- HRegion region=
- server.getFromOnlineRegions(regionInfo.getEncodedName());
- LOG.info("Flush " + region.toString() + " on " + server.getServerName() + ", " +
- region.flushcache() + ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());;
+ LOG.info("Starting flushes on " + server.getServerName() +
+ ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
+
+ for (HRegionInfo regionInfo : server.getOnlineRegions()) {
+ HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
+ flush(r, server);
}
- assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
- 0);
+ LOG.info("Post flush on " + server.getServerName());
+ long now = System.currentTimeMillis();
+ long timeout = now + 1000;
+ while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
+ timeout < System.currentTimeMillis()) {
+ Threads.sleep(10);
+ }
+ long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
+ if (size > 0) {
+ // If size > 0, see if its because the meta region got edits while
+ // our test was running....
+ for (HRegionInfo regionInfo : server.getOnlineRegions()) {
+ HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
+ long l = r.getMemstoreSize().longValue();
+ if (l > 0) {
+ // Only meta could have edits at this stage. Give it another flush
+ // clear them.
+ assertTrue(regionInfo.isMetaRegion());
+ LOG.info(r.toString() + " " + l + ", reflushing");
+ r.flushcache();
+ }
+ }
+ }
+ size = server.getRegionServerAccounting().getGlobalMemstoreSize();
+ assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
}
TEST_UTIL.shutdownMiniCluster();
}
+ /**
+ * Flush and log stats on flush
+ * @param r
+ * @param server
+ * @throws IOException
+ */
+ private void flush(final HRegion r, final HRegionServer server)
+ throws IOException {
+ LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
+ ", " + r.flushcache() + ", size=" +
+ server.getRegionServerAccounting().getGlobalMemstoreSize());
+ }
+
/** figure out how many regions are currently being served. */
private int getRegionCount() throws IOException {
int total = 0;
@@ -138,4 +175,3 @@ public class TestGlobalMemStoreSize {
}
}
}
-