You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:08:37 UTC
svn commit: r1181439 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
Author: nspiegelberg
Date: Tue Oct 11 02:08:37 2011
New Revision: 1181439
URL: http://svn.apache.org/viewvc?rev=1181439&view=rev
Log:
HBASE 3198/3208 import: off by one errors in log seq number
Summary:
Imported from open source.
Test Plan:
the new unit test is not passing yet. I am still investigating. Uploading for
review.
DiffCamp Revision: 179949
Reviewed By: nspiegelberg
CC: nspiegelberg, hbase@lists
Revert Plan:
OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181439&r1=1181438&r2=1181439&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 11 02:08:37 2011
@@ -569,20 +569,20 @@ public class HLog implements Syncable {
*/
private byte [][] cleanOldLogs() throws IOException {
Long oldestOutstandingSeqNum = getOldestOutstandingSeqNum();
- // Get the set of all log files whose final ID is older than or
- // equal to the oldest pending region operation
+ // Get the set of all log files whose last sequence number is smaller than
+ // the oldest edit's sequence number.
TreeSet<Long> sequenceNumbers =
new TreeSet<Long>(this.outputfiles.headMap(
- (Long.valueOf(oldestOutstandingSeqNum.longValue() + 1L))).keySet());
+ (Long.valueOf(oldestOutstandingSeqNum.longValue()))).keySet());
// Now remove old log files (if any)
int logsToRemove = sequenceNumbers.size();
if (logsToRemove > 0) {
if (LOG.isDebugEnabled()) {
// Find associated region; helps debugging.
byte [] oldestRegion = getOldestRegion(oldestOutstandingSeqNum);
- LOG.debug("Found " + logsToRemove + " hlogs to remove " +
- " out of total " + this.outputfiles.size() + "; " +
- "oldest outstanding seqnum is " + oldestOutstandingSeqNum +
+ LOG.debug("Found " + logsToRemove + " hlogs to remove" +
+ " out of total " + this.outputfiles.size() + ";" +
+ " oldest outstanding sequenceid is " + oldestOutstandingSeqNum +
" from region " + Bytes.toString(oldestRegion));
}
for (Long seq : sequenceNumbers) {
@@ -595,7 +595,7 @@ public class HLog implements Syncable {
int logCount = this.outputfiles.size() - logsToRemove;
if (logCount > this.maxLogs && this.outputfiles != null &&
this.outputfiles.size() > 0) {
- regions = findMemstoresWithEditsOlderThan(this.outputfiles.firstKey(),
+ regions = findMemstoresWithEditsEqualOrOlderThan(this.outputfiles.firstKey(),
this.lastSeqWritten);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < regions.length; i++) {
@@ -610,19 +610,19 @@ public class HLog implements Syncable {
}
/**
- * Return regions (memstores) that have edits that are less than the passed
- * <code>oldestWALseqid</code>.
+ * Return regions (memstores) that have edits that are equal or less than
+ * the passed <code>oldestWALseqid</code>.
* @param oldestWALseqid
* @param regionsToSeqids
* @return All regions whose seqid is < than <code>oldestWALseqid</code> (Not
* necessarily in order). Null if no regions found.
*/
- static byte [][] findMemstoresWithEditsOlderThan(final long oldestWALseqid,
+ static byte [][] findMemstoresWithEditsEqualOrOlderThan(final long oldestWALseqid,
final Map<byte [], Long> regionsToSeqids) {
// This method is static so it can be unit tested the easier.
List<byte []> regions = null;
for (Map.Entry<byte [], Long> e: regionsToSeqids.entrySet()) {
- if (e.getValue().longValue() < oldestWALseqid) {
+ if (e.getValue().longValue() <= oldestWALseqid) {
if (regions == null) regions = new ArrayList<byte []>();
regions.add(e.getKey());
}
@@ -673,7 +673,7 @@ public class HLog implements Syncable {
}
if (currentfilenum >= 0) {
oldFile = computeFilename();
- this.outputfiles.put(Long.valueOf(this.logSeqNum.get() - 1), oldFile);
+ this.outputfiles.put(Long.valueOf(this.logSeqNum.get()), oldFile);
}
}
return oldFile;
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1181439&r1=1181438&r2=1181439&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Tue Oct 11 02:08:37 2011
@@ -257,28 +257,30 @@ public class TestHLog {
}
/**
- * Test the findMemstoresWithEditsOlderThan method.
+ * Test the findMemstoresWithEditsEqualOrOlderThan method.
* @throws IOException
*/
@Test
- public void testFindMemstoresWithEditsOlderThan() throws IOException {
+ public void testFindMemstoresWithEditsEqualOrOlderThan() throws IOException {
Map<byte [], Long> regionsToSeqids = new HashMap<byte [], Long>();
for (int i = 0; i < 10; i++) {
Long l = Long.valueOf(i);
regionsToSeqids.put(l.toString().getBytes(), l);
}
byte [][] regions =
- HLog.findMemstoresWithEditsOlderThan(1, regionsToSeqids);
- assertEquals(1, regions.length);
- assertTrue(Bytes.equals(regions[0], "0".getBytes()));
- regions = HLog.findMemstoresWithEditsOlderThan(3, regionsToSeqids);
- int count = 3;
+ HLog.findMemstoresWithEditsEqualOrOlderThan(1, regionsToSeqids);
+ assertEquals(2, regions.length);
+ assertTrue(Bytes.equals(regions[0], "0".getBytes()) ||
+ Bytes.equals(regions[0], "1".getBytes()));
+ regions = HLog.findMemstoresWithEditsEqualOrOlderThan(3, regionsToSeqids);
+ int count = 4;
assertEquals(count, regions.length);
// Regions returned are not ordered.
for (int i = 0; i < count; i++) {
assertTrue(Bytes.equals(regions[i], "0".getBytes()) ||
Bytes.equals(regions[i], "1".getBytes()) ||
- Bytes.equals(regions[i], "2".getBytes()));
+ Bytes.equals(regions[i], "2".getBytes()) ||
+ Bytes.equals(regions[i], "3".getBytes()));
}
}
@@ -587,6 +589,64 @@ public class TestHLog {
assertEquals(COL_COUNT, visitor.increments);
}
+ @Test
+ public void testLogCleaning() throws Exception {
+ LOG.info("testLogCleaning");
+ final byte [] tableName = Bytes.toBytes("testLogCleaning");
+ final byte [] tableName2 = Bytes.toBytes("testLogCleaning2");
+
+ HLog log = new HLog(fs, dir, oldLogDir, conf, null);
+ HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
+ HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+ HRegionInfo hri2 = new HRegionInfo(new HTableDescriptor(tableName2),
+ HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+
+ // Add a single edit and make sure that rolling won't remove the file
+ // Before HBASE-3198 it used to delete it
+ addEdits(log, hri, tableName, 1);
+ log.rollWriter();
+ assertEquals(1, log.getNumLogFiles());
+
+ // See if there's anything wrong with more than 1 edit
+ addEdits(log, hri, tableName, 2);
+ log.rollWriter();
+ assertEquals(2, log.getNumLogFiles());
+
+ // Now mix edits from 2 regions, still no flushing
+ addEdits(log, hri, tableName, 1);
+ addEdits(log, hri2, tableName2, 1);
+ addEdits(log, hri, tableName, 1);
+ addEdits(log, hri2, tableName2, 1);
+ log.rollWriter();
+ assertEquals(3, log.getNumLogFiles());
+
+ // Flush the first region, we expect to see the first two files getting
+ // archived
+ long seqId = log.startCacheFlush();
+ log.completeCacheFlush(hri.getRegionName(), tableName, seqId, false);
+ log.rollWriter();
+ assertEquals(2, log.getNumLogFiles());
+
+ // Flush the second region, which removes all the remaining output files
+ // since the oldest was completely flushed and the two others only contain
+ // flush information
+ seqId = log.startCacheFlush();
+ log.completeCacheFlush(hri2.getRegionName(), tableName2, seqId, false);
+ log.rollWriter();
+ assertEquals(0, log.getNumLogFiles());
+ }
+
+ private void addEdits(HLog log, HRegionInfo hri, byte [] tableName,
+ int times) throws IOException {
+ final byte [] row = Bytes.toBytes("row");
+ for (int i = 0; i < times; i++) {
+ long timestamp = System.currentTimeMillis();
+ WALEdit cols = new WALEdit();
+ cols.add(new KeyValue(row, row, row, timestamp, row));
+ log.append(hri, tableName, cols, timestamp);
+ }
+ }
+
static class DumbLogEntriesVisitor implements LogEntryVisitor {
int increments = 0;