You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/03/21 00:59:23 UTC
svn commit: r1303222 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/regionserver/
Author: larsh
Date: Tue Mar 20 23:59:23 2012
New Revision: 1303222
URL: http://svn.apache.org/viewvc?rev=1303222&view=rev
Log:
HBASE-5569 Do not collect deleted KVs when they are still in use by a scanner.
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1303222&r1=1303221&r2=1303222&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Mar 20 23:59:23 2012
@@ -692,7 +692,7 @@ public class KeyValue implements Writabl
return "empty";
}
return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) +
- "/vlen=" + getValueLength();
+ "/vlen=" + getValueLength() + "/ts=" + memstoreTS;
}
/**
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1303222&r1=1303221&r2=1303222&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Tue Mar 20 23:59:23 2012
@@ -278,10 +278,15 @@ public class ScanQueryMatcher {
if (!keepDeletedCells) {
// first ignore delete markers if the scanner can do so, and the
// range does not include the marker
+ //
+ // during flushes and compactions also ignore delete markers newer
+ // than the readpoint of any open scanner, this prevents deleted
+ // rows that could still be seen by a scanner from being collected
boolean includeDeleteMarker = seePastDeleteMarkers ?
tr.withinTimeRange(timestamp) :
tr.withinOrAfterTimeRange(timestamp);
- if (includeDeleteMarker) {
+ if (includeDeleteMarker
+ && kv.getMemstoreTS() <= maxReadPointToTrackVersions) {
this.deletes.add(bytes, offset, qualLength, timestamp, type);
}
// Can't early out now, because DelFam come before any other keys
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java?rev=1303222&r1=1303221&r2=1303222&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java Tue Mar 20 23:59:23 2012
@@ -404,7 +404,7 @@ public class TestKeyValue extends TestCa
System.err.println("kv=" + kv);
System.err.println("kvFromKey=" + kvFromKey);
assertEquals(kvFromKey.toString(),
- kv.toString().replaceAll("=[0-9]+$", "=0"));
+ kv.toString().replaceAll("=[0-9]+", "=0"));
}
@org.junit.Rule
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java?rev=1303222&r1=1303221&r2=1303222&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java Tue Mar 20 23:59:23 2012
@@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.client.Pu
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.junit.experimental.categories.Category;
@@ -55,11 +54,9 @@ public class TestAtomicOperation extends
HRegion region = null;
private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final String DIR = TEST_UTIL.getDataTestDir("TestIncrement").toString();
+ private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
- private final int MAX_VERSIONS = 2;
-
// Test names
static final byte[] tableName = Bytes.toBytes("testtable");;
static final byte[] qual1 = Bytes.toBytes("qual1");
@@ -260,8 +257,8 @@ public class TestAtomicOperation extends
// create 100 threads, each will alternate between adding and
// removing a column
- int numThreads = 100;
- int opsPerThread = 1000;
+ int numThreads = 10;
+ int opsPerThread = 500;
AtomicOperation[] all = new AtomicOperation[numThreads];
AtomicLong timeStamps = new AtomicLong(0);
@@ -275,9 +272,11 @@ public class TestAtomicOperation extends
for (int i=0; i<numOps; i++) {
try {
// throw in some flushes
- if (r.nextFloat() < 0.001) {
- LOG.debug("flushing");
- region.flushcache();
+ if (i%10==0) {
+ synchronized(region) {
+ LOG.debug("flushing");
+ region.flushcache();
+ }
}
long ts = timeStamps.incrementAndGet();
RowMutations rm = new RowMutations(row);
@@ -342,7 +341,7 @@ public class TestAtomicOperation extends
// create 100 threads, each will alternate between adding and
// removing a column
- int numThreads = 100;
+ int numThreads = 10;
int opsPerThread = 1000;
AtomicOperation[] all = new AtomicOperation[numThreads];
@@ -358,9 +357,11 @@ public class TestAtomicOperation extends
for (int i=0; i<numOps; i++) {
try {
// throw in some flushes
- if (r.nextFloat() < 0.001) {
- LOG.debug("flushing");
- region.flushcache();
+ if (i%10==0) {
+ synchronized(region) {
+ LOG.debug("flushing");
+ region.flushcache();
+ }
}
long ts = timeStamps.incrementAndGet();
List<Mutation> mrm = new ArrayList<Mutation>();
@@ -386,6 +387,7 @@ public class TestAtomicOperation extends
RegionScanner rs = region.getScanner(s);
List<KeyValue> r = new ArrayList<KeyValue>();
while(rs.next(r));
+ rs.close();
if (r.size() != 1) {
LOG.debug(r);
failures.incrementAndGet();
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1303222&r1=1303221&r2=1303222&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Tue Mar 20 23:59:23 2012
@@ -131,6 +131,7 @@ public class TestCompaction extends HBas
r.delete(new Delete(results.get(0).getRow()), null, false);
if (!result) break;
} while(true);
+ s.close();
// Flush
r.flushcache();
// Major compact.