You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/05/26 14:50:45 UTC
svn commit: r1342892 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: mbautin
Date: Sat May 26 12:50:44 2012
New Revision: 1342892
URL: http://svn.apache.org/viewvc?rev=1342892&view=rev
Log:
[HBASE-4721] [0.89-fb] pull in HBASE 4721: keep deletes for a while even after major compaction
Author: aaiyer
Test Plan: mr unit tests
Reviewers: kannan, pkhemani
Differential Revision: https://phabricator.fb.com/D479953
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1342892&r1=1342891&r2=1342892&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Sat May 26 12:50:44 2012
@@ -47,7 +47,7 @@ public class ScanQueryMatcher {
/** Keeps track of deletes */
protected DeleteTracker deletes;
- protected boolean retainDeletesInOutput;
+ protected long retainDeletesInOutputUntil;
/** Keeps track of columns and versions */
protected ColumnTracker columns;
@@ -85,7 +85,7 @@ public class ScanQueryMatcher {
public ScanQueryMatcher(Scan scan, byte [] family,
NavigableSet<byte[]> columnSet, KeyValue.KeyComparator rowComparator,
int maxVersions, long readPointToUse,
- boolean retainDeletesInOutput,
+ long retainDeletesInOutputUntil,
long oldestUnexpiredTS) {
this.tr = scan.getTimeRange();
this.oldestStamp = oldestUnexpiredTS;
@@ -95,7 +95,7 @@ public class ScanQueryMatcher {
this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(),
family);
this.filter = scan.getFilter();
- this.retainDeletesInOutput = retainDeletesInOutput;
+ this.retainDeletesInOutputUntil = retainDeletesInOutputUntil;
this.maxReadPointToTrackVersions = readPointToUse;
// Single branch to deal with two types of reads (columns vs all in family)
@@ -130,7 +130,8 @@ public class ScanQueryMatcher {
// Deletes are included explicitly (for minor compaction).
this(scan, family, columns, rowComparator, maxVersions,
Long.MAX_VALUE, // max Readpoint to Track versions
- false, oldestUnexpiredTS);
+ Long.MAX_VALUE, // do not include deletes
+ oldestUnexpiredTS);
}
/**
@@ -203,7 +204,7 @@ public class ScanQueryMatcher {
this.deletes.add(bytes, offset, qualLength, timestamp, type);
// Can't early out now, because DelFam come before any other keys
}
- if (retainDeletesInOutput) {
+ if (timestamp >= this.retainDeletesInOutputUntil) {
return MatchCode.INCLUDE;
}
else {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1342892&r1=1342891&r2=1342892&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Sat May 26 12:50:44 2012
@@ -113,6 +113,7 @@ public class Store extends SchemaConfigu
final CacheConfig cacheConf;
// ttl in milliseconds.
protected long ttl;
+ private long timeToPurgeDeletes;
private final int minFilesToCompact;
private final int maxFilesToCompact;
private final long minCompactSize;
@@ -203,6 +204,12 @@ public class Store extends SchemaConfigu
// second -> ms adjust for user data
this.ttl *= 1000;
}
+ // used by ScanQueryMatcher
+ long timeToPurgeDeletes =
+ Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
+ LOG.info("time to purge deletes set to " + timeToPurgeDeletes +
+ "ms in store " + this);
+
this.memstore = new MemStore(this.comparator);
this.storeNameStr = getColumnFamilyName();
@@ -605,7 +612,8 @@ public class Store extends SchemaConfigu
scan.setMaxVersions(family.getMaxVersions());
InternalScanner scanner = new StoreScanner(this, scan,
MemStore.getSnapshotScanners(snapshot, this.comparator),
- this.region.getSmallestReadPoint(), true);
+ this.region.getSmallestReadPoint(),
+ Long.MIN_VALUE); // include all deletes
String fileName;
try {
@@ -1360,8 +1368,11 @@ public class Store extends SchemaConfigu
Scan scan = new Scan();
scan.setMaxVersions(family.getMaxVersions());
/* include deletes, unless we are doing a major compaction */
+ long retainDeletesUntil = (majorCompaction)?
+ (System.currentTimeMillis() - this.timeToPurgeDeletes)
+ : Long.MIN_VALUE;
scanner = new StoreScanner(this, scan, scanners, smallestReadPoint,
- !majorCompaction);
+ retainDeletesUntil);
int bytesWritten = 0;
// since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1342892&r1=1342891&r2=1342892&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Sat May 26 12:50:44 2012
@@ -104,7 +104,8 @@ class StoreScanner extends NonLazyKeyVal
new ScanQueryMatcher(scan, store.getFamily().getName(), columns,
store.comparator.getRawComparator(),
store.versionsToReturn(scan.getMaxVersions()), Long.MAX_VALUE,
- false, oldestUnexpiredTS);
+ Long.MAX_VALUE, // do not include the deletes
+ oldestUnexpiredTS);
// Pass columns to try to filter out unnecessary StoreFiles.
List<KeyValueScanner> scanners = getScannersNoCompaction();
@@ -146,14 +147,14 @@ class StoreScanner extends NonLazyKeyVal
*/
StoreScanner(Store store, Scan scan,
List<? extends KeyValueScanner> scanners, long smallestReadPoint,
- boolean retainDeletesInOutput) throws IOException {
+ long retainDeletesInOutputUntil) throws IOException {
this(store, false, scan, null, store.ttl);
matcher =
new ScanQueryMatcher(scan, store.getFamily().getName(), null,
store.comparator.getRawComparator(),
store.versionsToReturn(scan.getMaxVersions()), smallestReadPoint,
- retainDeletesInOutput, oldestUnexpiredTS);
+ retainDeletesInOutputUntil, oldestUnexpiredTS);
// Filter the list of scanners using Bloom filters, time range, TTL, etc.
scanners = selectScannersFrom(scanners);
@@ -173,11 +174,23 @@ class StoreScanner extends NonLazyKeyVal
final NavigableSet<byte[]> columns,
final List<KeyValueScanner> scanners)
throws IOException {
+ this(scan, colFamily, ttl, comparator, columns, scanners, Long.MAX_VALUE);
+ }
+
+ /** Constructor for testing. */
+ StoreScanner(final Scan scan, final byte [] colFamily, final long ttl,
+ final KeyValue.KVComparator comparator,
+ final NavigableSet<byte[]> columns,
+ final List<KeyValueScanner> scanners,
+ final long retainDeletesInOutputUntil)
+ throws IOException {
this(null, scan.getCacheBlocks(), scan, columns, ttl);
this.matcher =
new ScanQueryMatcher(scan, colFamily, columns,
comparator.getRawComparator(), scan.getMaxVersions(),
- Long.MAX_VALUE, false, oldestUnexpiredTS);
+ Long.MAX_VALUE,
+ retainDeletesInOutputUntil,
+ oldestUnexpiredTS);
// Seek all scanners to the initial key
for(KeyValueScanner scanner : scanners) {
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1342892&r1=1342891&r2=1342892&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java Sat May 26 12:50:44 2012
@@ -419,7 +419,7 @@ public class HFileReadWriteTest {
Scan scan = new Scan();
// Include deletes
- scanner = new StoreScanner(store, scan, scanners, Long.MAX_VALUE, true);
+ scanner = new StoreScanner(store, scan, scanners, Long.MAX_VALUE, Long.MIN_VALUE);
ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
while (scanner.next(kvs) || kvs.size() != 0) {
@@ -857,4 +857,4 @@ public class HFileReadWriteTest {
}
}
-}
\ No newline at end of file
+}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=1342892&r1=1342891&r2=1342892&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Sat May 26 12:50:44 2012
@@ -350,6 +350,59 @@ public class TestStoreScanner extends Te
assertEquals(false, scan.next(results));
}
+ public void testDeleteMarkerLongevity() throws Exception {
+ final long now = System.currentTimeMillis();
+ KeyValue [] kvs = new KeyValue[] {
+ /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
+ now-100, KeyValue.Type.DeleteFamily), // live
+ /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
+ now-1000, KeyValue.Type.DeleteFamily), // expired
+ /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now-50,
+ KeyValue.Type.Put, "v3"), // live
+ /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now-55,
+ KeyValue.Type.Delete, "dontcare"), // live
+ /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now-55,
+ KeyValue.Type.Put, "deleted-version v2"), // deleted
+ /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now-60,
+ KeyValue.Type.Put, "v1"), // live
+ /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now-65,
+ KeyValue.Type.Put, "v0"), // max-version reached
+ /*7*/ KeyValueTestUtil.create("R1", "cf", "a",
+ now-100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
+ /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now-600,
+ KeyValue.Type.DeleteColumn, "dont-care"), //expired
+ /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now-70,
+ KeyValue.Type.Put, "v2"), //live
+ /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now-750,
+ KeyValue.Type.Put, "v1"), //expired
+ /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now-500,
+ KeyValue.Type.Delete, "dontcare"), //expired
+ /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now-600,
+ KeyValue.Type.Put, "v1"), //expired
+ /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now-1000,
+ KeyValue.Type.Delete, "dontcare"), //expired
+ /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now-60,
+ KeyValue.Type.Put, "expired put"), //live
+ /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now-100,
+ KeyValue.Type.Delete, "not-expired delete"), //live
+ };
+ List<KeyValueScanner> scanners = scanFixture(kvs);
+ StoreScanner scanner =
+ new StoreScanner(new Scan().setMaxVersions(2), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ null, scanners, now - 200);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ results = new ArrayList<KeyValue>();
+ assertEquals(true, scanner.next(results));
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[2], results.get(1));
+ assertEquals(kvs[3], results.get(2));
+ assertEquals(kvs[5], results.get(3));
+ assertEquals(kvs[9], results.get(4));
+ assertEquals(kvs[14], results.get(5));
+ assertEquals(kvs[15], results.get(6));
+ assertEquals(7, results.size());
+ }
+
public void testDeleteColumn() throws IOException {
KeyValue [] kvs = new KeyValue[] {
KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
@@ -500,4 +553,4 @@ public class TestStoreScanner extends Te
assertEquals(false, scanner.next(results));
}
-}
\ No newline at end of file
+}