You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/02/02 08:53:37 UTC
[30/41] hbase git commit: HBASE-19895 Add keepDeletedCells option in
ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)
HBASE-19895 Add keepDeletedCells option in ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1125859
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1125859
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1125859
Branch: refs/heads/HBASE-19064
Commit: a11258599e7412ea4867ef850cd67bb9e7bd8e67
Parents: 99b9fff
Author: tedyu <yu...@gmail.com>
Authored: Thu Feb 1 18:27:18 2018 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Feb 1 18:27:18 2018 -0800
----------------------------------------------------------------------
.../regionserver/CustomizedScanInfoBuilder.java | 18 +++-
.../hadoop/hbase/regionserver/ScanInfo.java | 9 +-
.../hadoop/hbase/regionserver/ScanOptions.java | 5 ++
.../hbase/regionserver/TestCompaction.java | 2 +-
.../TestDefaultCompactSelection.java | 3 +-
.../hbase/regionserver/TestMajorCompaction.java | 88 +++++++++++++-------
6 files changed, 87 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java
index c3d5e57..b791010 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -31,6 +32,8 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
private Long ttl;
+ private KeepDeletedCells keepDeletedCells = null;
+
public CustomizedScanInfoBuilder(ScanInfo scanInfo) {
this.scanInfo = scanInfo;
}
@@ -56,14 +59,25 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
}
public ScanInfo build() {
- if (maxVersions == null && ttl == null) {
+ if (maxVersions == null && ttl == null && keepDeletedCells == null) {
return scanInfo;
}
- return scanInfo.customize(getMaxVersions(), getTTL());
+ return scanInfo.customize(getMaxVersions(), getTTL(), getKeepDeletedCells());
}
@Override
public String toString() {
return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]";
}
+
+ @Override
+ public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
+ this.keepDeletedCells = keepDeletedCells;
+ }
+
+ @Override
+ public KeepDeletedCells getKeepDeletedCells() {
+ return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
index 419afff..2fde311 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java
@@ -171,10 +171,11 @@ public class ScanInfo {
}
/**
- * Used for CP users for customizing max versions and ttl.
+ * Used for CP users for customizing max versions, ttl and keepDeletedCells.
*/
- ScanInfo customize(int maxVersions, long ttl) {
- return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, ttl, comparator,
- ttl, usePread, maxVersions, parallelSeekEnabled, ttl, newVersionBehavior);
+ ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
+ return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, timeToPurgeDeletes,
+ comparator, tableMaxRowSize, usePread, cellsPerTimeoutCheck, parallelSeekEnabled,
+ preadMaxBytes, newVersionBehavior);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java
index 5a35d51..aca857a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -59,4 +60,8 @@ public interface ScanOptions {
long getTTL();
void setTTL(long ttl);
+
+ void setKeepDeletedCells(KeepDeletedCells keepDeletedCells);
+
+ KeepDeletedCells getKeepDeletedCells();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 75110fd..6fe76d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -218,7 +218,7 @@ public class TestCompaction {
final int ttl = 1000;
for (HStore store : this.r.stores.values()) {
ScanInfo old = store.getScanInfo();
- ScanInfo si = old.customize(old.getMaxVersions(), ttl);
+ ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si);
}
Thread.sleep(ttl);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index 3c3bbd8..9c45a83 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -170,7 +170,8 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy {
public void testCompactionEmptyHFile() throws IOException {
// Set TTL
ScanInfo oldScanInfo = store.getScanInfo();
- ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600);
+ ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
+ oldScanInfo.getKeepDeletedCells());
store.setScanInfo(newScanInfo);
// Do not compact empty store file
List<HStoreFile> candidates = sfCreate(0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index e261886..ee717f9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
@@ -134,36 +135,20 @@ public class TestMajorCompaction {
* Test that on a major compaction, if all cells are expired or deleted, then
* we'll end up with no product. Make sure scanner over region returns
* right answer in this case - and that it just basically works.
- * @throws IOException
+ * @throws IOException exception encountered
*/
@Test
public void testMajorCompactingToNoOutput() throws IOException {
- createStoreFile(r);
- for (int i = 0; i < compactionThreshold; i++) {
- createStoreFile(r);
- }
- // Now delete everything.
- InternalScanner s = r.getScanner(new Scan());
- do {
- List<Cell> results = new ArrayList<>();
- boolean result = s.next(results);
- r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
- if (!result) break;
- } while(true);
- s.close();
- // Flush
- r.flush(true);
- // Major compact.
- r.compact(true);
- s = r.getScanner(new Scan());
- int counter = 0;
- do {
- List<Cell> results = new ArrayList<>();
- boolean result = s.next(results);
- if (!result) break;
- counter++;
- } while(true);
- assertEquals(0, counter);
+ testMajorCompactingWithDeletes(KeepDeletedCells.FALSE);
+ }
+
+ /**
+ * Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true
+ * @throws IOException exception encountered
+ */
+ @Test
+ public void testMajorCompactingWithKeepDeletedCells() throws IOException {
+ testMajorCompactingWithDeletes(KeepDeletedCells.TRUE);
}
/**
@@ -298,7 +283,7 @@ public class TestMajorCompaction {
final int ttl = 1000;
for (HStore store : r.getStores()) {
ScanInfo old = store.getScanInfo();
- ScanInfo si = old.customize(old.getMaxVersions(), ttl);
+ ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si);
}
Thread.sleep(1000);
@@ -471,7 +456,9 @@ public class TestMajorCompaction {
boolean result = s.next(results);
assertTrue(!results.isEmpty());
r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
- if (!result) break;
+ if (!result) {
+ break;
+ }
} while (true);
s.close();
// Flush
@@ -485,10 +472,51 @@ public class TestMajorCompaction {
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
- if (!result) break;
+ if (!result) {
+ break;
+ }
counter++;
} while (true);
s.close();
assertEquals(0, counter);
}
+
+ private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells)
+ throws IOException {
+ createStoreFile(r);
+ for (int i = 0; i < compactionThreshold; i++) {
+ createStoreFile(r);
+ }
+ // Now delete everything.
+ InternalScanner s = r.getScanner(new Scan());
+ int originalCount = 0;
+ do {
+ List<Cell> results = new ArrayList<>();
+ boolean result = s.next(results);
+ r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
+ if (!result) break;
+ originalCount++;
+ } while (true);
+ s.close();
+ // Flush
+ r.flush(true);
+
+ for (HStore store : this.r.stores.values()) {
+ ScanInfo old = store.getScanInfo();
+ ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells);
+ store.setScanInfo(si);
+ }
+ // Major compact.
+ r.compact(true);
+ s = r.getScanner(new Scan().setRaw(true));
+ int counter = 0;
+ do {
+ List<Cell> results = new ArrayList<>();
+ boolean result = s.next(results);
+ if (!result) break;
+ counter++;
+ } while (true);
+ assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter);
+
+ }
}