You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2020/01/22 08:35:43 UTC
[hbase] branch branch-2.2 updated: HBASE-23711 - Add test for
MinVersions and KeepDeletedCells TTL
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new f812abf HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL
f812abf is described below
commit f812abf64d339df92f423e558379be320b16b610
Author: Geoffrey Jacoby <gj...@apache.org>
AuthorDate: Wed Jan 22 00:24:11 2020 -0800
HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL
Signed-off-by: stack <st...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../hadoop/hbase/regionserver/TestMinVersions.java | 168 ++++++++++++++++++---
1 file changed, 150 insertions(+), 18 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
index 13ad7f8..f164e3a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
@@ -27,17 +27,23 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.TimestampsFilter;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
+import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -54,7 +60,7 @@ public class TestMinVersions {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMinVersions.class);
- HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
+ HBaseTestingUtility hbu = new HBaseTestingUtility();
private final byte[] T0 = Bytes.toBytes("0");
private final byte[] T1 = Bytes.toBytes("1");
private final byte[] T2 = Bytes.toBytes("2");
@@ -71,8 +77,14 @@ public class TestMinVersions {
*/
@Test
public void testGetClosestBefore() throws Exception {
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
+
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {
@@ -121,8 +133,15 @@ public class TestMinVersions {
@Test
public void testStoreMemStore() throws Exception {
// keep 3 versions minimum
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
+
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
@@ -176,8 +195,14 @@ public class TestMinVersions {
*/
@Test
public void testDelete() throws Exception {
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
@@ -235,8 +260,13 @@ public class TestMinVersions {
*/
@Test
public void testMemStore() throws Exception {
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
@@ -310,9 +340,14 @@ public class TestMinVersions {
*/
@Test
public void testBaseCase() throws Exception {
- // 1 version minimum, 1000 versions maximum, ttl = 1s
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
+ // 2 version minimum, 1000 versions maximum, ttl = 1s
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
HRegion region = hbu.createLocalHRegion(htd, null, null);
try {
@@ -403,10 +438,23 @@ public class TestMinVersions {
*/
@Test
public void testFilters() throws Exception {
- HTableDescriptor htd =
- hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
- HRegion region = hbu.createLocalHRegion(htd, null, null);
final byte [] c1 = COLUMNS[1];
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+ ColumnFamilyDescriptor cfd2 =
+ ColumnFamilyDescriptorBuilder.newBuilder(c1)
+ .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+ setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+ List<ColumnFamilyDescriptor> cfdList = new ArrayList();
+ cfdList.add(cfd);
+ cfdList.add(cfd2);
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
+ HRegion region = hbu.createLocalHRegion(htd, null, null);
// 2s in the past
long ts = EnvironmentEdgeManager.currentTime() - 2000;
@@ -476,12 +524,96 @@ public class TestMinVersions {
}
}
+ @Test
+ public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
+ int ttl = 4;
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(c0)
+ .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
+ setKeepDeletedCells(KeepDeletedCells.TTL).build();
+
+ TableDescriptor htd = TableDescriptorBuilder.
+ newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
+ HRegion region = hbu.createLocalHRegion(htd, null, null);
+
+ long startTS = EnvironmentEdgeManager.currentTime();
+ ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
+ injectEdge.setValue(startTS);
+ EnvironmentEdgeManager.injectEdge(injectEdge);
+
+ long ts = startTS - 2000;
+ // 1st version
+ Put p = new Put(T1, ts-3);
+ p.addColumn(c0, c0, T1);
+ region.put(p);
+
+ // 2nd version
+ p = new Put(T1, ts-2);
+ p.addColumn(c0, c0, T2);
+ region.put(p);
+
+ // 3rd version
+ p = new Put(T1, ts-1);
+ p.addColumn(c0, c0, T3);
+ region.put(p);
+
+ Get g;
+ Result r;
+
+ //check we can still see all versions before compaction
+ g = new Get(T1);
+ g.readAllVersions();
+ g.setTimeRange(0, ts);
+ r = region.get(g);
+ checkResult(r, c0, T3, T2, T1);
+
+ region.flush(true);
+ region.compact(true);
+ Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
+ long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
+ Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
+ //check that nothing was purged yet
+ g = new Get(T1);
+ g.readAllVersions();
+ g.setTimeRange(0, ts);
+ r = region.get(g);
+ checkResult(r, c0, T3, T2, T1);
+
+ g = new Get(T1);
+ g.readAllVersions();
+ g.setTimeRange(0, ts -1);
+ r = region.get(g);
+ checkResult(r, c0, T2, T1);
+
+ injectEdge.incValue(ttl * 1000);
+
+ region.flush(true);
+ region.compact(true);
+
+ //check that after compaction (which is after TTL) that only T1 was purged
+ g = new Get(T1);
+ g.readAllVersions();
+ g.setTimeRange(0, ts);
+ r = region.get(g);
+ checkResult(r, c0, T3, T2);
+
+ g = new Get(T1);
+ g.readAllVersions();
+ g.setTimestamp(ts -2);
+ r = region.get(g);
+ checkResult(r, c0, T2);
+ }
+
private void checkResult(Result r, byte[] col, byte[] ... vals) {
- assertEquals(r.size(), vals.length);
+ assertEquals(vals.length, r.size());
List<Cell> kvs = r.getColumnCells(col, col);
assertEquals(kvs.size(), vals.length);
for (int i=0;i<vals.length;i++) {
- assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
+ String expected = Bytes.toString(vals[i]);
+ String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
+ assertTrue(expected + " was expected but doesn't match " + actual,
+ CellUtil.matchingValue(kvs.get(i), vals[i]));
}
}