You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2018/10/20 15:10:33 UTC
hbase git commit: HBASE-21194 Add tests in TestCopyTable which
exercises MOB feature
Repository: hbase
Updated Branches:
refs/heads/master 7adf59010 -> b723ce105
HBASE-21194 Add tests in TestCopyTable which exercises MOB feature
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b723ce10
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b723ce10
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b723ce10
Branch: refs/heads/master
Commit: b723ce105174825a19b391e7d64ac2b32d0aad63
Parents: 7adf590
Author: Artem Ervits <ge...@gmail.com>
Authored: Fri Oct 19 14:25:25 2018 -0400
Committer: tedyu <yu...@gmail.com>
Committed: Sat Oct 20 08:10:27 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/TestCopyTable.java | 180 ++++++++++++++-----
.../hadoop/hbase/HBaseTestingUtility.java | 6 +-
.../apache/hadoop/hbase/mob/MobTestUtil.java | 13 ++
.../hbase/mob/compactions/TestMobCompactor.java | 39 ++--
4 files changed, 164 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
index 6f41671..ed6857d 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -30,15 +30,20 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.mob.MobTestUtil;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MapReduceTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.LauncherSecurityManager;
import org.apache.hadoop.util.ToolRunner;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -100,7 +105,7 @@ public class TestCopyTable {
if (bulkload) {
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
- "--bulkload", tableName1.getNameAsString() });
+ "--bulkload", tableName1.getNameAsString() });
} else {
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
@@ -121,9 +126,74 @@ public class TestCopyTable {
}
}
+ private void doCopyTableTestWithMob(boolean bulkload) throws Exception {
+ final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
+ final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
+ final byte[] FAMILY = Bytes.toBytes("mob");
+ final byte[] COLUMN1 = Bytes.toBytes("c1");
+
+ ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+
+ cfd.setMobEnabled(true);
+ cfd.setMobThreshold(5);
+ TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
+ .setColumnFamily(cfd.build())
+ .build();
+ TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
+ .setColumnFamily(cfd.build())
+ .build();
+
+ try (Table t1 = TEST_UTIL.createTable(desc1, null);
+ Table t2 = TEST_UTIL.createTable(desc2, null);) {
+
+ // put rows into the first table
+ for (int i = 0; i < 10; i++) {
+ Put p = new Put(Bytes.toBytes("row" + i));
+ p.addColumn(FAMILY, COLUMN1, COLUMN1);
+ t1.put(p);
+ }
+
+ CopyTable copy = new CopyTable();
+
+ int code;
+ if (bulkload) {
+ code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
+ copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
+ "--bulkload", tableName1.getNameAsString() });
+ } else {
+ code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
+ copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
+ tableName1.getNameAsString() });
+ }
+ assertEquals("copy job failed", 0, code);
+
+ // verify the data was copied into table 2
+ for (int i = 0; i < 10; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = t2.get(g);
+ assertEquals(1, r.size());
+ assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
+ assertEquals("compare row values between two tables",
+ t1.getDescriptor().getValue("row" + i),
+ t2.getDescriptor().getValue("row" + i));
+ }
+
+ assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1),
+ MobTestUtil.countMobRows(t2));
+ assertEquals("compare count of mob row values between two tables",
+ t1.getDescriptor().getValues().size(),
+ t2.getDescriptor().getValues().size());
+ assertTrue("The mob row count is 0 but should be > 0",
+ MobTestUtil.countMobRows(t2) > 0);
+
+ } finally {
+ TEST_UTIL.deleteTable(tableName1);
+ TEST_UTIL.deleteTable(tableName2);
+ }
+ }
+
/**
* Simple end-to-end test
- * @throws Exception
*/
@Test
public void testCopyTable() throws Exception {
@@ -138,56 +208,70 @@ public class TestCopyTable {
doCopyTableTest(true);
}
+ /**
+ * Simple end-to-end test on table with MOB
+ */
+ @Test
+ public void testCopyTableWithMob() throws Exception {
+ doCopyTableTestWithMob(false);
+ }
+
+ /**
+ * Simple end-to-end test with bulkload on table with MOB.
+ */
+ @Test
+ public void testCopyTableWithBulkloadWithMob() throws Exception {
+ doCopyTableTestWithMob(true);
+ }
+
@Test
public void testStartStopRow() throws Exception {
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
- final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
- final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
- final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
-
- Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
- Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);
-
- // put rows into the first table
- Put p = new Put(ROW0);
- p.addColumn(FAMILY, COLUMN1, COLUMN1);
- t1.put(p);
- p = new Put(ROW1);
- p.addColumn(FAMILY, COLUMN1, COLUMN1);
- t1.put(p);
- p = new Put(ROW2);
- p.addColumn(FAMILY, COLUMN1, COLUMN1);
- t1.put(p);
-
- CopyTable copy = new CopyTable();
- assertEquals(
- 0,
- ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
- copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1",
- "--stoprow=\\x01row2", tableName1.getNameAsString() }));
-
- // verify the data was copied into table 2
- // row1 exist, row0, row2 do not exist
- Get g = new Get(ROW1);
- Result r = t2.get(g);
- assertEquals(1, r.size());
- assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
-
- g = new Get(ROW0);
- r = t2.get(g);
- assertEquals(0, r.size());
-
- g = new Get(ROW2);
- r = t2.get(g);
- assertEquals(0, r.size());
-
- t1.close();
- t2.close();
- TEST_UTIL.deleteTable(tableName1);
- TEST_UTIL.deleteTable(tableName2);
+ final byte[] row0 = Bytes.toBytesBinary("\\x01row0");
+ final byte[] row1 = Bytes.toBytesBinary("\\x01row1");
+ final byte[] row2 = Bytes.toBytesBinary("\\x01row2");
+
+ try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
+ Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) {
+
+ // put rows into the first table
+ Put p = new Put(row0);
+ p.addColumn(FAMILY, COLUMN1, COLUMN1);
+ t1.put(p);
+ p = new Put(row1);
+ p.addColumn(FAMILY, COLUMN1, COLUMN1);
+ t1.put(p);
+ p = new Put(row2);
+ p.addColumn(FAMILY, COLUMN1, COLUMN1);
+ t1.put(p);
+
+ CopyTable copy = new CopyTable();
+ assertEquals(0, ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
+ copy, new String[]{"--new.name=" + tableName2, "--startrow=\\x01row1",
+ "--stoprow=\\x01row2", tableName1.getNameAsString()}));
+
+ // verify the data was copied into table 2
+ // row1 exist, row0, row2 do not exist
+ Get g = new Get(row1);
+ Result r = t2.get(g);
+ assertEquals(1, r.size());
+ assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
+
+ g = new Get(row0);
+ r = t2.get(g);
+ assertEquals(0, r.size());
+
+ g = new Get(row2);
+ r = t2.get(g);
+ assertEquals(0, r.size());
+
+ } finally {
+ TEST_UTIL.deleteTable(tableName1);
+ TEST_UTIL.deleteTable(tableName2);
+ }
}
/**
@@ -215,8 +299,8 @@ public class TestCopyTable {
long currentTime = System.currentTimeMillis();
String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
- "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
- "--versions=1", sourceTable.getNameAsString() };
+ "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
+ "--versions=1", sourceTable.getNameAsString() };
assertNull(t2.get(new Get(ROW1)).getRow());
assertTrue(runCopy(args));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index ab9a799..695daea 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2310,12 +2310,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
/**
* Return the number of rows in the given table.
+ * @param table to count rows
+ * @return count of rows
*/
- public int countRows(final Table table) throws IOException {
+ public static int countRows(final Table table) throws IOException {
return countRows(table, new Scan());
}
- public int countRows(final Table table, final Scan scan) throws IOException {
+ public static int countRows(final Table table, final Scan scan) throws IOException {
try (ResultScanner results = table.getScanner(scan)) {
int count = 0;
while (results.next() != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
index 76cb664..8972dba 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
@@ -24,6 +24,7 @@ import java.util.Random;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -104,4 +105,16 @@ public class MobTestUtil {
results.close();
Assert.assertEquals(expectedCount, count);
}
+
+ /**
+ * Gets the number of rows in the given table.
+ * @param table to get the scanner
+ * @return the number of rows
+ */
+ public static int countMobRows(final Table table) throws IOException {
+ Scan scan = new Scan();
+ // Do not retrieve the mob data when scanning
+ scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+ return HBaseTestingUtility.countRows(table, scan);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
index 8ed4fbb..86e07d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobFileName;
+import org.apache.hadoop.hbase.mob.MobTestUtil;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -295,7 +296,7 @@ public class TestMobCompactor {
int rowNumPerRegion = count * rowNumPerFile;
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
countMobCells(table));
assertEquals("Before deleting: mob file count", regionNum * count,
@@ -305,7 +306,7 @@ public class TestMobCompactor {
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("Before compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
assertEquals("Before compaction: family1 mob file count", regionNum * count,
@@ -322,7 +323,7 @@ public class TestMobCompactor {
compactor.compact();
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("After compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
// After the compaction, the files smaller than the mob compaction merge size
@@ -445,7 +446,7 @@ public class TestMobCompactor {
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("Before compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
assertEquals("Before compaction: family1 mob file count", regionNum * count,
@@ -462,7 +463,7 @@ public class TestMobCompactor {
compactor.compact();
assertEquals("After first compaction: mob rows count", regionNum
- * (rowNumPerRegion - delRowNum), countMobRows(table));
+ * (rowNumPerRegion - delRowNum), MobTestUtil.countMobRows(table));
assertEquals("After first compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
assertEquals("After first compaction: family1 mob file count", regionNum,
@@ -482,7 +483,7 @@ public class TestMobCompactor {
admin.enableTable(tableName);
assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
* rowNumPerRegion, countMobCells(table));
assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
@@ -500,7 +501,7 @@ public class TestMobCompactor {
compactor.compact();
assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
* rowNumPerRegion, countMobCells(table));
assertEquals("After second compaction: family1 mob file count", regionNum,
@@ -550,7 +551,7 @@ public class TestMobCompactor {
int rowNumPerRegion = count * rowNumPerFile;
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
countMobCells(table));
assertEquals("Before deleting: mob file count", regionNum * count,
@@ -559,7 +560,7 @@ public class TestMobCompactor {
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("Before compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
assertEquals("Before compaction: family1 mob file count", regionNum * count,
@@ -576,7 +577,7 @@ public class TestMobCompactor {
waitUntilMobCompactionFinished(tableName);
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
- countMobRows(table));
+ MobTestUtil.countMobRows(table));
assertEquals("After compaction: mob cells count", regionNum
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
assertEquals("After compaction: family1 mob file count", regionNum,
@@ -764,18 +765,6 @@ public class TestMobCompactor {
}
/**
- * Gets the number of rows in the given table.
- * @param table to get the scanner
- * @return the number of rows
- */
- private int countMobRows(final Table table) throws IOException {
- Scan scan = new Scan();
- // Do not retrieve the mob data when scanning
- scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
- return TEST_UTIL.countRows(table, scan);
- }
-
- /**
* Gets the number of cells in the given table.
* @param table to get the scanner
* @return the number of cells
@@ -923,13 +912,15 @@ public class TestMobCompactor {
for (int i = 0; i < 1000; i ++) {
Put put0 = new Put(Bytes.toBytes("r0" + i));
- put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151130Monday, Bytes.toBytes(mobValue0));
+ put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1),
+ tsFor20151130Monday, Bytes.toBytes(mobValue0));
pArray[i] = put0;
}
loadData(admin, bufMut, tableName, pArray);
Put put06 = new Put(mobKey06);
- put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151128Saturday, Bytes.toBytes(mobValue0));
+ put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1),
+ tsFor20151128Saturday, Bytes.toBytes(mobValue0));
loadData(admin, bufMut, tableName, new Put[] { put06 });