You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/10/27 19:40:31 UTC
hbase git commit: HBASE-14688 Cleanup MOB tests
Repository: hbase
Updated Branches:
refs/heads/master e04e7402c -> c91bfff58
HBASE-14688 Cleanup MOB tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c91bfff5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c91bfff5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c91bfff5
Branch: refs/heads/master
Commit: c91bfff5862fd38b3d301e3371d2f643d0c501ea
Parents: e04e740
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Oct 27 10:28:07 2015 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Oct 27 11:37:39 2015 -0700
----------------------------------------------------------------------
.../hadoop/hbase/HBaseTestingUtility.java | 7 +-
.../apache/hadoop/hbase/mob/MobTestUtil.java | 42 +++--
.../hadoop/hbase/mob/TestCachedMobFile.java | 31 ++--
.../hbase/mob/TestDefaultMobStoreFlusher.java | 181 ++++++-------------
.../hbase/mob/TestExpiredMobFileCleaner.java | 8 +-
.../hbase/mob/TestMobDataBlockEncoding.java | 19 +-
.../apache/hadoop/hbase/mob/TestMobFile.java | 16 +-
.../hbase/mob/compactions/TestMobCompactor.java | 28 +--
.../TestPartitionedMobCompactor.java | 95 +++-------
.../hbase/regionserver/TestDeleteMobTable.java | 155 +++++++---------
.../regionserver/TestMobStoreCompaction.java | 31 ++--
.../hbase/regionserver/TestMobStoreScanner.java | 114 ++++--------
.../hadoop/hbase/util/BaseTestHBaseFsck.java | 5 +-
13 files changed, 265 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index ab7b51c..eb1494e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1845,7 +1845,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
//
// ==========================================================================
-
+
/**
* Provide an existing table name to truncate.
* Scans the table and issues a delete for each row read.
@@ -2141,7 +2141,10 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Return the number of rows in the given table.
*/
public int countRows(final Table table) throws IOException {
- Scan scan = new Scan();
+ return countRows(table, new Scan());
+ }
+
+ public int countRows(final Table table, final Scan scan) throws IOException {
ResultScanner results = table.getScanner(scan);
int count = 0;
for (@SuppressWarnings("unused") Result res : results) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
index bc3354b..3e0e2d9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
@@ -19,11 +19,16 @@
package org.apache.hadoop.hbase.mob;
import java.io.IOException;
+import java.util.List;
import java.util.Random;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
@@ -73,15 +78,30 @@ public class MobTestUtil {
/**
* Compare two Cells only for their row family qualifier value
*/
- public static void assertCellEquals(Cell firstKeyValue,
- Cell secondKeyValue) {
- Assert.assertEquals(Bytes.toString(CellUtil.cloneRow(firstKeyValue)),
- Bytes.toString(CellUtil.cloneRow(secondKeyValue)));
- Assert.assertEquals(Bytes.toString(CellUtil.cloneFamily(firstKeyValue)),
- Bytes.toString(CellUtil.cloneFamily(secondKeyValue)));
- Assert.assertEquals(Bytes.toString(CellUtil.cloneQualifier(firstKeyValue)),
- Bytes.toString(CellUtil.cloneQualifier(secondKeyValue)));
- Assert.assertEquals(Bytes.toString(CellUtil.cloneValue(firstKeyValue)),
- Bytes.toString(CellUtil.cloneValue(secondKeyValue)));
- }
+ public static void assertCellEquals(Cell firstKeyValue, Cell secondKeyValue) {
+ Assert.assertArrayEquals(CellUtil.cloneRow(firstKeyValue),
+ CellUtil.cloneRow(secondKeyValue));
+ Assert.assertArrayEquals(CellUtil.cloneFamily(firstKeyValue),
+ CellUtil.cloneFamily(secondKeyValue));
+ Assert.assertArrayEquals(CellUtil.cloneQualifier(firstKeyValue),
+ CellUtil.cloneQualifier(secondKeyValue));
+ Assert.assertArrayEquals(CellUtil.cloneValue(firstKeyValue),
+ CellUtil.cloneValue(secondKeyValue));
+ }
+
+ public static void assertCellsValue(Table table, Scan scan,
+ byte[] expectedValue, int expectedCount) throws IOException {
+ ResultScanner results = table.getScanner(scan);
+ int count = 0;
+ for (Result res : results) {
+ List<Cell> cells = res.listCells();
+ for(Cell cell : cells) {
+ // Verify the value
+ Assert.assertArrayEquals(expectedValue, CellUtil.cloneValue(cell));
+ count++;
+ }
+ }
+ results.close();
+ Assert.assertEquals(expectedCount, count);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
index b38e7cb..4bb525d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -43,10 +44,9 @@ import org.junit.experimental.categories.Category;
@Category(SmallTests.class)
public class TestCachedMobFile extends TestCase{
static final Log LOG = LogFactory.getLog(TestCachedMobFile.class);
- private Configuration conf = HBaseConfiguration.create();
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Configuration conf = TEST_UTIL.getConfiguration();
private CacheConfig cacheConf = new CacheConfig(conf);
- private static final String TABLE = "tableName";
- private static final String FAMILY = "familyName";
private static final String FAMILY1 = "familyName1";
private static final String FAMILY2 = "familyName2";
private static final long EXPECTED_REFERENCE_ZERO = 0;
@@ -56,13 +56,11 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testOpenClose() throws Exception {
String caseName = getName();
- FileSystem fs = FileSystem.get(conf);
- Path testDir = FSUtils.getRootDir(conf);
- Path outputDir = new Path(new Path(testDir, TABLE),
- FAMILY);
+ Path testDir = TEST_UTIL.getDataTestDir();
+ FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
- .withOutputDir(outputDir).withFileContext(meta).build();
+ .withOutputDir(testDir).withFileContext(meta).build();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
@@ -79,17 +77,15 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testCompare() throws Exception {
String caseName = getName();
- FileSystem fs = FileSystem.get(conf);
- Path testDir = FSUtils.getRootDir(conf);
- Path outputDir1 = new Path(new Path(testDir, TABLE),
- FAMILY1);
+ Path testDir = TEST_UTIL.getDataTestDir();
+ FileSystem fs = testDir.getFileSystem(conf);
+ Path outputDir1 = new Path(testDir, FAMILY1);
HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
StoreFile.Writer writer1 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir1).withFileContext(meta).build();
MobTestUtil.writeStoreFile(writer1, caseName);
CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf);
- Path outputDir2 = new Path(new Path(testDir, TABLE),
- FAMILY2);
+ Path outputDir2 = new Path(testDir, FAMILY2);
StoreFile.Writer writer2 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir2)
.withFileContext(meta)
@@ -105,12 +101,11 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testReadKeyValue() throws Exception {
- FileSystem fs = FileSystem.get(conf);
- Path testDir = FSUtils.getRootDir(conf);
- Path outputDir = new Path(new Path(testDir, TABLE), "familyname");
+ Path testDir = TEST_UTIL.getDataTestDir();
+ FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
- .withOutputDir(outputDir).withFileContext(meta).build();
+ .withOutputDir(testDir).withFileContext(meta).build();
String caseName = getName();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
index b91d4d1..94a2ed4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
@@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hbase.mob;
-import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
@@ -26,9 +25,7 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -64,127 +61,67 @@ public class TestDefaultMobStoreFlusher {
}
@Test
- public void testFlushNonMobFile() throws InterruptedException {
- String TN = "testFlushNonMobFile";
- TableName tn = TableName.valueOf(TN);
- Table table = null;
- HBaseAdmin admin = null;
-
- try {
- HTableDescriptor desc = new HTableDescriptor(tn);
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setMaxVersions(4);
- desc.addFamily(hcd);
-
- admin = TEST_UTIL.getHBaseAdmin();
- admin.createTable(desc);
- table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
- .getTable(TableName.valueOf(TN));
-
- //Put data
- Put put0 = new Put(row1);
- put0.addColumn(family, qf1, 1, value1);
- table.put(put0);
-
- //Put more data
- Put put1 = new Put(row2);
- put1.addColumn(family, qf2, 1, value2);
- table.put(put1);
-
- //Flush
- admin.flush(tn);
-
- Scan scan = new Scan();
- scan.addColumn(family, qf1);
- scan.setMaxVersions(4);
- ResultScanner scanner = table.getScanner(scan);
-
- //Compare
- Result result = scanner.next();
- int size = 0;
- while (result != null) {
- size++;
- List<Cell> cells = result.getColumnCells(family, qf1);
- // Verify the cell size
- Assert.assertEquals(1, cells.size());
- // Verify the value
- Assert.assertEquals(Bytes.toString(value1),
- Bytes.toString(CellUtil.cloneValue(cells.get(0))));
- result = scanner.next();
- }
- scanner.close();
- Assert.assertEquals(1, size);
- admin.close();
- } catch (MasterNotRunningException e1) {
- e1.printStackTrace();
- } catch (ZooKeeperConnectionException e2) {
- e2.printStackTrace();
- } catch (IOException e3) {
- e3.printStackTrace();
- }
+ public void testFlushNonMobFile() throws Exception {
+ TableName tn = TableName.valueOf("testFlushNonMobFile");
+ HTableDescriptor desc = new HTableDescriptor(tn);
+ HColumnDescriptor hcd = new HColumnDescriptor(family);
+ hcd.setMaxVersions(4);
+ desc.addFamily(hcd);
+
+ testFlushFile(desc);
}
@Test
- public void testFlushMobFile() throws InterruptedException {
- String TN = "testFlushMobFile";
- TableName tn = TableName.valueOf(TN);
- Table table = null;
- Admin admin = null;
-
- try {
- HTableDescriptor desc = new HTableDescriptor(tn);
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- hcd.setMobEnabled(true);
- hcd.setMobThreshold(3L);
- hcd.setMaxVersions(4);
- desc.addFamily(hcd);
-
- Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
- admin = c.getAdmin();
- admin.createTable(desc);
- table = c.getTable(TableName.valueOf(TN));
-
- //put data
- Put put0 = new Put(row1);
- put0.addColumn(family, qf1, 1, value1);
- table.put(put0);
-
- //put more data
- Put put1 = new Put(row2);
- put1.addColumn(family, qf2, 1, value2);
- table.put(put1);
-
- //flush
- admin.flush(tn);
-
- //Scan
- Scan scan = new Scan();
- scan.addColumn(family, qf1);
- scan.setMaxVersions(4);
- ResultScanner scanner = table.getScanner(scan);
-
- //Compare
- Result result = scanner.next();
- int size = 0;
- while (result != null) {
- size++;
- List<Cell> cells = result.getColumnCells(family, qf1);
- // Verify the the cell size
- Assert.assertEquals(1, cells.size());
- // Verify the value
- Assert.assertEquals(Bytes.toString(value1),
- Bytes.toString(CellUtil.cloneValue(cells.get(0))));
- result = scanner.next();
- }
- scanner.close();
- Assert.assertEquals(1, size);
- admin.close();
- } catch (MasterNotRunningException e1) {
- e1.printStackTrace();
- } catch (ZooKeeperConnectionException e2) {
- e2.printStackTrace();
- } catch (IOException e3) {
- e3.printStackTrace();
- }
+ public void testFlushMobFile() throws Exception {
+ TableName tn = TableName.valueOf("testFlushMobFile");
+ HTableDescriptor desc = new HTableDescriptor(tn);
+ HColumnDescriptor hcd = new HColumnDescriptor(family);
+ hcd.setMobEnabled(true);
+ hcd.setMobThreshold(3L);
+ hcd.setMaxVersions(4);
+ desc.addFamily(hcd);
+
+ testFlushFile(desc);
}
+
+ private void testFlushFile(HTableDescriptor htd) throws Exception {
+ Table table = null;
+ try {
+ table = TEST_UTIL.createTable(htd, null);
+
+ //put data
+ Put put0 = new Put(row1);
+ put0.addColumn(family, qf1, 1, value1);
+ table.put(put0);
+
+ //put more data
+ Put put1 = new Put(row2);
+ put1.addColumn(family, qf2, 1, value2);
+ table.put(put1);
+
+ //flush
+ TEST_UTIL.flush(htd.getTableName());
+
+ //Scan
+ Scan scan = new Scan();
+ scan.addColumn(family, qf1);
+ scan.setMaxVersions(4);
+ ResultScanner scanner = table.getScanner(scan);
+
+ //Compare
+ int size = 0;
+ for (Result result: scanner) {
+ size++;
+ List<Cell> cells = result.getColumnCells(family, qf1);
+ // Verify the cell size
+ Assert.assertEquals(1, cells.size());
+ // Verify the value
+ Assert.assertArrayEquals(value1, CellUtil.cloneValue(cells.get(0)));
+ }
+ scanner.close();
+ Assert.assertEquals(1, size);
+ } finally {
+ table.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index dfaeca6..fff6f44 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import java.util.Random;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -124,7 +123,7 @@ public class TestExpiredMobFileCleaner {
public void testCleaner() throws Exception {
init();
- Path mobDirPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
+ Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
byte[] dummyData = makeDummyData(600);
long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000; // 3 days before
@@ -158,11 +157,6 @@ public class TestExpiredMobFileCleaner {
assertEquals("After cleanup without delay 2", secondFile, lastFile);
}
- private Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
- Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), familyName);
- return p;
- }
-
private int secondsOfDay() {
return 24 * 3600;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
index 15aa7d4..8ad6f95 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
@@ -18,11 +18,8 @@
*/
package org.apache.hadoop.hbase.mob;
-import java.util.List;
import java.util.Random;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -32,7 +29,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -117,19 +113,6 @@ public class TestMobDataBlockEncoding {
Scan scan = new Scan();
scan.setMaxVersions(4);
-
- ResultScanner results = table.getScanner(scan);
- int count = 0;
- for (Result res : results) {
- List<Cell> cells = res.listCells();
- for(Cell cell : cells) {
- // Verify the value
- Assert.assertEquals(Bytes.toString(value),
- Bytes.toString(CellUtil.cloneValue(cell)));
- count++;
- }
- }
- results.close();
- Assert.assertEquals(3, count);
+ MobTestUtil.assertCellsValue(table, scan, value, 3);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
index d05da24..e2dced9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
@@ -47,17 +47,14 @@ public class TestMobFile extends TestCase {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Configuration conf = TEST_UTIL.getConfiguration();
private CacheConfig cacheConf = new CacheConfig(conf);
- private final String TABLE = "tableName";
- private final String FAMILY = "familyName";
@Test
public void testReadKeyValue() throws Exception {
- FileSystem fs = FileSystem.get(conf);
- Path testDir = FSUtils.getRootDir(conf);
- Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
+ Path testDir = TEST_UTIL.getDataTestDir();
+ FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
- .withOutputDir(outputDir)
+ .withOutputDir(testDir)
.withFileContext(meta)
.build();
String caseName = getName();
@@ -106,12 +103,11 @@ public class TestMobFile extends TestCase {
@Test
public void testGetScanner() throws Exception {
- FileSystem fs = FileSystem.get(conf);
- Path testDir = FSUtils.getRootDir(conf);
- Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
+ Path testDir = TEST_UTIL.getDataTestDir();
+ FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
- .withOutputDir(outputDir)
+ .withOutputDir(testDir)
.withFileContext(meta)
.build();
MobTestUtil.writeStoreFile(writer, getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
index 6da7655..9922aff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -413,8 +413,8 @@ public class TestMobCompactor {
result = table.get(get);
cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
// the ref name is the new file
- Path mobFamilyPath = new Path(
- MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString());
+ Path mobFamilyPath =
+ MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString());
List<Path> paths = new ArrayList<Path>();
if (fs.exists(mobFamilyPath)) {
FileStatus[] files = fs.listStatus(mobFamilyPath);
@@ -495,13 +495,7 @@ public class TestMobCompactor {
Scan scan = new Scan();
// Do not retrieve the mob data when scanning
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
- ResultScanner results = table.getScanner(scan);
- int count = 0;
- for (Result res : results) {
- count++;
- }
- results.close();
- return count;
+ return TEST_UTIL.countRows(table, scan);
}
/**
@@ -532,8 +526,7 @@ public class TestMobCompactor {
*/
private int countFiles(TableName tableName, boolean isMobFile, String familyName)
throws IOException {
- Path mobDirPath = MobUtils.getMobFamilyPath(
- MobUtils.getMobRegionPath(conf, tableName), familyName);
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@@ -553,8 +546,7 @@ public class TestMobCompactor {
}
private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
- Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
- familyName);
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
boolean hasFiles = false;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@@ -579,8 +571,7 @@ public class TestMobCompactor {
* @return the number of the HFileLink
*/
private int countHFileLinks(String familyName) throws IOException {
- Path mobDirPath = MobUtils.getMobFamilyPath(
- MobUtils.getMobRegionPath(conf, tableName), familyName);
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@@ -601,8 +592,7 @@ public class TestMobCompactor {
* @return the number of files large than the size
*/
private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException {
- Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
- familyName);
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@@ -729,8 +719,8 @@ public class TestMobCompactor {
// Do not retrieve the mob data when scanning
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
ResultScanner results = table.getScanner(scan);
- Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
- tableName), familyName);
+ Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(),
+ tableName, familyName);
List<Path> actualFilePaths = new ArrayList<>();
List<Path> expectFilePaths = new ArrayList<>();
for (Result res : results) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
index aba63c6..6774ac3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
@@ -104,56 +104,27 @@ public class TestPartitionedMobCompactor {
@Test
public void testCompactionSelectWithAllFiles() throws Exception {
- resetConf();
String tableName = "testCompactionSelectWithAllFiles";
- init(tableName);
- int count = 10;
- // create 10 mob files.
- createStoreFiles(basePath, family, qf, count, Type.Put);
- // create 10 del files
- createStoreFiles(basePath, family, qf, count, Type.Delete);
- listFiles();
- long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD;
- List<String> expectedStartKeys = new ArrayList<>();
- for(FileStatus file : mobFiles) {
- if(file.getLen() < mergeSize) {
- String fileName = file.getPath().getName();
- String startKey = fileName.substring(0, 32);
- expectedStartKeys.add(startKey);
- }
- }
- testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
+ testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
+ CompactionType.ALL_FILES, false);
}
@Test
public void testCompactionSelectWithPartFiles() throws Exception {
- resetConf();
String tableName = "testCompactionSelectWithPartFiles";
- init(tableName);
- int count = 10;
- // create 10 mob files.
- createStoreFiles(basePath, family, qf, count, Type.Put);
- // create 10 del files
- createStoreFiles(basePath, family, qf, count, Type.Delete);
- listFiles();
- long mergeSize = 4000;
- List<String> expectedStartKeys = new ArrayList<>();
- for(FileStatus file : mobFiles) {
- if(file.getLen() < 4000) {
- String fileName = file.getPath().getName();
- String startKey = fileName.substring(0, 32);
- expectedStartKeys.add(startKey);
- }
- }
- // set the mob compaction mergeable threshold
- conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
- testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
+ testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false);
}
@Test
public void testCompactionSelectWithForceAllFiles() throws Exception {
- resetConf();
String tableName = "testCompactionSelectWithForceAllFiles";
+ testCompactionAtMergeSize(tableName, Long.MAX_VALUE, CompactionType.ALL_FILES, true);
+ }
+
+ private void testCompactionAtMergeSize(final String tableName,
+ final long mergeSize, final CompactionType type, final boolean isForceAllFiles)
+ throws Exception {
+ resetConf();
init(tableName);
int count = 10;
// create 10 mob files.
@@ -161,51 +132,41 @@ public class TestPartitionedMobCompactor {
// create 10 del files
createStoreFiles(basePath, family, qf, count, Type.Delete);
listFiles();
- long mergeSize = 4000;
List<String> expectedStartKeys = new ArrayList<>();
for(FileStatus file : mobFiles) {
- String fileName = file.getPath().getName();
- String startKey = fileName.substring(0, 32);
- expectedStartKeys.add(startKey);
+ if(file.getLen() < mergeSize) {
+ String fileName = file.getPath().getName();
+ String startKey = fileName.substring(0, 32);
+ expectedStartKeys.add(startKey);
+ }
}
// set the mob compaction mergeable threshold
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
- testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
+ testSelectFiles(tableName, type, isForceAllFiles, expectedStartKeys);
}
@Test
public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
- resetConf();
String tableName = "testCompactDelFilesWithDefaultBatchSize";
- init(tableName);
- // create 20 mob files.
- createStoreFiles(basePath, family, qf, 20, Type.Put);
- // create 13 del files
- createStoreFiles(basePath, family, qf, 13, Type.Delete);
- listFiles();
- testCompactDelFiles(tableName, 1, 13, false);
+ testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE,
+ MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
}
@Test
public void testCompactDelFilesWithSmallBatchSize() throws Exception {
- resetConf();
String tableName = "testCompactDelFilesWithSmallBatchSize";
- init(tableName);
- // create 20 mob files.
- createStoreFiles(basePath, family, qf, 20, Type.Put);
- // create 13 del files
- createStoreFiles(basePath, family, qf, 13, Type.Delete);
- listFiles();
-
- // set the mob compaction batch size
- conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4);
- testCompactDelFiles(tableName, 1, 13, false);
+ testCompactDelFilesAtBatchSize(tableName, 4, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
}
@Test
public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
- resetConf();
String tableName = "testCompactDelFilesWithSmallBatchSize";
+ testCompactDelFilesAtBatchSize(tableName, 4, 2);
+ }
+
+ private void testCompactDelFilesAtBatchSize(String tableName, int batchSize,
+ int delfileMaxCount) throws Exception {
+ resetConf();
init(tableName);
// create 20 mob files.
createStoreFiles(basePath, family, qf, 20, Type.Put);
@@ -214,10 +175,10 @@ public class TestPartitionedMobCompactor {
listFiles();
// set the max del file count
- conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
+ conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, delfileMaxCount);
// set the mob compaction batch size
- conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2);
- testCompactDelFiles(tableName, 4, 13, false);
+ conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
+ testCompactDelFiles(tableName, 1, 13, false);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
index 6312653..779fdca 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
@@ -74,132 +73,110 @@ public class TestDeleteMobTable {
return mobVal;
}
- @Test
- public void testDeleteMobTable() throws Exception {
- byte[] tableName = Bytes.toBytes("testDeleteMobTable");
- TableName tn = TableName.valueOf(tableName);
- HTableDescriptor htd = new HTableDescriptor(tn);
+ private HTableDescriptor createTableDescriptor(TableName tableName, boolean hasMob) {
+ HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
- hcd.setMobEnabled(true);
- hcd.setMobThreshold(0);
+ if (hasMob) {
+ hcd.setMobEnabled(true);
+ hcd.setMobThreshold(0);
+ }
htd.addFamily(hcd);
- HBaseAdmin admin = null;
- Table table = null;
+ return htd;
+ }
+
+ private Table createTableWithOneFile(HTableDescriptor htd) throws IOException {
+ Table table = TEST_UTIL.createTable(htd, null);
try {
- admin = TEST_UTIL.getHBaseAdmin();
- admin.createTable(htd);
- table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
+ // insert data
byte[] value = generateMobValue(10);
-
byte[] row = Bytes.toBytes("row");
Put put = new Put(row);
put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
table.put(put);
- admin.flush(tn);
+ // create an hfile
+ TEST_UTIL.getHBaseAdmin().flush(htd.getTableName());
+ } catch (IOException e) {
+ table.close();
+ throw e;
+ }
+ return table;
+ }
+
+ @Test
+ public void testDeleteMobTable() throws Exception {
+ TableName tn = TableName.valueOf("testDeleteMobTable");
+ HTableDescriptor htd = createTableDescriptor(tn, true);
+ HColumnDescriptor hcd = htd.getFamily(FAMILY);
+ String fileName = null;
+ Table table = createTableWithOneFile(htd);
+ try {
// the mob file exists
Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
- String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
+ fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertTrue(mobTableDirExist(tn));
- table.close();
-
- admin.disableTable(tn);
- admin.deleteTable(tn);
-
- Assert.assertFalse(admin.tableExists(tn));
- Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
- Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
- Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
- Assert.assertFalse(mobTableDirExist(tn));
} finally {
- if (admin != null) {
- admin.close();
- }
+ table.close();
+ TEST_UTIL.deleteTable(tn);
}
+
+ Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn));
+ Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
+ Assert.assertFalse(mobTableDirExist(tn));
}
@Test
public void testDeleteNonMobTable() throws Exception {
- byte[] tableName = Bytes.toBytes("testDeleteNonMobTable");
- TableName tn = TableName.valueOf(tableName);
- HTableDescriptor htd = new HTableDescriptor(tn);
- HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
- htd.addFamily(hcd);
- HBaseAdmin admin = null;
- Table table = null;
- try {
- admin = TEST_UTIL.getHBaseAdmin();
- admin.createTable(htd);
- table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
- byte[] value = generateMobValue(10);
-
- byte[] row = Bytes.toBytes("row");
- Put put = new Put(row);
- put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
- table.put(put);
-
- admin.flush(tn);
- table.close();
+ TableName tn = TableName.valueOf("testDeleteNonMobTable");
+ HTableDescriptor htd = createTableDescriptor(tn, false);
+ HColumnDescriptor hcd = htd.getFamily(FAMILY);
+ Table table = createTableWithOneFile(htd);
+ try {
// the mob file doesn't exist
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertFalse(mobTableDirExist(tn));
-
- admin.disableTable(tn);
- admin.deleteTable(tn);
-
- Assert.assertFalse(admin.tableExists(tn));
- Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
- Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
- Assert.assertFalse(mobTableDirExist(tn));
} finally {
- if (admin != null) {
- admin.close();
- }
+ table.close();
+ TEST_UTIL.deleteTable(tn);
}
+
+ Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn));
+ Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertFalse(mobTableDirExist(tn));
}
@Test
public void testMobFamilyDelete() throws Exception {
TableName tn = TableName.valueOf("testMobFamilyDelete");
- HTableDescriptor htd = new HTableDescriptor(tn);
- HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
- hcd.setMobEnabled(true);
- hcd.setMobThreshold(0);
- htd.addFamily(hcd);
+ HTableDescriptor htd = createTableDescriptor(tn, true);
+ HColumnDescriptor hcd = htd.getFamily(FAMILY);
htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family2")));
- HBaseAdmin admin = null;
- Table table = null;
+
+ Table table = createTableWithOneFile(htd);
try {
- admin = TEST_UTIL.getHBaseAdmin();
- admin.createTable(htd);
- table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
- byte[] value = generateMobValue(10);
- byte[] row = Bytes.toBytes("row");
- Put put = new Put(row);
- put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
- table.put(put);
- admin.flush(tn);
// the mob file exists
Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertTrue(mobTableDirExist(tn));
- admin.deleteColumnFamily(tn, FAMILY);
+
+ TEST_UTIL.getHBaseAdmin().deleteColumnFamily(tn, FAMILY);
+
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
- Assert.assertFalse(mobColumnFamilyDirExist(tn));
+ Assert.assertFalse(mobColumnFamilyDirExist(tn, hcd.getNameAsString()));
} finally {
table.close();
- if (admin != null) {
- admin.close();
- }
TEST_UTIL.deleteTable(tn);
}
}
@@ -209,9 +186,8 @@ public class TestDeleteMobTable {
Path mobFileDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
if (fs.exists(mobFileDir)) {
return fs.listStatus(mobFileDir).length;
- } else {
- return 0;
}
+ return 0;
}
private int countArchiveMobFiles(TableName tn, String familyName)
@@ -221,9 +197,8 @@ public class TestDeleteMobTable {
MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
if (fs.exists(storePath)) {
return fs.listStatus(storePath).length;
- } else {
- return 0;
}
+ return 0;
}
private boolean mobTableDirExist(TableName tn) throws IOException {
@@ -232,12 +207,9 @@ public class TestDeleteMobTable {
return fs.exists(tableDir);
}
- private boolean mobColumnFamilyDirExist(TableName tn) throws IOException {
+ private boolean mobColumnFamilyDirExist(TableName tn, String familyName) throws IOException {
FileSystem fs = TEST_UTIL.getTestFileSystem();
- Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
- HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tn);
- Path mobFamilyDir = new Path(tableDir, new Path(mobRegionInfo.getEncodedName(),
- Bytes.toString(FAMILY)));
+ Path mobFamilyDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
return fs.exists(mobFamilyDir);
}
@@ -256,8 +228,7 @@ public class TestDeleteMobTable {
ResultScanner rs = table.getScanner(scan);
Result r = rs.next();
Assert.assertNotNull(r);
- byte[] value = r.getValue(FAMILY, QF);
- String fileName = Bytes.toString(value, Bytes.SIZEOF_INT, value.length - Bytes.SIZEOF_INT);
+ String fileName = MobUtils.getMobFileName(r.getColumnLatestCell(FAMILY, QF));
Path filePath = new Path(
MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName), fileName);
FileSystem fs = TEST_UTIL.getTestFileSystem();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
index 7cac301..f9ffc88 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
@@ -278,8 +278,7 @@ public class TestMobStoreCompaction {
}
private int countMobFiles() throws IOException {
- Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
- hcd.getNameAsString());
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
if (fs.exists(mobDirPath)) {
FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
return files.length;
@@ -289,8 +288,7 @@ public class TestMobStoreCompaction {
private long countMobCellsInMetadata() throws IOException {
long mobCellsCount = 0;
- Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
- hcd.getNameAsString());
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
Configuration copyOfConf = new Configuration(conf);
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
CacheConfig cacheConfig = new CacheConfig(copyOfConf);
@@ -357,9 +355,16 @@ public class TestMobStoreCompaction {
private int countRows() throws IOException {
Scan scan = new Scan();
- // Do not retrieve the mob data when scanning
InternalScanner scanner = region.getScanner(scan);
+ try {
+ return countRows(scanner);
+ } finally {
+ scanner.close();
+ }
+ }
+ private int countRows(InternalScanner scanner) throws IOException {
+ // Do not retrieve the mob data when scanning
int scannedCount = 0;
List<Cell> results = new ArrayList<Cell>();
boolean hasMore = true;
@@ -368,8 +373,6 @@ public class TestMobStoreCompaction {
scannedCount += results.size();
results.clear();
}
- scanner.close();
-
return scannedCount;
}
@@ -423,8 +426,7 @@ public class TestMobStoreCompaction {
Configuration copyOfConf = new Configuration(conf);
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
CacheConfig cacheConfig = new CacheConfig(copyOfConf);
- Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
- hcd.getNameAsString());
+ Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
List<StoreFile> sfs = new ArrayList<>();
int numDelfiles = 0;
int size = 0;
@@ -436,6 +438,7 @@ public class TestMobStoreCompaction {
numDelfiles++;
}
}
+
List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false,
HConstants.LATEST_TIMESTAMP);
Scan scan = new Scan();
@@ -446,12 +449,10 @@ public class TestMobStoreCompaction {
CellComparator.COMPARATOR);
StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
scanners, 0L, HConstants.LATEST_TIMESTAMP);
- List<Cell> results = new ArrayList<>();
- boolean hasMore = true;
- while (hasMore) {
- hasMore = scanner.next(results);
- size += results.size();
- results.clear();
+ try {
+ size += countRows(scanner);
+ } finally {
+ scanner.close();
}
}
// assert the number of the existing del files
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
index 8250b47..f3af63a4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobTestUtil;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -196,12 +197,12 @@ public class TestMobStoreScanner {
table.put(put4);
Result result = rs.next();
Cell cell = result.getColumnLatestCell(family, qf1);
- Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(cell)));
+ Assert.assertArrayEquals(value1, CellUtil.cloneValue(cell));
admin.flush(tn);
result = rs.next();
cell = result.getColumnLatestCell(family, qf1);
- Assert.assertEquals("value2", Bytes.toString(CellUtil.cloneValue(cell)));
+ Assert.assertArrayEquals(value2, CellUtil.cloneValue(cell));
}
@Test
@@ -213,7 +214,7 @@ public class TestMobStoreScanner {
get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
Result result = table.get(get);
Cell cell = result.getColumnLatestCell(family, qf1);
- Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
+ Assert.assertEquals(0, cell.getValueLength());
}
@Test
@@ -249,8 +250,7 @@ public class TestMobStoreScanner {
private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
throws IOException {
- Path regionDir = MobUtils.getMobRegionPath(conf, table);
- Path famDir = new Path(regionDir, family);
+ Path famDir = MobUtils.getMobFamilyPath(conf, table, family);
FileStatus[] hfFss = fs.listStatus(famDir);
for (FileStatus hfs : hfFss) {
if (!hfs.isDirectory()) {
@@ -262,44 +262,21 @@ public class TestMobStoreScanner {
private void testGetFromFiles(boolean reversed) throws Exception {
TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
- setUp(defaultThreshold, tn);
- long ts1 = System.currentTimeMillis();
- long ts2 = ts1 + 1;
- long ts3 = ts1 + 2;
- byte [] value = generateMobValue((int)defaultThreshold+1);
-
- Put put1 = new Put(row1);
- put1.addColumn(family, qf1, ts3, value);
- put1.addColumn(family, qf2, ts2, value);
- put1.addColumn(family, qf3, ts1, value);
- table.put(put1);
-
- admin.flush(tn);
-
- Scan scan = new Scan();
- setScan(scan, reversed, false);
-
- ResultScanner results = table.getScanner(scan);
- int count = 0;
- for (Result res : results) {
- List<Cell> cells = res.listCells();
- for(Cell cell : cells) {
- // Verify the value
- Assert.assertEquals(Bytes.toString(value),
- Bytes.toString(CellUtil.cloneValue(cell)));
- count++;
- }
- }
- results.close();
- Assert.assertEquals(3, count);
+ testGet(tn, reversed, true);
}
private void testGetFromMemStore(boolean reversed) throws Exception {
- setUp(defaultThreshold, TableName.valueOf("testGetFromMemStore" + reversed));
+ TableName tn = TableName.valueOf("testGetFromMemStore" + reversed);
+ testGet(tn, reversed, false);
+ }
+
+ private void testGet(TableName tableName, boolean reversed, boolean doFlush)
+ throws Exception {
+ setUp(defaultThreshold, tableName);
long ts1 = System.currentTimeMillis();
long ts2 = ts1 + 1;
long ts3 = ts1 + 2;
- byte [] value = generateMobValue((int)defaultThreshold+1);;
+ byte [] value = generateMobValue((int)defaultThreshold+1);
Put put1 = new Put(row1);
put1.addColumn(family, qf1, ts3, value);
@@ -307,22 +284,13 @@ public class TestMobStoreScanner {
put1.addColumn(family, qf3, ts1, value);
table.put(put1);
+ if (doFlush) {
+ admin.flush(tableName);
+ }
+
Scan scan = new Scan();
setScan(scan, reversed, false);
-
- ResultScanner results = table.getScanner(scan);
- int count = 0;
- for (Result res : results) {
- List<Cell> cells = res.listCells();
- for(Cell cell : cells) {
- // Verify the value
- Assert.assertEquals(Bytes.toString(value),
- Bytes.toString(CellUtil.cloneValue(cell)));
- count++;
- }
- }
- results.close();
- Assert.assertEquals(3, count);
+ MobTestUtil.assertCellsValue(table, scan, value, 3);
}
private void testGetReferences(boolean reversed) throws Exception {
@@ -426,8 +394,8 @@ public class TestMobStoreScanner {
// Get the files in the mob path
Path mobFamilyPath;
- mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tn),
- hcd.getNameAsString());
+ mobFamilyPath = MobUtils.getMobFamilyPath(
+ TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString());
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
FileStatus[] files = fs.listStatus(mobFamilyPath);
@@ -458,19 +426,7 @@ public class TestMobStoreScanner {
// Scan from archive
Scan scan = new Scan();
setScan(scan, reversed, false);
- ResultScanner results = table.getScanner(scan);
- int count = 0;
- for (Result res : results) {
- List<Cell> cells = res.listCells();
- for(Cell cell : cells) {
- // Verify the value
- Assert.assertEquals(Bytes.toString(value),
- Bytes.toString(CellUtil.cloneValue(cell)));
- count++;
- }
- }
- results.close();
- Assert.assertEquals(3, count);
+ MobTestUtil.assertCellsValue(table, scan, value, 3);
}
/**
@@ -478,12 +434,9 @@ public class TestMobStoreScanner {
*/
private static void assertNotMobReference(Cell cell, byte[] row, byte[] family,
byte[] value) throws IOException {
- Assert.assertEquals(Bytes.toString(row),
- Bytes.toString(CellUtil.cloneRow(cell)));
- Assert.assertEquals(Bytes.toString(family),
- Bytes.toString(CellUtil.cloneFamily(cell)));
- Assert.assertTrue(Bytes.toString(value).equals(
- Bytes.toString(CellUtil.cloneValue(cell))));
+ Assert.assertArrayEquals(row, CellUtil.cloneRow(cell));
+ Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell));
+ Assert.assertArrayEquals(value, CellUtil.cloneValue(cell));
}
/**
@@ -491,20 +444,15 @@ public class TestMobStoreScanner {
*/
private static void assertIsMobReference(Cell cell, byte[] row, byte[] family,
byte[] value, TableName tn) throws IOException {
- Assert.assertEquals(Bytes.toString(row),
- Bytes.toString(CellUtil.cloneRow(cell)));
- Assert.assertEquals(Bytes.toString(family),
- Bytes.toString(CellUtil.cloneFamily(cell)));
- Assert.assertFalse(Bytes.toString(value).equals(
- Bytes.toString(CellUtil.cloneValue(cell))));
+ Assert.assertArrayEquals(row, CellUtil.cloneRow(cell));
+ Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell));
+ Assert.assertFalse(Bytes.equals(value, CellUtil.cloneValue(cell)));
byte[] referenceValue = CellUtil.cloneValue(cell);
- String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
- referenceValue.length - Bytes.SIZEOF_INT);
+ String fileName = MobUtils.getMobFileName(cell);
int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT);
Assert.assertEquals(value.length, valLen);
- Path mobFamilyPath;
- mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
- tn), hcd.getNameAsString());
+ Path mobFamilyPath = MobUtils.getMobFamilyPath(
+ TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString());
Path targetPath = new Path(mobFamilyPath, fileName);
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
Assert.assertTrue(fs.exists(targetPath));
http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
index f90a8c9..7459a7d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
@@ -131,7 +131,7 @@ public class BaseTestHBaseFsck {
Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
-
+
/**
* Create a new region in META.
*/
@@ -633,8 +633,7 @@ public class BaseTestHBaseFsck {
* @throws IOException
*/
Path getFlushedMobFile(FileSystem fs, TableName table) throws IOException {
- Path regionDir = MobUtils.getMobRegionPath(conf, table);
- Path famDir = new Path(regionDir, FAM_STR);
+ Path famDir = MobUtils.getMobFamilyPath(conf, table, FAM_STR);
// keep doing this until we get a legit hfile
while (true) {