You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/11/29 07:20:56 UTC
[1/3] hbase git commit: HBASE-16912 Cleanup for HBaseTestingUtility.
Repository: hbase
Updated Branches:
refs/heads/hbase-14439 a36f80c7a -> 6d1813a2f
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index 5995e28..3872784 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
+import static java.awt.SystemColor.info;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -71,6 +72,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
@@ -95,8 +98,6 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.AsyncFSWALProvider;
-import org.apache.hadoop.hbase.wal.FSHLogProvider;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKey;
@@ -120,7 +121,7 @@ public abstract class AbstractTestWALReplay {
private static final Log LOG = LogFactory.getLog(AbstractTestWALReplay.class);
static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
- private Path hbaseRootDir = null;
+ private Path hbaseRootDir;
private String logName;
private Path oldLogDir;
private Path logDir;
@@ -183,10 +184,6 @@ public abstract class AbstractTestWALReplay {
}
}
- /**
- *
- * @throws Exception
- */
@Test
public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
final TableName tableName =
@@ -281,383 +278,352 @@ public abstract class AbstractTestWALReplay {
}
}
-// /**
-// * Tests for hbase-2727.
-// * @throws Exception
-// * @see <a href="https://issues.apache.org/jira/browse/HBASE-2727">HBASE-2727</a>
-// */
-// @Test
-// public void test2727() throws Exception {
-// // Test being able to have > 1 set of edits in the recovered.edits directory.
-// // Ensure edits are replayed properly.
-// final TableName tableName =
-// TableName.valueOf("test2727");
-//
-// MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
-// HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
-// deleteDir(basedir);
-//
-// HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// Region region2 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region2);
-// final byte [] rowName = tableName.getName();
-//
-// WAL wal1 = createWAL(this.conf, hbaseRootDir, logName);
-// // Add 1k to each family.
-// final int countPerFamily = 1000;
-//
-// NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(
-// Bytes.BYTES_COMPARATOR);
-// for(byte[] fam : htd.getFamiliesKeys()) {
-// scopes.put(fam, 0);
-// }
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
-// wal1, htd, mvcc, scopes);
-// }
-// wal1.shutdown();
-// runWALSplit(this.conf);
-//
-// WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
-// // Add 1k to each family.
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
-// ee, wal2, htd, mvcc, scopes);
-// }
-// wal2.shutdown();
-// runWALSplit(this.conf);
-//
-// WAL wal3 = createWAL(this.conf, hbaseRootDir, logName);
-// try {
-// HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
-// long seqid = region.getOpenSeqNum();
-// // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
-// // When opened, this region would apply 6k edits, and increment the sequenceId by 1
-// assertTrue(seqid > mvcc.getWritePoint());
-// assertEquals(seqid - 1, mvcc.getWritePoint());
-// LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
-// + mvcc.getReadPoint());
-//
-// // TODO: Scan all.
-// region.close();
-// } finally {
-// wal3.close();
-// }
-// }
-
-// /**
-// * Test case of HRegion that is only made out of bulk loaded files. Assert
-// * that we don't 'crash'.
-// * @throws IOException
-// * @throws IllegalAccessException
-// * @throws NoSuchFieldException
-// * @throws IllegalArgumentException
-// * @throws SecurityException
-// */
-// @Test
-// public void testRegionMadeOfBulkLoadedFilesOnly()
-// throws IOException, SecurityException, IllegalArgumentException,
-// NoSuchFieldException, IllegalAccessException, InterruptedException {
-// final TableName tableName =
-// TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
-// deleteDir(basedir);
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// Region region2 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region2);
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// Region region = HRegion.openHRegion(hri, htd, wal, this.conf);
-//
-// byte [] family = htd.getFamilies().iterator().next().getName();
-// Path f = new Path(basedir, "hfile");
-// HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
-// Bytes.toBytes("z"), 10);
-// List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
-// hfs.add(Pair.newPair(family, f.toString()));
-// region.bulkLoadHFiles(hfs, true, null);
-//
-// // Add an edit so something in the WAL
-// byte [] row = tableName.getName();
-// region.put((new Put(row)).addColumn(family, family, family));
-// wal.sync();
-// final int rowsInsertedCount = 11;
-//
-// assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
-//
-// // Now 'crash' the region by stealing its wal
-// final Configuration newConf = HBaseConfiguration.create(this.conf);
-// User user = HBaseTestingUtility.getDifferentUser(newConf,
-// tableName.getNameAsString());
-// user.runAs(new PrivilegedExceptionAction() {
-// @Override
-// public Object run() throws Exception {
-// runWALSplit(newConf);
-// WAL wal2 = createWAL(newConf, hbaseRootDir, logName);
-//
-// HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
-// hbaseRootDir, hri, htd, wal2);
-// long seqid2 = region2.getOpenSeqNum();
-// assertTrue(seqid2 > -1);
-// assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
-//
-// // I can't close wal1. Its been appropriated when we split.
-// region2.close();
-// wal2.close();
-// return null;
-// }
-// });
-// }
-
-// /**
-// * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
-// * files) and an edit in the memstore.
-// * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
-// * from being replayed"
-// * @throws IOException
-// * @throws IllegalAccessException
-// * @throws NoSuchFieldException
-// * @throws IllegalArgumentException
-// * @throws SecurityException
-// */
-// @Test
-// public void testCompactedBulkLoadedFiles()
-// throws IOException, SecurityException, IllegalArgumentException,
-// NoSuchFieldException, IllegalAccessException, InterruptedException {
-// final TableName tableName =
-// TableName.valueOf("testCompactedBulkLoadedFiles");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
-// deleteDir(basedir);
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// HRegion region2 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region2);
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
-//
-// // Add an edit so something in the WAL
-// byte [] row = tableName.getName();
-// byte [] family = htd.getFamilies().iterator().next().getName();
-// region.put((new Put(row)).addColumn(family, family, family));
-// wal.sync();
-//
-// List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
-// for (int i = 0; i < 3; i++) {
-// Path f = new Path(basedir, "hfile"+i);
-// HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
-// Bytes.toBytes(i + "50"), 10);
-// hfs.add(Pair.newPair(family, f.toString()));
-// }
-// region.bulkLoadHFiles(hfs, true, null);
-// final int rowsInsertedCount = 31;
-// assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
-//
-// // major compact to turn all the bulk loaded files into one normal file
-// region.compact(true);
-// assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
-//
-// // Now 'crash' the region by stealing its wal
-// final Configuration newConf = HBaseConfiguration.create(this.conf);
-// User user = HBaseTestingUtility.getDifferentUser(newConf,
-// tableName.getNameAsString());
-// user.runAs(new PrivilegedExceptionAction() {
-// @Override
-// public Object run() throws Exception {
-// runWALSplit(newConf);
-// WAL wal2 = createWAL(newConf, hbaseRootDir, logName);
-//
-// HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
-// hbaseRootDir, hri, htd, wal2);
-// long seqid2 = region2.getOpenSeqNum();
-// assertTrue(seqid2 > -1);
-// assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
-//
-// // I can't close wal1. Its been appropriated when we split.
-// region2.close();
-// wal2.close();
-// return null;
-// }
-// });
-// }
-//
-//
-// /**
-// * Test writing edits into an HRegion, closing it, splitting logs, opening
-// * Region again. Verify seqids.
-// * @throws IOException
-// * @throws IllegalAccessException
-// * @throws NoSuchFieldException
-// * @throws IllegalArgumentException
-// * @throws SecurityException
-// */
-// @Test
-// public void testReplayEditsWrittenViaHRegion()
-// throws IOException, SecurityException, IllegalArgumentException,
-// NoSuchFieldException, IllegalAccessException, InterruptedException {
-// final TableName tableName =
-// TableName.valueOf("testReplayEditsWrittenViaHRegion");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
-// deleteDir(basedir);
-// final byte[] rowName = tableName.getName();
-// final int countPerFamily = 10;
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// HRegion region3 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region3);
-// // Write countPerFamily edits into the three families. Do a flush on one
-// // of the families during the load of edits so its seqid is not same as
-// // others to test we do right thing when different seqids.
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
-// long seqid = region.getOpenSeqNum();
-// boolean first = true;
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
-// if (first) {
-// // If first, so we have at least one family w/ different seqid to rest.
-// region.flush(true);
-// first = false;
-// }
-// }
-// // Now assert edits made it in.
-// final Get g = new Get(rowName);
-// Result result = region.get(g);
-// assertEquals(countPerFamily * htd.getFamilies().size(),
-// result.size());
-// // Now close the region (without flush), split the log, reopen the region and assert that
-// // replay of log has the correct effect, that our seqids are calculated correctly so
-// // all edits in logs are seen as 'stale'/old.
-// region.close(true);
-// wal.shutdown();
-// runWALSplit(this.conf);
-// WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
-// long seqid2 = region2.getOpenSeqNum();
-// assertTrue(seqid + result.size() < seqid2);
-// final Result result1b = region2.get(g);
-// assertEquals(result.size(), result1b.size());
-//
-// // Next test. Add more edits, then 'crash' this region by stealing its wal
-// // out from under it and assert that replay of the log adds the edits back
-// // correctly when region is opened again.
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
-// }
-// // Get count of edits.
-// final Result result2 = region2.get(g);
-// assertEquals(2 * result.size(), result2.size());
-// wal2.sync();
-// final Configuration newConf = HBaseConfiguration.create(this.conf);
-// User user = HBaseTestingUtility.getDifferentUser(newConf,
-// tableName.getNameAsString());
-// user.runAs(new PrivilegedExceptionAction() {
-// @Override
-// public Object run() throws Exception {
-// runWALSplit(newConf);
-// FileSystem newFS = FileSystem.get(newConf);
-// // Make a new wal for new region open.
-// WAL wal3 = createWAL(newConf, hbaseRootDir, logName);
-// final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
-// HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
-// @Override
-// protected boolean restoreEdit(Store s, Cell cell) {
-// boolean b = super.restoreEdit(s, cell);
-// countOfRestoredEdits.incrementAndGet();
-// return b;
-// }
-// };
-// long seqid3 = region3.initialize();
-// Result result3 = region3.get(g);
-// // Assert that count of cells is same as before crash.
-// assertEquals(result2.size(), result3.size());
-// assertEquals(htd.getFamilies().size() * countPerFamily,
-// countOfRestoredEdits.get());
-//
-// // I can't close wal1. Its been appropriated when we split.
-// region3.close();
-// wal3.close();
-// return null;
-// }
-// });
-// }
-//
-// /**
-// * Test that we recover correctly when there is a failure in between the
-// * flushes. i.e. Some stores got flushed but others did not.
-// *
-// * Unfortunately, there is no easy hook to flush at a store level. The way
-// * we get around this is by flushing at the region level, and then deleting
-// * the recently flushed store file for one of the Stores. This would put us
-// * back in the situation where all but that store got flushed and the region
-// * died.
-// *
-// * We restart Region again, and verify that the edits were replayed.
-// *
-// * @throws IOException
-// * @throws IllegalAccessException
-// * @throws NoSuchFieldException
-// * @throws IllegalArgumentException
-// * @throws SecurityException
-// */
-// @Test
-// public void testReplayEditsAfterPartialFlush()
-// throws IOException, SecurityException, IllegalArgumentException,
-// NoSuchFieldException, IllegalAccessException, InterruptedException {
-// final TableName tableName =
-// TableName.valueOf("testReplayEditsWrittenViaHRegion");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
-// deleteDir(basedir);
-// final byte[] rowName = tableName.getName();
-// final int countPerFamily = 10;
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// HRegion region3 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region3);
-// // Write countPerFamily edits into the three families. Do a flush on one
-// // of the families during the load of edits so its seqid is not same as
-// // others to test we do right thing when different seqids.
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
-// long seqid = region.getOpenSeqNum();
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
-// }
-//
-// // Now assert edits made it in.
-// final Get g = new Get(rowName);
-// Result result = region.get(g);
-// assertEquals(countPerFamily * htd.getFamilies().size(),
-// result.size());
-//
-// // Let us flush the region
-// region.flush(true);
-// region.close(true);
-// wal.shutdown();
-//
-// // delete the store files in the second column family to simulate a failure
-// // in between the flushcache();
-// // we have 3 families. killing the middle one ensures that taking the maximum
-// // will make us fail.
-// int cf_count = 0;
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// cf_count++;
-// if (cf_count == 2) {
-// region.getRegionStorage().deleteFamily(hcd.getNameAsString());
-// }
-// }
-//
-//
-// // Let us try to split and recover
-// runWALSplit(this.conf);
-// WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
-// long seqid2 = region2.getOpenSeqNum();
-// assertTrue(seqid + result.size() < seqid2);
-//
-// final Result result1b = region2.get(g);
-// assertEquals(result.size(), result1b.size());
-// }
+ /**
+ * Tests for hbase-2727.
+ * @throws Exception
+ * @see <a href="https://issues.apache.org/jira/browse/HBASE-2727">HBASE-2727</a>
+ */
+ @Test
+ public void test2727() throws Exception {
+ // Test being able to have > 1 set of edits in the recovered.edits directory.
+ // Ensure edits are replayed properly.
+ final TableName tableName = TableName.valueOf("test2727");
+
+ MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ //Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ //deleteDir(basedir);
+
+ HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ HRegion region2 = TEST_UTIL.createLocalHRegion(hri, htd, conf);
+ HBaseTestingUtility.closeRegionAndWAL(region2);
+ final byte [] rowName = tableName.getName();
+
+ WAL wal1 = createWAL(this.conf, hbaseRootDir, logName);
+ // Add 1k to each family.
+ final int countPerFamily = 1000;
+
+ NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(
+ Bytes.BYTES_COMPARATOR);
+ for(byte[] fam : htd.getFamiliesKeys()) {
+ scopes.put(fam, 0);
+ }
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
+ wal1, htd, mvcc, scopes);
+ }
+ wal1.shutdown();
+ runWALSplit(this.conf);
+
+ WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
+ // Add 1k to each family.
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
+ ee, wal2, htd, mvcc, scopes);
+ }
+ wal2.shutdown();
+ runWALSplit(this.conf);
+
+ WAL wal3 = createWAL(this.conf, hbaseRootDir, logName);
+ try {
+ HRegion region = HRegion.openHRegion(hri, htd, wal3, conf);
+ long seqid = region.getOpenSeqNum();
+ // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
+ // When opened, this region would apply 6k edits, and increment the sequenceId by 1
+ assertTrue(seqid > mvcc.getWritePoint());
+ assertEquals(seqid - 1, mvcc.getWritePoint());
+ LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
+ + mvcc.getReadPoint());
+
+ // TODO: Scan all.
+ region.close();
+ } finally {
+ wal3.close();
+ }
+ }
+
+ /**
+ * Test case of HRegion that is only made out of bulk loaded files. Assert
+ * that we don't 'crash'.
+ */
+ @Test
+ public void testRegionMadeOfBulkLoadedFilesOnly() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
+ deleteDir(basedir);
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region2 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region2);
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ Region region = HRegion.openHRegion(hri, htd, wal, conf);
+
+ byte [] family = htd.getFamilies().iterator().next().getName();
+ Path f = new Path(basedir, "hfile");
+ HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
+ Bytes.toBytes("z"), 10);
+ List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
+ hfs.add(Pair.newPair(family, f.toString()));
+ region.bulkLoadHFiles(hfs, true, null);
+
+ // Add an edit so something in the WAL
+ byte [] row = tableName.getName();
+ region.put((new Put(row)).addColumn(family, family, family));
+ wal.sync();
+ final int rowsInsertedCount = 11;
+
+ assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
+
+ // Now 'crash' the region by stealing its wal
+ final Configuration newConf = HBaseConfiguration.create(this.conf);
+ User user = HBaseTestingUtility.getDifferentUser(newConf,
+ tableName.getNameAsString());
+ user.runAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() throws Exception {
+ runWALSplit(newConf);
+ WAL wal2 = createWAL(newConf, hbaseRootDir, logName);
+
+ HRegion region2 = HRegion.openHRegion(hri, htd, wal2, newConf);
+ long seqid2 = region2.getOpenSeqNum();
+ assertTrue(seqid2 > -1);
+ assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
+
+ // I can't close wal1. Its been appropriated when we split.
+ region2.close();
+ wal2.close();
+ return null;
+ }
+ });
+ }
+
+ /**
+ * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
+ * files) and an edit in the memstore.
+ * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
+ * from being replayed"
+ */
+ @Test
+ public void testCompactedBulkLoadedFiles() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testCompactedBulkLoadedFiles");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
+ deleteDir(basedir);
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region2 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region2);
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
+
+ // Add an edit so something in the WAL
+ byte [] row = tableName.getName();
+ byte [] family = htd.getFamilies().iterator().next().getName();
+ region.put((new Put(row)).addColumn(family, family, family));
+ wal.sync();
+
+ List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
+ for (int i = 0; i < 3; i++) {
+ Path f = new Path(basedir, "hfile"+i);
+ HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
+ Bytes.toBytes(i + "50"), 10);
+ hfs.add(Pair.newPair(family, f.toString()));
+ }
+ region.bulkLoadHFiles(hfs, true, null);
+ final int rowsInsertedCount = 31;
+ assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
+
+ // major compact to turn all the bulk loaded files into one normal file
+ region.compact(true);
+ assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
+
+ // Now 'crash' the region by stealing its wal
+ final Configuration newConf = HBaseConfiguration.create(this.conf);
+ User user = HBaseTestingUtility.getDifferentUser(newConf,
+ tableName.getNameAsString());
+ user.runAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() throws Exception {
+ runWALSplit(newConf);
+ WAL wal2 = createWAL(newConf, hbaseRootDir, logName);
+
+ HRegion region2 = HRegion.openHRegion(hri, htd, wal2, newConf);
+ long seqid2 = region2.getOpenSeqNum();
+ assertTrue(seqid2 > -1);
+ assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
+
+ // I can't close wal1. Its been appropriated when we split.
+ region2.close();
+ wal2.close();
+ return null;
+ }
+ });
+ }
+
+
+ /**
+ * Test writing edits into an HRegion, closing it, splitting logs, opening
+ * Region again. Verify seqids.
+ */
+ @Test
+ public void testReplayEditsWrittenViaHRegion() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testReplayEditsWrittenViaHRegion");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ deleteDir(basedir);
+ final byte[] rowName = tableName.getName();
+ final int countPerFamily = 10;
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region3 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region3);
+ // Write countPerFamily edits into the three families. Do a flush on one
+ // of the families during the load of edits so its seqid is not same as
+ // others to test we do right thing when different seqids.
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region = HRegion.openHRegion(hri, htd, wal, conf);
+ long seqid = region.getOpenSeqNum();
+ boolean first = true;
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
+ if (first) {
+ // If first, so we have at least one family w/ different seqid to rest.
+ region.flush(true);
+ first = false;
+ }
+ }
+ // Now assert edits made it in.
+ final Get g = new Get(rowName);
+ Result result = region.get(g);
+ assertEquals(countPerFamily * htd.getFamilies().size(),
+ result.size());
+ // Now close the region (without flush), split the log, reopen the region and assert that
+ // replay of log has the correct effect, that our seqids are calculated correctly so
+ // all edits in logs are seen as 'stale'/old.
+ region.close(true);
+ wal.shutdown();
+ runWALSplit(this.conf);
+ WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region2 = HRegion.openHRegion(hri, htd, wal2, conf);
+ long seqid2 = region2.getOpenSeqNum();
+ assertTrue(seqid + result.size() < seqid2);
+ final Result result1b = region2.get(g);
+ assertEquals(result.size(), result1b.size());
+
+ // Next test. Add more edits, then 'crash' this region by stealing its wal
+ // out from under it and assert that replay of the log adds the edits back
+ // correctly when region is opened again.
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
+ }
+ // Get count of edits.
+ final Result result2 = region2.get(g);
+ assertEquals(2 * result.size(), result2.size());
+ wal2.sync();
+ final Configuration newConf = HBaseConfiguration.create(this.conf);
+ User user = HBaseTestingUtility.getDifferentUser(newConf,
+ tableName.getNameAsString());
+ user.runAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() throws Exception {
+ runWALSplit(newConf);
+ FileSystem newFS = FileSystem.get(newConf);
+ // Make a new wal for new region open.
+ WAL wal3 = createWAL(newConf, hbaseRootDir, logName);
+ final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
+ RegionStorage regionStorage = RegionStorage.open(newConf, hri, true);
+ HRegion region3 = new HRegion(regionStorage, htd, wal3, null) {
+ @Override
+ protected boolean restoreEdit(Store s, Cell cell) {
+ boolean b = super.restoreEdit(s, cell);
+ countOfRestoredEdits.incrementAndGet();
+ return b;
+ }
+ };
+ long seqid3 = region3.initialize();
+ Result result3 = region3.get(g);
+ // Assert that count of cells is same as before crash.
+ assertEquals(result2.size(), result3.size());
+ assertEquals(htd.getFamilies().size() * countPerFamily,
+ countOfRestoredEdits.get());
+
+ // I can't close wal1. Its been appropriated when we split.
+ region3.close();
+ wal3.close();
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Test that we recover correctly when there is a failure in between the
+ * flushes. i.e. Some stores got flushed but others did not.
+ *
+ * Unfortunately, there is no easy hook to flush at a store level. The way
+ * we get around this is by flushing at the region level, and then deleting
+ * the recently flushed store file for one of the Stores. This would put us
+ * back in the situation where all but that store got flushed and the region
+ * died.
+ *
+ * We restart Region again, and verify that the edits were replayed.
+ */
+ @Test
+ public void testReplayEditsAfterPartialFlush() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testReplayEditsWrittenViaHRegion");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ deleteDir(basedir);
+ final byte[] rowName = tableName.getName();
+ final int countPerFamily = 10;
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region3 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region3);
+ // Write countPerFamily edits into the three families. Do a flush on one
+ // of the families during the load of edits so its seqid is not same as
+ // others to test we do right thing when different seqids.
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region = HRegion.openHRegion(hri, htd, wal, conf);
+ long seqid = region.getOpenSeqNum();
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
+ }
+
+ // Now assert edits made it in.
+ final Get g = new Get(rowName);
+ Result result = region.get(g);
+ assertEquals(countPerFamily * htd.getFamilies().size(),
+ result.size());
+
+ // Let us flush the region
+ region.flush(true);
+ region.close(true);
+ wal.shutdown();
+
+ // delete the store files in the second column family to simulate a failure
+ // in between the flushcache();
+ // we have 3 families. killing the middle one ensures that taking the maximum
+ // will make us fail.
+ int cf_count = 0;
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ cf_count++;
+ if (cf_count == 2) {
+ region.getRegionStorage().deleteFamily(hcd.getNameAsString(), false);
+ }
+ }
+
+
+ // Let us try to split and recover
+ runWALSplit(this.conf);
+ WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region2 = HRegion.openHRegion(hri, htd, wal2, conf);
+ long seqid2 = region2.getOpenSeqNum();
+ assertTrue(seqid + result.size() < seqid2);
+
+ final Result result1b = region2.get(g);
+ assertEquals(result.size(), result1b.size());
+ }
// StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
@@ -680,91 +646,89 @@ public abstract class AbstractTestWALReplay {
};
-// /**
-// * Test that we could recover the data correctly after aborting flush. In the
-// * test, first we abort flush after writing some data, then writing more data
-// * and flush again, at last verify the data.
-// * @throws IOException
-// */
-// @Test
-// public void testReplayEditsAfterAbortingFlush() throws IOException {
-// final TableName tableName =
-// TableName.valueOf("testReplayEditsAfterAbortingFlush");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
-// deleteDir(basedir);
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// HRegion region3 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region3);
-// // Write countPerFamily edits into the three families. Do a flush on one
-// // of the families during the load of edits so its seqid is not same as
-// // others to test we do right thing when different seqids.
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
-// Mockito.doReturn(false).when(rsServices).isAborted();
-// when(rsServices.getServerName()).thenReturn(ServerName.valueOf("foo", 10, 10));
-// Configuration customConf = new Configuration(this.conf);
-// customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
-// CustomStoreFlusher.class.getName());
-// HRegion region =
-// HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
-// int writtenRowCount = 10;
-// List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
-// htd.getFamilies());
-// for (int i = 0; i < writtenRowCount; i++) {
-// Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
-// put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
-// Bytes.toBytes("val"));
-// region.put(put);
-// }
-//
-// // Now assert edits made it in.
-// RegionScanner scanner = region.getScanner(new Scan());
-// assertEquals(writtenRowCount, getScannedCount(scanner));
-//
-// // Let us flush the region
-// CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
-// try {
-// region.flush(true);
-// fail("Injected exception hasn't been thrown");
-// } catch (Throwable t) {
-// LOG.info("Expected simulated exception when flushing region,"
-// + t.getMessage());
-// // simulated to abort server
-// Mockito.doReturn(true).when(rsServices).isAborted();
-// region.setClosing(false); // region normally does not accept writes after
-// // DroppedSnapshotException. We mock around it for this test.
-// }
-// // writing more data
-// int moreRow = 10;
-// for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
-// Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
-// put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
-// Bytes.toBytes("val"));
-// region.put(put);
-// }
-// writtenRowCount += moreRow;
-// // call flush again
-// CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
-// try {
-// region.flush(true);
-// } catch (IOException t) {
-// LOG.info("Expected exception when flushing region because server is stopped,"
-// + t.getMessage());
-// }
-//
-// region.close(true);
-// wal.shutdown();
-//
-// // Let us try to split and recover
-// runWALSplit(this.conf);
-// WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
-// Mockito.doReturn(false).when(rsServices).isAborted();
-// HRegion region2 =
-// HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
-// scanner = region2.getScanner(new Scan());
-// assertEquals(writtenRowCount, getScannedCount(scanner));
-// }
+ /**
+ * Test that we could recover the data correctly after aborting flush. In the
+ * test, first we abort flush after writing some data, then writing more data
+ * and flush again, at last verify the data.
+ * @throws IOException
+ */
+ @Test
+ public void testReplayEditsAfterAbortingFlush() throws IOException {
+ final TableName tableName =
+ TableName.valueOf("testReplayEditsAfterAbortingFlush");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ deleteDir(basedir);
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region3 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region3);
+ // Write countPerFamily edits into the three families. Do a flush on one
+ // of the families during the load of edits so its seqid is not same as
+ // others to test we do right thing when different seqids.
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
+ Mockito.doReturn(false).when(rsServices).isAborted();
+ when(rsServices.getServerName()).thenReturn(ServerName.valueOf("foo", 10, 10));
+ Configuration customConf = new Configuration(this.conf);
+ customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
+ CustomStoreFlusher.class.getName());
+ HRegion region = HRegion.openHRegion(hri, htd, wal, customConf, rsServices, null);
+ int writtenRowCount = 10;
+ List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
+ htd.getFamilies());
+ for (int i = 0; i < writtenRowCount; i++) {
+ Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
+ put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
+ Bytes.toBytes("val"));
+ region.put(put);
+ }
+
+ // Now assert edits made it in.
+ RegionScanner scanner = region.getScanner(new Scan());
+ assertEquals(writtenRowCount, getScannedCount(scanner));
+
+ // Let us flush the region
+ CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
+ try {
+ region.flush(true);
+ fail("Injected exception hasn't been thrown");
+ } catch (Throwable t) {
+ LOG.info("Expected simulated exception when flushing region,"
+ + t.getMessage());
+ // simulated to abort server
+ Mockito.doReturn(true).when(rsServices).isAborted();
+ region.setClosing(false); // region normally does not accept writes after
+ // DroppedSnapshotException. We mock around it for this test.
+ }
+ // writing more data
+ int moreRow = 10;
+ for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
+ Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
+ put.addColumn(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
+ Bytes.toBytes("val"));
+ region.put(put);
+ }
+ writtenRowCount += moreRow;
+ // call flush again
+ CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
+ try {
+ region.flush(true);
+ } catch (IOException t) {
+ LOG.info("Expected exception when flushing region because server is stopped,"
+ + t.getMessage());
+ }
+
+ region.close(true);
+ wal.shutdown();
+
+ // Let us try to split and recover
+ runWALSplit(this.conf);
+ WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
+ Mockito.doReturn(false).when(rsServices).isAborted();
+ HRegion region2 = HRegion.openHRegion(hri, htd, wal2, this.conf, rsServices, null);
+ scanner = region2.getScanner(new Scan());
+ assertEquals(writtenRowCount, getScannedCount(scanner));
+ }
private int getScannedCount(RegionScanner scanner) throws IOException {
int scannedCount = 0;
@@ -780,315 +744,315 @@ public abstract class AbstractTestWALReplay {
return scannedCount;
}
-// /**
-// * Create an HRegion with the result of a WAL split and test we only see the
-// * good edits
-// * @throws Exception
-// */
-// @Test
-// public void testReplayEditsWrittenIntoWAL() throws Exception {
-// final TableName tableName =
-// TableName.valueOf("testReplayEditsWrittenIntoWAL");
-// final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
-// deleteDir(basedir);
-//
-// final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
-// HRegion region2 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region2);
-// final WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// final byte[] rowName = tableName.getName();
-// final byte[] regionName = hri.getEncodedNameAsBytes();
-//
-// // Add 1k to each family.
-// final int countPerFamily = 1000;
-// Set<byte[]> familyNames = new HashSet<byte[]>();
-// NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(
-// Bytes.BYTES_COMPARATOR);
-// for(byte[] fam : htd.getFamiliesKeys()) {
-// scopes.put(fam, 0);
-// }
-// for (HColumnDescriptor hcd: htd.getFamilies()) {
-// addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
-// ee, wal, htd, mvcc, scopes);
-// familyNames.add(hcd.getName());
-// }
-//
-// // Add a cache flush, shouldn't have any effect
-// wal.startCacheFlush(regionName, familyNames);
-// wal.completeCacheFlush(regionName);
-//
-// // Add an edit to another family, should be skipped.
-// WALEdit edit = new WALEdit();
-// long now = ee.currentTime();
-// edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
-// now, rowName));
-// wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit,
-// true);
-//
-// // Delete the c family to verify deletes make it over.
-// edit = new WALEdit();
-// now = ee.currentTime();
-// edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now, KeyValue.Type.DeleteFamily));
-// wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit,
-// true);
-//
-// // Sync.
-// wal.sync();
-// // Make a new conf and a new fs for the splitter to run on so we can take
-// // over old wal.
-// final Configuration newConf = HBaseConfiguration.create(this.conf);
-// User user = HBaseTestingUtility.getDifferentUser(newConf,
-// ".replay.wal.secondtime");
-// user.runAs(new PrivilegedExceptionAction<Void>() {
-// @Override
-// public Void run() throws Exception {
-// runWALSplit(newConf);
-// FileSystem newFS = FileSystem.get(newConf);
-// // 100k seems to make for about 4 flushes during HRegion#initialize.
-// newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
-// // Make a new wal for new region.
-// WAL newWal = createWAL(newConf, hbaseRootDir, logName);
-// final AtomicInteger flushcount = new AtomicInteger(0);
-// try {
-// final HRegion region =
-// new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
-// @Override
-// protected FlushResult internalFlushcache(final WAL wal, final long myseqid,
-// final Collection<Store> storesToFlush, MonitoredTask status,
-// boolean writeFlushWalMarker)
-// throws IOException {
-// LOG.info("InternalFlushCache Invoked");
-// FlushResult fs = super.internalFlushcache(wal, myseqid, storesToFlush,
-// Mockito.mock(MonitoredTask.class), writeFlushWalMarker);
-// flushcount.incrementAndGet();
-// return fs;
-// }
-// };
-// // The seq id this region has opened up with
-// long seqid = region.initialize();
-//
-// // The mvcc readpoint of from inserting data.
-// long writePoint = mvcc.getWritePoint();
-//
-// // We flushed during init.
-// assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
-// assertTrue((seqid - 1) == writePoint);
-//
-// Get get = new Get(rowName);
-// Result result = region.get(get);
-// // Make sure we only see the good edits
-// assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
-// result.size());
-// region.close();
-// } finally {
-// newWal.close();
-// }
-// return null;
-// }
-// });
-// }
-
-// @Test
-// // the following test is for HBASE-6065
-// public void testSequentialEditLogSeqNum() throws IOException {
-// final TableName tableName = TableName.valueOf(currentTest.getMethodName());
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir =
-// FSUtils.getTableDir(this.hbaseRootDir, tableName);
-// deleteDir(basedir);
-// final byte[] rowName = tableName.getName();
-// final int countPerFamily = 10;
-// final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
-//
-// // Mock the WAL
-// MockWAL wal = createMockWAL();
-//
-// HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
-// for (HColumnDescriptor hcd : htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
-// }
-//
-// // Let us flush the region
-// // But this time completeflushcache is not yet done
-// region.flush(true);
-// for (HColumnDescriptor hcd : htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
-// }
-// long lastestSeqNumber = region.getReadPoint(null);
-// // get the current seq no
-// wal.doCompleteCacheFlush = true;
-// // allow complete cache flush with the previous seq number got after first
-// // set of edits.
-// wal.completeCacheFlush(hri.getEncodedNameAsBytes());
-// wal.shutdown();
-// FileStatus[] listStatus = wal.getFiles();
-// assertNotNull(listStatus);
-// assertTrue(listStatus.length > 0);
-// WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
-// this.fs, this.conf, null, null, null, mode, wals);
-// FileStatus[] listStatus1 = this.fs.listStatus(
-// new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
-// "recovered.edits")), new PathFilter() {
-// @Override
-// public boolean accept(Path p) {
-// if (WALSplitter.isSequenceIdFile(p)) {
-// return false;
-// }
-// return true;
-// }
-// });
-// int editCount = 0;
-// for (FileStatus fileStatus : listStatus1) {
-// editCount = Integer.parseInt(fileStatus.getPath().getName());
-// }
-// // The sequence number should be same
-// assertEquals(
-// "The sequence number of the recoverd.edits and the current edit seq should be same",
-// lastestSeqNumber, editCount);
-// }
-
-// /**
-// * testcase for https://issues.apache.org/jira/browse/HBASE-15252
-// */
-// @Test
-// public void testDatalossWhenInputError() throws IOException, InstantiationException,
-// IllegalAccessException {
-// final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
-// deleteDir(basedir);
-// final byte[] rowName = tableName.getName();
-// final int countPerFamily = 10;
-// final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
-// HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// Path regionDir = region1.getRegionStorage().getRegionDir();
-// HBaseTestingUtility.closeRegionAndWAL(region1);
-//
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
-// for (HColumnDescriptor hcd : htd.getFamilies()) {
-// addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
-// }
-// // Now assert edits made it in.
-// final Get g = new Get(rowName);
-// Result result = region.get(g);
-// assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
-// // Now close the region (without flush), split the log, reopen the region and assert that
-// // replay of log has the correct effect.
-// region.close(true);
-// wal.shutdown();
-//
-// runWALSplit(this.conf);
-//
-// // here we let the DFSInputStream throw an IOException just after the WALHeader.
-// Path editFile = WALSplitter.getSplitEditFilesSorted(this.fs, regionDir).first();
-// FSDataInputStream stream = fs.open(editFile);
-// stream.seek(ProtobufLogReader.PB_WAL_MAGIC.length);
-// Class<? extends AbstractFSWALProvider.Reader> logReaderClass =
-// conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
-// AbstractFSWALProvider.Reader.class);
-// AbstractFSWALProvider.Reader reader = logReaderClass.newInstance();
-// reader.init(this.fs, editFile, conf, stream);
-// final long headerLength = stream.getPos();
-// reader.close();
-// FileSystem spyFs = spy(this.fs);
-// doAnswer(new Answer<FSDataInputStream>() {
-//
-// @Override
-// public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
-// FSDataInputStream stream = (FSDataInputStream) invocation.callRealMethod();
-// Field field = FilterInputStream.class.getDeclaredField("in");
-// field.setAccessible(true);
-// final DFSInputStream in = (DFSInputStream) field.get(stream);
-// DFSInputStream spyIn = spy(in);
-// doAnswer(new Answer<Integer>() {
-//
-// private long pos;
-//
-// @Override
-// public Integer answer(InvocationOnMock invocation) throws Throwable {
-// if (pos >= headerLength) {
-// throw new IOException("read over limit");
-// }
-// int b = (Integer) invocation.callRealMethod();
-// if (b > 0) {
-// pos += b;
-// }
-// return b;
-// }
-// }).when(spyIn).read(any(byte[].class), any(int.class), any(int.class));
-// doAnswer(new Answer<Void>() {
-//
-// @Override
-// public Void answer(InvocationOnMock invocation) throws Throwable {
-// invocation.callRealMethod();
-// in.close();
-// return null;
-// }
-// }).when(spyIn).close();
-// field.set(stream, spyIn);
-// return stream;
-// }
-// }).when(spyFs).open(eq(editFile));
-//
-// WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
-// HRegion region2;
-// try {
-// // log replay should fail due to the IOException, otherwise we may lose data.
-// region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2);
-// assertEquals(result.size(), region2.get(g).size());
-// } catch (IOException e) {
-// assertEquals("read over limit", e.getMessage());
-// }
-// region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2);
-// assertEquals(result.size(), region2.get(g).size());
-// }
-//
-// /**
-// * testcase for https://issues.apache.org/jira/browse/HBASE-14949.
-// */
-// private void testNameConflictWhenSplit(boolean largeFirst) throws IOException {
-// final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
-// final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
-// final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
-// final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
-// deleteDir(basedir);
-//
-// final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
-// NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
-// for (byte[] fam : htd.getFamiliesKeys()) {
-// scopes.put(fam, 0);
-// }
-// HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-// HBaseTestingUtility.closeRegionAndWAL(region);
-// final byte[] family = htd.getColumnFamilies()[0].getName();
-// final byte[] rowName = tableName.getName();
-// FSWALEntry entry1 = createFSWALEntry(htd, hri, 1L, rowName, family, ee, mvcc, 1, scopes);
-// FSWALEntry entry2 = createFSWALEntry(htd, hri, 2L, rowName, family, ee, mvcc, 2, scopes);
-//
-// Path largeFile = new Path(logDir, "wal-1");
-// Path smallFile = new Path(logDir, "wal-2");
-// writerWALFile(largeFile, Arrays.asList(entry1, entry2));
-// writerWALFile(smallFile, Arrays.asList(entry2));
-// FileStatus first, second;
-// if (largeFirst) {
-// first = fs.getFileStatus(largeFile);
-// second = fs.getFileStatus(smallFile);
-// } else {
-// first = fs.getFileStatus(smallFile);
-// second = fs.getFileStatus(largeFile);
-// }
-// WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null,
-// RecoveryMode.LOG_SPLITTING, wals);
-// WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null,
-// RecoveryMode.LOG_SPLITTING, wals);
-// WAL wal = createWAL(this.conf, hbaseRootDir, logName);
-// region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
-// assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
-// assertEquals(2, region.get(new Get(rowName)).size());
-// }
+ /**
+ * Create an HRegion with the result of a WAL split and test we only see the
+ * good edits
+ */
+ @Test
+ public void testReplayEditsWrittenIntoWAL() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testReplayEditsWrittenIntoWAL");
+ final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ deleteDir(basedir);
+
+ final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
+ Region region2 = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region2);
+ final WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ final byte[] rowName = tableName.getName();
+ final byte[] regionName = hri.getEncodedNameAsBytes();
+
+ // Add 1k to each family.
+ final int countPerFamily = 1000;
+ Set<byte[]> familyNames = new HashSet<byte[]>();
+ NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(
+ Bytes.BYTES_COMPARATOR);
+ for(byte[] fam : htd.getFamiliesKeys()) {
+ scopes.put(fam, 0);
+ }
+ for (HColumnDescriptor hcd: htd.getFamilies()) {
+ addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
+ ee, wal, htd, mvcc, scopes);
+ familyNames.add(hcd.getName());
+ }
+
+ // Add a cache flush, shouldn't have any effect
+ wal.startCacheFlush(regionName, familyNames);
+ wal.completeCacheFlush(regionName);
+
+ // Add an edit to another family, should be skipped.
+ WALEdit edit = new WALEdit();
+ long now = ee.currentTime();
+ edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
+ now, rowName));
+ wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit,
+ true);
+
+ // Delete the c family to verify deletes make it over.
+ edit = new WALEdit();
+ now = ee.currentTime();
+ edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now, KeyValue.Type.DeleteFamily));
+ wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit,
+ true);
+
+ // Sync.
+ wal.sync();
+ // Make a new conf and a new fs for the splitter to run on so we can take
+ // over old wal.
+ final Configuration newConf = HBaseConfiguration.create(this.conf);
+ User user = HBaseTestingUtility.getDifferentUser(newConf,
+ ".replay.wal.secondtime");
+ user.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ runWALSplit(newConf);
+ FileSystem newFS = FileSystem.get(newConf);
+ // 100k seems to make for about 4 flushes during HRegion#initialize.
+ newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
+ // Make a new wal for new region.
+ WAL newWal = createWAL(newConf, hbaseRootDir, logName);
+ final AtomicInteger flushcount = new AtomicInteger(0);
+ try {
+ RegionStorage regionStorage = RegionStorage.open(newConf, hri, true);
+ final HRegion region = new HRegion(regionStorage, htd, newWal, null) {
+ @Override
+ protected FlushResult internalFlushcache(final WAL wal, final long myseqid,
+ final Collection<Store> storesToFlush, MonitoredTask status,
+ boolean writeFlushWalMarker)
+ throws IOException {
+ LOG.info("InternalFlushCache Invoked");
+ FlushResult fs = super.internalFlushcache(wal, myseqid, storesToFlush,
+ Mockito.mock(MonitoredTask.class), writeFlushWalMarker);
+ flushcount.incrementAndGet();
+ return fs;
+ }
+ };
+ // The seq id this region has opened up with
+ long seqid = region.initialize();
+
+ // The mvcc readpoint of from inserting data.
+ long writePoint = mvcc.getWritePoint();
+
+ // We flushed during init.
+ assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
+ assertTrue((seqid - 1) == writePoint);
+
+ Get get = new Get(rowName);
+ Result result = region.get(get);
+ // Make sure we only see the good edits
+ assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
+ result.size());
+ region.close();
+ } finally {
+ newWal.close();
+ }
+ return null;
+ }
+ });
+ }
+
+ @Test
+ // the following test is for HBASE-6065
+ public void testSequentialEditLogSeqNum() throws IOException {
+ final TableName tableName = TableName.valueOf(currentTest.getMethodName());
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir =
+ FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ deleteDir(basedir);
+ final byte[] rowName = tableName.getName();
+ final int countPerFamily = 10;
+ final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
+
+ // Mock the WAL
+ MockWAL wal = createMockWAL();
+
+ HRegion region = HRegion.openHRegion(hri, htd, wal, conf);
+ for (HColumnDescriptor hcd : htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
+ }
+
+ // Let us flush the region
+ // But this time completeflushcache is not yet done
+ region.flush(true);
+ for (HColumnDescriptor hcd : htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
+ }
+ long lastestSeqNumber = region.getReadPoint(null);
+ // get the current seq no
+ wal.doCompleteCacheFlush = true;
+ // allow complete cache flush with the previous seq number got after first
+ // set of edits.
+ wal.completeCacheFlush(hri.getEncodedNameAsBytes());
+ wal.shutdown();
+ FileStatus[] listStatus = wal.getFiles();
+ assertNotNull(listStatus);
+ assertTrue(listStatus.length > 0);
+ WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
+ this.fs, this.conf, null, null, null, mode, wals);
+ FileStatus[] listStatus1 = this.fs.listStatus(
+ new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
+ "recovered.edits")), new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ if (WALSplitter.isSequenceIdFile(p)) {
+ return false;
+ }
+ return true;
+ }
+ });
+ int editCount = 0;
+ for (FileStatus fileStatus : listStatus1) {
+ editCount = Integer.parseInt(fileStatus.getPath().getName());
+ }
+ // The sequence number should be same
+ assertEquals(
+ "The sequence number of the recoverd.edits and the current edit seq should be same",
+ lastestSeqNumber, editCount);
+ }
+
+ /**
+ * testcase for https://issues.apache.org/jira/browse/HBASE-15252
+ */
+ @Test
+ public void testDatalossWhenInputError() throws IOException, InstantiationException,
+ IllegalAccessException {
+ final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ deleteDir(basedir);
+ final byte[] rowName = tableName.getName();
+ final int countPerFamily = 10;
+ final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
+ HRegion region1 = TEST_UTIL.createLocalHRegion(hri, htd);
+ // todo(fsredo): remove this after phase 1?
+ Path regionDir = ((LegacyPathIdentifier)region1.getRegionStorage().getRegionContainer()).path;
+ HBaseTestingUtility.closeRegionAndWAL(region1);
+
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region = HRegion.openHRegion(hri, htd, wal, conf);
+ for (HColumnDescriptor hcd : htd.getFamilies()) {
+ addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
+ }
+ // Now assert edits made it in.
+ final Get g = new Get(rowName);
+ Result result = region.get(g);
+ assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
+ // Now close the region (without flush), split the log, reopen the region and assert that
+ // replay of log has the correct effect.
+ region.close(true);
+ wal.shutdown();
+
+ runWALSplit(this.conf);
+
+ // here we let the DFSInputStream throw an IOException just after the WALHeader.
+ Path editFile = WALSplitter.getSplitEditFilesSorted(this.fs, regionDir).first();
+ FSDataInputStream stream = fs.open(editFile);
+ stream.seek(ProtobufLogReader.PB_WAL_MAGIC.length);
+ Class<? extends AbstractFSWALProvider.Reader> logReaderClass =
+ conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
+ AbstractFSWALProvider.Reader.class);
+ AbstractFSWALProvider.Reader reader = logReaderClass.newInstance();
+ reader.init(this.fs, editFile, conf, stream);
+ final long headerLength = stream.getPos();
+ reader.close();
+ FileSystem spyFs = spy(this.fs);
+ doAnswer(new Answer<FSDataInputStream>() {
+
+ @Override
+ public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
+ FSDataInputStream stream = (FSDataInputStream) invocation.callRealMethod();
+ Field field = FilterInputStream.class.getDeclaredField("in");
+ field.setAccessible(true);
+ final DFSInputStream in = (DFSInputStream) field.get(stream);
+ DFSInputStream spyIn = spy(in);
+ doAnswer(new Answer<Integer>() {
+
+ private long pos;
+
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ if (pos >= headerLength) {
+ throw new IOException("read over limit");
+ }
+ int b = (Integer) invocation.callRealMethod();
+ if (b > 0) {
+ pos += b;
+ }
+ return b;
+ }
+ }).when(spyIn).read(any(byte[].class), any(int.class), any(int.class));
+ doAnswer(new Answer<Void>() {
+
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ invocation.callRealMethod();
+ in.close();
+ return null;
+ }
+ }).when(spyIn).close();
+ field.set(stream, spyIn);
+ return stream;
+ }
+ }).when(spyFs).open(eq(editFile));
+
+ WAL wal2 = createWAL(this.conf, hbaseRootDir, logName);
+ HRegion region2;
+ try {
+ // log replay should fail due to the IOException, otherwise we may lose data.
+ region2 = HRegion.openHRegion(hri, htd, wal2, conf);
+ assertEquals(result.size(), region2.get(g).size());
+ } catch (IOException e) {
+ assertEquals("read over limit", e.getMessage());
+ }
+ region2 = HRegion.openHRegion(hri, htd, wal2, conf);
+ assertEquals(result.size(), region2.get(g).size());
+ }
+
+ /**
+ * testcase for https://issues.apache.org/jira/browse/HBASE-14949.
+ */
+ private void testNameConflictWhenSplit(boolean largeFirst) throws IOException {
+ final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
+ final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ deleteDir(basedir);
+
+ final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
+ NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
+ for (byte[] fam : htd.getFamiliesKeys()) {
+ scopes.put(fam, 0);
+ }
+ Region region = TEST_UTIL.createLocalHRegion(hri, htd);
+ HBaseTestingUtility.closeRegionAndWAL(region);
+ final byte[] family = htd.getColumnFamilies()[0].getName();
+ final byte[] rowName = tableName.getName();
+ FSWALEntry entry1 = createFSWALEntry(htd, hri, 1L, rowName, family, ee, mvcc, 1, scopes);
+ FSWALEntry entry2 = createFSWALEntry(htd, hri, 2L, rowName, family, ee, mvcc, 2, scopes);
+
+ Path largeFile = new Path(logDir, "wal-1");
+ Path smallFile = new Path(logDir, "wal-2");
+ writerWALFile(largeFile, Arrays.asList(entry1, entry2));
+ writerWALFile(smallFile, Arrays.asList(entry2));
+ FileStatus first, second;
+ if (largeFirst) {
+ first = fs.getFileStatus(largeFile);
+ second = fs.getFileStatus(smallFile);
+ } else {
+ first = fs.getFileStatus(smallFile);
+ second = fs.getFileStatus(largeFile);
+ }
+ WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null,
+ RecoveryMode.LOG_SPLITTING, wals);
+ WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null,
+ RecoveryMode.LOG_SPLITTING, wals);
+ WAL wal = createWAL(this.conf, hbaseRootDir, logName);
+ region = HRegion.openHRegion(hri, htd, wal, conf);
+ assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
+ assertEquals(2, region.get(new Get(rowName)).size());
+ }
@Test
public void testNameConflictWhenSplit0() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
index 661af14..bda24a5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
@@ -63,9 +64,9 @@ public class TestMergeTable {
* Hand-makes regions of a mergeable size and adds the hand-made regions to
* hand-made meta. The hand-made regions are created offline. We then start
* up mini cluster, disables the hand-made table and starts in on merging.
- * @throws Exception
*/
- @Test (timeout=300000) public void testMergeTable() throws Exception {
+ @Test (timeout=300000)
+ public void testMergeTable() throws Exception {
// Table we are manually creating offline.
HTableDescriptor desc = new HTableDescriptor(org.apache.hadoop.hbase.TableName.valueOf(Bytes.toBytes("test")));
desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
@@ -77,13 +78,7 @@ public class TestMergeTable {
// Startup hdfs. Its in here we'll be putting our manually made regions.
UTIL.startMiniDFSCluster(1);
// Create hdfs hbase rootdir.
- Path rootdir = UTIL.createRootDir();
- FileSystem fs = FileSystem.get(UTIL.getConfiguration());
- if (fs.exists(rootdir)) {
- if (fs.delete(rootdir, true)) {
- LOG.info("Cleaned up existing " + rootdir);
- }
- }
+ UTIL.createRootDir();
// Now create three data regions: The first is too large to merge since it
// will be > 64 MB in size. The second two will be smaller and will be
@@ -96,16 +91,17 @@ public class TestMergeTable {
// Create regions and populate them at same time. Create the tabledir
// for them first.
- new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir).createTableDescriptor(desc);
+ MasterStorage masterStorage = MasterStorage.open(UTIL.getConfiguration(), true);
+ masterStorage.createTableDescriptor(desc, false);
HRegion [] regions = {
- createRegion(desc, null, row_70001, 1, 70000, rootdir),
- createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir),
- createRegion(desc, row_80001, null, 80001, 11000, rootdir)
+ createRegion(desc, null, row_70001, 1, 70000),
+ createRegion(desc, row_70001, row_80001, 70001, 10000),
+ createRegion(desc, row_80001, null, 80001, 11000)
};
// Now create the root and meta regions and insert the data regions
// created above into hbase:meta
- setupMeta(rootdir, regions);
+ setupMeta(regions);
try {
LOG.info("Starting mini zk cluster");
UTIL.startMiniZKCluster();
@@ -137,11 +133,10 @@ public class TestMergeTable {
}
private HRegion createRegion(final HTableDescriptor desc,
- byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)
+ byte [] startKey, byte [] endKey, int firstRow, int nrows)
throws IOException {
HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, rootdir, UTIL.getConfiguration(),
- desc);
+ HRegion region = UTIL.createLocalHRegion(hri, desc);
LOG.info("Created region " + region.getRegionInfo().getRegionNameAsString());
for(int i = firstRow; i < firstRow + nrows; i++) {
Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i)));
@@ -157,16 +152,13 @@ public class TestMergeTable {
return region;
}
- protected void setupMeta(Path rootdir, final HRegion [] regions)
- throws IOException {
- HRegion meta =
- HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, rootdir,
- UTIL.getConfiguration(), UTIL.getMetaTableDescriptor());
- for (HRegion r: regions) {
+ protected void setupMeta(final HRegion[] regions) throws IOException {
+ HRegion meta = UTIL.createLocalHRegion(HRegionInfo.FIRST_META_REGIONINFO,
+ UTIL.getMetaTableDescriptor());
+ for (HRegion r : regions) {
HRegion.addRegionToMETA(meta, r);
}
HBaseTestingUtility.closeRegionAndWAL(meta);
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index 1924c9e..87f73b4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -149,14 +150,13 @@ public class TestMergeTool extends HBaseTestCase {
try {
// Create meta region
createMetaRegion();
- new FSTableDescriptors(this.conf, this.fs, testDir).createTableDescriptor(this.desc);
+ MasterStorage masterStorage = MasterStorage.open(conf, true);
+ masterStorage.createTableDescriptor(desc, false);
/*
* Create the regions we will merge
*/
for (int i = 0; i < sourceRegions.length; i++) {
- regions[i] =
- HBaseTestingUtility.createRegionAndWAL(this.sourceRegions[i], testDir, this.conf,
- this.desc);
+ regions[i] = TEST_UTIL.createLocalHRegion(sourceRegions[i], desc);
/*
* Insert data
*/
[2/3] hbase git commit: HBASE-16912 Cleanup for HBaseTestingUtility.
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index acd1d72..3590624 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -35,8 +35,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
index edd7847..163007a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
@@ -27,7 +27,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -55,18 +54,17 @@ import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
@Category({RegionServerTests.class, MediumTests.class})
-public class TestBlocksRead {
+public class TestBlocksRead {
private static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
@Rule public TestName testName = new TestName();
- static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL,
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static BlockCache blockCache;
+
+ private static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL,
BloomType.ROW, BloomType.NONE };
- private static BlockCache blockCache;
Region region = null;
- private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString();
- private Configuration conf = TEST_UTIL.getConfiguration();
@BeforeClass
public static void setUp() throws Exception {
@@ -80,16 +78,10 @@ public class TestBlocksRead {
}
/**
- * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)}
- * @param tableName
- * @param callingMethod
- * @param conf
- * @param family
- * @throws IOException
+ * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(Region)}
* @return created and initialized region.
*/
- private Region initHRegion(byte[] tableName, String callingMethod,
- Configuration conf, String family) throws IOException {
+ private Region initHRegion(byte[] tableName, String family) throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor familyDesc;
for (int i = 0; i < BLOOM_TYPE.length; i++) {
@@ -101,9 +93,8 @@ public class TestBlocksRead {
}
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- Path path = new Path(DIR + callingMethod);
- Region r = HBaseTestingUtility.createRegionAndWAL(info, path, conf, htd);
- blockCache = new CacheConfig(conf).getBlockCache();
+ Region r = TEST_UTIL.createLocalHRegion(info, htd);
+ blockCache = new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache();
return r;
}
@@ -213,7 +204,7 @@ public class TestBlocksRead {
byte[] TABLE = Bytes.toBytes("testBlocksRead");
String FAMILY = "cf1";
Cell kvs[];
- this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, FAMILY);
try {
putData(FAMILY, "row", "col1", 1);
@@ -269,7 +260,7 @@ public class TestBlocksRead {
byte[] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
String FAMILY = "cf1";
Cell kvs[];
- this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, FAMILY);
try {
// File 1
@@ -376,7 +367,7 @@ public class TestBlocksRead {
byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
String FAMILY = "cf1";
- this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, FAMILY);
try {
putData(FAMILY, "row", "col1", 1);
@@ -420,7 +411,7 @@ public class TestBlocksRead {
byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
String FAMILY = "cf1";
Cell kvs[];
- this.region = initHRegion(TABLE, testName.getMethodName(), conf, FAMILY);
+ this.region = initHRegion(TABLE, FAMILY);
try {
deleteFamily(FAMILY, "row", 200);
for (int i = 0; i < 100; i++) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 93f6dcb..3b99224 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -32,16 +32,12 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -59,7 +55,6 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
index e9ff8ec..20c2779 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
@@ -262,7 +262,7 @@ public class TestFailedAppendAndSync {
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
public static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, WAL wal)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
index 7ee3f0b..67a2eeb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
@@ -24,9 +24,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -43,7 +40,6 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.wal.WAL;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -72,17 +68,14 @@ public class TestGetClosestAtOrBefore {
private static final byte[] T40 = Bytes.toBytes("040");
private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static Configuration conf = UTIL.getConfiguration();
@Test
public void testUsingMetaAndBinary() throws IOException {
- FileSystem filesystem = FileSystem.get(conf);
- Path rootdir = UTIL.getDataTestDirOnTestFS();
// Up flush size else we bind up when we use default catalog flush of 16k.
UTIL.getMetaTableDescriptor().setMemStoreFlushSize(64 * 1024 * 1024);
- Region mr = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO,
- rootdir, this.conf, UTIL.getMetaTableDescriptor());
+ Region mr = UTIL.createLocalHRegion(HRegionInfo.FIRST_META_REGIONINFO,
+ UTIL.getMetaTableDescriptor());
try {
// Write rows for three tables 'A', 'B', and 'C'.
for (char c = 'A'; c < 'D'; c++) {
@@ -144,7 +137,7 @@ public class TestGetClosestAtOrBefore {
findRow(mr, 'C', 46, -1);
findRow(mr, 'C', 43, -1);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(mr);
+ UTIL.destroyRegion(mr);
}
}
@@ -162,9 +155,7 @@ public class TestGetClosestAtOrBefore {
TableName tableb = TableName.valueOf("" + table);
// Find the row.
byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
- byte [] metaKey = HRegionInfo.createRegionName(
- tableb, tofindBytes,
- HConstants.NINES, false);
+ byte [] metaKey = HRegionInfo.createRegionName( tableb, tofindBytes, HConstants.NINES, false);
LOG.info("find=" + new String(metaKey));
Result r = UTIL.getClosestRowBefore(mr, metaKey, HConstants.CATALOG_FAMILY);
if (answer == -1) {
@@ -291,9 +282,7 @@ public class TestGetClosestAtOrBefore {
} finally {
if (region != null) {
try {
- WAL wal = ((HRegion)region).getWAL();
- ((HRegion)region).close();
- wal.close();
+ UTIL.destroyRegion(region);
} catch (Exception e) {
e.printStackTrace();
}
@@ -349,9 +338,7 @@ public class TestGetClosestAtOrBefore {
} finally {
if (region != null) {
try {
- WAL wal = ((HRegion)region).getWAL();
- ((HRegion)region).close();
- wal.close();
+ UTIL.destroyRegion(region);
} catch (Exception e) {
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index c9b4217..cbb79ae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.hbase.regionserver;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.security.PrivilegedExceptionAction;
@@ -31,7 +29,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
@@ -49,8 +46,6 @@ import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
@@ -102,17 +97,10 @@ import org.apache.hadoop.hbase.filter.NullComparator;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.ServerProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
-import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
@@ -120,12 +108,10 @@ import org.apache.hadoop.hbase.regionserver.Region.RowLock;
import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -139,14 +125,11 @@ import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.FaultyFSLog;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKey;
-import org.apache.hadoop.hbase.wal.WALProvider;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
-import org.apache.hadoop.hbase.wal.WALSplitter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -252,7 +235,6 @@ public class TestHRegion {
/**
* Test that I can use the max flushed sequence id after the close.
- * @throws IOException
*/
@Test
public void testSequenceId() throws IOException {
@@ -287,7 +269,6 @@ public class TestHRegion {
* case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
* in current memstore. The fix is removing all conditions except abort check so we ensure 2
* flushes for region close."
- * @throws IOException
*/
@Test
public void testCloseCarryingSnapshot() throws IOException {
@@ -475,7 +456,6 @@ public class TestHRegion {
* much smaller than expected. In extreme case, if the error accumulates to even bigger than
* HRegion's memstore size limit, any further flush is skipped because flush does not do anything
* if memstoreSize is not larger than 0."
- * @throws Exception
*/
@Test
public void testFlushSizeAccounting() throws Exception {
@@ -4251,7 +4231,7 @@ public class TestHRegion {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = TEST_UTIL.createLocalHRegion(info, htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
int num_unique_rows = 10;
int duplicate_multiplier = 2;
@@ -4293,8 +4273,7 @@ public class TestHRegion {
assertEquals(num_unique_rows, reader.getFilterEntries());
}
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.region);
- this.region = null;
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -4309,7 +4288,7 @@ public class TestHRegion {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = TEST_UTIL.createLocalHRegion(info, htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
// For row:0, col:0: insert versions 1 through 5.
byte row[] = Bytes.toBytes("row:" + 0);
@@ -4336,8 +4315,7 @@ public class TestHRegion {
checkOneCell(kvs[2], FAMILY, 0, 0, 2);
checkOneCell(kvs[3], FAMILY, 0, 0, 1);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.region);
- this.region = null;
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -4357,7 +4335,7 @@ public class TestHRegion {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = TEST_UTIL.createLocalHRegion(info, htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
// Insert some data
byte row[] = Bytes.toBytes("row1");
@@ -4379,8 +4357,7 @@ public class TestHRegion {
Cell[] keyValues = region.get(get).rawCells();
assertTrue(keyValues.length == 0);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(this.region);
- this.region = null;
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -4993,14 +4970,6 @@ public class TestHRegion {
// }
// }
- static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
- Configuration confForWAL = new Configuration(conf);
- confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
- return new WALFactory(confForWAL,
- Collections.<WALActionsListener>singletonList(new MetricsWAL()),
- "hregion-" + RandomStringUtils.randomNumeric(8));
- }
-
// @Test
// public void testCompactionFromPrimary() throws IOException {
// Path rootDir = new Path(dir + "testRegionReplicaSecondary");
@@ -5203,7 +5172,7 @@ public class TestHRegion {
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
byte[]... families) throws IOException {
@@ -5212,7 +5181,7 @@ public class TestHRegion {
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
boolean isReadOnly, byte[]... families) throws IOException {
@@ -5231,7 +5200,7 @@ public class TestHRegion {
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
@@ -6042,9 +6011,7 @@ public class TestHRegion {
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
// open the region w/o rss and wal and flush some files
- HRegion region =
- HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
- .getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(hri, htd);
assertNotNull(region);
// create a file in fam1 for the region before opening in OpenRegionHandler
@@ -6094,7 +6061,7 @@ public class TestHRegion {
assertEquals(0, store.getStoreFileCount()); // no store files
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -6112,21 +6079,23 @@ public class TestHRegion {
@Test
public void testFlushedFileWithNoTags() throws Exception {
- TableName tableName = TableName.valueOf(getClass().getSimpleName());
- HTableDescriptor htd = new HTableDescriptor(tableName);
- htd.addFamily(new HColumnDescriptor(fam1));
- HRegionInfo info = new HRegionInfo(tableName, null, null, false);
- Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName());
- region = HBaseTestingUtility.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), htd);
- Put put = new Put(Bytes.toBytes("a-b-0-0"));
- put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
- region.put(put);
- region.flush(true);
- Store store = region.getStore(fam1);
- Collection<StoreFile> storefiles = store.getStorefiles();
- for (StoreFile sf : storefiles) {
- assertFalse("Tags should not be present "
- ,sf.getReader().getHFileReader().getFileContext().isIncludesTags());
+ try {
+ TableName tableName = TableName.valueOf(getClass().getSimpleName());
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(fam1));
+ HRegionInfo info = new HRegionInfo(tableName, null, null, false);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
+ Put put = new Put(Bytes.toBytes("a-b-0-0"));
+ put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
+ region.put(put);
+ region.flush(true);
+ Store store = region.getStore(fam1);
+ Collection<StoreFile> storefiles = store.getStorefiles();
+ for (StoreFile sf : storefiles) {
+ assertFalse("Tags should not be present ", sf.getReader().getHFileReader().getFileContext().isIncludesTags());
+ }
+ } finally {
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -6146,9 +6115,7 @@ public class TestHRegion {
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
// open the region w/o rss and wal and flush some files
- HRegion region =
- HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
- .getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(hri, htd);
assertNotNull(region);
// create a file in fam1 for the region before opening in OpenRegionHandler
@@ -6215,15 +6182,13 @@ public class TestHRegion {
assertEquals(0, store.getStoreFileCount()); // no store files
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
/**
* Utility method to setup a WAL mock.
* Needs to do the bit where we close latch on the WALKey on append else test hangs.
- * @return
- * @throws IOException
*/
private WAL mockWAL() throws IOException {
WAL wal = mock(WAL.class);
@@ -6373,9 +6338,9 @@ public class TestHRegion {
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(new HRegionInfo(htd.getTableName(),
+ Region region = TEST_UTIL.createLocalHRegion(new HRegionInfo(htd.getTableName(),
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
- TEST_UTIL.getDataTestDir(), conf, htd);
+ htd, conf);
assertNotNull(region);
try {
long now = EnvironmentEdgeManager.currentTime();
@@ -6452,7 +6417,7 @@ public class TestHRegion {
// Increment with a TTL of 5 seconds
Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
incr.setTTL(5000);
- region.increment(incr); // 2
+ region.increment(incr, HConstants.NO_NONCE, HConstants.NO_NONCE); // 2
// New value should be 2
r = region.get(new Get(row));
@@ -6477,7 +6442,7 @@ public class TestHRegion {
assertNull(r.getValue(fam1, q1));
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index de561d5..1370143 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -41,11 +41,13 @@ import com.google.protobuf.ByteString;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -80,6 +82,8 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -119,14 +123,12 @@ public class TestHRegionReplayEvents {
Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")};
// Test names
- protected byte[] tableName;
- protected String method;
+ protected TableName tableName;
protected final byte[] row = Bytes.toBytes("rowA");
protected final byte[] row2 = Bytes.toBytes("rowB");
protected byte[] cq = Bytes.toBytes("cq");
// per test fields
- private Path rootDir;
private HTableDescriptor htd;
private long time;
private RegionServerServices rss;
@@ -140,14 +142,10 @@ public class TestHRegionReplayEvents {
public void setup() throws IOException {
TEST_UTIL = HBaseTestingUtility.createLocalHTU();
CONF = TEST_UTIL.getConfiguration();
- dir = TEST_UTIL.getDataTestDir("TestHRegionReplayEvents").toString();
- method = name.getMethodName();
- tableName = Bytes.toBytes(name.getMethodName());
- rootDir = new Path(dir + method);
- TEST_UTIL.getConfiguration().set(HConstants.HBASE_DIR, rootDir.toString());
- method = name.getMethodName();
-
- htd = new HTableDescriptor(TableName.valueOf(method));
+ TEST_UTIL.createRootDir();
+ tableName = TableName.valueOf(name.getMethodName());
+
+ htd = new HTableDescriptor(tableName);
for (byte[] family : families) {
htd.addFamily(new HColumnDescriptor(family));
}
@@ -161,7 +159,9 @@ public class TestHRegionReplayEvents {
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
false, time, 1);
- wals = TestHRegion.createWALFactory(CONF, rootDir);
+ wals = new WALFactory(CONF,
+ Collections.singletonList(new MetricsWAL()),
+ "hregion-" + RandomStringUtils.randomNumeric(8));
walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes(),
primaryHri.getTable().getNamespace());
walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes(),
@@ -178,13 +178,13 @@ public class TestHRegionReplayEvents {
string+"-"+string, 1);
when(rss.getExecutorService()).thenReturn(es);
-// primaryRegion = HRegion.createHRegion(CONF, rootDir, htd, primaryHri, walPrimary);
+ primaryRegion = HRegion.createHRegion(CONF, htd, primaryHri, walPrimary);
primaryRegion.close();
List<Region> regions = new ArrayList<Region>();
regions.add(primaryRegion);
when(rss.getOnlineRegions()).thenReturn(regions);
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
secondaryRegion = HRegion.openHRegion(secondaryHri, htd, null, CONF, rss, null);
reader = null;
@@ -197,10 +197,10 @@ public class TestHRegionReplayEvents {
}
if (primaryRegion != null) {
- HBaseTestingUtility.closeRegionAndWAL(primaryRegion);
+ TEST_UTIL.destroyRegion(primaryRegion);
}
if (secondaryRegion != null) {
- HBaseTestingUtility.closeRegionAndWAL(secondaryRegion);
+ TEST_UTIL.destroyRegion(secondaryRegion);
}
EnvironmentEdgeManagerTestHelper.reset();
@@ -208,10 +208,6 @@ public class TestHRegionReplayEvents {
TEST_UTIL.cleanupTestDir();
}
- String getName() {
- return name.getMethodName();
- }
-
// Some of the test cases are as follows:
// 1. replay flush start marker again
// 2. replay flush with smaller seqId than what is there in memstore snapshot
@@ -824,7 +820,7 @@ public class TestHRegionReplayEvents {
// close the region and open again.
primaryRegion.close();
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
// now replay the edits and the flush marker
reader = createWALReaderForPrimary();
@@ -904,7 +900,7 @@ public class TestHRegionReplayEvents {
// close the region and open again.
primaryRegion.close();
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
// now replay the edits and the flush marker
reader = createWALReaderForPrimary();
@@ -983,7 +979,7 @@ public class TestHRegionReplayEvents {
// close the region and open again.
primaryRegion.close();
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
// now replay the edits and the flush marker
reader = createWALReaderForPrimary();
@@ -1082,11 +1078,8 @@ public class TestHRegionReplayEvents {
public void testSeqIdsFromReplay() throws IOException {
// test the case where seqId's coming from replayed WALEdits are made persisted with their
// original seqIds and they are made visible through mvcc read point upon replay
- String method = name.getMethodName();
- byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
-
- HRegion region = initHRegion(tableName, method, family);
+ HRegion region = initHRegion(tableName, family);
try {
// replay an entry that is bigger than current read point
long readPoint = region.getMVCC().getReadPoint();
@@ -1327,7 +1320,7 @@ public class TestHRegionReplayEvents {
disableReads(secondaryRegion);
primaryRegion.close();
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
reader = createWALReaderForPrimary();
while (true) {
@@ -1349,6 +1342,7 @@ public class TestHRegionReplayEvents {
}
@Test
+ // TODO(fsredo): fix this test.
public void testRefreshStoreFiles() throws IOException {
assertEquals(0, primaryRegion.getStoreFileList(families).size());
assertEquals(0, secondaryRegion.getStoreFileList(families).size());
@@ -1364,7 +1358,7 @@ public class TestHRegionReplayEvents {
// refresh the store file list, and ensure that the files are picked up.
secondaryRegion.refreshStoreFiles();
assertPathListsEqual(primaryRegion.getStoreFileList(families),
- secondaryRegion.getStoreFileList(families));
+ secondaryRegion.getStoreFileList(families));
assertEquals(families.length, secondaryRegion.getStoreFileList(families).size());
LOG.info("-- Verifying edits from secondary");
@@ -1477,7 +1471,7 @@ public class TestHRegionReplayEvents {
// close the region and open again.
primaryRegion.close();
-// primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
+ primaryRegion = HRegion.openHRegion(primaryHri, htd, walPrimary, CONF, rss, null);
// bulk load a file into primary region
Random random = new Random();
@@ -1660,16 +1654,9 @@ public class TestHRegionReplayEvents {
}
}
- private static HRegion initHRegion(byte[] tableName,
- String callingMethod, byte[]... families) throws IOException {
- return initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
- callingMethod, TEST_UTIL.getConfiguration(), false, Durability.SYNC_WAL, null, families);
- }
-
- private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
- String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
- WAL wal, byte[]... families) throws IOException {
- return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
- isReadOnly, durability, wal, families);
+ private static HRegion initHRegion(TableName tableName, byte[]... families) throws IOException {
+ return TEST_UTIL.createLocalHRegion(tableName, HConstants.EMPTY_START_ROW,
+ HConstants.EMPTY_END_ROW, false, Durability.SYNC_WAL, null,
+ families);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
index d66899b..d705f3b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
@@ -62,7 +62,7 @@ public class TestHRegionWithInMemoryFlush extends TestHRegion{
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
@Override
public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
@@ -78,8 +78,6 @@ public class TestHRegionWithInMemoryFlush extends TestHRegion{
/**
* Splits twice and verifies getting from each of the split regions.
- *
- * @throws Exception
*/
@Override
public void testBasicSplit() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 3ef89ad..5a96c22 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -110,16 +110,13 @@ public class TestMajorCompaction {
@After
public void tearDown() throws Exception {
- WAL wal = ((HRegion)r).getWAL();
- ((HRegion)r).close();
- wal.close();
+ UTIL.destroyRegion(r);
}
/**
* Test that on a major compaction, if all cells are expired or deleted, then
* we'll end up with no product. Make sure scanner over region returns
* right answer in this case - and that it just basically works.
- * @throws IOException
*/
@Test
public void testMajorCompactingToNoOutput() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
index 1bd20c6..d6084c7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
@@ -90,9 +90,7 @@ public class TestMinorCompaction {
@After
public void tearDown() throws Exception {
- WAL wal = ((HRegion)r).getWAL();
- ((HRegion)r).close();
- wal.close();
+ UTIL.destroyRegion(r);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
index 6bfaa59..ce759fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
@@ -21,7 +21,6 @@ import com.google.common.hash.Hashing;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -69,8 +68,6 @@ public class TestPerColumnFamilyFlush {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static final Path DIR = TEST_UTIL.getDataTestDir("TestHRegion");
-
public static final TableName TABLENAME = TableName.valueOf("TestPerColumnFamilyFlush", "t1");
public static final byte[][] FAMILIES = { Bytes.toBytes("f1"), Bytes.toBytes("f2"),
@@ -82,14 +79,13 @@ public class TestPerColumnFamilyFlush {
public static final byte[] FAMILY3 = FAMILIES[2];
- private HRegion initHRegion(String callingMethod, Configuration conf) throws IOException {
+ private HRegion initHRegion(Configuration conf) throws IOException {
HTableDescriptor htd = new HTableDescriptor(TABLENAME);
for (byte[] family : FAMILIES) {
htd.addFamily(new HColumnDescriptor(family));
}
HRegionInfo info = new HRegionInfo(TABLENAME, null, null, false);
- Path path = new Path(DIR, callingMethod);
- return HBaseTestingUtility.createRegionAndWAL(info, path, conf, htd);
+ return TEST_UTIL.createLocalHRegion(info, htd, conf);
}
// A helper function to create puts.
@@ -130,7 +126,7 @@ public class TestPerColumnFamilyFlush {
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
100 * 1024);
// Intialize the region
- Region region = initHRegion("testSelectiveFlushWhenEnabled", conf);
+ Region region = initHRegion(conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
region.put(createPut(1, i));
@@ -273,7 +269,7 @@ public class TestPerColumnFamilyFlush {
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushAllStoresPolicy.class.getName());
// Intialize the HRegion
- HRegion region = initHRegion("testSelectiveFlushWhenNotEnabled", conf);
+ HRegion region = initHRegion(conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
region.put(createPut(1, i));
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
index 1583bf8..c04a564 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Scan;
@@ -83,9 +84,8 @@ public class TestRegionIncrement {
private HRegion getRegion(final Configuration conf, final String tableName) throws IOException {
WAL wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
TEST_UTIL.getDataTestDir().toString(), conf);
- return (HRegion)TEST_UTIL.createLocalHRegion(Bytes.toBytes(tableName),
- HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, tableName, conf,
- false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
+ return TEST_UTIL.createLocalHRegion(TableName.valueOf(tableName), HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.EMPTY_BYTE_ARRAY, false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
}
private void closeRegion(final HRegion region) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
index 57d9365..1c77b79 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.RegionStorage;
import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -81,38 +82,28 @@ public class TestRegionMergeTransaction {
@Before
public void setup() throws IOException {
- this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
- this.fs.delete(this.testdir, true);
+ fs = FileSystem.get(TEST_UTIL.getConfiguration());
final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
FSUtils.setRootDir(walConf, this.testdir);
this.wals = new WALFactory(walConf, null, TestRegionMergeTransaction.class.getName());
- this.region_a = createRegion(this.testdir, this.wals, STARTROW_A, STARTROW_B);
- this.region_b = createRegion(this.testdir, this.wals, STARTROW_B, STARTROW_C);
- this.region_c = createRegion(this.testdir, this.wals, STARTROW_C, ENDROW);
+ this.region_a = createRegion(STARTROW_A, STARTROW_B);
+ this.region_b = createRegion(STARTROW_B, STARTROW_C);
+ this.region_c = createRegion(STARTROW_C, ENDROW);
assert region_a != null && region_b != null && region_c != null;
TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
}
@After
public void teardown() throws IOException {
-// for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
-// if (region != null && !region.isClosed()) region.close();
-// if (this.fs.exists(region.getRegionStorage().getRegionDir())
-// && !this.fs.delete(region.getRegionStorage().getRegionDir(), true)) {
-// throw new IOException("Failed deleting of "
-// + region.getRegionStorage().getRegionDir());
-// }
-// }
-// if (this.wals != null) {
-// this.wals.close();
-// }
-// this.fs.delete(this.testdir, true);
+ TEST_UTIL.destroyRegion(region_a);
+ TEST_UTIL.destroyRegion(region_b);
+ TEST_UTIL.destroyRegion(region_c);
+ TEST_UTIL.cleanupTestDir();
}
/**
* Test straight prepare works. Tries to merge on {@link #region_a} and
* {@link #region_b}
- * @throws IOException
*/
@Test
public void testPrepare() throws IOException {
@@ -258,7 +249,7 @@ public class TestRegionMergeTransaction {
Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration(), cp);
HRegion mergedRegion = (HRegion)mt.execute(mockServer, null);
// Do some assertions about execution.
- // TODO move tests to rely on RegionStorage
+ // TODO(fsredo) move tests to rely on RegionStorage
final Path mergesdir = ((LegacyPathIdentifier)mt.getMergesDir()).path;
assertTrue(this.fs.exists(mergesdir));
// Assert region_a and region_b is closed.
@@ -379,11 +370,8 @@ public class TestRegionMergeTransaction {
// Make sure that merged region is still in the filesystem, that
// they have not been removed; this is supposed to be the case if we go
// past point of no return.
-// Path tableDir = this.region_a.getRegionStorage().getRegionDir()
-// .getParent();
-// Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
-// .getEncodedName());
-// assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
+ assertTrue(RegionStorage.open(TEST_UTIL.getConfiguration(), mt.getMergedRegionInfo(), false)
+ .exists());
}
@Test
@@ -435,21 +423,17 @@ public class TestRegionMergeTransaction {
private class MockedFailedMergedRegionOpen extends IOException {
}
- private HRegion createRegion(final Path testdir, final WALFactory wals,
- final byte[] startrow, final byte[] endrow)
- throws IOException {
+ private HRegion createRegion(final byte[] startrow, final byte[] endrow) throws IOException {
// Make a region with start and end keys.
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
HColumnDescriptor hcd = new HColumnDescriptor(CF);
htd.addFamily(hcd);
HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
- HRegion a = HBaseTestingUtility.createRegionAndWAL(hri, testdir,
- TEST_UTIL.getConfiguration(), htd);
+ Region a = TEST_UTIL.createLocalHRegion(hri, htd);
HBaseTestingUtility.closeRegionAndWAL(a);
-// return HRegion.openHRegion(testdir, hri, htd,
-// wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
-// TEST_UTIL.getConfiguration());
- return null;
+ return HRegion.openHRegion(hri, htd,
+ wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
+ TEST_UTIL.getConfiguration());
}
private int countRows(final HRegion r) throws IOException {
@@ -472,11 +456,8 @@ public class TestRegionMergeTransaction {
/**
* Load region with rows from 'aaa' to 'zzz', skip the rows which are out of
* range of the region
- * @param r Region
- * @param f Family
* @param flush flush the cache if true
* @return Count of rows loaded.
- * @throws IOException
*/
private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
index 3e02243..fa8f681 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
@@ -41,7 +41,6 @@ public class TestResettingCounters {
@Test
public void testResettingCounters() throws Exception {
-
HBaseTestingUtility htu = new HBaseTestingUtility();
Configuration conf = htu.getConfiguration();
FileSystem fs = FileSystem.get(conf);
@@ -62,14 +61,7 @@ public class TestResettingCounters {
for (byte [] family : families) htd.addFamily(new HColumnDescriptor(family));
HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false);
- String testDir = htu.getDataTestDir() + "/TestResettingCounters/";
- Path path = new Path(testDir);
- if (fs.exists(path)) {
- if (!fs.delete(path, true)) {
- throw new IOException("Failed delete of " + path);
- }
- }
- Region region = HBaseTestingUtility.createRegionAndWAL(hri, path, conf, htd);
+ Region region = htu.createLocalHRegion(hri, htd);
try {
Increment odd = new Increment(rows[0]);
odd.setDurability(Durability.SKIP_WAL);
@@ -100,9 +92,8 @@ public class TestResettingCounters {
assertEquals(6, Bytes.toLong(CellUtil.cloneValue(kvs[i])));
}
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ htu.destroyRegion(region);
}
- HBaseTestingUtility.closeRegionAndWAL(region);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java
index 4d3a1c3..a4470b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
@@ -40,22 +39,20 @@ import java.io.IOException;
*/
@Category({RegionServerTests.class, MediumTests.class})
public class TestRowTooBig {
- private final static HBaseTestingUtility HTU = HBaseTestingUtility.createLocalHTU();
- private static Path rootRegionDir;
+ private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static final HTableDescriptor TEST_HTD =
new HTableDescriptor(TableName.valueOf(TestRowTooBig.class.getSimpleName()));
@BeforeClass
public static void before() throws Exception {
- HTU.startMiniCluster();
- HTU.getConfiguration().setLong(HConstants.TABLE_MAX_ROWSIZE_KEY,
+ TEST_UTIL.startMiniCluster();
+ TEST_UTIL.getConfiguration().setLong(HConstants.TABLE_MAX_ROWSIZE_KEY,
10 * 1024 * 1024L);
- rootRegionDir = HTU.getDataTestDirOnTestFS("TestRowTooBig");
}
@AfterClass
public static void after() throws Exception {
- HTU.shutdownMiniCluster();
+ TEST_UTIL.shutdownMiniCluster();
}
/**
@@ -67,7 +64,6 @@ public class TestRowTooBig {
* OOME happened before we actually get to reading results, but
* during seeking, as each StoreFile gets it's own scanner,
* and each scanner seeks after the first KV.
- * @throws IOException
*/
@Test(expected = RowTooBigException.class)
public void testScannersSeekOnFewLargeCells() throws IOException {
@@ -85,8 +81,7 @@ public class TestRowTooBig {
final HRegionInfo hri =
new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW,
HConstants.EMPTY_END_ROW);
- Region region =
- HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, HTU.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(hri, htd);
try {
// Add 5 cells to memstore
for (int i = 0; i < 5 ; i++) {
@@ -101,7 +96,7 @@ public class TestRowTooBig {
Get get = new Get(row1);
region.get(get);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
@@ -113,8 +108,6 @@ public class TestRowTooBig {
* - try to Get whole row.
*
* OOME happened in StoreScanner.next(..).
- *
- * @throws IOException
*/
@Test(expected = RowTooBigException.class)
public void testScanAcrossManySmallColumns() throws IOException {
@@ -132,8 +125,7 @@ public class TestRowTooBig {
final HRegionInfo hri =
new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW,
HConstants.EMPTY_END_ROW);
- Region region =
- HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, HTU.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(hri, htd);
try {
// Add to memstore
for (int i = 0; i < 10; i++) {
@@ -150,7 +142,7 @@ public class TestRowTooBig {
Get get = new Get(row1);
region.get(get);
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
index 1b42754..4daa43b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
@@ -104,7 +104,7 @@ public class TestScanner {
private static final long START_CODE = Long.MAX_VALUE;
- private HRegion region;
+ private Region region;
private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
final private byte[] col1;
@@ -123,7 +123,6 @@ public class TestScanner {
/**
* Test basic stop row filter works.
- * @throws Exception
*/
@Test
public void testStopRow() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index e9bb468..bebdf1a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -289,7 +289,7 @@ public class TestWALLockup {
/**
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
public static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, WAL wal)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index 74826b0..c85c539 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -63,7 +63,7 @@ public class TestWalAndCompactingMemStoreFlush {
public static final byte[] FAMILY2 = FAMILIES[1];
public static final byte[] FAMILY3 = FAMILIES[2];
- private HRegion initHRegion(String callingMethod, Configuration conf) throws IOException {
+ private HRegion initHRegion(Configuration conf) throws IOException {
int i=0;
HTableDescriptor htd = new HTableDescriptor(TABLENAME);
for (byte[] family : FAMILIES) {
@@ -75,8 +75,7 @@ public class TestWalAndCompactingMemStoreFlush {
}
HRegionInfo info = new HRegionInfo(TABLENAME, null, null, false);
- Path path = new Path(DIR, callingMethod);
- return HBaseTestingUtility.createRegionAndWAL(info, path, conf, htd);
+ return TEST_UTIL.createLocalHRegion(info, htd, conf);
}
// A helper function to create puts.
@@ -132,7 +131,7 @@ public class TestWalAndCompactingMemStoreFlush {
conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.25);
// Intialize the region
- Region region = initHRegion("testSelectiveFlushWhenEnabled", conf);
+ Region region = initHRegion(conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
@@ -385,7 +384,7 @@ public class TestWalAndCompactingMemStoreFlush {
conf.setInt("hbase.hregion.compacting.memstore.type",1);
// Intialize the region
- Region region = initHRegion("testSelectiveFlushWhenEnabled", conf);
+ Region region = initHRegion(conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
@@ -629,7 +628,7 @@ public class TestWalAndCompactingMemStoreFlush {
conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.5);
// Intialize the HRegion
- HRegion region = initHRegion("testSelectiveFlushWhenNotEnabled", conf);
+ HRegion region = initHRegion(conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
region.put(createPut(1, i));
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
index 3420635..985f884 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -164,8 +165,6 @@ public abstract class AbstractTestFSWAL {
/**
* helper method to simulate region flush for a WAL.
- * @param wal
- * @param regionEncodedName
*/
protected void flushRegion(WAL wal, byte[] regionEncodedName, Set<byte[]> flushedFamilyNames) {
wal.startCacheFlush(regionEncodedName, flushedFamilyNames);
@@ -175,7 +174,6 @@ public abstract class AbstractTestFSWAL {
/**
* tests the log comparator. Ensure that we are not mixing meta logs with non-meta logs (throws
* exception if we do). Comparison is based on the timestamp present in the wal name.
- * @throws Exception
*/
@Test
public void testWALComparator() throws Exception {
@@ -343,7 +341,6 @@ public abstract class AbstractTestFSWAL {
* slowing appends in the background ring buffer thread while in foreground we call flush. The
* addition of the sync over HRegion in flush should fix an issue where flush was returning before
* all of its appends had made it out to the WAL (HBASE-11109).
- * @throws IOException
* @see <a href="https://issues.apache.org/jira/browse/HBASE-11109">HBASE-11109</a>
*/
@Test
@@ -354,8 +351,7 @@ public abstract class AbstractTestFSWAL {
final byte[] rowName = tableName.getName();
final HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor("f"));
- HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(),
- TEST_UTIL.getConfiguration(), htd);
+ Region r = TEST_UTIL.createLocalHRegion(hri, htd);
HBaseTestingUtility.closeRegionAndWAL(r);
final int countPerFamily = 10;
final AtomicBoolean goslow = new AtomicBoolean(false);
@@ -375,9 +371,7 @@ public abstract class AbstractTestFSWAL {
}
}
});
-// HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(),
-// TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
- HRegion region = null;
+ HRegion region = HRegion.openHRegion(hri, htd, wal, TEST_UTIL.getConfiguration());
EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
try {
List<Put> puts = null;
[3/3] hbase git commit: HBASE-16912 Cleanup for HBaseTestingUtility.
Posted by bu...@apache.org.
HBASE-16912 Cleanup for HBaseTestingUtility.
- Moves most uses of createRegionAndWAL() to not use the path directly.
Most testcases don't need to know the exact path of region, but they use it
to ensure isolation among test functions by cleaning up directory in a
@Before function. The solution here is to change the createRegionAndWal()
function to use a random dir on each invocation.
- Of the tests changed, failing ones are TestHRegion and 1 test in
TestHRegionReplayEvents.
- Override getRegionInfoForStorage() in LegacyRegionStorage to fix 11 test
failures in TestHRegionReplayEvents. (1 test still failing).
- Rename variables 'rfs' to 'regionStorage' where they appear.
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d1813a2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d1813a2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d1813a2
Branch: refs/heads/hbase-14439
Commit: 6d1813a2f42d452c22cd3c0dff96d8d1b0cc2618
Parents: a36f80c
Author: Apekshit Sharma <ap...@apache.org>
Authored: Mon Nov 7 13:59:21 2016 -0800
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Nov 29 01:11:33 2016 -0600
----------------------------------------------------------------------
.../apache/hadoop/hbase/fs/RegionStorage.java | 16 +-
.../hbase/fs/legacy/LegacyRegionStorage.java | 9 +-
.../hadoop/hbase/regionserver/HRegion.java | 128 +-
.../org/apache/hadoop/hbase/HBaseTestCase.java | 20 +-
.../hadoop/hbase/HBaseTestingUtility.java | 202 +--
.../hbase/client/TestIntraRowPagination.java | 10 +-
.../coprocessor/TestCoprocessorInterface.java | 16 +-
.../coprocessor/TestRegionObserverStacking.java | 14 +-
.../hbase/filter/TestColumnPrefixFilter.java | 11 +-
.../hbase/filter/TestDependentColumnFilter.java | 12 +-
.../apache/hadoop/hbase/filter/TestFilter.java | 13 +-
.../hbase/filter/TestFilterFromRegionSide.java | 5 +-
.../filter/TestInvocationRecordFilter.java | 15 +-
.../filter/TestMultipleColumnPrefixFilter.java | 17 +-
.../hbase/io/encoding/TestPrefixTree.java | 9 +-
.../encoding/TestSeekBeforeWithReverseScan.java | 9 +-
.../io/hfile/TestScannerFromBucketCache.java | 40 +-
.../TestScannerSelectionUsingKeyRange.java | 3 +-
.../io/hfile/TestScannerSelectionUsingTTL.java | 3 +-
.../hadoop/hbase/master/TestMasterFailover.java | 14 +-
.../hadoop/hbase/mob/TestMobFileCache.java | 7 +-
.../hbase/regionserver/TestAtomicOperation.java | 2 -
.../hbase/regionserver/TestBlocksRead.java | 35 +-
.../hbase/regionserver/TestCompaction.java | 5 -
.../regionserver/TestFailedAppendAndSync.java | 2 +-
.../regionserver/TestGetClosestAtOrBefore.java | 25 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 103 +-
.../regionserver/TestHRegionReplayEvents.java | 69 +-
.../TestHRegionWithInMemoryFlush.java | 4 +-
.../hbase/regionserver/TestMajorCompaction.java | 5 +-
.../hbase/regionserver/TestMinorCompaction.java | 4 +-
.../regionserver/TestPerColumnFamilyFlush.java | 12 +-
.../hbase/regionserver/TestRegionIncrement.java | 6 +-
.../TestRegionMergeTransaction.java | 53 +-
.../regionserver/TestResettingCounters.java | 13 +-
.../hbase/regionserver/TestRowTooBig.java | 24 +-
.../hadoop/hbase/regionserver/TestScanner.java | 3 +-
.../hbase/regionserver/TestWALLockup.java | 2 +-
.../TestWalAndCompactingMemStoreFlush.java | 11 +-
.../regionserver/wal/AbstractTestFSWAL.java | 12 +-
.../regionserver/wal/AbstractTestWALReplay.java | 1520 +++++++++---------
.../hadoop/hbase/util/TestMergeTable.java | 40 +-
.../apache/hadoop/hbase/util/TestMergeTool.java | 8 +-
43 files changed, 1074 insertions(+), 1457 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
index 350e16c..d18902f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.*;
+import org.junit.Before;
@InterfaceAudience.Private
public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
@@ -139,8 +140,8 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
// ==========================================================================
// NOOOOO
// ==========================================================================
- // TODO refactor to checkRegionInfoInStorage or remove
- public abstract void checkRegionInfoOnFilesystem() throws IOException;
+ // TODO refactor to checkRegionInfoInStorage (done) or remove
+ public abstract void checkRegionInfoOnStorage() throws IOException;
/**
* Get an opaque handle to the backing storage associated with this region.
*
@@ -151,7 +152,7 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
public abstract IDENTIFIER getTempContainer();
- public HRegionInfo getRegionInfoForFS() { return hri; }
+ public HRegionInfo getRegionInfoFromStorage() { return hri; }
/**
* If region exists on the Storage
@@ -161,7 +162,8 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
public abstract boolean exists() throws IOException;
/**
- * Retrieve a referene to the backing storage associated with a particular family within this region.
+ * Retrieve a reference to the backing storage associated with a particular family within this
+ * region.
*/
public abstract IDENTIFIER getStoreContainer(final String familyName);
@@ -252,14 +254,14 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
LOG.debug("skipping override of the default FS, since the root container is not a LegacyPathIdentifier.");
}
- RegionStorage rfs = getInstance(conf, fs, rootContainer, regionInfo);
+ RegionStorage regionStorage = getInstance(conf, fs, rootContainer, regionInfo);
if (bootstrap) {
// TODO: are bootstrap and create two different things?
// should switch to bootstrap & read-only
// legacy region wants to recover the .regioninfo :(
- rfs.bootstrap();
+ regionStorage.bootstrap();
}
- return rfs;
+ return regionStorage;
}
public static void destroy(Configuration conf, HRegionInfo regionInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
index 00f279b..8081839 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
@@ -795,10 +795,10 @@ public class LegacyRegionStorage extends RegionStorage<LegacyPathIdentifier> {
cleanupMergesContainer();
// if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
- checkRegionInfoOnFilesystem();
+ checkRegionInfoOnStorage();
}
- public void checkRegionInfoOnFilesystem() throws IOException {
+ public void checkRegionInfoOnStorage() throws IOException {
writeRegionInfoFileContent(getConfiguration(), getFileSystem(),
LegacyLayout.getRegionInfoFile(getRegionContainer().path),
getRegionInfoFileContent(getRegionInfo()));
@@ -808,4 +808,9 @@ public class LegacyRegionStorage extends RegionStorage<LegacyPathIdentifier> {
protected void destroy() throws IOException {
fsWithRetries.deleteDir(regionDir);
}
+
+ @Override
+ public HRegionInfo getRegionInfoFromStorage() {
+ return regionInfoForFs;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index d0a4c94..1c06186 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -290,7 +290,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
private final WAL wal;
- private final RegionStorage fs;
+ private final RegionStorage regionStorage;
protected final Configuration conf;
private final Configuration baseConf;
private final int rowLockWaitDuration;
@@ -601,53 +601,21 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* extensions. Instances of HRegion should be instantiated with the
* {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
*
- * @param tableDir qualified path of directory where region should be located,
- * usually the table directory.
* @param wal The WAL is the outbound log for any updates to the HRegion
* The wal file is a logfile from the previous execution that's
* custom-computed for this HRegion. The HRegionServer computes and sorts the
* appropriate wal info for this HRegion. If there is a previous wal file
* (implying that the HRegion has been written-to before), then read it from
* the supplied path.
- * @param fs is the filesystem.
- * @param confParam is global configuration settings.
- * @param regionInfo - HRegionInfo that describes the region
- * is new), then read them from the supplied path.
* @param htd the table descriptor
* @param rsServices reference to {@link RegionServerServices} or null
- * @deprecated Use other constructors.
*/
- @Deprecated
- @VisibleForTesting
- private HRegion(final Path tableDir, final WAL wal, final FileSystem fs,
- final Configuration confParam, final HRegionInfo regionInfo,
- final HTableDescriptor htd, final RegionServerServices rsServices)
- throws IOException {
- this(RegionStorage.open(confParam, fs, new LegacyPathIdentifier(tableDir), regionInfo, false), htd, wal, rsServices);
- }
-
- /**
- * HRegion constructor. This constructor should only be used for testing and
- * extensions. Instances of HRegion should be instantiated with the
- * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
- *
- * @param fs is the filesystem.
- * @param wal The WAL is the outbound log for any updates to the HRegion
- * The wal file is a logfile from the previous execution that's
- * custom-computed for this HRegion. The HRegionServer computes and sorts the
- * appropriate wal info for this HRegion. If there is a previous wal file
- * (implying that the HRegion has been written-to before), then read it from
- * the supplied path.
- * @param confParam is global configuration settings.
- * @param htd the table descriptor
- * @param rsServices reference to {@link RegionServerServices} or null
- */
- public HRegion(final RegionStorage rfs, final HTableDescriptor htd, final WAL wal,
+ public HRegion(final RegionStorage regionStorage, final HTableDescriptor htd, final WAL wal,
final RegionServerServices rsServices) {
- this(rfs, wal, rfs.getConfiguration(), htd, rsServices);
+ this(regionStorage, wal, regionStorage.getConfiguration(), htd, rsServices);
}
- private HRegion(final RegionStorage fs, final WAL wal, final Configuration confParam,
+ private HRegion(final RegionStorage regionStorage, final WAL wal, final Configuration confParam,
final HTableDescriptor htd, final RegionServerServices rsServices) {
if (htd == null) {
throw new IllegalArgumentException("Need table descriptor");
@@ -658,7 +626,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
this.wal = wal;
- this.fs = fs;
+ this.regionStorage = regionStorage;
// 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
this.baseConf = confParam;
@@ -827,14 +795,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Write HRI to a file in case we need to recover hbase:meta
status.setStatus("Writing region info on filesystem");
- fs.checkRegionInfoOnFilesystem();
+ regionStorage.checkRegionInfoOnStorage();
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
long maxSeqId = initializeStores(reporter, status);
this.mvcc.advanceTo(maxSeqId);
if (ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {
- final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+ final StorageIdentifier regionContainer = this.regionStorage.getRegionContainer();
/*
* TODO either move wal replay stuff to not rely on details from RegionStorage,
* implement a WALStorage abstraction, or make a "Recovered Edits visitor".
@@ -842,13 +810,13 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (regionContainer instanceof LegacyPathIdentifier) {
// Recover any edits if available.
maxSeqId = Math.max(maxSeqId,
- replayRecoveredEditsIfAny(((LegacyPathIdentifier)this.fs.getRegionContainer()).path, maxSeqIdInStores,
+ replayRecoveredEditsIfAny(((LegacyPathIdentifier)this.regionStorage.getRegionContainer()).path, maxSeqIdInStores,
reporter, status));
// Make sure mvcc is up to max.
this.mvcc.advanceTo(maxSeqId);
} else {
- LOG.debug("Skipping check for recovered edits, because RegionStorage implementation '" + this.fs.getClass() +
- "' doesn't return Paths for the region container.");
+ LOG.debug("Skipping check for recovered edits, because RegionStorage implementation '" +
+ this.regionStorage.getClass() + "' doesn't return Paths for the region container.");
}
}
this.lastReplayedOpenRegionSeqId = maxSeqId;
@@ -860,14 +828,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (this.writestate.writesEnabled) {
// Remove temporary data left over from old regions
status.setStatus("Cleaning up temporary data from old regions");
- fs.cleanupTempContainer();
+ regionStorage.cleanupTempContainer();
status.setStatus("Cleaning up detritus from prior splits");
// Get rid of any splits or merges that were lost in-progress. Clean out
// these directories here on open. We may be opening a region that was
// being split but we crashed in the middle of it all.
- fs.cleanupAnySplitDetritus();
- fs.cleanupMergesContainer();
+ regionStorage.cleanupAnySplitDetritus();
+ regionStorage.cleanupMergesContainer();
}
// Initialize split policy
@@ -889,17 +857,17 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// is opened before recovery completes. So we add a safety bumper to avoid new sequence number
// overlaps used sequence numbers
if (this.writestate.writesEnabled) {
- final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+ final StorageIdentifier regionContainer = this.regionStorage.getRegionContainer();
/*
* TODO more WAL replay stuff that needs to get pulled out of the notion of region storage
*/
if (regionContainer instanceof LegacyPathIdentifier) {
- nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(),
+ nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.regionStorage.getFileSystem(),
((LegacyPathIdentifier)regionContainer).path, nextSeqid,
(this.recovering ? (this.flushPerChanges + 10000000) : 1));
} else {
LOG.debug("Skipping region sequence id checkpointing, because RegionStorage implementation '" +
- this.fs.getClass() + "' doesn't return Paths for the region container.");
+ this.regionStorage.getClass() + "' doesn't return Paths for the region container.");
nextSeqid++;
}
} else {
@@ -1051,7 +1019,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionEventDesc,
mvcc);
- final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+ final StorageIdentifier regionContainer = this.regionStorage.getRegionContainer();
/*
* TODO more WAL stuff to move out of region storage
*/
@@ -1059,13 +1027,13 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Store SeqId in HDFS when a region closes
// checking region folder exists is due to many tests which delete the table folder while a
// table is still online
- if (this.fs.getFileSystem().exists(((LegacyPathIdentifier)regionContainer).path)) {
- WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), ((LegacyPathIdentifier)regionContainer).path,
+ if (this.regionStorage.getFileSystem().exists(((LegacyPathIdentifier)regionContainer).path)) {
+ WALSplitter.writeRegionSequenceIdFile(this.regionStorage.getFileSystem(), ((LegacyPathIdentifier)regionContainer).path,
mvcc.getReadPoint(), 0);
}
} else {
LOG.debug("skipping WAL sequence ID checkpointing because the RegionStorage implementation, '" +
- this.fs.getClass() + "' doesn't return Paths for the region container.");
+ this.regionStorage.getClass() + "' doesn't return Paths for the region container.");
}
}
@@ -1175,7 +1143,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
@Override
public HRegionInfo getRegionInfo() {
- return this.fs.getRegionInfo();
+ return this.regionStorage.getRegionInfo();
}
/**
@@ -1741,12 +1709,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
/** @return {@link FileSystem} being used by this region */
public FileSystem getFilesystem() {
- return fs.getFileSystem();
+ return regionStorage.getFileSystem();
}
/** @return the {@link RegionStorage} used by this region */
public RegionStorage getRegionStorage() {
- return this.fs;
+ return this.regionStorage;
}
@Override
@@ -4011,7 +3979,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
long seqid = minSeqIdForTheRegion;
- FileSystem fs = this.fs.getFileSystem();
+ FileSystem fs = this.regionStorage.getFileSystem();
NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir);
if (LOG.isDebugEnabled()) {
LOG.debug("Found " + (files == null ? 0 : files.size())
@@ -4113,7 +4081,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
String msg = "Replaying edits from " + edits;
LOG.info(msg);
MonitoredTask status = TaskMonitor.get().createStatus(msg);
- FileSystem fs = this.fs.getFileSystem();
+ FileSystem fs = this.regionStorage.getFileSystem();
status.setStatus("Opening recovered edits");
WAL.Reader reader = null;
@@ -4944,7 +4912,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
for (String storeFile : storeFiles) {
StoreFileInfo storeFileInfo = null;
try {
- storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile);
+ storeFileInfo = regionStorage.getStoreFileInfo(Bytes.toString(family), storeFile);
store.bulkLoadHFile(storeFileInfo);
} catch(FileNotFoundException ex) {
LOG.warn(getRegionInfo().getEncodedName() + " : "
@@ -5119,7 +5087,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (!RegionReplicaUtil.isDefaultReplica(this.getRegionInfo()) &&
Bytes.equals(encodedRegionName,
- this.fs.getRegionInfoForFS().getEncodedNameAsBytes())) {
+ this.regionStorage.getRegionInfoFromStorage().getEncodedNameAsBytes())) {
return;
}
@@ -5143,7 +5111,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
/*
- * @param fs
* @param p File to check.
* @return True if file was zero-length (and if so, we'll delete it in here).
* @throws IOException
@@ -5557,7 +5524,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Note the size of the store file
try {
- storeFilesSizes.put(name, this.fs.getStoreFileLen(commitedStoreFile));
+ storeFilesSizes.put(name, this.regionStorage.getStoreFileLen(commitedStoreFile));
} catch (IOException e) {
LOG.warn("Failed to find the size of hfile " + commitedStoreFile);
storeFilesSizes.put(name, 0L);
@@ -6339,14 +6306,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
static HRegion newHRegion(Configuration conf,
HTableDescriptor htd, HRegionInfo regionInfo, WAL wal, RegionServerServices rsServices)
throws IOException {
- RegionStorage rfs = RegionStorage.open(conf, regionInfo, false);
- return newHRegion(rfs, htd, wal, rsServices);
+ RegionStorage regionStorage = RegionStorage.open(conf, regionInfo, false);
+ return newHRegion(regionStorage, htd, wal, rsServices);
}
- private static HRegion newHRegion(RegionStorage rfs, HTableDescriptor htd, WAL wal,
+ private static HRegion newHRegion(RegionStorage regionStorage, HTableDescriptor htd, WAL wal,
RegionServerServices rsServices) throws IOException {
try {
- Configuration conf = rfs.getConfiguration();
+ Configuration conf = regionStorage.getConfiguration();
@SuppressWarnings("unchecked")
Class<? extends HRegion> regionClass =
@@ -6356,7 +6323,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
regionClass.getConstructor(RegionStorage.class, HTableDescriptor.class,
WAL.class, RegionServerServices.class);
- return c.newInstance(rfs, htd, wal, rsServices);
+ return c.newInstance(regionStorage, htd, wal, rsServices);
} catch (Throwable e) {
// todo: what should I throw here?
throw new IllegalStateException("Could not instantiate a region instance.", e);
@@ -6367,7 +6334,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* Convenience method creating new HRegions. Used by createTable.
*
* @param info Info for region to create.
- * @param rootDir Root directory for HBase instance
* @param wal shared WAL
* @param initialize - true to initialize the region
* @return new HRegion
@@ -6379,8 +6345,22 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
LOG.info("creating HRegion " + info.getTable().getNameAsString()
+ " HTD == " + hTableDescriptor +
" Table name == " + info.getTable().getNameAsString());
- RegionStorage rfs = RegionStorage.open(conf, info, true);
- HRegion region = HRegion.newHRegion(rfs, hTableDescriptor, wal, null);
+ RegionStorage regionStorage = RegionStorage.open(conf, info, true);
+ HRegion region = HRegion.newHRegion(regionStorage, hTableDescriptor, wal, null);
+ if (initialize) region.initialize(null);
+ return region;
+ }
+
+ @VisibleForTesting
+ public static HRegion createHRegion(final Configuration conf,
+ final HTableDescriptor hTableDescriptor, final HRegionInfo info,
+ final Path dir, final WAL wal, final boolean initialize) throws IOException {
+ LOG.info("creating HRegion " + info.getTable().getNameAsString()
+ + " HTD == " + hTableDescriptor +
+ " Table name == " + info.getTable().getNameAsString());
+ RegionStorage regionStorage = RegionStorage.open(conf, FSUtils.getCurrentFileSystem(conf),
+ new LegacyPathIdentifier(dir), info, true);
+ HRegion region = HRegion.newHRegion(regionStorage, hTableDescriptor, wal, null);
if (initialize) region.initialize(null);
return region;
}
@@ -6436,8 +6416,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
*/
public static HRegion openHRegion(final HRegionInfo info,
final HTableDescriptor htd, final WAL wal, final Configuration conf,
- final RegionServerServices rsServices,
- final CancelableProgressable reporter)
+ final RegionServerServices rsServices, final CancelableProgressable reporter)
throws IOException {
if (info == null) throw new IllegalArgumentException("Passed region info is null");
if (LOG.isDebugEnabled()) {
@@ -6450,7 +6429,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
/**
* TODO remove after refactoring TableSnapshotScanner and TableSnapshotInputFormatImpl to use a RegionStorage impl instead of specifying a different root dir manually.
*/
- public static HRegion openHRegion(final FileSystem fs, final Path rootDir, final HRegionInfo info, HTableDescriptor htd, Configuration conf) throws IOException {
+ public static HRegion openHRegion(final FileSystem fs, final Path rootDir, final HRegionInfo info,
+ HTableDescriptor htd, Configuration conf) throws IOException {
if (info == null) throw new IllegalArgumentException("Passed region info is null");
if (LOG.isDebugEnabled()) {
LOG.debug("Opening region: " + info);
@@ -6552,7 +6532,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
*/
HRegion createDaughterRegionFromSplits(final HRegionInfo hri) throws IOException {
// Move the files from the temporary .splits to the final /table/region directory
- fs.commitDaughterRegion(hri);
+ regionStorage.commitDaughterRegion(hri);
// Create the daughter HRegion instance
HRegion r = HRegion.newHRegion(this.getBaseConf(),
@@ -6580,7 +6560,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
r.writeRequestsCount.add(this.getWriteRequestsCount()
+ region_b.getWriteRequestsCount());
- this.fs.commitMergedRegion(mergedRegionInfo);
+ this.regionStorage.commitMergedRegion(mergedRegionInfo);
return r;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index 153f36b..e44bd07 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -136,7 +136,6 @@ public abstract class HBaseTestCase extends TestCase {
/**
* @see HBaseTestingUtility#getBaseTestDir
- * @param testName
* @return directory to use for this test
*/
protected Path getUnitTestdir(String testName) {
@@ -145,12 +144,7 @@ public abstract class HBaseTestCase extends TestCase {
/**
* You must call close on the returned region and then close on the log file it created. Do
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL.
- * @param desc
- * @param startKey
- * @param endKey
- * @return An {@link HRegion}
- * @throws IOException
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} to close both the region and the WAL.
*/
public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
byte [] endKey)
@@ -162,7 +156,7 @@ public abstract class HBaseTestCase extends TestCase {
byte [] endKey, Configuration conf)
throws IOException {
HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
- return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc);
+ return testUtil.createRegionAndWAL(hri, desc, conf, testDir);
}
protected HRegion openClosedRegion(final HRegion closedRegion)
@@ -171,7 +165,7 @@ public abstract class HBaseTestCase extends TestCase {
}
/**
- * Create a table of name <code>name</code> with {@link COLUMNS} for
+ * Create a table of name <code>name</code> with {@link #COLUMNS} for
* families.
* @param name Name to give table.
* @return Column descriptor.
@@ -181,7 +175,7 @@ public abstract class HBaseTestCase extends TestCase {
}
/**
- * Create a table of name <code>name</code> with {@link COLUMNS} for
+ * Create a table of name <code>name</code> with {@link #COLUMNS} for
* families.
* @param name Name to give table.
* @param versions How many versions to allow per column.
@@ -194,7 +188,7 @@ public abstract class HBaseTestCase extends TestCase {
}
/**
- * Create a table of name <code>name</code> with {@link COLUMNS} for
+ * Create a table of name <code>name</code> with {@link #COLUMNS} for
* families.
* @param name Name to give table.
* @param versions How many versions to allow per column.
@@ -422,8 +416,8 @@ public abstract class HBaseTestCase extends TestCase {
*/
protected void createMetaRegion() throws IOException {
FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
- meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir,
- conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
+ meta = testUtil.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO,
+ fsTableDescriptors.get(TableName.META_TABLE_NAME), conf, testDir);
}
protected void closeRootAndMeta() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 45d8aae..d996010 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -81,6 +81,8 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
@@ -361,16 +363,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
- * Close both the region {@code r} and it's underlying WAL. For use in tests.
+ * Close both the region {@code region} and it's underlying WAL.
+ * To also delete the data, use {@link #destroyRegion} instead.
*/
- public static void closeRegionAndWAL(final Region r) throws IOException {
- closeRegionAndWAL((HRegion)r);
- }
-
- /**
- * Close both the HRegion {@code r} and it's underlying WAL. For use in tests.
- */
- public static void closeRegionAndWAL(final HRegion r) throws IOException {
+ public static void closeRegionAndWAL(final Region region) throws IOException {
+ HRegion r = (HRegion) region;
if (r == null) return;
r.close();
if (r.getWAL() == null) return;
@@ -378,6 +375,15 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
+ * Closes the region to clean up resources and also destroys the data.
+ * To only close the region but not delete data, use {@link #closeRegionAndWAL(Region)}.
+ */
+ public void destroyRegion(final Region r) throws IOException {
+ closeRegionAndWAL(r);
+ RegionStorage.destroy(getConfiguration(), r.getRegionInfo());
+ }
+
+ /**
* Returns this classes's instance of {@link Configuration}. Be careful how
* you use the returned Configuration since {@link Connection} instances
* can be shared. The Map of Connections is keyed by the Configuration. If
@@ -1120,7 +1126,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Starts the hbase cluster up again after shutting it down previously in a
* test. Use this if you want to keep dfs/zk up and just stop/start hbase.
* @param servers number of region servers
- * @throws IOException
*/
public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
if(connection != null){
@@ -1266,7 +1271,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Note : Directory will be made irrespective of whether path has been fetched or not.
* If directory already exists, it will be overwritten
* @return Fully qualified path to hbase root dir
- * @throws IOException
*/
public Path createRootDir(boolean create) throws IOException {
FileSystem fs = FileSystem.get(this.conf);
@@ -1281,7 +1285,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Same as {@link HBaseTestingUtility#createRootDir(boolean create)}
* except that <code>create</code> flag is false.
* @return Fully qualified path to hbase root dir
- * @throws IOException
*/
public Path createRootDir() throws IOException {
return createRootDir(false);
@@ -1300,7 +1303,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Flushes all caches in the mini hbase cluster
- * @throws IOException
*/
public void flush() throws IOException {
getMiniHBaseCluster().flushcache();
@@ -1308,7 +1310,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Flushes all caches in the mini hbase cluster
- * @throws IOException
*/
public void flush(TableName tableName) throws IOException {
getMiniHBaseCluster().flushcache(tableName);
@@ -1316,7 +1317,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Compact all regions in the mini hbase cluster
- * @throws IOException
*/
public void compact(boolean major) throws IOException {
getMiniHBaseCluster().compact(major);
@@ -1324,7 +1324,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Compact all of a table's reagion in the mini hbase cluster
- * @throws IOException
*/
public void compact(TableName tableName, boolean major) throws IOException {
getMiniHBaseCluster().compact(tableName, major);
@@ -1332,10 +1331,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param family
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, String family)
throws IOException{
@@ -1344,10 +1340,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, String[] families)
throws IOException {
@@ -1360,10 +1353,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param family
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[] family)
throws IOException{
@@ -1372,11 +1362,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table with multiple regions.
- * @param tableName
- * @param family
- * @param numRegions
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createMultiRegionTable(TableName tableName, byte[] family, int numRegions)
throws IOException {
@@ -1390,10 +1376,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families)
throws IOException {
@@ -1402,10 +1385,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table with multiple regions.
- * @param tableName
- * @param families
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createMultiRegionTable(TableName tableName, byte[][] families) throws IOException {
return createTable(tableName, families, KEYS_FOR_HBA_CREATE_TABLE);
@@ -1413,11 +1393,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param splitKeys
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families, byte[][] splitKeys)
throws IOException {
@@ -1438,11 +1414,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param htd
- * @param families
- * @param c Configuration to use
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(HTableDescriptor htd, byte[][] families, Configuration c)
throws IOException {
@@ -1451,12 +1423,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param htd
- * @param families
- * @param splitKeys
- * @param c Configuration to use
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys,
Configuration c) throws IOException {
@@ -1477,10 +1444,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param htd
- * @param splitRows
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(HTableDescriptor htd, byte[][] splitRows)
throws IOException {
@@ -1493,12 +1457,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param splitKeys
- * @param c Configuration to use
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families, byte[][] splitKeys,
final Configuration c) throws IOException {
@@ -1507,11 +1466,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param family
- * @param numVersions
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[] family, int numVersions)
throws IOException {
@@ -1520,11 +1475,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param numVersions
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families, int numVersions)
throws IOException {
@@ -1533,12 +1484,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param numVersions
- * @param splitKeys
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families, int numVersions,
byte[][] splitKeys) throws IOException {
@@ -1556,11 +1502,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table with multiple regions.
- * @param tableName
- * @param families
- * @param numVersions
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createMultiRegionTable(TableName tableName, byte[][] families, int numVersions)
throws IOException {
@@ -1569,12 +1511,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param numVersions
- * @param blockSize
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families,
int numVersions, int blockSize) throws IOException {
@@ -1613,11 +1550,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param families
- * @param numVersions
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[][] families,
int[] numVersions)
@@ -1639,11 +1572,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table.
- * @param tableName
- * @param family
- * @param splitRows
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createTable(TableName tableName, byte[] family, byte[][] splitRows)
throws IOException {
@@ -1659,10 +1588,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create a table with multiple regions.
- * @param tableName
- * @param family
* @return An HTable instance for the created table.
- * @throws IOException
*/
public Table createMultiRegionTable(TableName tableName, byte[] family) throws IOException {
return createTable(tableName, family, KEYS_FOR_HBA_CREATE_TABLE);
@@ -1809,73 +1735,45 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
/**
* Create an HRegion that writes to the local tmp dirs
- * @param desc
- * @param startKey
- * @param endKey
- * @return
- * @throws IOException
*/
- public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey,
- byte [] endKey)
- throws IOException {
+ public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey, byte [] endKey)
+ throws IOException {
HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
return createLocalHRegion(hri, desc);
}
/**
* Create an HRegion that writes to the local tmp dirs. Creates the WAL for you. Be sure to call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when you're finished with it.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when you're finished with it.
*/
public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc) throws IOException {
- return createRegionAndWAL(info, getDataTestDir(), getConfiguration(), desc);
+ return createLocalHRegion(info, desc, conf);
}
/**
- * Create an HRegion that writes to the local tmp dirs with specified wal
- * @param info regioninfo
- * @param desc table descriptor
- * @param wal wal for this region.
- * @return created hregion
- * @throws IOException
+ * Create an HRegion that writes to the local tmp dirs. Creates the WAL for you. Be sure to call
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when you're finished with it.
*/
- public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
- throws IOException {
-// return HRegion.createHRegion(getConfiguration(), getDataTestDir(), desc, info, wal);
- return null;
+ public HRegion createLocalHRegion(final HRegionInfo info, final HTableDescriptor htd,
+ Configuration conf) throws IOException {
+ // TODO: maybe we can change it to not necessarily have the wal within region dir.
+ Path dir =
+ ((LegacyPathIdentifier)RegionStorage.open(conf, info, false).getRegionContainer()).path;
+ WAL wal = createWal(conf, dir, info);
+ return HRegion.createHRegion(conf, htd, info, wal, true);
}
/**
- * @param tableName
- * @param startKey
- * @param stopKey
- * @param callingMethod
- * @param conf
- * @param isReadOnly
- * @param families
- * @throws IOException
- * @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
- * @deprecated use
- * {@link #createLocalHRegion(TableName, byte[], byte[], boolean, Durability, WAL, byte[]...)}
+ * Create an HRegion that writes to the local tmp dirs with specified wal
*/
- @Deprecated
- public HRegion createLocalHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
- String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
- WAL wal, byte[]... families) throws IOException {
- return this
- .createLocalHRegion(TableName.valueOf(tableName), startKey, stopKey, isReadOnly, durability,
- wal, families);
+ public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
+ throws IOException {
+ return HRegion.createHRegion(getConfiguration(), desc, info, wal, true);
}
/**
- * @param tableName
- * @param startKey
- * @param stopKey
- * @param isReadOnly
- * @param families
* @return A region on which you must call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
- * @throws IOException
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} when done.
*/
public HRegion createLocalHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
boolean isReadOnly, Durability durability, WAL wal, byte[]... families) throws IOException {
@@ -1884,10 +1782,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
public HRegion createLocalHRegionWithInMemoryFlags(TableName tableName, byte[] startKey,
- byte[] stopKey,
- boolean isReadOnly, Durability durability, WAL wal, boolean[] compactedMemStore,
- byte[]... families)
- throws IOException {
+ byte[] stopKey, boolean isReadOnly, Durability durability, WAL wal,
+ boolean[] compactedMemStore, byte[]... families) throws IOException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setReadOnly(isReadOnly);
int i=0;
@@ -1917,7 +1813,6 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Scans the table and issues a delete for each row read.
* @param tableName existing table
* @return HTable to that new table
- * @throws IOException
*/
public Table deleteTableData(TableName tableName) throws IOException {
Table table = getConnection().getTable(tableName);
@@ -2312,11 +2207,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Create rows in hbase:meta for regions of the specified table with the specified
* start keys. The first startKey should be a 0 length byte array if you
* want to form a proper range of regions.
- * @param conf
- * @param htd
- * @param startKeys
* @return list of region info for regions added to meta
- * @throws IOException
*/
public List<HRegionInfo> createMultiRegionsInMeta(final Configuration conf,
final HTableDescriptor htd, byte [][] startKeys)
@@ -2354,25 +2245,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
}
- /**
- * Create a region with it's own WAL. Be sure to call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to clean up all resources.
- */
- public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
- final Configuration conf, final HTableDescriptor htd) throws IOException {
- return createRegionAndWAL(info, rootDir, conf, htd, true);
- }
/**
* Create a region with it's own WAL. Be sure to call
- * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to clean up all resources.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(Region)} to clean up all resources.
*/
- public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
- final Configuration conf, final HTableDescriptor htd, boolean initialize)
- throws IOException {
- WAL wal = createWal(conf, rootDir, info);
-// return HRegion.createHRegion(conf, rootDir, htd, info, wal, initialize);
- return null;
+ // TODO: should be gone when fsredo phase 1 is complete.
+ public HRegion createRegionAndWAL(final HRegionInfo info, final HTableDescriptor htd,
+ Configuration conf, Path dir) throws IOException {
+ WAL wal = createWal(conf, dir, info);
+ return HRegion.createHRegion(conf, htd, info, dir, wal, true);
}
/**
@@ -3842,7 +3724,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
htd.addFamily(hcd);
HRegionInfo info =
new HRegionInfo(TableName.valueOf(tableName), null, null, false);
- return createRegionAndWAL(info, getDataTestDir(), getConfiguration(), htd);
+ return createLocalHRegion(info, htd);
}
public void setFileSystemURI(String fsURI) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java
index 1f6dc98..76a7a4c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HTestConst;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -44,12 +44,9 @@ public class TestIntraRowPagination {
/**
* Test from client side for scan with maxResultPerCF set
- *
- * @throws Exception
*/
@Test
public void testScanLimitAndOffset() throws Exception {
- //byte [] TABLE = HTestConst.DEFAULT_TABLE_BYTES;
byte [][] ROWS = HTestConst.makeNAscii(HTestConst.DEFAULT_ROW_BYTES, 2);
byte [][] FAMILIES = HTestConst.makeNAscii(HTestConst.DEFAULT_CF_BYTES, 3);
byte [][] QUALIFIERS = HTestConst.makeNAscii(HTestConst.DEFAULT_QUALIFIER_BYTES, 10);
@@ -60,8 +57,7 @@ public class TestIntraRowPagination {
HColumnDescriptor hcd = new HColumnDescriptor(family);
htd.addFamily(hcd);
}
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
Put put;
Scan scan;
@@ -101,7 +97,7 @@ public class TestIntraRowPagination {
TestScannersFromClientSide.verifyResult(result, kvListExp, toLog,
"Testing scan with storeOffset and storeLimit");
} finally {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index b2ef1bd..85c2978 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -38,7 +38,6 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -77,7 +76,6 @@ public class TestCoprocessorInterface {
@Rule public TestName name = new TestName();
private static final Log LOG = LogFactory.getLog(TestCoprocessorInterface.class);
private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
- static final Path DIR = TEST_UTIL.getDataTestDir();
private static class CustomScanner implements RegionScanner {
@@ -294,8 +292,7 @@ public class TestCoprocessorInterface {
byte [][] families = { fam1, fam2, fam3 };
Configuration hc = initSplit();
- Region region = initHRegion(tableName, name.getMethodName(), hc,
- new Class<?>[]{}, families);
+ Region region = initHRegion(tableName, hc, new Class<?>[]{}, families);
for (int i = 0; i < 3; i++) {
HBaseTestCase.addContent(region, fam3);
@@ -372,8 +369,7 @@ public class TestCoprocessorInterface {
byte [][] families = { fam1, fam2, fam3 };
Configuration hc = initSplit();
- Region region = initHRegion(tableName, name.getMethodName(), hc,
- new Class<?>[]{CoprocessorImpl.class}, families);
+ Region region = initHRegion(tableName, hc, new Class<?>[]{CoprocessorImpl.class}, families);
for (int i = 0; i < 3; i++) {
HBaseTestCase.addContent(region, fam3);
region.flush(true);
@@ -446,16 +442,14 @@ public class TestCoprocessorInterface {
return r;
}
- Region initHRegion (TableName tableName, String callingMethod,
- Configuration conf, Class<?> [] implClasses, byte [][] families)
- throws IOException {
+ private Region initHRegion(TableName tableName, Configuration conf, Class<?> [] implClasses,
+ byte [][] families) throws IOException {
HTableDescriptor htd = new HTableDescriptor(tableName);
for(byte [] family : families) {
htd.addFamily(new HColumnDescriptor(family));
}
HRegionInfo info = new HRegionInfo(tableName, null, null, false);
- Path path = new Path(DIR + callingMethod);
- Region r = HBaseTestingUtility.createRegionAndWAL(info, path, conf, htd);
+ Region r = TEST_UTIL.createLocalHRegion(info, htd, conf);
// this following piece is a hack.
RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
index 723edcb..45ef885 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
@@ -44,9 +44,7 @@ import org.junit.experimental.categories.Category;
@Category({CoprocessorTests.class, SmallTests.class})
public class TestRegionObserverStacking extends TestCase {
- private static HBaseTestingUtility TEST_UTIL
- = new HBaseTestingUtility();
- static final Path DIR = TEST_UTIL.getDataTestDir();
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
public static class ObserverA extends BaseRegionObserver {
long id;
@@ -94,15 +92,14 @@ public class TestRegionObserverStacking extends TestCase {
}
}
- HRegion initHRegion (byte [] tableName, String callingMethod,
- Configuration conf, byte [] ... families) throws IOException {
+ private HRegion initHRegion(byte [] tableName, Configuration conf, byte [] ... families)
+ throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
for(byte [] family : families) {
htd.addFamily(new HColumnDescriptor(family));
}
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- Path path = new Path(DIR + callingMethod);
- HRegion r = HBaseTestingUtility.createRegionAndWAL(info, path, conf, htd);
+ HRegion r = TEST_UTIL.createLocalHRegion(info, htd);
// this following piece is a hack. currently a coprocessorHost
// is secretly loaded at OpenRegionHandler. we don't really
// start a region server here, so just manually create cphost
@@ -119,8 +116,7 @@ public class TestRegionObserverStacking extends TestCase {
byte[][] FAMILIES = new byte[][] { A } ;
Configuration conf = HBaseConfiguration.create();
- HRegion region = initHRegion(TABLE, getClass().getName(),
- conf, FAMILIES);
+ HRegion region = initHRegion(TABLE, conf, FAMILIES);
RegionCoprocessorHost h = region.getCoprocessorHost();
h.load(ObserverA.class, Coprocessor.PRIORITY_HIGHEST, conf);
h.load(ObserverB.class, Coprocessor.PRIORITY_USER, conf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
index 828842d..531194e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
@@ -38,8 +38,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -49,8 +49,7 @@ import org.junit.experimental.categories.Category;
@Category({FilterTests.class, SmallTests.class})
public class TestColumnPrefixFilter {
- private final static HBaseTestingUtility TEST_UTIL = new
- HBaseTestingUtility();
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Test
public void testColumnPrefixFilter() throws IOException {
@@ -58,8 +57,7 @@ public class TestColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestColumnPrefixFilter"));
htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
@@ -121,8 +119,7 @@ public class TestColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestColumnPrefixFilter"));
htd.addFamily((new HColumnDescriptor(family)).setMaxVersions(3));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
try {
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
index 3a635b8..6cd53e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
@@ -87,14 +87,13 @@ public class TestDependentColumnFilter {
hcd1.setMaxVersions(3);
htd.addFamily(hcd1);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ this.region = TEST_UTIL.createLocalHRegion(info, htd);
addData();
}
@After
public void tearDown() throws Exception {
- HBaseTestingUtility.closeRegionAndWAL(this.region);
+ TEST_UTIL.destroyRegion(region);
}
private void addData() throws IOException {
@@ -138,11 +137,6 @@ public class TestDependentColumnFilter {
/**
* This shouldn't be confused with TestFilter#verifyScan
* as expectedKeys is not the per row total, but the scan total
- *
- * @param s
- * @param expectedRows
- * @param expectedCells
- * @throws IOException
*/
private void verifyScan(Scan s, long expectedRows, long expectedCells)
throws IOException {
@@ -225,8 +219,6 @@ public class TestDependentColumnFilter {
/**
* Test that the filter correctly drops rows without a corresponding timestamp
- *
- * @throws Exception
*/
@Test
public void testFilterDropping() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 6b22164..567ee0d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -144,8 +144,7 @@ public class TestFilter {
htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[0]));
htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[1]));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ this.region = TEST_UTIL.createLocalHRegion(info, htd);
// Insert first half
for(byte [] ROW : ROWS_ONE) {
@@ -219,7 +218,7 @@ public class TestFilter {
@After
public void tearDown() throws Exception {
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
@Test
@@ -556,8 +555,6 @@ public class TestFilter {
* {@link Filter#filterRow()} method.
*
* See HBASE-2258.
- *
- * @throws Exception
*/
@Test
public void testWhileMatchFilterWithFilterRow() throws Exception {
@@ -1493,8 +1490,7 @@ public class TestFilter {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestFilter"));
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- Region testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ Region testRegion = TEST_UTIL.createLocalHRegion(info, htd);
for(int i=0; i<5; i++) {
Put p = new Put(Bytes.toBytes((char)('a'+i) + "row"));
@@ -2054,8 +2050,7 @@ public class TestFilter {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testNestedFilterListWithSCVF"));
htd.addFamily(new HColumnDescriptor(FAMILIES[0]));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- Region testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ Region testRegion = TEST_UTIL.createLocalHRegion(info, htd);
for(int i=0; i<10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.setDurability(Durability.SKIP_WAL);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
index 0a287ce..bee5edc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
@@ -83,8 +83,7 @@ public class TestFilterFromRegionSide {
htd.addFamily(hcd);
}
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- REGION = HBaseTestingUtility
- .createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+ REGION = TEST_UTIL.createLocalHRegion(info, htd);
for(Put put:createPuts(ROWS, FAMILIES, QUALIFIERS, VALUE)){
REGION.put(put);
}
@@ -111,7 +110,7 @@ public class TestFilterFromRegionSide {
@AfterClass
public static void tearDownAfterClass() throws Exception {
- REGION.close();
+ TEST_UTIL.destroyRegion(REGION);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
index 8291e52..ee77a41 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
@@ -69,8 +69,7 @@ public class TestInvocationRecordFilter {
TableName.valueOf(TABLE_NAME_BYTES));
htd.addFamily(new HColumnDescriptor(FAMILY_NAME_BYTES));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getConfiguration(), htd);
+ this.region = TEST_UTIL.createLocalHRegion(info, htd);
Put put = new Put(ROW_BYTES);
for (int i = 0; i < 10; i += 2) {
@@ -82,6 +81,11 @@ public class TestInvocationRecordFilter {
this.region.flush(true);
}
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.destroyRegion(region);
+ }
+
@Test
public void testFilterInvocation() throws Exception {
List<Integer> selectQualifiers = new ArrayList<Integer>();
@@ -149,13 +153,6 @@ public class TestInvocationRecordFilter {
expectedValues.equals(actualValues));
}
- @After
- public void tearDown() throws Exception {
- WAL wal = ((HRegion)region).getWAL();
- ((HRegion)region).close();
- wal.close();
- }
-
/**
* Filter which gives the list of keyvalues for which the filter is invoked.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java
index 7b700b7..9db23e5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java
@@ -38,8 +38,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -61,8 +61,7 @@ public class TestMultipleColumnPrefixFilter {
htd.addFamily(hcd);
// HRegionInfo info = new HRegionInfo(htd, null, null, false);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
- getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
@@ -113,7 +112,7 @@ public class TestMultipleColumnPrefixFilter {
;
assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
@Test
@@ -128,8 +127,7 @@ public class TestMultipleColumnPrefixFilter {
hcd2.setMaxVersions(3);
htd.addFamily(hcd2);
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
- getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
@@ -186,7 +184,7 @@ public class TestMultipleColumnPrefixFilter {
;
assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
@Test
@@ -195,8 +193,7 @@ public class TestMultipleColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
- getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd);
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
@@ -243,7 +240,7 @@ public class TestMultipleColumnPrefixFilter {
assertEquals(results1.size(), results2.size());
- HBaseTestingUtility.closeRegionAndWAL(region);
+ TEST_UTIL.destroyRegion(region);
}
List<String> generateRandomWords(int numberOfWords, String suffix) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
index e31a73b..b1872dd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
@@ -26,7 +26,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -49,6 +48,7 @@ import org.junit.experimental.categories.Category;
@Category({ IOTests.class, SmallTests.class })
public class TestPrefixTree {
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final String row4 = "a-b-B-2-1402397300-1402416535";
private static final byte[] row4_bytes = Bytes.toBytes(row4);
@@ -63,7 +63,6 @@ public class TestPrefixTree {
private final static byte[] qual1 = Bytes.toBytes("qf_1");
private final static byte[] qual2 = Bytes.toBytes("qf_2");
- private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
private Region region;
@@ -73,14 +72,12 @@ public class TestPrefixTree {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor(fam).setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE));
HRegionInfo info = new HRegionInfo(tableName, null, null, false);
- Path path = testUtil.getDataTestDir(getClass().getSimpleName());
- region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(), htd);
+ region = TEST_UTIL.createLocalHRegion(info, htd);
}
@After
public void tearDown() throws Exception {
- HBaseTestingUtility.closeRegionAndWAL(region);
- testUtil.cleanupTestDir();
+ TEST_UTIL.destroyRegion(region);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
index 2826694..c377270 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -46,7 +45,7 @@ import org.junit.experimental.categories.Category;
@Category({ IOTests.class, SmallTests.class })
public class TestSeekBeforeWithReverseScan {
- private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Region region;
@@ -59,14 +58,12 @@ public class TestSeekBeforeWithReverseScan {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor(cfName).setDataBlockEncoding(DataBlockEncoding.FAST_DIFF));
HRegionInfo info = new HRegionInfo(tableName, null, null, false);
- Path path = testUtil.getDataTestDir(getClass().getSimpleName());
- region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(), htd);
+ region = TEST_UTIL.createLocalHRegion(info, htd);
}
@After
public void tearDown() throws Exception {
- HBaseTestingUtility.closeRegionAndWAL(region);
- testUtil.cleanupTestDir();
+ TEST_UTIL.destroyRegion(region);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
index 898f3bb..3da6396 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -60,7 +60,7 @@ public class TestScannerFromBucketCache {
@Rule
public TestName name = new TestName();
- HRegion region = null;
+ Region region = null;
private HBaseTestingUtility test_util;
public Configuration conf;
private final int MAX_VERSIONS = 2;
@@ -334,37 +334,15 @@ public class TestScannerFromBucketCache {
return actual;
}
- private static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
+ /**
+ * Be sure to call {@link HBaseTestingUtility#closeRegionAndWAL(Region)} to clean up resources.
+ */
+ private static Region initHRegion(TableName tableName, String callingMethod, Configuration conf,
HBaseTestingUtility test_util, byte[]... families) throws IOException {
- return initHRegion(tableName, null, null, callingMethod, conf, test_util, false, families);
- }
-
- private static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
- String callingMethod, Configuration conf, HBaseTestingUtility test_util, boolean isReadOnly,
- byte[]... families) throws IOException {
Path logDir = test_util.getDataTestDirOnTestFS(callingMethod + ".log");
- HRegionInfo hri = new HRegionInfo(tableName, startKey, stopKey);
+ HRegionInfo hri = new HRegionInfo(tableName);
final WAL wal = HBaseTestingUtility.createWal(conf, logDir, hri);
- return initHRegion(tableName, startKey, stopKey, callingMethod, conf, test_util, isReadOnly,
- Durability.SYNC_WAL, wal, families);
- }
-
- /**
- * @param tableName
- * @param startKey
- * @param stopKey
- * @param callingMethod
- * @param conf
- * @param isReadOnly
- * @param families
- * @throws IOException
- * @return A region on which you must call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)}
- * when done.
- */
- private static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
- String callingMethod, Configuration conf, HBaseTestingUtility test_util, boolean isReadOnly,
- Durability durability, WAL wal, byte[]... families) throws IOException {
- return test_util.createLocalHRegion(tableName, startKey, stopKey, isReadOnly, durability, wal,
- families);
+ return test_util.createLocalHRegion(tableName, null, null, false, Durability.SYNC_WAL, wal,
+ families);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
index 9c6bb38..63b89c2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
@@ -99,8 +99,7 @@ public class TestScannerSelectionUsingKeyRange {
HTableDescriptor htd = new HTableDescriptor(TABLE);
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(TABLE);
- Region region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), conf,
- htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd, conf);
for (int iFile = 0; iFile < NUM_FILES; ++iFile) {
for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
index 08b259d..a3007c1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
@@ -106,8 +106,7 @@ public class TestScannerSelectionUsingTTL {
HTableDescriptor htd = new HTableDescriptor(TABLE);
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(TABLE);
- Region region = HBaseTestingUtility.createRegionAndWAL(info,
- TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, htd);
+ Region region = TEST_UTIL.createLocalHRegion(info, htd, conf);
long ts = EnvironmentEdgeManager.currentTime();
long version = 0; //make sure each new set of Put's have a new ts
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index a20327d..8315b75 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -62,17 +62,15 @@ import org.junit.experimental.categories.Category;
public class TestMasterFailover {
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
- HRegion createRegion(final HRegionInfo hri, final Path rootdir, final Configuration c,
- final HTableDescriptor htd)
+ void createRegion(HBaseTestingUtility hbtu, final HRegionInfo hri, final HTableDescriptor htd)
throws IOException {
- HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, rootdir, c, htd);
+ Region r = hbtu.createLocalHRegion(hri, htd);
// The above call to create a region will create an wal file. Each
// log file create will also create a running thread to do syncing. We need
// to close out this log else we will have a running thread trying to sync
// the file system continuously which is ugly when dfs is taken away at the
// end of the test.
HBaseTestingUtility.closeRegionAndWAL(r);
- return r;
}
// TODO: Next test to add is with testing permutations of the RIT or the RS
@@ -233,7 +231,7 @@ public class TestMasterFailover {
fstd.createTableDescriptor(offlineTable);
HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
- createRegion(hriOffline, rootdir, conf, offlineTable);
+ createRegion(TEST_UTIL, hriOffline, offlineTable);
MetaTableAccessor.addRegionToMeta(master.getConnection(), hriOffline);
log("Regions in hbase:meta and namespace have been created");
@@ -263,7 +261,7 @@ public class TestMasterFailover {
stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
HRegionInfo failedClose = new HRegionInfo(offlineTable.getTableName(), null, null);
- createRegion(failedClose, rootdir, conf, offlineTable);
+ createRegion(TEST_UTIL, failedClose, offlineTable);
MetaTableAccessor.addRegionToMeta(master.getConnection(), failedClose);
oldState = new RegionState(failedClose, State.PENDING_CLOSE);
@@ -271,7 +269,7 @@ public class TestMasterFailover {
stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
HRegionInfo failedOpen = new HRegionInfo(offlineTable.getTableName(), null, null);
- createRegion(failedOpen, rootdir, conf, offlineTable);
+ createRegion(TEST_UTIL, failedOpen, offlineTable);
MetaTableAccessor.addRegionToMeta(master.getConnection(), failedOpen);
// Simulate a region transitioning to failed open when the region server reports the
@@ -282,7 +280,7 @@ public class TestMasterFailover {
HRegionInfo failedOpenNullServer = new HRegionInfo(offlineTable.getTableName(), null, null);
LOG.info("Failed open NUll server " + failedOpenNullServer.getEncodedName());
- createRegion(failedOpenNullServer, rootdir, conf, offlineTable);
+ createRegion(TEST_UTIL, failedOpenNullServer, offlineTable);
MetaTableAccessor.addRegionToMeta(master.getConnection(), failedOpenNullServer);
// Simulate a region transitioning to failed open when the master couldn't find a plan for
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1813a2/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
index 9abe040..cfc4257 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.regionserver.HMobStore;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -47,7 +47,7 @@ import org.junit.experimental.categories.Category;
public class TestMobFileCache extends TestCase {
static final Log LOG = LogFactory.getLog(TestMobFileCache.class);
private HBaseTestingUtility UTIL;
- private HRegion region;
+ private Region region;
private Configuration conf;
private MobCacheConfig mobCacheConf;
private MobFileCache mobFileCache;
@@ -95,8 +95,7 @@ public class TestMobFileCache extends TestCase {
@Override
protected void tearDown() throws Exception {
- region.close();
- region.getFilesystem().delete(UTIL.getDataTestDir(), true);
+ UTIL.destroyRegion(region);
}
/**