You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/12/14 00:15:06 UTC
[38/50] [abbrv] hbase git commit: HBASE-17437 Support specifying a
WAL directory outside of the root directory (Yishan Yang and Zach York)
HBASE-17588 Remove unused imports brought in by HBASE-17437 (Zach York)
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 760cdc1..1fcb241 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.regionserver.wal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -88,6 +89,8 @@ public class TestFSHLog {
protected static Configuration conf;
protected static FileSystem fs;
protected static Path dir;
+ protected static Path rootDir;
+ protected static Path walRootDir;
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Rule
@@ -99,8 +102,10 @@ public class TestFSHLog {
for (FileStatus dir : entries) {
fs.delete(dir.getPath(), true);
}
- final Path hbaseDir = TEST_UTIL.createRootDir();
- dir = new Path(hbaseDir, currentTest.getMethodName());
+ rootDir = TEST_UTIL.createRootDir();
+ walRootDir = TEST_UTIL.createWALRootDir();
+ dir = new Path(walRootDir, currentTest.getMethodName());
+ assertNotEquals(rootDir, walRootDir);
}
@After
@@ -133,6 +138,8 @@ public class TestFSHLog {
@AfterClass
public static void tearDownAfterClass() throws Exception {
+ fs.delete(rootDir, true);
+ fs.delete(walRootDir, true);
TEST_UTIL.shutdownMiniCluster();
}
@@ -144,7 +151,7 @@ public class TestFSHLog {
// test to see whether the coprocessor is loaded or not.
FSHLog log = null;
try {
- log = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(),
+ log = new FSHLog(fs, walRootDir, dir.toString(),
HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
WALCoprocessorHost host = log.getCoprocessorHost();
Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName());
@@ -195,7 +202,7 @@ public class TestFSHLog {
FSHLog wal1 = null;
FSHLog walMeta = null;
try {
- wal1 = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(),
+ wal1 = new FSHLog(fs, walRootDir, dir.toString(),
HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
LOG.debug("Log obtained is: " + wal1);
Comparator<Path> comp = wal1.LOG_NAME_COMPARATOR;
@@ -205,7 +212,7 @@ public class TestFSHLog {
assertTrue(comp.compare(p1, p1) == 0);
// comparing with different filenum.
assertTrue(comp.compare(p1, p2) < 0);
- walMeta = new FSHLog(fs, FSUtils.getRootDir(conf), dir.toString(),
+ walMeta = new FSHLog(fs, walRootDir, dir.toString(),
HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null,
DefaultWALProvider.META_WAL_PROVIDER_ID);
Comparator<Path> compMeta = walMeta.LOG_NAME_COMPARATOR;
@@ -253,7 +260,7 @@ public class TestFSHLog {
LOG.debug("testFindMemStoresEligibleForFlush");
Configuration conf1 = HBaseConfiguration.create(conf);
conf1.setInt("hbase.regionserver.maxlogs", 1);
- FSHLog wal = new FSHLog(fs, FSUtils.getRootDir(conf1), dir.toString(),
+ FSHLog wal = new FSHLog(fs, walRootDir, dir.toString(),
HConstants.HREGION_OLDLOGDIR_NAME, conf1, null, true, null, null);
HTableDescriptor t1 =
new HTableDescriptor(TableName.valueOf("t1")).addFamily(new HColumnDescriptor("row"));
@@ -330,7 +337,7 @@ public class TestFSHLog {
@Test(expected=IOException.class)
public void testFailedToCreateWALIfParentRenamed() throws IOException {
final String name = "testFailedToCreateWALIfParentRenamed";
- FSHLog log = new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME,
+ FSHLog log = new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME,
conf, null, true, null, null);
long filenum = System.currentTimeMillis();
Path path = log.computeFilename(filenum);
@@ -359,13 +366,13 @@ public class TestFSHLog {
final byte[] rowName = tableName.getName();
final HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor("f"));
- HRegion r = HRegion.createHRegion(hri, TEST_UTIL.getDefaultRootDirPath(),
+ HRegion r = HRegion.createHRegion(hri, rootDir,
TEST_UTIL.getConfiguration(), htd);
HRegion.closeHRegion(r);
final int countPerFamily = 10;
final MutableBoolean goslow = new MutableBoolean(false);
// subclass and doctor a method.
- FSHLog wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDefaultRootDirPath(),
+ FSHLog wal = new FSHLog(FileSystem.get(conf), walRootDir,
testName, conf) {
@Override
void atHeadOfRingBufferEventHandlerAppend() {
@@ -377,7 +384,7 @@ public class TestFSHLog {
}
};
HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(),
- TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
+ TEST_UTIL.getTestFileSystem(), rootDir, hri, htd, wal);
EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
try {
List<Put> puts = null;
@@ -430,7 +437,7 @@ public class TestFSHLog {
SecurityException, IllegalArgumentException, IllegalAccessException {
final String name = "testSyncRunnerIndexOverflow";
FSHLog log =
- new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME, conf,
+ new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME, conf,
null, true, null, null);
try {
Field ringBufferEventHandlerField = FSHLog.class.getDeclaredField("ringBufferEventHandler");
@@ -468,7 +475,7 @@ public class TestFSHLog {
final CountDownLatch putFinished = new CountDownLatch(1);
try (FSHLog log =
- new FSHLog(fs, FSUtils.getRootDir(conf), name, HConstants.HREGION_OLDLOGDIR_NAME, conf,
+ new FSHLog(fs, walRootDir, name, HConstants.HREGION_OLDLOGDIR_NAME, conf,
null, true, null, null)) {
log.registerWALActionsListener(new WALActionsListener.Base() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
index 2ccf12b..3f15c2a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
@@ -71,7 +71,8 @@ public class TestLogRollAbort {
/* For the split-then-roll test */
private static final Path HBASEDIR = new Path("/hbase");
- private static final Path OLDLOGDIR = new Path(HBASEDIR, HConstants.HREGION_OLDLOGDIR_NAME);
+ private static final Path HBASELOGDIR = new Path("/hbaselog");
+ private static final Path OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME);
// Need to override this setup so we can edit the config before it gets sent
// to the HDFS & HBase cluster startup.
@@ -112,6 +113,7 @@ public class TestLogRollAbort {
// disable region rebalancing (interferes with log watching)
cluster.getMaster().balanceSwitch(false);
FSUtils.setRootDir(conf, HBASEDIR);
+ FSUtils.setWALRootDir(conf, HBASELOGDIR);
}
@After
@@ -183,7 +185,7 @@ public class TestLogRollAbort {
public void testLogRollAfterSplitStart() throws IOException {
LOG.info("Verify wal roll after split starts will fail.");
String logName = "testLogRollAfterSplitStart";
- Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
+ Path thisTestsDir = new Path(HBASELOGDIR, DefaultWALProvider.getWALDirectoryName(logName));
final WALFactory wals = new WALFactory(conf, null, logName);
try {
@@ -220,7 +222,7 @@ public class TestLogRollAbort {
LOG.debug("Renamed region directory: " + rsSplitDir);
LOG.debug("Processing the old log files.");
- WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);
+ WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);
LOG.debug("Trying to roll the WAL.");
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
index 3eba637..bac1b6f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
@@ -20,14 +20,19 @@ package org.apache.hadoop.hbase.regionserver.wal;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -53,21 +58,27 @@ public class TestWALActionsListener {
new HBaseTestingUtility();
private final static byte[] SOME_BYTES = Bytes.toBytes("t");
- private static FileSystem fs;
private static Configuration conf;
+ private static Path rootDir;
+ private static Path walRootDir;
+ private static FileSystem fs;
+ private static FileSystem walFs;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
conf.setInt("hbase.regionserver.maxlogs", 5);
- fs = FileSystem.get(conf);
- FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
+ rootDir = TEST_UTIL.createRootDir();
+ walRootDir = TEST_UTIL.createWALRootDir();
+ fs = FSUtils.getRootDirFileSystem(conf);
+ walFs = FSUtils.getWALFileSystem(conf);
}
@Before
public void setUp() throws Exception {
- fs.delete(new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_LOGDIR_NAME), true);
- fs.delete(new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_OLDLOGDIR_NAME), true);
+ fs.delete(rootDir, true);
+ walFs.delete(new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME), true);
+ walFs.delete(new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME), true);
}
@After
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index b478b2d..2622f6d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
@@ -121,6 +122,7 @@ public class TestWALReplay {
static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
private Path hbaseRootDir = null;
+ private Path hbaseWALRootDir = null;
private String logName;
private Path oldLogDir;
private Path logDir;
@@ -142,8 +144,12 @@ public class TestWALReplay {
TEST_UTIL.startMiniCluster(3);
Path hbaseRootDir =
TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
- LOG.info("hbase.rootdir=" + hbaseRootDir);
+ Path hbaseWALRootDir =
+ TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbaselog"));
+ LOG.info(HConstants.HBASE_DIR + "=" + hbaseRootDir);
+ LOG.info(HFileSystem.HBASE_WAL_DIR + "=" + hbaseWALRootDir);
FSUtils.setRootDir(conf, hbaseRootDir);
+ FSUtils.setWALRootDir(conf, hbaseWALRootDir);
}
@AfterClass
@@ -156,12 +162,16 @@ public class TestWALReplay {
this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
this.hbaseRootDir = FSUtils.getRootDir(this.conf);
- this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ this.hbaseWALRootDir = FSUtils.getWALRootDir(this.conf);
+ this.oldLogDir = new Path(this.hbaseWALRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
this.logName = DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName() + "-manual");
- this.logDir = new Path(this.hbaseRootDir, logName);
+ this.logDir = new Path(this.hbaseWALRootDir, logName);
if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
}
+ if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseWALRootDir)) {
+ TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true);
+ }
this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
this.wals = new WALFactory(conf, null, currentTest.getMethodName());
@@ -171,6 +181,7 @@ public class TestWALReplay {
public void tearDown() throws Exception {
this.wals.close();
TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
+ TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true);
}
/*
@@ -296,11 +307,11 @@ public class TestWALReplay {
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
deleteDir(basedir);
HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
+ HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
final byte [] rowName = tableName.getName();
@@ -326,7 +337,7 @@ public class TestWALReplay {
WAL wal3 = createWAL(this.conf);
try {
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal3);
long seqid = region.getOpenSeqNum();
// The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
// When opened, this region would apply 6k edits, and increment the sequenceId by 1
@@ -358,13 +369,13 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
+ final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString());
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
+ HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
byte [] family = htd.getFamilies().iterator().next().getName();
Path f = new Path(basedir, "hfile");
@@ -393,7 +404,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(newConf);
HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
- hbaseRootDir, hri, htd, wal2);
+ hbaseWALRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid2 > -1);
assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
@@ -424,14 +435,14 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testCompactedBulkLoadedFiles");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
+ final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString());
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region2 = HRegion.createHRegion(hri,
- hbaseRootDir, this.conf, htd);
+ hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
// Add an edit so something in the WAL
byte [] row = tableName.getName();
@@ -465,7 +476,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(newConf);
HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
- hbaseRootDir, hri, htd, wal2);
+ hbaseWALRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid2 > -1);
assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
@@ -495,19 +506,19 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsWrittenViaHRegion");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region3 = HRegion.createHRegion(hri,
- hbaseRootDir, this.conf, htd);
+ hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region3);
// Write countPerFamily edits into the three families. Do a flush on one
// of the families during the load of edits so its seqid is not same as
// others to test we do right thing when different seqids.
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
long seqid = region.getOpenSeqNum();
boolean first = true;
for (HColumnDescriptor hcd: htd.getFamilies()) {
@@ -530,7 +541,7 @@ public class TestWALReplay {
wal.shutdown();
runWALSplit(this.conf);
WAL wal2 = createWAL(this.conf);
- HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
+ HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid + result.size() < seqid2);
final Result result1b = region2.get(g);
@@ -605,19 +616,19 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsWrittenViaHRegion");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region3 = HRegion.createHRegion(hri,
- hbaseRootDir, this.conf, htd);
+ hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region3);
// Write countPerFamily edits into the three families. Do a flush on one
// of the families during the load of edits so its seqid is not same as
// others to test we do right thing when different seqids.
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
long seqid = region.getOpenSeqNum();
for (HColumnDescriptor hcd: htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
@@ -650,7 +661,7 @@ public class TestWALReplay {
// Let us try to split and recover
runWALSplit(this.conf);
WAL wal2 = createWAL(this.conf);
- HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
+ HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid + result.size() < seqid2);
@@ -690,10 +701,10 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsAfterAbortingFlush");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
+ HRegion region3 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
region3.close();
region3.getWAL().close();
// Write countPerFamily edits into the three families. Do a flush on one
@@ -707,7 +718,7 @@ public class TestWALReplay {
customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
CustomStoreFlusher.class.getName());
HRegion region =
- HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
+ HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal, customConf, rsServices, null);
int writtenRowCount = 10;
List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
htd.getFamilies());
@@ -761,7 +772,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(this.conf);
Mockito.doReturn(false).when(rsServices).isAborted();
HRegion region2 =
- HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
+ HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal2, this.conf, rsServices, null);
scanner = region2.getScanner(new Scan());
assertEquals(writtenRowCount, getScannedCount(scanner));
}
@@ -791,12 +802,12 @@ public class TestWALReplay {
TableName.valueOf("testReplayEditsWrittenIntoWAL");
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region2 = HRegion.createHRegion(hri,
- hbaseRootDir, this.conf, htd);
+ hbaseWALRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
final WAL wal = createWAL(this.conf);
final byte[] rowName = tableName.getName();
@@ -890,7 +901,7 @@ public class TestWALReplay {
final TableName tableName = TableName.valueOf(currentTest.getMethodName());
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
final Path basedir =
- FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
@@ -899,7 +910,7 @@ public class TestWALReplay {
// Mock the WAL
MockWAL wal = createMockWAL();
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
for (HColumnDescriptor hcd : htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
}
@@ -920,10 +931,10 @@ public class TestWALReplay {
FileStatus[] listStatus = wal.getFiles();
assertNotNull(listStatus);
assertTrue(listStatus.length > 0);
- WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
+ WALSplitter.splitLogFile(hbaseWALRootDir, listStatus[0],
this.fs, this.conf, null, null, null, mode, wals);
FileStatus[] listStatus1 = this.fs.listStatus(
- new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
+ new Path(FSUtils.getTableDir(hbaseWALRootDir, tableName), new Path(hri.getEncodedName(),
"recovered.edits")), new PathFilter() {
@Override
public boolean accept(Path p) {
@@ -951,17 +962,17 @@ public class TestWALReplay {
IllegalAccessException {
final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
- HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
+ HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd);
Path regionDir = region1.getRegionFileSystem().getRegionDir();
HBaseTestingUtility.closeRegionAndWAL(region1);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
for (HColumnDescriptor hcd : htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
}
@@ -1031,12 +1042,12 @@ public class TestWALReplay {
HRegion region2;
try {
// log replay should fail due to the IOException, otherwise we may lose data.
- region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2);
+ region2 = HRegion.openHRegion(conf, spyFs, hbaseWALRootDir, hri, htd, wal2);
assertEquals(result.size(), region2.get(g).size());
} catch (IOException e) {
assertEquals("read over limit", e.getMessage());
}
- region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2);
+ region2 = HRegion.openHRegion(conf, fs, hbaseWALRootDir, hri, htd, wal2);
assertEquals(result.size(), region2.get(g).size());
}
@@ -1047,11 +1058,11 @@ public class TestWALReplay {
final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
+ HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd);
HBaseTestingUtility.closeRegionAndWAL(region);
final byte[] family = htd.getColumnFamilies()[0].getName();
final byte[] rowName = tableName.getName();
@@ -1070,12 +1081,12 @@ public class TestWALReplay {
first = fs.getFileStatus(smallFile);
second = fs.getFileStatus(largeFile);
}
- WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null,
+ WALSplitter.splitLogFile(hbaseWALRootDir, first, fs, conf, null, null, null,
RecoveryMode.LOG_SPLITTING, wals);
- WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null,
+ WALSplitter.splitLogFile(hbaseWALRootDir, second, fs, conf, null, null, null,
RecoveryMode.LOG_SPLITTING, wals);
WAL wal = createWAL(this.conf);
- region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
+ region = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal);
assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
assertEquals(2, region.get(new Get(rowName)).size());
}
@@ -1093,9 +1104,9 @@ public class TestWALReplay {
static class MockWAL extends FSHLog {
boolean doCompleteCacheFlush = false;
- public MockWAL(FileSystem fs, Path rootDir, String logName, Configuration conf)
+ public MockWAL(FileSystem fs, Path walRootDir, String logName, Configuration conf)
throws IOException {
- super(fs, rootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
+ super(fs, walRootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
}
@Override
@@ -1115,7 +1126,7 @@ public class TestWALReplay {
}
private MockWAL createMockWAL() throws IOException {
- MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
+ MockWAL wal = new MockWAL(fs, hbaseWALRootDir, logName, conf);
// Set down maximum recovery so we dfsclient doesn't linger retrying something
// long gone.
HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
@@ -1222,7 +1233,7 @@ public class TestWALReplay {
*/
private Path runWALSplit(final Configuration c) throws IOException {
List<Path> splits = WALSplitter.split(
- hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c, wals);
+ hbaseWALRootDir, logDir, oldLogDir, FSUtils.getWALFileSystem(c), c, wals);
// Split should generate only 1 file since there's only 1 region
assertEquals("splits=" + splits, 1, splits.size());
// Make sure the file exists
@@ -1237,7 +1248,7 @@ public class TestWALReplay {
* @throws IOException
*/
private WAL createWAL(final Configuration c) throws IOException {
- FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
+ FSHLog wal = new FSHLog(FSUtils.getWALFileSystem(c), hbaseWALRootDir, logName, c);
// Set down maximum recovery so we dfsclient doesn't linger retrying something
// long gone.
HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
index 96f4180..5fa9633 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
@@ -39,8 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -393,6 +394,54 @@ public class TestFSUtils {
verifyFileInDirWithStoragePolicy("1772");
}
+ @Test
+ public void testSetWALRootDir() throws Exception {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Configuration conf = htu.getConfiguration();
+ Path p = new Path("file:///hbase/root");
+ FSUtils.setWALRootDir(conf, p);
+ assertEquals(p.toString(), conf.get(HFileSystem.HBASE_WAL_DIR));
+ }
+
+ @Test
+ public void testGetWALRootDir() throws IOException {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Configuration conf = htu.getConfiguration();
+ Path root = new Path("file:///hbase/root");
+ Path walRoot = new Path("file:///hbase/logroot");
+ FSUtils.setRootDir(conf, root);
+ assertEquals(FSUtils.getRootDir(conf), root);
+ assertEquals(FSUtils.getWALRootDir(conf), root);
+ FSUtils.setWALRootDir(conf, walRoot);
+ assertEquals(FSUtils.getWALRootDir(conf), walRoot);
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testGetWALRootDirIllegalWALDir() throws IOException {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Configuration conf = htu.getConfiguration();
+ Path root = new Path("file:///hbase/root");
+ Path invalidWALDir = new Path("file:///hbase/root/logroot");
+ FSUtils.setRootDir(conf, root);
+ FSUtils.setWALRootDir(conf, invalidWALDir);
+ FSUtils.getWALRootDir(conf);
+ }
+
+ @Test
+ public void testRemoveWALRootPath() throws Exception {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Configuration conf = htu.getConfiguration();
+ FSUtils.setRootDir(conf, new Path("file:///user/hbase"));
+ Path testFile = new Path(FSUtils.getRootDir(conf), "test/testfile");
+ Path tmpFile = new Path("file:///test/testfile");
+ assertEquals(FSUtils.removeWALRootPath(testFile, conf), "test/testfile");
+ assertEquals(FSUtils.removeWALRootPath(tmpFile, conf), tmpFile.toString());
+ FSUtils.setWALRootDir(conf, new Path("file:///user/hbaseLogDir"));
+ assertEquals(FSUtils.removeWALRootPath(testFile, conf), testFile.toString());
+ Path logFile = new Path(FSUtils.getWALRootDir(conf), "test/testlog");
+ assertEquals(FSUtils.removeWALRootPath(logFile, conf), "test/testlog");
+ }
+
private void cleanupFile(FileSystem fileSys, Path name) throws IOException {
assertTrue(fileSys.exists(name));
assertTrue(fileSys.delete(name, true));
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
index 6b1ca03..8e8e73a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
@@ -101,7 +101,7 @@ public class IOTestProvider implements WALProvider {
providerId = DEFAULT_PROVIDER_ID;
}
final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
- log = new IOTestWAL(FileSystem.get(conf), FSUtils.getRootDir(conf),
+ log = new IOTestWAL(FSUtils.getWALFileSystem(conf), FSUtils.getWALRootDir(conf),
DefaultWALProvider.getWALDirectoryName(factory.factoryId),
HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners,
true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
index e928a4d..fa88294 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
@@ -66,6 +66,8 @@ public class TestDefaultWALProvider {
protected static Configuration conf;
protected static FileSystem fs;
+ protected static FileSystem walFs;
+ protected static Path walRootDir;
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected MultiVersionConcurrencyControl mvcc;
@@ -79,6 +81,7 @@ public class TestDefaultWALProvider {
for (FileStatus dir : entries) {
fs.delete(dir.getPath(), true);
}
+ walFs.delete(walRootDir, true);
}
@After
@@ -104,13 +107,15 @@ public class TestDefaultWALProvider {
TEST_UTIL.startMiniDFSCluster(3);
// Set up a working space for our tests.
- TEST_UTIL.createRootDir();
+ walRootDir = TEST_UTIL.createWALRootDir();
conf = TEST_UTIL.getConfiguration();
- fs = TEST_UTIL.getDFSCluster().getFileSystem();
+ fs = FSUtils.getRootDirFileSystem(conf);
+ walFs = FSUtils.getWALFileSystem(conf);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
+ walFs.delete(walRootDir, true);
TEST_UTIL.shutdownMiniCluster();
}
@@ -121,13 +126,12 @@ public class TestDefaultWALProvider {
@Test
public void testGetServerNameFromWALDirectoryName() throws IOException {
ServerName sn = ServerName.valueOf("hn", 450, 1398);
- String hl = FSUtils.getRootDir(conf) + "/" +
+ String hl = walRootDir + "/" +
DefaultWALProvider.getWALDirectoryName(sn.toString());
// Must not throw exception
assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, null));
- assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
- FSUtils.getRootDir(conf).toUri().toString()));
+ assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, walRootDir.toUri().toString()));
assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, ""));
assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, " "));
assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl));
@@ -136,7 +140,7 @@ public class TestDefaultWALProvider {
final String wals = "/WALs/";
ServerName parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
- FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
+ walRootDir.toUri().toString() + wals + sn +
"/localhost%2C32984%2C1343316388997.1343316390417");
assertEquals("standard", sn, parsed);
@@ -144,7 +148,7 @@ public class TestDefaultWALProvider {
assertEquals("subdir", sn, parsed);
parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
- FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
+ walRootDir.toUri().toString() + wals + sn +
"-splitting/localhost%3A57020.1340474893931");
assertEquals("split", sn, parsed);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index 878bb32..9b4a968 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -84,6 +84,7 @@ public class TestWALFactory {
private static MiniDFSCluster cluster;
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected static Path hbaseDir;
+ protected static Path hbaseWALDir;
protected FileSystem fs;
protected Path dir;
@@ -142,6 +143,7 @@ public class TestWALFactory {
cluster = TEST_UTIL.getDFSCluster();
hbaseDir = TEST_UTIL.createRootDir();
+ hbaseWALDir = TEST_UTIL.createWALRootDir();
}
@AfterClass
@@ -164,12 +166,12 @@ public class TestWALFactory {
final TableName tableName = TableName.valueOf(currentTest.getMethodName());
final byte [] rowName = tableName.getName();
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
- final Path logdir = new Path(hbaseDir,
+ final Path logdir = new Path(hbaseWALDir,
DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
- Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ Path oldLogDir = new Path(hbaseWALDir, HConstants.HREGION_OLDLOGDIR_NAME);
final int howmany = 3;
HRegionInfo[] infos = new HRegionInfo[3];
- Path tabledir = FSUtils.getTableDir(hbaseDir, tableName);
+ Path tabledir = FSUtils.getTableDir(hbaseWALDir, tableName);
fs.mkdirs(tabledir);
for(int i = 0; i < howmany; i++) {
infos[i] = new HRegionInfo(tableName,
@@ -203,7 +205,7 @@ public class TestWALFactory {
}
}
wals.shutdown();
- List<Path> splits = WALSplitter.split(hbaseDir, logdir, oldLogDir, fs, conf, wals);
+ List<Path> splits = WALSplitter.split(hbaseWALDir, logdir, oldLogDir, fs, conf, wals);
verifySplits(splits, howmany);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java
new file mode 100644
index 0000000..e4c0ae8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+@Category(MediumTests.class)
+public class TestWALRootDir {
+ private static final Log LOG = LogFactory.getLog(TestWALRootDir.class);
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static Configuration conf;
+ private static FileSystem fs;
+ private static FileSystem walFs;
+ static final TableName tableName = TableName.valueOf("TestWALWALDir");
+ private static final byte [] rowName = Bytes.toBytes("row");
+ private static final byte [] family = Bytes.toBytes("column");
+ private static HTableDescriptor htd;
+ private static Path walRootDir;
+ private static Path rootDir;
+ private static WALFactory wals;
+
+ @Before
+ public void setUp() throws Exception {
+ cleanup();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+ TEST_UTIL.startMiniDFSCluster(1);
+ rootDir = TEST_UTIL.createRootDir();
+ walRootDir = TEST_UTIL.createWALRootDir();
+ fs = FSUtils.getRootDirFileSystem(conf);
+ walFs = FSUtils.getWALFileSystem(conf);
+ htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(family));
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cleanup();
+ TEST_UTIL.shutdownMiniDFSCluster();
+ }
+
+ @Test
+ public void testWALRootDir() throws Exception {
+ HRegionInfo regionInfo = new HRegionInfo(tableName);
+ wals = new WALFactory(conf, null, "testWALRootDir");
+ WAL log = wals.getWAL(regionInfo.getEncodedNameAsBytes(), regionInfo.getTable().getNamespace());
+
+ assertEquals(1, getWALFiles(walFs, walRootDir).size());
+ byte [] value = Bytes.toBytes("value");
+ WALEdit edit = new WALEdit();
+ edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"),
+ System.currentTimeMillis(), value));
+ long txid = log.append(htd,regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 0), edit, true);
+ log.sync(txid);
+ assertEquals("Expect 1 log have been created", 1, getWALFiles(walFs, walRootDir).size());
+ log.rollWriter();
+ //Create 1 more WAL
+ assertEquals(2, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
+ edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"),
+ System.currentTimeMillis(), value));
+ txid = log.append(htd, regionInfo, getWalKey(System.currentTimeMillis(), regionInfo, 1), edit, true);
+ log.sync(txid);
+ log.rollWriter();
+ log.shutdown();
+
+ assertEquals("Expect 3 logs in WALs dir", 3, getWALFiles(walFs, new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME)).size());
+ }
+
+ protected WALKey getWalKey(final long time, HRegionInfo hri, final long startPoint) {
+ return new WALKey(hri.getEncodedNameAsBytes(), tableName, time, new MultiVersionConcurrencyControl(startPoint));
+ }
+
+ private List<FileStatus> getWALFiles(FileSystem fs, Path dir)
+ throws IOException {
+ List<FileStatus> result = new ArrayList<FileStatus>();
+ LOG.debug("Scanning " + dir.toString() + " for WAL files");
+
+ FileStatus[] files = fs.listStatus(dir);
+ if (files == null) return Collections.emptyList();
+ for (FileStatus file : files) {
+ if (file.isDirectory()) {
+ // recurse into sub directories
+ result.addAll(getWALFiles(fs, file.getPath()));
+ } else {
+ String name = file.getPath().toString();
+ if (!name.startsWith(".")) {
+ result.add(file);
+ }
+ }
+ }
+ return result;
+ }
+
+ private static void cleanup() throws Exception{
+ walFs.delete(walRootDir, true);
+ fs.delete(rootDir, true);
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index 4c77d25..3ae155c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -118,6 +118,7 @@ public class TestWALSplit {
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Path HBASEDIR;
+ private Path HBASELOGDIR;
private Path WALDIR;
private Path OLDLOGDIR;
private Path CORRUPTDIR;
@@ -180,8 +181,9 @@ public class TestWALSplit {
LOG.info("Cleaning up cluster for new test.");
fs = TEST_UTIL.getDFSCluster().getFileSystem();
HBASEDIR = TEST_UTIL.createRootDir();
- OLDLOGDIR = new Path(HBASEDIR, HConstants.HREGION_OLDLOGDIR_NAME);
- CORRUPTDIR = new Path(HBASEDIR, HConstants.CORRUPT_DIR_NAME);
+ HBASELOGDIR = TEST_UTIL.createWALRootDir();
+ OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME);
+ CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME);
TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);
REGIONS.clear();
Collections.addAll(REGIONS, "bbb", "ccc");
@@ -189,7 +191,7 @@ public class TestWALSplit {
this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
wals = new WALFactory(conf, null, name.getMethodName());
- WALDIR = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(name.getMethodName()));
+ WALDIR = new Path(HBASELOGDIR, DefaultWALProvider.getWALDirectoryName(name.getMethodName()));
//fs.mkdirs(WALDIR);
}
@@ -205,6 +207,7 @@ public class TestWALSplit {
} finally {
wals = null;
fs.delete(HBASEDIR, true);
+ fs.delete(HBASELOGDIR, true);
}
}
@@ -1111,7 +1114,7 @@ public class TestWALSplit {
useDifferentDFSClient();
WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);
- final Path corruptDir = new Path(FSUtils.getRootDir(conf), HConstants.CORRUPT_DIR_NAME);
+ final Path corruptDir = new Path(FSUtils.getWALRootDir(conf), HConstants.CORRUPT_DIR_NAME);
assertEquals(1, fs.listStatus(corruptDir).length);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/979438d2/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index 8af1882..5a3dd87 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -346,7 +346,7 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
}
if (verify) {
LOG.info("verifying written log entries.");
- Path dir = new Path(FSUtils.getRootDir(getConf()),
+ Path dir = new Path(FSUtils.getWALRootDir(getConf()),
DefaultWALProvider.getWALDirectoryName("wals"));
long editCount = 0;
FileStatus [] fsss = fs.listStatus(dir);