You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/12/14 00:15:08 UTC
[40/50] [abbrv] hbase git commit: HBASE-20723 Custom hbase.wal.dir
results in data loss because we write recovered edits into a different place
than where the recovering region server looks for them
HBASE-20723 Custom hbase.wal.dir results in data loss because we write recovered edits into a different place than where the recovering region server looks for them
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2702f23a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2702f23a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2702f23a
Branch: refs/heads/branch-1.3
Commit: 2702f23a4addaa459d6f221c0008be0c556f672d
Parents: 979438d
Author: tedyu <yu...@gmail.com>
Authored: Sat Jun 16 10:34:47 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Dec 12 19:12:42 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/wal/DefaultWALProvider.java | 8 +--
.../apache/hadoop/hbase/wal/WALSplitter.java | 60 ++++++++--------
.../hbase/regionserver/wal/TestWALReplay.java | 72 ++++++++++----------
.../apache/hadoop/hbase/wal/TestWALFactory.java | 3 +-
.../apache/hadoop/hbase/wal/TestWALSplit.java | 8 +--
5 files changed, 77 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2702f23a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index 01234a1..9cb5da0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -18,9 +18,6 @@
*/
package org.apache.hadoop.hbase.wal;
-import java.io.Closeable;
-import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -370,8 +367,9 @@ public class DefaultWALProvider implements WALProvider {
ProtobufLogWriter.class, Writer.class);
Writer writer = null;
try {
- writer = logWriterClass.newInstance();
- writer.init(fs, path, conf, overwritable);
+ writer = logWriterClass.getDeclaredConstructor().newInstance();
+ FileSystem rootFs = FileSystem.get(path.toUri(), conf);
+ writer.init(rootFs, path, conf, overwritable);
return writer;
} catch (Exception e) {
LOG.debug("Error instantiating log writer.", e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/2702f23a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 50bb79f..c88e6d3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -140,7 +140,7 @@ public class WALSplitter {
public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
// Parameters for split process
- protected final Path rootDir;
+ protected final Path walDir;
protected final FileSystem fs;
protected final Configuration conf;
@@ -187,14 +187,14 @@ public class WALSplitter {
public final static String SPLIT_WRITER_CREATION_BOUNDED = "hbase.split.writer.creation.bounded";
@VisibleForTesting
- WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,
+ WALSplitter(final WALFactory factory, Configuration conf, Path walDir,
FileSystem fs, LastSequenceId idChecker,
CoordinatedStateManager csm, RecoveryMode mode) {
this.conf = HBaseConfiguration.create(conf);
String codecClassName = conf
.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);
- this.rootDir = rootDir;
+ this.walDir = walDir;
this.fs = fs;
this.sequenceIdChecker = idChecker;
this.csm = (BaseCoordinatedStateManager)csm;
@@ -244,10 +244,10 @@ public class WALSplitter {
* @return false if it is interrupted by the progress-able.
* @throws IOException
*/
- public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,
+ public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,
Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,
CoordinatedStateManager cp, RecoveryMode mode, final WALFactory factory) throws IOException {
- WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker, cp, mode);
+ WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker, cp, mode);
return s.splitLogFile(logfile, reporter);
}
@@ -315,7 +315,7 @@ public class WALSplitter {
in = getReader(logfile, skipErrors, reporter);
} catch (CorruptedLogFileException e) {
LOG.warn("Could not get reader, corrupted log file " + logPath, e);
- ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);
+ ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);
isCorrupted = true;
}
if (in == null) {
@@ -406,7 +406,7 @@ public class WALSplitter {
throw iie;
} catch (CorruptedLogFileException e) {
LOG.warn("Could not parse, corrupted log file " + logPath, e);
- csm.getSplitLogWorkerCoordination().markCorrupted(rootDir,
+ csm.getSplitLogWorkerCoordination().markCorrupted(walDir,
logfile.getPath().getName(), fs);
isCorrupted = true;
} catch (IOException e) {
@@ -537,18 +537,19 @@ public class WALSplitter {
* <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.
* This method also ensures existence of RECOVERED_EDITS_DIR under the region
* creating it if necessary.
- * @param fs
* @param logEntry
- * @param rootDir HBase root dir.
- * @param fileBeingSplit the file being split currently. Used to generate tmp file name.
+ * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.
+ * @param conf
* @return Path to file into which to dump split log edits.
* @throws IOException
*/
@SuppressWarnings("deprecation")
@VisibleForTesting
- static Path getRegionSplitEditsPath(final FileSystem fs,
- final Entry logEntry, final Path rootDir, String fileBeingSplit)
+ static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,
+ Configuration conf)
throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ Path rootDir = FSUtils.getRootDir(conf);
Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTablename());
String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());
Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);
@@ -582,7 +583,7 @@ public class WALSplitter {
// Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
// region's replayRecoveredEdits will not delete it
String fileName = formatRecoveredEditsFileName(logEntry.getKey().getLogSeqNum());
- fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileBeingSplit);
+ fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);
return new Path(dir, fileName);
}
@@ -1354,7 +1355,8 @@ public class WALSplitter {
}
// delete the one with fewer wal entries
- void deleteOneWithFewerEntries(WriterAndPath wap, Path dst) throws IOException {
+ void deleteOneWithFewerEntries(FileSystem rootFs, WriterAndPath wap, Path dst)
+ throws IOException {
long dstMinLogSeqNum = -1L;
try (WAL.Reader reader = walFactory.createReader(fs, dst)) {
WAL.Entry entry = reader.next();
@@ -1378,8 +1380,8 @@ public class WALSplitter {
}
} else {
LOG.warn("Found existing old edits file and we have less entries. Deleting " + wap.p
- + ", length=" + fs.getFileStatus(wap.p).getLen());
- if (!fs.delete(wap.p, false)) {
+ + ", length=" + rootFs.getFileStatus(wap.p).getLen());
+ if (!rootFs.delete(wap.p, false)) {
LOG.warn("Failed deleting of " + wap.p);
throw new IOException("Failed deleting of " + wap.p);
}
@@ -1463,6 +1465,7 @@ public class WALSplitter {
if (LOG.isTraceEnabled()) {
LOG.trace("Closing " + wap.p);
}
+ FileSystem rootFs = FileSystem.get(conf);
try {
wap.w.close();
} catch (IOException ioe) {
@@ -1477,7 +1480,7 @@ public class WALSplitter {
}
if (wap.editsWritten == 0) {
// just remove the empty recovered.edits file
- if (fs.exists(wap.p) && !fs.delete(wap.p, false)) {
+ if (rootFs.exists(wap.p) && !rootFs.delete(wap.p, false)) {
LOG.warn("Failed deleting empty " + wap.p);
throw new IOException("Failed deleting empty " + wap.p);
}
@@ -1487,14 +1490,14 @@ public class WALSplitter {
Path dst = getCompletedRecoveredEditsFilePath(wap.p,
regionMaximumEditLogSeqNum.get(encodedRegionName));
try {
- if (!dst.equals(wap.p) && fs.exists(dst)) {
- deleteOneWithFewerEntries(wap, dst);
+ if (!dst.equals(wap.p) && rootFs.exists(dst)) {
+ deleteOneWithFewerEntries(rootFs, wap, dst);
}
// Skip the unit tests which create a splitter that reads and
// writes the data without touching disk.
// TestHLogSplit#testThreading is an example.
- if (fs.exists(wap.p)) {
- if (!fs.rename(wap.p, dst)) {
+ if (rootFs.exists(wap.p)) {
+ if (!rootFs.rename(wap.p, dst)) {
throw new IOException("Failed renaming " + wap.p + " to " + dst);
}
LOG.info("Rename " + wap.p + " to " + dst);
@@ -1568,7 +1571,7 @@ public class WALSplitter {
if (blacklistedRegions.contains(region)) {
return null;
}
- ret = createWAP(region, entry, rootDir);
+ ret = createWAP(region, entry);
if (ret == null) {
blacklistedRegions.add(region);
return null;
@@ -1583,17 +1586,18 @@ public class WALSplitter {
/**
* @return a path with a write for that path. caller should close.
*/
- WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir) throws IOException {
- Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir,
- fileBeingSplit.getPath().getName());
+ WriterAndPath createWAP(byte[] region, Entry entry) throws IOException {
+ Path regionedits = getRegionSplitEditsPath(entry,
+ fileBeingSplit.getPath().getName(), conf);
if (regionedits == null) {
return null;
}
- if (fs.exists(regionedits)) {
+ FileSystem rootFs = FileSystem.get(conf);
+ if (rootFs.exists(regionedits)) {
LOG.warn("Found old edits file. It could be the "
+ "result of a previous failed split attempt. Deleting " + regionedits + ", length="
- + fs.getFileStatus(regionedits).getLen());
- if (!fs.delete(regionedits, false)) {
+ + rootFs.getFileStatus(regionedits).getLen());
+ if (!rootFs.delete(regionedits, false)) {
LOG.warn("Failed delete of old " + regionedits);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2702f23a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 2622f6d..a196f57 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -307,11 +307,11 @@ public class TestWALReplay {
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
+ Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
deleteDir(basedir);
HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
+ HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
final byte [] rowName = tableName.getName();
@@ -337,7 +337,7 @@ public class TestWALReplay {
WAL wal3 = createWAL(this.conf);
try {
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal3);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
long seqid = region.getOpenSeqNum();
// The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
// When opened, this region would apply 6k edits, and increment the sequenceId by 1
@@ -369,13 +369,13 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString());
+ final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region2 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
+ HRegion region2 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
byte [] family = htd.getFamilies().iterator().next().getName();
Path f = new Path(basedir, "hfile");
@@ -404,7 +404,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(newConf);
HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
- hbaseWALRootDir, hri, htd, wal2);
+ hbaseRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid2 > -1);
assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
@@ -435,14 +435,14 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testCompactedBulkLoadedFiles");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = new Path(this.hbaseWALRootDir, tableName.getNameAsString());
+ final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region2 = HRegion.createHRegion(hri,
- hbaseWALRootDir, this.conf, htd);
+ hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
// Add an edit so something in the WAL
byte [] row = tableName.getName();
@@ -476,7 +476,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(newConf);
HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
- hbaseWALRootDir, hri, htd, wal2);
+ hbaseRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid2 > -1);
assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
@@ -506,19 +506,19 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsWrittenViaHRegion");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region3 = HRegion.createHRegion(hri,
- hbaseWALRootDir, this.conf, htd);
+ hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region3);
// Write countPerFamily edits into the three families. Do a flush on one
// of the families during the load of edits so its seqid is not same as
// others to test we do right thing when different seqids.
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
long seqid = region.getOpenSeqNum();
boolean first = true;
for (HColumnDescriptor hcd: htd.getFamilies()) {
@@ -541,7 +541,7 @@ public class TestWALReplay {
wal.shutdown();
runWALSplit(this.conf);
WAL wal2 = createWAL(this.conf);
- HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal2);
+ HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid + result.size() < seqid2);
final Result result1b = region2.get(g);
@@ -616,19 +616,19 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsWrittenViaHRegion");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region3 = HRegion.createHRegion(hri,
- hbaseWALRootDir, this.conf, htd);
+ hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region3);
// Write countPerFamily edits into the three families. Do a flush on one
// of the families during the load of edits so its seqid is not same as
// others to test we do right thing when different seqids.
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
long seqid = region.getOpenSeqNum();
for (HColumnDescriptor hcd: htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
@@ -661,7 +661,7 @@ public class TestWALReplay {
// Let us try to split and recover
runWALSplit(this.conf);
WAL wal2 = createWAL(this.conf);
- HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal2);
+ HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
long seqid2 = region2.getOpenSeqNum();
assertTrue(seqid + result.size() < seqid2);
@@ -701,10 +701,10 @@ public class TestWALReplay {
final TableName tableName =
TableName.valueOf("testReplayEditsAfterAbortingFlush");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
- HRegion region3 = HRegion.createHRegion(hri, hbaseWALRootDir, this.conf, htd);
+ HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
region3.close();
region3.getWAL().close();
// Write countPerFamily edits into the three families. Do a flush on one
@@ -718,7 +718,7 @@ public class TestWALReplay {
customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
CustomStoreFlusher.class.getName());
HRegion region =
- HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal, customConf, rsServices, null);
+ HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
int writtenRowCount = 10;
List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
htd.getFamilies());
@@ -772,7 +772,7 @@ public class TestWALReplay {
WAL wal2 = createWAL(this.conf);
Mockito.doReturn(false).when(rsServices).isAborted();
HRegion region2 =
- HRegion.openHRegion(this.hbaseWALRootDir, hri, htd, wal2, this.conf, rsServices, null);
+ HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
scanner = region2.getScanner(new Scan());
assertEquals(writtenRowCount, getScannedCount(scanner));
}
@@ -802,12 +802,12 @@ public class TestWALReplay {
TableName.valueOf("testReplayEditsWrittenIntoWAL");
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
HRegion region2 = HRegion.createHRegion(hri,
- hbaseWALRootDir, this.conf, htd);
+ hbaseRootDir, this.conf, htd);
HRegion.closeHRegion(region2);
final WAL wal = createWAL(this.conf);
final byte[] rowName = tableName.getName();
@@ -901,7 +901,7 @@ public class TestWALReplay {
final TableName tableName = TableName.valueOf(currentTest.getMethodName());
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
final Path basedir =
- FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
+ FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
@@ -910,7 +910,7 @@ public class TestWALReplay {
// Mock the WAL
MockWAL wal = createMockWAL();
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
for (HColumnDescriptor hcd : htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
}
@@ -934,7 +934,7 @@ public class TestWALReplay {
WALSplitter.splitLogFile(hbaseWALRootDir, listStatus[0],
this.fs, this.conf, null, null, null, mode, wals);
FileStatus[] listStatus1 = this.fs.listStatus(
- new Path(FSUtils.getTableDir(hbaseWALRootDir, tableName), new Path(hri.getEncodedName(),
+ new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
"recovered.edits")), new PathFilter() {
@Override
public boolean accept(Path p) {
@@ -962,17 +962,17 @@ public class TestWALReplay {
IllegalAccessException {
final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(this.hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
final byte[] rowName = tableName.getName();
final int countPerFamily = 10;
final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
- HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd);
+ HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.hbaseWALRootDir, this.conf, htd);
Path regionDir = region1.getRegionFileSystem().getRegionDir();
HBaseTestingUtility.closeRegionAndWAL(region1);
WAL wal = createWAL(this.conf);
- HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
for (HColumnDescriptor hcd : htd.getFamilies()) {
addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
}
@@ -1042,12 +1042,12 @@ public class TestWALReplay {
HRegion region2;
try {
// log replay should fail due to the IOException, otherwise we may lose data.
- region2 = HRegion.openHRegion(conf, spyFs, hbaseWALRootDir, hri, htd, wal2);
+ region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2);
assertEquals(result.size(), region2.get(g).size());
} catch (IOException e) {
assertEquals("read over limit", e.getMessage());
}
- region2 = HRegion.openHRegion(conf, fs, hbaseWALRootDir, hri, htd, wal2);
+ region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2);
assertEquals(result.size(), region2.get(g).size());
}
@@ -1058,11 +1058,11 @@ public class TestWALReplay {
final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
- final Path basedir = FSUtils.getTableDir(hbaseWALRootDir, tableName);
+ final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
deleteDir(basedir);
final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
- HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseWALRootDir, this.hbaseWALRootDir, this.conf, htd);
+ HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.hbaseWALRootDir, this.conf, htd);
HBaseTestingUtility.closeRegionAndWAL(region);
final byte[] family = htd.getColumnFamilies()[0].getName();
final byte[] rowName = tableName.getName();
@@ -1086,7 +1086,7 @@ public class TestWALReplay {
WALSplitter.splitLogFile(hbaseWALRootDir, second, fs, conf, null, null, null,
RecoveryMode.LOG_SPLITTING, wals);
WAL wal = createWAL(this.conf);
- region = HRegion.openHRegion(conf, this.fs, hbaseWALRootDir, hri, htd, wal);
+ region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
assertEquals(2, region.get(new Get(rowName)).size());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2702f23a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index 9b4a968..3c0bade 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -118,6 +118,7 @@ public class TestWALFactory {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ FSUtils.setWALRootDir(TEST_UTIL.getConfiguration(), new Path("file:///tmp/wal"));
// Make block sizes small.
TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
// needed for testAppendClose()
@@ -171,7 +172,7 @@ public class TestWALFactory {
Path oldLogDir = new Path(hbaseWALDir, HConstants.HREGION_OLDLOGDIR_NAME);
final int howmany = 3;
HRegionInfo[] infos = new HRegionInfo[3];
- Path tabledir = FSUtils.getTableDir(hbaseWALDir, tableName);
+ Path tabledir = FSUtils.getTableDir(hbaseDir, tableName);
fs.mkdirs(tabledir);
for(int i = 0; i < howmany; i++) {
infos[i] = new HRegionInfo(tableName,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2702f23a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index 3ae155c..0d0bbec 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -385,8 +385,8 @@ public class TestWALSplit {
new Entry(new WALKey(encoded,
TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
new WALEdit());
- Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,
- FILENAME_BEING_SPLIT);
+ Path p = WALSplitter.getRegionSplitEditsPath(entry,
+ FILENAME_BEING_SPLIT, conf);
String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
}
@@ -411,8 +411,8 @@ public class TestWALSplit {
assertEquals(parent.getName(), HConstants.RECOVERED_EDITS_DIR);
fs.createNewFile(parent); // create a recovered.edits file
- Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,
- FILENAME_BEING_SPLIT);
+ Path p = WALSplitter.getRegionSplitEditsPath(entry,
+ FILENAME_BEING_SPLIT, conf);
String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
WALFactory.createRecoveredEditsWriter(fs, p, conf).close();