You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/12/17 06:20:46 UTC
hbase git commit: HBASE-12645 HBaseTestingUtility is using ${$HOME}
for rootDir (Varun Saxena)
Repository: hbase
Updated Branches:
refs/heads/branch-1 063570535 -> e82f048d7
HBASE-12645 HBaseTestingUtility is using ${$HOME} for rootDir (Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e82f048d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e82f048d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e82f048d
Branch: refs/heads/branch-1
Commit: e82f048d7b17b871f813bf6f4aa7ecda226d1b19
Parents: 0635705
Author: stack <st...@apache.org>
Authored: Tue Dec 16 16:41:27 2014 -0800
Committer: stack <st...@apache.org>
Committed: Tue Dec 16 21:20:32 2014 -0800
----------------------------------------------------------------------
.../hadoop/hbase/HBaseTestingUtility.java | 122 ++++++++++++++++---
.../hbase/client/TestTableSnapshotScanner.java | 2 +-
.../TableSnapshotInputFormatTestBase.java | 2 +-
.../replication/TestReplicationSource.java | 8 +-
.../apache/hadoop/hbase/util/TestMergeTool.java | 5 +-
5 files changed, 111 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index bf61208..5a72965 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -444,7 +444,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
- * Sets up a path in test filesystem to be used by tests
+ * Sets up a path in test filesystem to be used by tests.
+ * Creates a new directory if not already setup.
*/
private void setupDataTestDirOnTestFS() throws IOException {
if (dataTestDirOnTestFS != null) {
@@ -452,22 +453,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
+ dataTestDirOnTestFS.toString());
return;
}
+ dataTestDirOnTestFS = getNewDataTestDirOnTestFS();
+ }
+ /**
+ * Sets up a new path in test filesystem to be used by tests.
+ */
+ private Path getNewDataTestDirOnTestFS() throws IOException {
//The file system can be either local, mini dfs, or if the configuration
//is supplied externally, it can be an external cluster FS. If it is a local
//file system, the tests should use getBaseTestDir, otherwise, we can use
//the working directory, and create a unique sub dir there
FileSystem fs = getTestFileSystem();
+ Path newDataTestDir = null;
if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
File dataTestDir = new File(getDataTestDir().toString());
if (deleteOnExit()) dataTestDir.deleteOnExit();
- dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath());
+ newDataTestDir = new Path(dataTestDir.getAbsolutePath());
} else {
Path base = getBaseTestDirOnTestFS();
String randomStr = UUID.randomUUID().toString();
- dataTestDirOnTestFS = new Path(base, randomStr);
- if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS);
+ newDataTestDir = new Path(base, randomStr);
+ if (deleteOnExit()) fs.deleteOnExit(newDataTestDir);
}
+ return newDataTestDir;
}
/**
@@ -742,6 +751,19 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
+ * Start up a minicluster of hbase, dfs, and zookeeper.
+ * Set the <code>create</code> flag to create root or data directory path or not
+ * (will overwrite if dir already exists)
+ * @throws Exception
+ * @return Mini hbase cluster instance created.
+ * @see {@link #shutdownMiniDFSCluster()}
+ */
+ public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create)
+ throws Exception {
+ return startMiniCluster(1, numSlaves, create);
+ }
+
+ /**
* Start up a minicluster of hbase, optionally dfs, and zookeeper.
* Modifies Configuration. Homes the cluster data directory under a random
* subdirectory in a directory under System property test.build.data.
@@ -756,9 +778,21 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
*/
public MiniHBaseCluster startMiniCluster(final int numSlaves)
throws Exception {
- return startMiniCluster(1, numSlaves);
+ return startMiniCluster(1, numSlaves, false);
}
+ /**
+ * Start minicluster. Whether to create a new root or data dir path even if such a path
+ * has been created earlier is decided based on flag <code>create</code>
+ * @throws Exception
+ * @see {@link #shutdownMiniCluster()}
+ * @return Mini hbase cluster instance created.
+ */
+ public MiniHBaseCluster startMiniCluster(final int numMasters,
+ final int numSlaves, boolean create)
+ throws Exception {
+ return startMiniCluster(numMasters, numSlaves, null, create);
+ }
/**
* start minicluster
@@ -769,7 +803,14 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
public MiniHBaseCluster startMiniCluster(final int numMasters,
final int numSlaves)
throws Exception {
- return startMiniCluster(numMasters, numSlaves, null);
+ return startMiniCluster(numMasters, numSlaves, null, false);
+ }
+
+ public MiniHBaseCluster startMiniCluster(final int numMasters,
+ final int numSlaves, final String[] dataNodeHosts, boolean create)
+ throws Exception {
+ return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
+ null, null, create);
}
/**
@@ -798,7 +839,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
*/
public MiniHBaseCluster startMiniCluster(final int numMasters,
final int numSlaves, final String[] dataNodeHosts) throws Exception {
- return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, null, null);
+ return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
+ null, null);
}
/**
@@ -845,15 +887,27 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
}
+ public MiniHBaseCluster startMiniCluster(final int numMasters,
+ final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
+ Class<? extends HMaster> masterClass,
+ Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+ throws Exception {
+ return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts,
+ masterClass, regionserverClass, false);
+ }
+
/**
* Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom
* number of datanodes.
* @param numDataNodes Number of data nodes.
+ * @param create Set this flag to create a new
+ * root or data directory path or not (will overwrite if exists already).
*/
public MiniHBaseCluster startMiniCluster(final int numMasters,
final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
Class<? extends HMaster> masterClass,
- Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+ Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
+ boolean create)
throws Exception {
if (dataNodeHosts != null && dataNodeHosts.length != 0) {
numDataNodes = dataNodeHosts.length;
@@ -881,12 +935,13 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
// Start the MiniHBaseCluster
- return startMiniHBaseCluster(numMasters, numSlaves, masterClass, regionserverClass);
+ return startMiniHBaseCluster(numMasters, numSlaves, masterClass,
+ regionserverClass, create);
}
public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
throws IOException, InterruptedException{
- return startMiniHBaseCluster(numMasters, numSlaves, null, null);
+ return startMiniHBaseCluster(numMasters, numSlaves, null, null, false);
}
/**
@@ -895,6 +950,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Usually you won't want this. You'll usually want {@link #startMiniCluster()}.
* @param numMasters
* @param numSlaves
+ * @param create Whether to create a
+ * root or data directory path or not; will overwrite if exists already.
* @return Reference to the hbase mini hbase cluster.
* @throws IOException
* @throws InterruptedException
@@ -902,10 +959,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
*/
public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
final int numSlaves, Class<? extends HMaster> masterClass,
- Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+ Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
+ boolean create)
throws IOException, InterruptedException {
// Now do the mini hbase cluster. Set the hbase.rootdir in config.
- createRootDir();
+ createRootDir(create);
// These settings will make the server waits until this exact number of
// regions servers are connected.
@@ -1028,14 +1086,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
- * Returns the path to the default root dir the minicluster uses.
+ * Returns the path to the default root dir the minicluster uses. If getNewDirPathIfExists
+ * is true, a new root directory path is fetched irrespective of whether it has been fetched
+ * before or not. If false, previous path is used.
+ * Note: this does not cause the root dir to be created.
+ * @return Fully qualified path for the default hbase root dir
+ * @throws IOException
+ */
+ public Path getDefaultRootDirPath(boolean create) throws IOException {
+ if (!create) {
+ return getDataTestDirOnTestFS();
+ } else {
+ return getNewDataTestDirOnTestFS();
+ }
+ }
+
+ /**
+ * Same as {{@link HBaseTestingUtility#getDefaultRootDirPath(boolean getNewDirPathIfExists)}
+ * except that getNewDirPathIfExists flag is false.
* Note: this does not cause the root dir to be created.
* @return Fully qualified path for the default hbase root dir
* @throws IOException
*/
public Path getDefaultRootDirPath() throws IOException {
- FileSystem fs = FileSystem.get(this.conf);
- return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+ return getDefaultRootDirPath(false);
}
/**
@@ -1043,12 +1117,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* version file. Normally you won't make use of this method. Root hbasedir
* is created for you as part of mini cluster startup. You'd only use this
* method if you were doing manual operation.
+ * @param getNewDirPathIfExists This flag decides whether to get a new
+ * root or data directory path or not, if it has been fetched already.
+ * Note : Directory will be made irrespective of whether path has been fetched or not.
+ * If directory already exists, it will be overwritten
* @return Fully qualified path to hbase root dir
* @throws IOException
*/
- public Path createRootDir() throws IOException {
+ public Path createRootDir(boolean getNewDirPathIfExists) throws IOException {
FileSystem fs = FileSystem.get(this.conf);
- Path hbaseRootdir = getDefaultRootDirPath();
+ Path hbaseRootdir = getDefaultRootDirPath(getNewDirPathIfExists);
FSUtils.setRootDir(this.conf, hbaseRootdir);
fs.mkdirs(hbaseRootdir);
FSUtils.setVersion(fs, hbaseRootdir);
@@ -1056,6 +1134,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
+ * Same as {@link HBaseTestingUtility#createRootDir(boolean getNewDirPathIfExists)}
+ * except that getNewDirPathIfExists flag is false.
+ * @return Fully qualified path to hbase root dir
+ * @throws IOException
+ */
+ public Path createRootDir() throws IOException {
+ return createRootDir(false);
+ }
+
+ /**
* Flushes all caches in the mini hbase cluster
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
index 10f7b9f..3e915e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
@@ -56,7 +56,7 @@ public class TestTableSnapshotScanner {
public void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
- UTIL.startMiniCluster(NUM_REGION_SERVERS);
+ UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
fs = rootDir.getFileSystem(UTIL.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 7c55ad0..1f8ccae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -50,7 +50,7 @@ public abstract class TableSnapshotInputFormatTestBase {
public void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
- UTIL.startMiniCluster(NUM_REGION_SERVERS);
+ UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
fs = rootDir.getFileSystem(UTIL.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
index 1fb0312..458819d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
@@ -62,12 +62,10 @@ public class TestReplicationSource {
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniDFSCluster(1);
FS = TEST_UTIL.getDFSCluster().getFileSystem();
- FSUtils.setRootDir(TEST_UTIL.getConfiguration(), FS.getHomeDirectory());
- oldLogDir = new Path(FS.getHomeDirectory(),
- HConstants.HREGION_OLDLOGDIR_NAME);
+ Path rootDir = TEST_UTIL.createRootDir();
+ oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true);
- logDir = new Path(FS.getHomeDirectory(),
- HConstants.HREGION_LOGDIR_NAME);
+ logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
if (FS.exists(logDir)) FS.delete(logDir, true);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e82f048d/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
index ca09b39..d4c7fa8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java
@@ -135,10 +135,7 @@ public class TestMergeTool extends HBaseTestCase {
this.fs = this.dfsCluster.getFileSystem();
System.out.println("fs=" + this.fs);
FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
- Path parentdir = fs.getHomeDirectory();
- FSUtils.setRootDir(conf, parentdir);
- fs.mkdirs(parentdir);
- FSUtils.setVersion(fs, parentdir);
+ TEST_UTIL.createRootDir();
// Note: we must call super.setUp after starting the mini cluster or
// we will end up with a local file system