You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/04/02 23:03:40 UTC
svn commit: r1584182 - in
/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase:
HBaseTestingUtility.java client/TestHTableClientScanner.java
thrift/TestHBCpp.java
Author: liyin
Date: Wed Apr 2 21:03:40 2014
New Revision: 1584182
URL: http://svn.apache.org/r1584182
Log:
[HBASE-10869] Add FS_TYPE in HBaseTestingUtility configuration
Author: daviddeng
Summary:
Motivation of doing this is two-folded:
1. local file system is more stable than `MiniDFSCluster`, so for cases only testing on HBase logic, using this may avoid unstable problems caused by DFS unstable problem.
1. Starting of local file systme is much faster.
Currently only a few of the testcases is switch to LFS mode for testing. Some other diff may switch more.
Implementaiton
For `FS_TYPE_DFS`, same as before.
For `FS_TYPE_LFS`, use `LocalFileSystem`
Test Plan:
`TestHBCpp`
`TestHTableClientScanner`
Reviewers: liyintang, manukranthk, fan, aaiyer, gauravm
Reviewed By: gauravm
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1238164
Task ID: 3789093
Modified:
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableClientScanner.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestHBCpp.java
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1584182&r1=1584181&r2=1584182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Wed Apr 2 21:03:40 2014
@@ -1,5 +1,5 @@
/**
- * Copyright 2009 The Apache Software Foundation
+ * Copyright 2014 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -19,8 +19,35 @@
*/
package org.apache.hadoop.hbase;
-import com.google.common.base.Preconditions;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Jdk14Logger;
@@ -74,30 +101,7 @@ import org.apache.hadoop.security.UnixUs
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zookeeper.ZooKeeper;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.UnknownHostException;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-
-import static org.junit.Assert.*;
+import com.google.common.base.Preconditions;
/**
* Facility for testing HBase. Added as tool to abet junit4 testing. Replaces
@@ -369,6 +373,11 @@ public class HBaseTestingUtility {
return startMiniCluster(numMasters, numSlaves, MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
}
+ public static final String FS_TYPE_KEY = "mini.cluster.fs.type";
+ public static final String FS_TYPE_DFS = "dfs";
+ public static final String FS_TYPE_LFS = "lfs";
+ public static final String MINICLUSTER_FS_TYPE_DEFAULT = FS_TYPE_DFS;
+
/**
* Start up a minicluster of hbase, optionally dfs, and zookeeper.
* Modifies Configuration. Homes the cluster data directory under a random
@@ -398,16 +407,41 @@ public class HBaseTestingUtility {
// minidfs reads home from system property.
setupClusterTestBuildDir();
System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.getPath());
- // Bring up mini dfs cluster. This spews a bunch of warnings about missing
- // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
- startMiniDFSCluster(numSlaves);
-
- // Mangle conf so fs parameter points to minidfs we just started up
- FileSystem fs = this.dfsCluster.getFileSystem();
- this.conf.set("fs.defaultFS", fs.getUri().toString());
- // Do old style too just to be safe.
- this.conf.set("fs.default.name", fs.getUri().toString());
- this.dfsCluster.waitClusterUp();
+
+ // Initialize the file-system.
+ FileSystem fs = null;
+ Path hbaseRootdir = null;
+
+ String fsType = this.conf.get(FS_TYPE_KEY,
+ MINICLUSTER_FS_TYPE_DEFAULT);
+ switch (fsType) {
+ case FS_TYPE_DFS:
+ // Bring up mini dfs cluster. This spawns a bunch of warnings about
+ // missing scheme. Complaints are 'Scheme is undefined for
+ // build/test/data/dfs/name1'.
+ startMiniDFSCluster(numSlaves);
+ // Mangle conf so fs parameter points to minidfs we just started up
+ fs = this.dfsCluster.getFileSystem();
+ this.conf.set("fs.defaultFS", fs.getUri().toString());
+ // Do old style too just to be safe.
+ this.conf.set("fs.default.name", fs.getUri().toString());
+ this.dfsCluster.waitClusterUp();
+ hbaseRootdir = fs.getHomeDirectory();
+ break;
+
+ case FS_TYPE_LFS:
+ this.conf.set("fs.defaultFS", "file:///");
+ // Do old style too just to be safe.
+ this.conf.set("fs.default.name", "file:///");
+ fs = FileSystem.get(this.conf);
+ hbaseRootdir = new Path("file", "",
+ new File(this.clusterTestBuildDir, "fs").getAbsolutePath());
+ break;
+
+ default:
+ throw new IllegalArgumentException("conf[" + FS_TYPE_KEY + "] = "
+ + fsType);
+ }
// Start up a zk cluster.
if (this.zkCluster == null) {
@@ -415,7 +449,6 @@ public class HBaseTestingUtility {
}
// Now do the mini hbase cluster. Set the hbase.rootdir in config.
- Path hbaseRootdir = fs.makeQualified(fs.getHomeDirectory());
this.conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
fs.mkdirs(hbaseRootdir);
FSUtils.setVersion(fs, hbaseRootdir);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableClientScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableClientScanner.java?rev=1584182&r1=1584181&r2=1584182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableClientScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableClientScanner.java Wed Apr 2 21:03:40 2014
@@ -45,6 +45,9 @@ public class TestHTableClientScanner {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().set(
+ HBaseTestingUtility.FS_TYPE_KEY,
+ HBaseTestingUtility.FS_TYPE_LFS);
TEST_UTIL.startMiniCluster(SLAVES);
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestHBCpp.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestHBCpp.java?rev=1584182&r1=1584181&r2=1584182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestHBCpp.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestHBCpp.java Wed Apr 2 21:03:40 2014
@@ -58,6 +58,9 @@ public class TestHBCpp {
TEST_UTIL.getConfiguration().setBoolean(
HConstants.REGION_SERVER_WRITE_THRIFT_INFO_TO_META, true);
+ TEST_UTIL.getConfiguration().set(HBaseTestingUtility.FS_TYPE_KEY,
+ HBaseTestingUtility.FS_TYPE_LFS);
+
TEST_UTIL.startMiniCluster();
// create the table as SimpleClient assumes.
byte[] tableName = Bytes.toBytes("t1");
@@ -70,7 +73,7 @@ public class TestHBCpp {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
+ @Test(timeout = 120000L)
/**
* Spawn the current version of client unit tests from fbcode.
*/