You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/04/07 00:13:18 UTC
svn commit: r1310612 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/cli...
Author: mbautin
Date: Fri Apr 6 22:13:16 2012
New Revision: 1310612
URL: http://svn.apache.org/viewvc?rev=1310612&view=rev
Log:
[jira] [HBASE-5708] [89-fb] Improving robustness of map-reduce-related and other unit tests
Summary:
Some map-reduce-based tests are failing when executed concurrently in 89-fb
because mini-map-reduce cluster uses /tmp/hadoop-<username> for temporary
data.
Test Plan:
Run unit tests
Run map-reduce-based unit tests 50 times each
Reviewers: kannan, kranganathan, liyintang, pkhemani
Reviewed By: liyintang
CC: stack
Differential Revision: https://reviews.facebook.net/D2601
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
hbase/branches/0.89-fb/src/test/resources/hbase-site.xml
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Fri Apr 6 22:13:16 2012
@@ -479,6 +479,8 @@ public final class HConstants {
/** Delay when waiting for a variable (HBASE-4712) */
public static final int VARIABLE_WAIT_TIME_MS = 40;
+ public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Fri Apr 6 22:13:16 2012
@@ -95,6 +95,10 @@ public class OldLogsCleaner extends Chor
protected void chore() {
try {
FileStatus[] files = this.fs.listStatus(this.oldLogDir);
+ if (files == null) {
+ // We don't have any files to process.
+ return;
+ }
int nbDeletedLog = 0;
for (FileStatus file : files) {
Path filePath = file.getPath();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Fri Apr 6 22:13:16 2012
@@ -1789,7 +1789,7 @@ public class RegionManager {
LoadBalancer() {
Configuration conf = master.getConfiguration();
- float confSlop = conf.getFloat("hbase.regions.slop", (float)0.3);
+ float confSlop = conf.getFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 0.3);
this.slop = confSlop <= 0 ? 1 : confSlop;
this.maxRegToClose = conf.getInt("hbase.regions.close.max", -1);
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Fri Apr 6 22:13:16 2012
@@ -1005,6 +1005,10 @@ public class SplitLogManager implements
byte[] newData = RecoverableZooKeeper.removeMetaData(data);
tot_mgr_get_data_result.incrementAndGet();
if (rc != 0) {
+ if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+ LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+ return;
+ }
if (rc == KeeperException.Code.NONODE.intValue()) {
tot_mgr_get_data_nonode.incrementAndGet();
// The task znode has been deleted. Must be some pending delete
@@ -1093,6 +1097,10 @@ public class SplitLogManager implements
@Override
public void processResult(int rc, String path, Object ctx, String name) {
if (rc != 0) {
+ if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+ LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+ return;
+ }
Long retry_count = (Long)ctx;
LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path +
" retry=" + retry_count);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Apr 6 22:13:16 2012
@@ -1582,21 +1582,25 @@ public class HRegionServer implements HR
case MSG_REGION_OPEN:
// Open a region
+ boolean requeued = false;
if (!haveRootRegion.get() && !info.isRootRegion()) {
// root region is not online yet. requeue this task
LOG.info("putting region open request back into queue because" +
" root region is not yet available");
try {
toDo.put(e);
+ requeued = true;
} catch (InterruptedException ex) {
LOG.warn("insertion into toDo queue was interrupted", ex);
break;
}
}
- if (e.msg.getMessage() != null && e.msg.getMessage().length > 0) {
- openRegion(info, new String(e.msg.getMessage()));
- } else {
- openRegion(info, null);
+ if (!requeued) {
+ if (e.msg.getMessage() != null && e.msg.getMessage().length > 0) {
+ openRegion(info, new String(e.msg.getMessage()));
+ } else {
+ openRegion(info, null);
+ }
}
break;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Fri Apr 6 22:13:16 2012
@@ -19,7 +19,6 @@
*/
package org.apache.hadoop.hbase;
-import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Iterator;
@@ -53,9 +52,6 @@ import org.apache.hadoop.hdfs.MiniDFSClu
public abstract class HBaseTestCase extends TestCase {
private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
- /** configuration parameter name for test directory */
- public static final String TEST_DIRECTORY_KEY = "test.build.data";
-
protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
@@ -73,9 +69,7 @@ public abstract class HBaseTestCase exte
protected String START_KEY;
protected static final int MAXVERSIONS = 3;
- static {
- initialize();
- }
+ protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
public volatile HBaseConfiguration conf;
@@ -147,8 +141,7 @@ public abstract class HBaseTestCase exte
}
protected Path getUnitTestdir(String testName) {
- return new Path(
- conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
+ return testUtil.getTestDir(testName);
}
protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
@@ -600,21 +593,6 @@ public abstract class HBaseTestCase exte
}
/**
- * Initializes parameters used in the test environment:
- *
- * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
- * Sets the boolean debugging if "DEBUGGING" is set in the environment.
- * If debugging is enabled, reconfigures logging so that the root log level is
- * set to WARN and the logging level for the package is set to DEBUG.
- */
- public static void initialize() {
- if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
- System.setProperty(TEST_DIRECTORY_KEY, new File(
- "build/hbase/test").getAbsolutePath());
- }
- }
-
- /**
* Common method to close down a MiniDFSCluster and the associated file system
*
* @param cluster
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri Apr 6 22:13:16 2012
@@ -26,6 +26,7 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hbase.client.De
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.NoServerForRegionException;
import org.apache.hadoop.hbase.client.Put;
@@ -73,7 +73,6 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.RegionSplitter;
@@ -85,6 +84,7 @@ import org.apache.hadoop.hdfs.Distribute
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zookeeper.ZooKeeper;
@@ -114,10 +114,17 @@ public class HBaseTestingUtility {
private MiniDFSCluster dfsCluster = null;
private MiniHBaseCluster hbaseCluster = null;
private MiniMRCluster mrCluster = null;
- // If non-null, then already a cluster running.
+
+ /** The root directory for all mini-cluster test data for this testing utility instance. */
private File clusterTestBuildDir = null;
+
+ /** If there is a mini cluster running for this testing utility instance. */
+ private boolean miniClusterRunning;
+
private HBaseAdmin hbaseAdmin = null;
+ private String hadoopLogDir;
+
/**
* System property key to get test directory value.
*/
@@ -128,6 +135,9 @@ public class HBaseTestingUtility {
*/
public static final String DEFAULT_TEST_DIRECTORY = "target/build/data";
+ /** Filesystem URI used for map-reduce mini-cluster setup */
+ private static String fsURI;
+
/** Compression algorithms to use in parameterized JUnit 4 tests */
public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
Arrays.asList(new Object[][] {
@@ -182,25 +192,13 @@ public class HBaseTestingUtility {
}
/**
- * Makes sure the test directory is set up so that {@link #getTestDir()}
- * returns a valid directory. Useful in unit tests that do not run a
- * mini-cluster.
- */
- public void initTestDir() {
- if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
- clusterTestBuildDir = setupClusterTestBuildDir();
- System.setProperty(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
- }
- }
-
- /**
* @return Where to write test data on local filesystem; usually
* {@link #DEFAULT_TEST_DIRECTORY}
* @see #setupClusterTestBuildDir()
*/
- public static Path getTestDir() {
- return new Path(System.getProperty(TEST_DIRECTORY_KEY,
- DEFAULT_TEST_DIRECTORY));
+ public Path getTestDir() {
+ setupClusterTestBuildDir();
+ return new Path(clusterTestBuildDir.getPath());
}
/**
@@ -209,7 +207,7 @@ public class HBaseTestingUtility {
* {@link #getTestDir()}.
* @see #setupClusterTestBuildDir()
*/
- public static Path getTestDir(final String subdirName) {
+ public Path getTestDir(final String subdirName) {
return new Path(getTestDir(), subdirName);
}
@@ -223,33 +221,18 @@ public class HBaseTestingUtility {
* single instance only is how the minidfscluster works.
* @return The calculated cluster test build directory.
*/
- File setupClusterTestBuildDir() {
- String randomStr = UUID.randomUUID().toString();
- String dirStr = getTestDir(randomStr).toString();
- File dir = new File(dirStr).getAbsoluteFile();
- // Have it cleaned up on exit
- dir.deleteOnExit();
- return dir;
- }
-
- /**
- * @throws IOException If a cluster -- zk, dfs, or hbase -- already running.
- */
- void isRunningCluster() throws IOException {
- if (this.clusterTestBuildDir == null) return;
- throw new IOException("Cluster already running at " +
- this.clusterTestBuildDir);
- }
+ public void setupClusterTestBuildDir() {
+ if (clusterTestBuildDir != null) {
+ return;
+ }
- /**
- * Start a minidfscluster.
- * @param servers How many DNs to start.
- * @throws Exception
- * @see {@link #shutdownMiniDFSCluster()}
- * @return The mini dfs cluster created.
- */
- public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
- return startMiniDFSCluster(servers, null);
+ String randomStr = UUID.randomUUID().toString();
+ String dirStr = new Path(DEFAULT_TEST_DIRECTORY, randomStr).toString();
+ clusterTestBuildDir = new File(dirStr).getAbsoluteFile();
+ clusterTestBuildDir.mkdirs();
+ clusterTestBuildDir.deleteOnExit();
+ conf.set(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
+ LOG.info("Created new mini-cluster data directory: " + clusterTestBuildDir);
}
/**
@@ -260,31 +243,44 @@ public class HBaseTestingUtility {
* @see {@link #shutdownMiniDFSCluster()}
* @return The mini dfs cluster created.
*/
- public MiniDFSCluster startMiniDFSCluster(int servers, final File dir)
- throws IOException {
- // This does the following to home the minidfscluster
- // base_dir = new File(System.getProperty("test.build.data", "build/test/data"), "dfs/");
- // Some tests also do this:
- // System.getProperty("test.cache.data", "build/test/cache");
- if (dir == null) this.clusterTestBuildDir = setupClusterTestBuildDir();
- else this.clusterTestBuildDir = dir;
- System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.toString());
- System.setProperty("test.cache.data", this.clusterTestBuildDir.toString());
+ public MiniDFSCluster startMiniDFSCluster(int servers) throws IOException {
+ createDirsAndSetProperties();
this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
true, null, null, null, null);
return this.dfsCluster;
}
+ public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException {
+ createDirsAndSetProperties();
+ dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
+ null, null, null);
+ return dfsCluster;
+ }
+
+ /** This is used before starting HDFS and map-reduce mini-clusters */
+ private void createDirsAndSetProperties() {
+ setupClusterTestBuildDir();
+ System.setProperty(TEST_DIRECTORY_KEY, clusterTestBuildDir.getPath());
+ createDirAndSetProperty("cache_data", "test.cache.data");
+ createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
+ hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
+ createDirAndSetProperty("mapred_output", "mapred.output.dir");
+ createDirAndSetProperty("mapred_local", "mapred.local.dir");
+ createDirAndSetProperty("mapred_system", "mapred.system.dir");
+ createDirAndSetProperty("mapred_temp", "mapred.temp.dir");
+ }
+
/**
- * Shuts down instance created by call to {@link #startMiniDFSCluster(int, File)}
+ * Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
* or does nothing.
+ * @throws IOException
* @throws Exception
*/
- public void shutdownMiniDFSCluster() throws Exception {
+ public void shutdownMiniDFSCluster() throws IOException {
if (this.dfsCluster != null) {
- FileSystem.closeAll();
// The below throws an exception per dn, AsynchronousCloseException.
this.dfsCluster.shutdown();
+ dfsCluster = null;
}
}
@@ -296,8 +292,8 @@ public class HBaseTestingUtility {
* @return zk cluster started.
*/
public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
- return startMiniZKCluster(setupClusterTestBuildDir());
-
+ setupClusterTestBuildDir();
+ return startMiniZKCluster(clusterTestBuildDir);
}
private MiniZooKeeperCluster startMiniZKCluster(final File dir)
@@ -373,14 +369,18 @@ public class HBaseTestingUtility {
final int numSlaves) throws IOException, InterruptedException {
LOG.info("Starting up minicluster");
// If we already put up a cluster, fail.
- isRunningCluster();
+ if (miniClusterRunning) {
+ throw new IllegalStateException("A mini-cluster is already running");
+ }
+ miniClusterRunning = true;
+
// Make a new random dir to home everything in. Set it as system property.
// minidfs reads home from system property.
- this.clusterTestBuildDir = setupClusterTestBuildDir();
+ setupClusterTestBuildDir();
System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.getPath());
// Bring up mini dfs cluster. This spews a bunch of warnings about missing
// scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
- startMiniDFSCluster(numSlaves, this.clusterTestBuildDir);
+ startMiniDFSCluster(numSlaves);
// Mangle conf so fs parameter points to minidfs we just started up
FileSystem fs = this.dfsCluster.getFileSystem();
@@ -445,23 +445,14 @@ public class HBaseTestingUtility {
this.hbaseCluster.shutdown();
// Wait till hbase is down before going on to shutdown zk.
this.hbaseCluster.join();
+ hbaseCluster = null;
}
shutdownMiniZKCluster();
- if (this.dfsCluster != null) {
- // The below throws an exception per dn, AsynchronousCloseException.
- this.dfsCluster.shutdown();
- }
- // Clean up our directory.
- if (this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
- // Need to use deleteDirectory because File.delete required dir is empty.
- if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
- new Path(this.clusterTestBuildDir.toString()))) {
- LOG.warn("Failed delete of " + this.clusterTestBuildDir.toString());
- }
- }
- clusterTestBuildDir = null;
+ shutdownMiniDFSCluster();
+
+ cleanupTestDir();
+ miniClusterRunning = false;
LOG.info("Minicluster is down");
- clusterTestBuildDir = null;
}
/**
@@ -845,29 +836,76 @@ public class HBaseTestingUtility {
*
* @throws IOException When starting the cluster fails.
*/
- public void startMiniMapReduceCluster() throws IOException {
+ public MiniMRCluster startMiniMapReduceCluster() throws IOException {
startMiniMapReduceCluster(2);
+ return mrCluster;
+ }
+
+ /**
+ * Tasktracker has a bug where changing the hadoop.log.dir system property
+ * will not change its internal static LOG_DIR variable.
+ */
+ private void forceChangeTaskLogDir() {
+ Field logDirField;
+ try {
+ logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
+ logDirField.setAccessible(true);
+
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
+
+ logDirField.set(null, new File(hadoopLogDir, "userlogs"));
+ } catch (SecurityException e) {
+ throw new RuntimeException(e);
+ } catch (NoSuchFieldException e) {
+ // TODO Auto-generated catch block
+ throw new RuntimeException(e);
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
}
/**
- * Starts a <code>MiniMRCluster</code>.
+ * Starts a <code>MiniMRCluster</code>. Call {@link #setFileSystemURI(String)} to use a different
+ * filesystem.
*
* @param servers The number of <code>TaskTracker</code>'s to start.
* @throws IOException When starting the cluster fails.
*/
- public void startMiniMapReduceCluster(final int servers) throws IOException {
+ private void startMiniMapReduceCluster(final int servers) throws IOException {
+ if (mrCluster != null) {
+ throw new IllegalStateException("MiniMRCluster is already running");
+ }
LOG.info("Starting mini mapreduce cluster...");
// These are needed for the new and improved Map/Reduce framework
Configuration c = getConfiguration();
- System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir"));
- c.set("mapred.output.dir", c.get("hadoop.tmp.dir"));
+
+ setupClusterTestBuildDir();
+ createDirsAndSetProperties();
+
+ forceChangeTaskLogDir();
+
+ // Allow the user to override FS URI for this map-reduce cluster to use.
mrCluster = new MiniMRCluster(servers,
- FileSystem.get(c).getUri().toString(), 1);
+ fsURI != null ? fsURI : FileSystem.get(c).getUri().toString(), 1);
+
LOG.info("Mini mapreduce cluster started");
c.set("mapred.job.tracker",
mrCluster.createJobConf().get("mapred.job.tracker"));
}
+ private String createDirAndSetProperty(final String relPath, String property) {
+ String path = clusterTestBuildDir.getPath() + "/" + relPath;
+ System.setProperty(property, path);
+ conf.set(property, path);
+ new File(path).mkdirs();
+ LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
+ return path;
+ }
+
/**
* Stops the previously started <code>MiniMRCluster</code>.
*/
@@ -875,6 +913,7 @@ public class HBaseTestingUtility {
LOG.info("Stopping mini mapreduce cluster...");
if (mrCluster != null) {
mrCluster.shutdown();
+ mrCluster = null;
}
// Restore configuration to point to local jobtracker
conf.set("mapred.job.tracker", "local");
@@ -920,15 +959,14 @@ public class HBaseTestingUtility {
int sessionTimeout = nodeZK.getSessionTimeout();
byte[] password = nodeZK.getSessionPassword();
long sessionID = nodeZK.getSessionID();
- final long sleep = sessionTimeout * 10L;
- final int maxRetryNum = 50;
+ final int sleep = 1000;
+ final int maxRetryNum = 60 * 1000 / sleep; // Wait for a total of up to one minute.
int retryNum = maxRetryNum;
ZooKeeper zk = new ZooKeeper(quorumServers,
sessionTimeout, EmptyWatcher.instance, sessionID, password);
+ LOG.debug("Closing ZK");
zk.close();
- Thread.sleep(sleep);
- LOG.debug("ZooKeeper is closed");
while (!nodeZK.isAborted() && retryNum != 0) {
Thread.sleep(sleep);
@@ -937,6 +975,7 @@ public class HBaseTestingUtility {
if (retryNum == 0) {
fail("ZooKeeper is not aborted after " + maxRetryNum + " attempts.");
}
+ LOG.debug("ZooKeeper is closed");
}
/**
@@ -1022,10 +1061,6 @@ public class HBaseTestingUtility {
return FileSystem.get(conf);
}
- public void cleanupTestDir() throws IOException {
- getTestDir().getFileSystem(conf).delete(getTestDir(), true);
- }
-
public void waitTableAvailable(byte[] table, long timeoutMillis)
throws InterruptedException, IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
@@ -1471,4 +1506,20 @@ REGION_LOOP:
return region;
}
+ /**
+ * Useful for tests that do not start a full cluster, e.g. those that use a mini MR cluster only.
+ */
+ public void cleanupTestDir() throws IOException {
+ if (this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
+ // Need to use deleteDirectory because File.delete required dir is empty.
+ if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf),
+ new Path(this.clusterTestBuildDir.toString()))) {
+ LOG.warn("Failed delete of " + this.clusterTestBuildDir.toString());
+ }
+ }
+ }
+
+ public void setFileSystemURI(String fsURI) {
+ this.fsURI = fsURI;
+ }
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java Fri Apr 6 22:13:16 2012
@@ -59,6 +59,7 @@ public class TestFullLogReconstruction {
c.setInt("ipc.client.connect.max.retries", 1);
c.setInt("dfs.client.block.recovery.retries", 1);
c.setInt("hbase.regionserver.flushlogentries", 1);
+ c.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
TEST_UTIL.startMiniCluster(NUM_REGIONSERVER);
}
@@ -118,8 +119,10 @@ public class TestFullLogReconstruction {
}
TEST_UTIL.expireRegionServerSession(0);
+
// wait for meta region comes online
- Thread.sleep(1000);
+ TEST_UTIL.waitTableAvailable(HConstants.META_TABLE_NAME, 10000);
+
scan = new Scan();
results = table.getScanner(scan);
int newCount = 0;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java Fri Apr 6 22:13:16 2012
@@ -61,10 +61,11 @@ public class TestZooKeeper {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Test we can first start the ZK cluster by itself
+ conf = TEST_UTIL.getConfiguration();
TEST_UTIL.startMiniZKCluster();
- TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
+ conf.setBoolean("dfs.support.append", true);
+ conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 1000);
TEST_UTIL.startMiniCluster(NUM_REGIONSERVER);
- conf = TEST_UTIL.getConfiguration();
}
/**
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java Fri Apr 6 22:13:16 2012
@@ -3822,7 +3822,7 @@ public class TestFromClientSide {
// Create multiple regions for this table
TEST_UTIL.createMultiRegions(table, FAMILY);
- Path tempPath = new Path(HBaseTestingUtility.getTestDir(), "regions.dat");
+ Path tempPath = new Path(TEST_UTIL.getTestDir(), "regions.dat");
final String tempFileName = tempPath.toString();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java Fri Apr 6 22:13:16 2012
@@ -38,8 +38,8 @@ public class TestColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd, null, null, false);
- HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
- getTestDir(), TEST_UTIL.getConfiguration());
+ HRegion region = HRegion.createHRegion(info,
+ TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
String row;
Put p;
row = "001";
@@ -73,8 +73,8 @@ public class TestColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd, null, null, false);
- HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
- getTestDir(), TEST_UTIL.getConfiguration());
+ HRegion region = HRegion.createHRegion(info,
+ TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
@@ -129,8 +129,8 @@ public class TestColumnPrefixFilter {
HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter");
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd, null, null, false);
- HRegion region = HRegion.createHRegion(info, HBaseTestingUtility.
- getTestDir(), TEST_UTIL.getConfiguration());
+ HRegion region = HRegion.createHRegion(info,
+ TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
List<String> rows = generateRandomWords(100, "row");
List<String> columns = generateRandomWords(10000, "column");
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java Fri Apr 6 22:13:16 2012
@@ -59,7 +59,7 @@ public class TestHalfStoreFileReader {
@Test
public void testHalfScanAndReseek() throws IOException {
HBaseTestingUtility test_util = new HBaseTestingUtility();
- String root_dir = HBaseTestingUtility.getTestDir("TestHalfStoreFile").toString();
+ String root_dir = test_util.getTestDir("TestHalfStoreFile").toString();
Path p = new Path(root_dir, "test");
Configuration conf = test_util.getConfiguration();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Fri Apr 6 22:13:16 2012
@@ -260,7 +260,7 @@ public class TestCacheOnWrite {
}
public void writeStoreFile() throws IOException {
- Path storeFileParentDir = new Path(HBaseTestingUtility.getTestDir(),
+ Path storeFileParentDir = new Path(TEST_UTIL.getTestDir(),
"test_cache_on_write");
StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs,
DATA_BLOCK_SIZE)
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java Fri Apr 6 22:13:16 2012
@@ -127,7 +127,7 @@ public class TestFixedFileTrailer {
}
// Now check what happens if the trailer is corrupted.
- Path trailerPath = new Path(HBaseTestingUtility.getTestDir(), "trailer_"
+ Path trailerPath = new Path(util.getTestDir(), "trailer_"
+ version);
{
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java Fri Apr 6 22:13:16 2012
@@ -51,8 +51,9 @@ import org.apache.hadoop.io.Writable;
public class TestHFile extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestHFile.class);
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static String ROOT_DIR =
- HBaseTestingUtility.getTestDir("TestHFile").toString();
+ TEST_UTIL.getTestDir("TestHFile").toString();
private final int minBlockSize = 512;
private static String localFormatter = "%010d";
private Map<String, Long> startingMetrics;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Fri Apr 6 22:13:16 2012
@@ -100,7 +100,7 @@ public class TestHFileBlock {
@Before
public void setUp() throws IOException {
fs = FileSystem.get(TEST_UTIL.getConfiguration());
- TEST_UTIL.initTestDir();
+ TEST_UTIL.setupClusterTestBuildDir();
}
public void writeTestBlockContents(DataOutputStream dos) throws IOException {
@@ -249,7 +249,7 @@ public class TestHFileBlock {
for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
for (boolean pread : new boolean[] { false, true }) {
byte[] block = createTestV1Block(algo);
- Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v1_"
+ Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v1_"
+ algo);
LOG.info("Creating temporary file at " + path);
FSDataOutputStream os = fs.create(path);
@@ -283,7 +283,7 @@ public class TestHFileBlock {
public void testReaderV2() throws IOException {
for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
for (boolean pread : new boolean[] { false, true }) {
- Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v2_"
+ Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v2_"
+ algo);
FSDataOutputStream os = fs.create(path);
HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
@@ -343,7 +343,7 @@ public class TestHFileBlock {
for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
for (boolean pread : new boolean[] { false, true }) {
for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
- Path path = new Path(HBaseTestingUtility.getTestDir(), "blocks_v2_"
+ Path path = new Path(TEST_UTIL.getTestDir(), "blocks_v2_"
+ algo + "_" + encoding.toString());
FSDataOutputStream os = fs.create(path);
HFileDataBlockEncoder dataBlockEncoder =
@@ -470,7 +470,7 @@ public class TestHFileBlock {
for (boolean cacheOnWrite : BOOLEAN_VALUES) {
Random rand = defaultRandom();
LOG.info("Compression algorithm: " + algo + ", pread=" + pread);
- Path path = new Path(HBaseTestingUtility.getTestDir(), "prev_offset");
+ Path path = new Path(TEST_UTIL.getTestDir(), "prev_offset");
List<Long> expectedOffsets = new ArrayList<Long>();
List<Long> expectedPrevOffsets = new ArrayList<Long>();
List<BlockType> expectedTypes = new ArrayList<BlockType>();
@@ -626,7 +626,7 @@ public class TestHFileBlock {
public void testConcurrentReading() throws Exception {
for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
Path path =
- new Path(HBaseTestingUtility.getTestDir(), "concurrent_reading");
+ new Path(TEST_UTIL.getTestDir(), "concurrent_reading");
Random rand = defaultRandom();
List<Long> offsets = new ArrayList<Long>();
List<BlockType> types = new ArrayList<BlockType>();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Fri Apr 6 22:13:16 2012
@@ -114,7 +114,7 @@ public class TestHFileBlockIndex {
@Test
public void testBlockIndex() throws IOException {
- path = new Path(HBaseTestingUtility.getTestDir(), "block_index_" + compr);
+ path = new Path(TEST_UTIL.getTestDir(), "block_index_" + compr);
writeWholeIndex();
readIndex();
}
@@ -461,7 +461,7 @@ public class TestHFileBlockIndex {
*/
@Test
public void testHFileWriterAndReader() throws IOException {
- Path hfilePath = new Path(HBaseTestingUtility.getTestDir(),
+ Path hfilePath = new Path(TEST_UTIL.getTestDir(),
"hfile_for_block_index");
CacheConfig cacheConf = new CacheConfig(conf);
BlockCache blockCache = cacheConf.getBlockCache();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Fri Apr 6 22:13:16 2012
@@ -74,14 +74,6 @@ public class TestHFileDataBlockEncoder {
}
/**
- * Cleanup after JUnit test.
- */
- @After
- public void tearDown() throws IOException {
- TEST_UTIL.cleanupTestDir();
- }
-
- /**
* Test putting and taking out blocks into cache with different
* encoding options.
*/
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java Fri Apr 6 22:13:16 2012
@@ -73,8 +73,9 @@ class RawKeyValue implements Comparable<
* instead.</p>
*/
public class TestHFilePerformance extends TestCase {
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static String ROOT_DIR =
- HBaseTestingUtility.getTestDir("TestHFilePerformance").toString();
+ TEST_UTIL.getTestDir("TestHFilePerformance").toString();
private FileSystem fs;
private Configuration conf;
private long startTimeEpoch;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java Fri Apr 6 22:13:16 2012
@@ -60,6 +60,8 @@ public class TestHFileSeek extends TestC
private RandomDistribution.DiscreteRNG keyLenGen;
private KVGenerator kvGen;
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
@Override
public void setUp() throws IOException {
if (options == null) {
@@ -239,7 +241,7 @@ public class TestHFileSeek extends TestC
int maxWordLen = 20;
String rootDir =
- HBaseTestingUtility.getTestDir("TestTFileSeek").toString();
+ TEST_UTIL.getTestDir("TestTFileSeek").toString();
String file = "TestTFileSeek";
// String compress = "lzo"; DISABLED
String compress = "none";
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java Fri Apr 6 22:13:16 2012
@@ -68,7 +68,7 @@ public class TestHFileWriterV2 {
@Test
public void testHFileFormatV2() throws IOException {
- Path hfilePath = new Path(HBaseTestingUtility.getTestDir(),
+ Path hfilePath = new Path(TEST_UTIL.getTestDir(),
"testHFileFormatV2");
final Compression.Algorithm COMPRESS_ALGO = Compression.Algorithm.GZ;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java Fri Apr 6 22:13:16 2012
@@ -40,7 +40,7 @@ public class TestReseekTo {
@Test
public void testReseekTo() throws Exception {
- Path ncTFile = new Path(HBaseTestingUtility.getTestDir(), "basic.hfile");
+ Path ncTFile = new Path(TEST_UTIL.getTestDir(), "basic.hfile");
FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
HFile.Writer writer = HFile.getWriterFactory(
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java Fri Apr 6 22:13:16 2012
@@ -63,7 +63,7 @@ public class TestScannerSelectionUsingTT
private static String FAMILY = "myCF";
private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
- private static final int TTL_SECONDS = 2;
+ private static final int TTL_SECONDS = 10;
private static final int TTL_MS = TTL_SECONDS * 1000;
private static final int NUM_EXPIRED_FILES = 2;
@@ -103,7 +103,7 @@ public class TestScannerSelectionUsingTT
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd, null, null, false);
HRegion region = HRegion.createHRegion(
- info, HBaseTestingUtility.getTestDir(), TEST_UTIL.getConfiguration());
+ info, TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
if (iFile == NUM_EXPIRED_FILES) {
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapred/TestLegacyTableMapReduce.java Fri Apr 6 22:13:16 2012
@@ -19,20 +19,23 @@
*/
package org.apache.hadoop.hbase.mapred;
-import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.NavigableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MultiRegionTable;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.JobClient;
@@ -126,7 +129,9 @@ public class TestLegacyTableMapReduce ex
}
private void runTestOnTable(HTable table) throws IOException {
- MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
+ HBaseTestingUtility testUtil = new HBaseTestingUtility();
+ testUtil.setupClusterTestBuildDir();
+ MiniMRCluster mrCluster = testUtil.startMiniMapReduceCluster();
JobConf jobConf = null;
try {
@@ -148,9 +153,6 @@ public class TestLegacyTableMapReduce ex
verify(Bytes.toString(table.getTableName()));
} finally {
mrCluster.shutdown();
- if (jobConf != null) {
- FileUtil.fullyDelete(new File(jobConf.get("hadoop.tmp.dir")));
- }
}
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Fri Apr 6 22:13:16 2012
@@ -56,12 +56,10 @@ import org.apache.hadoop.hbase.client.Ro
import org.apache.hadoop.hbase.client.RowMutation;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
-import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -74,7 +72,6 @@ import org.apache.hadoop.mapreduce.Recor
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -158,11 +155,6 @@ public class TestHFileOutputFormat {
}
}
- @Before
- public void cleanupDir() throws IOException {
- util.cleanupTestDir();
- }
-
private void setupRandomGeneratorMapper(Job job) {
job.setInputFormatClass(NMapInputFormat.class);
job.setMapperClass(RandomKVGeneratingMapper.class);
@@ -255,7 +247,7 @@ public class TestHFileOutputFormat {
conf.setInt("io.sort.mb", 20);
conf.setInt("mapred.map.tasks", 1);
- Path dir = HBaseTestingUtility.getTestDir("testRowSortReducer");
+ Path dir = util.getTestDir("testRowSortReducer");
try {
Job job = new Job(conf);
@@ -403,7 +395,7 @@ public class TestHFileOutputFormat {
RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
TaskAttemptContext context = null;
Path dir =
- HBaseTestingUtility.getTestDir("test_LATEST_TIMESTAMP_isReplaced");
+ util.getTestDir("test_LATEST_TIMESTAMP_isReplaced");
try {
Job job = new Job(conf);
FileOutputFormat.setOutputPath(job, dir);
@@ -442,7 +434,7 @@ public class TestHFileOutputFormat {
@Test
public void testWritingPEData() throws Exception {
Configuration conf = util.getConfiguration();
- Path testDir = HBaseTestingUtility.getTestDir("testWritingPEData");
+ Path testDir = util.getTestDir("testWritingPEData");
FileSystem fs = testDir.getFileSystem(conf);
// Set down this value or we OOME in eclipse.
@@ -496,18 +488,21 @@ public class TestHFileOutputFormat {
@Test
public void testMRIncrementalLoad() throws Exception {
+ LOG.info("\nStarting test testMRIncrementalLoad\n");
doIncrementalLoadTest(false);
}
@Test
public void testMRIncrementalLoadWithSplit() throws Exception {
+ LOG.info("\nStarting test testMRIncrementalLoadWithSplit\n");
doIncrementalLoadTest(true);
}
private void doIncrementalLoadTest(
boolean shouldChangeRegions) throws Exception {
+ util = new HBaseTestingUtility();
Configuration conf = util.getConfiguration();
- Path testDir = HBaseTestingUtility.getTestDir("testLocalMRIncrementalLoad");
+ Path testDir = util.getTestDir("testLocalMRIncrementalLoad");
byte[][] startKeys = generateRandomStartKeys(5);
try {
@@ -566,9 +561,7 @@ public class TestHFileOutputFormat {
expectedRows, util.countRows(table));
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
- int count = 0;
for (Result res : results) {
- count++;
assertEquals(FAMILIES.length, res.raw().length);
KeyValue first = res.raw()[0];
for (KeyValue kv : res.raw()) {
@@ -692,7 +685,7 @@ public class TestHFileOutputFormat {
RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
TaskAttemptContext context = null;
Path dir =
- HBaseTestingUtility.getTestDir("testColumnFamilyCompression");
+ util.getTestDir("testColumnFamilyCompression");
HTable table = Mockito.mock(HTable.class);
@@ -812,7 +805,7 @@ public class TestHFileOutputFormat {
RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
TaskAttemptContext context = null;
Path dir =
- HBaseTestingUtility.getTestDir("testColumnFamilyBloomFilter");
+ util.getTestDir("testColumnFamilyBloomFilter");
HTable table = Mockito.mock(HTable.class);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java Fri Apr 6 22:13:16 2012
@@ -90,7 +90,7 @@ public class TestLoadIncrementalHFiles {
private void runTest(String testName, byte[][][] hfileRanges)
throws Exception {
- Path dir = HBaseTestingUtility.getTestDir(testName);
+ Path dir = util.getTestDir(testName);
FileSystem fs = util.getTestFileSystem();
dir = dir.makeQualified(fs);
Path familyDir = new Path(dir, Bytes.toString(FAMILY));
@@ -126,7 +126,7 @@ public class TestLoadIncrementalHFiles {
@Test
public void testSplitStoreFile() throws IOException {
- Path dir = HBaseTestingUtility.getTestDir("testSplitHFile");
+ Path dir = util.getTestDir("testSplitHFile");
FileSystem fs = util.getTestFileSystem();
Path testIn = new Path(dir, "testhfile");
HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java Fri Apr 6 22:13:16 2012
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -118,7 +119,9 @@ public class TestTableMapReduce extends
private void runTestOnTable(HTable table)
throws IOException, InterruptedException, ClassNotFoundException {
- MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
+ HBaseTestingUtility testUtil = new HBaseTestingUtility();
+ testUtil.setFileSystemURI(fs.getUri().toString());
+ MiniMRCluster mrCluster = testUtil.startMiniMapReduceCluster();
LOG.debug("ZK client port before runing MR job: " +
ZooKeeperWrapper.getZKClientPort(conf));
@@ -147,10 +150,7 @@ public class TestTableMapReduce extends
verify(Bytes.toString(table.getTableName()));
} finally {
mrCluster.shutdown();
- if (job != null) {
- FileUtil.fullyDelete(
- new File(job.getConfiguration().get("hadoop.tmp.dir")));
- }
+ testUtil.cleanupTestDir();
}
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java Fri Apr 6 22:13:16 2012
@@ -31,6 +31,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.master.SplitLogManager.Task;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
@@ -70,6 +71,12 @@ public class TestSplitLogManager {
private volatile boolean stopped = false;
+ /**
+ * Additional amount of time we wait for events to happen. Added where unit
+ * test failures have been observed.
+ */
+ private static final int EXTRA_TOLERANCE_MS = 200;
+
private Stoppable stopper = new Stoppable() {
@Override
public boolean isStopped() {
@@ -100,7 +107,9 @@ public class TestSplitLogManager {
@Before
public void setup() throws Exception {
conf = TEST_UTIL.getConfiguration();
- zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests");
+ // Use a different ZK wrapper instance for each tests.
+ zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests-" +
+ UUID.randomUUID().toString());
zkw.deleteChildrenRecursively(zkw.parentZNode);
zkw.createZNodeIfNotExists(zkw.parentZNode);
assertTrue(zkw.checkExists(zkw.parentZNode) != -1);
@@ -217,7 +226,7 @@ public class TestSplitLogManager {
assertTrue((task.last_update <= curt) &&
(task.last_update > (curt - 1000)));
LOG.info("waiting for manager to resubmit the orphan task");
- waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+ waitForCounter(tot_mgr_resubmit, 0, 1, to + 300);
assertTrue(task.isUnassigned());
waitForCounter(tot_mgr_rescan, 0, 1, to + 100);
}
@@ -271,18 +280,18 @@ public class TestSplitLogManager {
zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker1"));
waitForCounter(tot_mgr_heartbeat, 0, 1, 1000);
- waitForCounter(tot_mgr_resubmit, 0, 1, to + 100);
+ waitForCounter(tot_mgr_resubmit, 0, 1, to + EXTRA_TOLERANCE_MS);
int version1 = zkw.checkExists(tasknode);
assertTrue(version1 > version);
zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker2"));
waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
- waitForCounter(tot_mgr_resubmit, 1, 2, to + 100);
+ waitForCounter(tot_mgr_resubmit, 1, 2, to + EXTRA_TOLERANCE_MS);
int version2 = zkw.checkExists(tasknode);
assertTrue(version2 > version1);
zkw.setData(tasknode, TaskState.TASK_OWNED.get("worker3"));
waitForCounter(tot_mgr_heartbeat, 1, 2, 1000);
- waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + 100);
- Thread.sleep(to + 100);
+ waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + EXTRA_TOLERANCE_MS);
+ Thread.sleep(to + EXTRA_TOLERANCE_MS);
assertEquals(2L, tot_mgr_resubmit.get());
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java Fri Apr 6 22:13:16 2012
@@ -43,7 +43,9 @@ public class TestBlocksRead extends HBas
}
HRegion region = null;
- private final String DIR = HBaseTestingUtility.getTestDir() + "/TestHRegion/";
+
+ private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+ private final String DIR = testUtil.getTestDir() + "/TestHRegion/";
/**
* @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java Fri Apr 6 22:13:16 2012
@@ -65,7 +65,7 @@ public class TestColumnSeeking {
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd, null, null, false);
HRegion region =
- HRegion.createHRegion(info, HBaseTestingUtility.getTestDir(), TEST_UTIL
+ HRegion.createHRegion(info, TEST_UTIL.getTestDir(), TEST_UTIL
.getConfiguration());
List<String> rows = generateRandomWords(10, "row");
@@ -169,7 +169,7 @@ public class TestColumnSeeking {
htd.addFamily(new HColumnDescriptor(family));
HRegionInfo info = new HRegionInfo(htd, null, null, false);
HRegion region =
- HRegion.createHRegion(info, HBaseTestingUtility.getTestDir(), TEST_UTIL
+ HRegion.createHRegion(info, TEST_UTIL.getTestDir(), TEST_UTIL
.getConfiguration());
List<String> rows = generateRandomWords(10, "row");
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java Fri Apr 6 22:13:16 2012
@@ -52,7 +52,7 @@ public class TestCompactSelection extend
private Configuration conf;
private Store store;
private static final String DIR
- = HBaseTestingUtility.getTestDir() + "/TestCompactSelection/";
+ = TEST_UTIL.getTestDir() + "/TestCompactSelection/";
private static Path TEST_FILE;
private static final int minFiles = 3;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Fri Apr 6 22:13:16 2012
@@ -295,7 +295,7 @@ public class TestCompoundBloomFilter {
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConf, fs,
BLOCK_SIZES[t])
- .withOutputDir(HBaseTestingUtility.getTestDir())
+ .withOutputDir(TEST_UTIL.getTestDir())
.withBloomType(bt)
.build();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Fri Apr 6 22:13:16 2012
@@ -67,7 +67,7 @@ public class TestFSErrorsExposed {
@Test
public void testHFileScannerThrowsErrors() throws IOException {
Path hfilePath = new Path(new Path(
- HBaseTestingUtility.getTestDir("internalScannerExposesErrors"),
+ util.getTestDir("internalScannerExposesErrors"),
"regionname"), "familyname");
FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
@@ -114,7 +114,7 @@ public class TestFSErrorsExposed {
@Test
public void testStoreFileScannerThrowsErrors() throws IOException {
Path hfilePath = new Path(new Path(
- HBaseTestingUtility.getTestDir("internalScannerExposesErrors"),
+ util.getTestDir("internalScannerExposesErrors"),
"regionname"), "familyname");
FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Fri Apr 6 22:13:16 2012
@@ -85,7 +85,8 @@ public class TestHRegion extends HBaseTe
private static final String COLUMN_FAMILY = "MyCF";
HRegion region = null;
- private final String DIR = HBaseTestingUtility.getTestDir() +
+ private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final String DIR = TEST_UTIL.getTestDir() +
"/TestHRegion/";
private final int MAX_VERSIONS = 2;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java Fri Apr 6 22:13:16 2012
@@ -354,7 +354,7 @@ public class TestMultiColumnScanner {
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd, null, null, false);
HRegion region = HRegion.createHRegion(
- info, HBaseTestingUtility.getTestDir(), TEST_UTIL.getConfiguration());
+ info, TEST_UTIL.getTestDir(), TEST_UTIL.getConfiguration());
return region;
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerResets.java Fri Apr 6 22:13:16 2012
@@ -34,7 +34,8 @@ public class TestScannerResets extends H
}
HRegion region = null;
- private final String DIR = HBaseTestingUtility.getTestDir() + "/TestScannerResets/";
+ private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final String DIR = TEST_UTIL.getTestDir() + "/TestScannerResets/";
/**
* @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Fri Apr 6 22:13:16 2012
@@ -89,7 +89,8 @@ public class TestStore extends TestCase
long id = System.currentTimeMillis();
Get get = new Get(row);
- private static final String DIR = HBaseTestingUtility.getTestDir() + "/TestStore/";
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final String DIR = TEST_UTIL.getTestDir() + "/TestStore/";
private static final int MAX_VERSION = 4;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Fri Apr 6 22:13:16 2012
@@ -68,6 +68,7 @@ public class TestStoreFile extends HBase
private MiniDFSCluster cluster;
private CacheConfig cacheConf;
private Map<String, Long> startingMetrics;
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Override
public void setUp() throws Exception {
@@ -352,7 +353,7 @@ public class TestStoreFile extends HBase
}
private static String ROOT_DIR =
- HBaseTestingUtility.getTestDir("TestStoreFile").toString();
+ TEST_UTIL.getTestDir("TestStoreFile").toString();
private static String localFormatter = "%010d";
private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs)
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Fri Apr 6 22:13:16 2012
@@ -338,10 +338,9 @@ public class TestHLog {
}
// Now call sync to send the data to HDFS datanodes
wal.sync(true);
- int namenodePort = cluster.getNameNodePort();
+ int namenodePort = cluster.getNameNodePort();
final Path walPath = wal.computeFilename();
-
// Stop the cluster. (ensure restart since we're sharing MiniDFSCluster)
try {
cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
@@ -361,10 +360,12 @@ public class TestHLog {
LOG.error("Waiting for cluster to go down");
Thread.sleep(1000);
}
- cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
+ LOG.info("Waiting a few seconds before re-starting HDFS");
+ Thread.sleep(5000);
+ cluster = TEST_UTIL.startMiniDFSClusterForTestHLog(namenodePort);
cluster.waitActive();
fs = cluster.getFileSystem();
- LOG.info("START second instance.");
+ LOG.info("STARTED second instance.");
}
// set the lease period to be 1 second so that the
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java Fri Apr 6 22:13:16 2012
@@ -43,7 +43,7 @@ public class TestHLogMethods {
*/
@Test public void testGetSplitEditFilesSorted() throws IOException {
FileSystem fs = FileSystem.get(util.getConfiguration());
- Path regiondir = HBaseTestingUtility.getTestDir("regiondir");
+ Path regiondir = util.getTestDir("regiondir");
fs.delete(regiondir, true);
fs.mkdirs(regiondir);
Path recoverededits = HLog.getRegionDirRecoveredEditsDir(regiondir);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java Fri Apr 6 22:13:16 2012
@@ -112,9 +112,9 @@ public class TestReplicationSourceManage
server.getZooKeeperWrapper(), conf,
REPLICATING, "123456789");
fs = FileSystem.get(conf);
- oldLogDir = new Path(HBaseTestingUtility.getTestDir(),
+ oldLogDir = new Path(utility.getTestDir(),
HConstants.HREGION_OLDLOGDIR_NAME);
- logDir = new Path(HBaseTestingUtility.getTestDir(),
+ logDir = new Path(utility.getTestDir(),
HConstants.HREGION_LOGDIR_NAME);
manager = new ReplicationSourceManager(helper,
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java Fri Apr 6 22:13:16 2012
@@ -75,6 +75,9 @@ public class TestMiniClusterLoadSequenti
this.isMultiPut = isMultiPut;
this.dataBlockEncoding = dataBlockEncoding;
conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
+
+ // We don't want any region reassignments by the load balancer during the test.
+ conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, 10.0f);
}
@Parameters
Modified: hbase/branches/0.89-fb/src/test/resources/hbase-site.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/resources/hbase-site.xml?rev=1310612&r1=1310611&r2=1310612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/resources/hbase-site.xml (original)
+++ hbase/branches/0.89-fb/src/test/resources/hbase-site.xml Fri Apr 6 22:13:16 2012
@@ -142,7 +142,7 @@
<property>
<name>zookeeper.session.timeout</name>
- <value>1000</value>
+ <value>60000</value>
</property>
<property>
@@ -150,4 +150,11 @@
<value>false</value>
</property>
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>100</value>
+ <description>
+ Use a lot of retries in unit tests.
+ </description>
+ </property>
</configuration>