svn commit: r1326046 - in /hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase: MiniHBaseCluster.java regionserver/TestFSErrorsExposed.java

[mb...@apache.org]

Author: mbautin
Date: Sat Apr 14 02:46:49 2012
New Revision: 1326046

URL: http://svn.apache.org/viewvc?rev=1326046&view=rev
Log:
[jira] [HBASE-5763] [89-fb] Fix random failures in TestFSErrorsExposed

Summary: TestFSErrorsExposed frequently fails due to unclean mini-cluster
shutdown. Bringing datanodes back up, waiting for some time, and preemptively
killing all regionservers and the master before shutdown.

Test Plan: Run TestFSErrorsExposed 100 times

Reviewers: liyintang, kannan, pritamdamania, amirshim

Reviewed By: amirshim

CC: stack

Differential Revision: https://reviews.facebook.net/D2739

Modified:
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=1326046&r1=1326045&r2=1326046&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Sat Apr 14 02:46:49 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.security.UnixUserGroupInformation;
@@ -577,4 +578,12 @@ public class MiniHBaseCluster {
     return hbaseCluster.getActiveMaster();
   }
 
+  public void killAll() {
+    for (RegionServerThread rst : getRegionServerThreads()) {
+      rst.getRegionServer().kill();
+    }
+    for (HMaster master : getMasters()) {
+      master.stop("killMiniHBaseCluster");
+    }
+  }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1326046&r1=1326045&r2=1326046&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Sat Apr 14 02:46:49 2012
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.Test;
 
 /**
@@ -159,9 +160,10 @@ public class TestFSErrorsExposed {
    * removes the data from HDFS underneath it, and ensures that
    * errors are bubbled to the client.
    */
-  @Test
+  @Test(timeout=5 * 60 * 1000)
   public void testFullSystemBubblesFSErrors() throws Exception {
     try {
+      util.getConfiguration().setInt("hbase.client.retries.number", 3);
       util.startMiniCluster(1);
       byte[] tableName = Bytes.toBytes("table");
       byte[] fam = Bytes.toBytes("fam");
@@ -193,7 +195,11 @@ public class TestFSErrorsExposed {
         assertTrue(e.getMessage().contains("Could not seek"));
       }
 
+      // Restart data nodes so that HBase can shut down cleanly.
+      util.getDFSCluster().restartDataNodes();
+
     } finally {
+      util.getMiniHBaseCluster().killAll();
       util.shutdownMiniCluster();
     }
   }