You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/07/12 19:24:43 UTC
svn commit: r1502626 - in
/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase: ./
backup/ client/ coprocessor/ io/encoding/ regionserver/
Author: stack
Date: Fri Jul 12 17:24:43 2013
New Revision: 1502626
URL: http://svn.apache.org/r1502626
Log:
HBASE-8934 Fix bunch of flaky tests
Modified:
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri Jul 12 17:24:43 2013
@@ -1070,6 +1070,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(getConfiguration(), tableName);
}
@@ -1094,6 +1096,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(c, tableName);
}
@@ -1116,6 +1120,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(c, tableName);
}
@@ -1149,6 +1155,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(new Configuration(getConfiguration()), tableName);
}
@@ -1170,6 +1178,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(new Configuration(getConfiguration()), tableName);
}
@@ -1193,6 +1203,8 @@ public class HBaseTestingUtility extends
i++;
}
getHBaseAdmin().createTable(desc);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(new Configuration(getConfiguration()), tableName);
}
@@ -1210,6 +1222,8 @@ public class HBaseTestingUtility extends
HColumnDescriptor hcd = new HColumnDescriptor(family);
desc.addFamily(hcd);
getHBaseAdmin().createTable(desc, splitRows);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(getConfiguration(), tableName);
}
@@ -1229,6 +1243,8 @@ public class HBaseTestingUtility extends
desc.addFamily(hcd);
}
getHBaseAdmin().createTable(desc, splitRows);
+ // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+ waitUntilAllRegionsAssigned(tableName);
return new HTable(getConfiguration(), tableName);
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java Fri Jul 12 17:24:43 2013
@@ -248,9 +248,23 @@ public class TestHFileArchiving {
UTIL.deleteTable(TABLE_NAME);
LOG.debug("Deleted table");
- // then get the files in the archive directory.
+ assertArchiveFiles(fs, storeFiles, 30000);
+ }
+
+ private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
+ long end = System.currentTimeMillis() + timeout;
Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
- List<String> archivedFiles = getAllFileNames(fs, archiveDir);
+ List<String> archivedFiles = new ArrayList<String>();
+
+ // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
+ // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
+ while (System.currentTimeMillis() < end) {
+ archivedFiles = getAllFileNames(fs, archiveDir);
+ if (archivedFiles.size() >= storeFiles.size()) {
+ break;
+ }
+ }
+
Collections.sort(storeFiles);
Collections.sort(archivedFiles);
@@ -267,6 +281,7 @@ public class TestHFileArchiving {
archivedFiles.containsAll(storeFiles));
}
+
/**
* Test that the store files are archived when a column family is removed.
* @throws Exception
@@ -307,23 +322,8 @@ public class TestHFileArchiving {
// then delete the table so the hfiles get archived
UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
- // then get the files in the archive directory.
- Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
- List<String> archivedFiles = getAllFileNames(fs, archiveDir);
- Collections.sort(storeFiles);
- Collections.sort(archivedFiles);
-
- LOG.debug("Store files:");
- for (int i = 0; i < storeFiles.size(); i++) {
- LOG.debug(i + " - " + storeFiles.get(i));
- }
- LOG.debug("Archive files:");
- for (int i = 0; i < archivedFiles.size(); i++) {
- LOG.debug(i + " - " + archivedFiles.get(i));
- }
-
- assertTrue("Archived files are missing some of the store files!",
- archivedFiles.containsAll(storeFiles));
+ assertArchiveFiles(fs, storeFiles, 30000);
+ UTIL.deleteTable(TABLE_NAME);
}
/**
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java Fri Jul 12 17:24:43 2013
@@ -39,7 +39,21 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.exceptions.InvalidFamilyOperationException;
@@ -761,7 +775,7 @@ public class TestAdmin {
}
ladmin.close();
}
-
+
@Test
public void testTableAvailableWithRandomSplitKeys() throws Exception {
byte[] tableName = Bytes.toBytes("testTableAvailableWithRandomSplitKeys");
@@ -1077,7 +1091,7 @@ public class TestAdmin {
new HColumnDescriptor("/cfamily/name");
}
- @Test(timeout=36000)
+ @Test(timeout=300000)
public void testEnableDisableAddColumnDeleteColumn() throws Exception {
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
byte [] tableName = Bytes.toBytes("testMasterAdmin");
@@ -1320,10 +1334,15 @@ public class TestAdmin {
.getServerName().getServerName());
}
}
- Thread.sleep(1000);
- onlineRegions = ProtobufUtil.getOnlineRegions(rs);
+ boolean isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
+ long timeout = System.currentTimeMillis() + 10000;
+ while ((System.currentTimeMillis() < timeout) && (isInList)) {
+ Thread.sleep(100);
+ isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
+ }
+
assertFalse("The region should not be present in online regions list.",
- onlineRegions.contains(info));
+ isInList);
}
@Test
@@ -1372,7 +1391,7 @@ public class TestAdmin {
}
boolean isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
- long timeout = System.currentTimeMillis() + 2000;
+ long timeout = System.currentTimeMillis() + 10000;
while ((System.currentTimeMillis() < timeout) && (isInList)) {
Thread.sleep(100);
isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java Fri Jul 12 17:24:43 2013
@@ -166,7 +166,14 @@ public class TestScannerTimeout {
scan.setCaching(SCANNER_CACHING);
LOG.info("************ TEST3686A");
MetaReader.fullScanMetaAndPrint(TEST_UTIL.getHBaseCluster().getMaster().getCatalogTracker());
- HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
+ // Set a very high timeout, we want to test what happens when a RS
+ // fails but the region is recovered before the lease times out.
+ // Since the RS is already created, this conf is client-side only for
+ // this new table
+ Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+ conf.setInt(
+ HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, SCANNER_TIMEOUT*100);
+ HTable table = new HTable(conf, TABLE_NAME);
LOG.info("START ************ TEST3686A---22");
ResultScanner r = table.getScanner(scan);
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java Fri Jul 12 17:24:43 2013
@@ -77,7 +77,7 @@ public class TestRegionServerCoprocessor
TEST_UTIL.shutdownMiniCluster();
}
- @Test(timeout=30000)
+ @Test(timeout=60000)
public void testExceptionFromCoprocessorDuringPut()
throws IOException, InterruptedException {
// Set watches on the zookeeper nodes for all of the regionservers in the
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java Fri Jul 12 17:24:43 2013
@@ -69,7 +69,7 @@ public class TestChangingEncoding {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final Configuration conf = TEST_UTIL.getConfiguration();
- private static final int TIMEOUT_MS = 240000;
+ private static final int TIMEOUT_MS = 600000;
private HBaseAdmin admin;
private HColumnDescriptor hcd;
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java?rev=1502626&r1=1502625&r2=1502626&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java Fri Jul 12 17:24:43 2013
@@ -211,7 +211,7 @@ public class TestCompactionState {
}
return count;
}
-
+
private static void loadData(final HTable ht, final byte[][] families,
final int rows, final int flushes) throws IOException {
List<Put> puts = new ArrayList<Put>(rows);