You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/03/24 17:59:07 UTC
[1/2] hbase git commit: HBASE-13321 Fix flaky TestHBaseFsck
Repository: hbase
Updated Branches:
refs/heads/branch-1 266e883da -> c305b4436
refs/heads/master 80d230e1f -> fcc09dcd3
HBASE-13321 Fix flaky TestHBaseFsck
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fcc09dcd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fcc09dcd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fcc09dcd
Branch: refs/heads/master
Commit: fcc09dcd38b8543a9dd40212206cb4231620dac9
Parents: 80d230e
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Mar 24 16:39:17 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Mar 24 16:39:17 2015 +0000
----------------------------------------------------------------------
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 109 ++++++++++++++++---
1 file changed, 91 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fcc09dcd/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index ac79656..618763c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -84,6 +84,10 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
@@ -98,6 +102,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.SplitTransaction;
import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
+import org.apache.hadoop.hbase.security.access.AccessControlClient;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
@@ -150,6 +155,9 @@ public class TestHBaseFsck {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ MasterSyncObserver.class.getName());
+
conf.setInt("hbase.regionserver.handler.count", 2);
conf.setInt("hbase.regionserver.metahandler.count", 2);
@@ -173,6 +181,9 @@ public class TestHBaseFsck {
admin = connection.getAdmin();
admin.setBalancerRunning(false, true);
+
+ TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
+ TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
}
@AfterClass
@@ -192,7 +203,10 @@ public class TestHBaseFsck {
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
TableName table = TableName.valueOf("tableBadMetaAssign");
- TEST_UTIL.createTable(table, FAM);
+ HTableDescriptor desc = new HTableDescriptor(table);
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
+ desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
+ createTable(TEST_UTIL, desc, null);
// We created 1 table, should be fine
assertNoErrors(doFsck(conf, false));
@@ -411,7 +425,8 @@ public class TestHBaseFsck {
desc.setRegionReplication(replicaCount);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
- admin.createTable(desc, SPLITS);
+ createTable(TEST_UTIL, desc, SPLITS);
+
tbl = (HTable) connection.getTable(tablename, tableExecutorService);
List<Put> puts = new ArrayList<Put>();
for (byte[] row : ROWKEYS) {
@@ -442,15 +457,14 @@ public class TestHBaseFsck {
* @param tablename
* @throws IOException
*/
- void cleanupTable(TableName tablename) throws IOException {
+ void cleanupTable(TableName tablename) throws Exception {
if (tbl != null) {
tbl.close();
tbl = null;
}
((ClusterConnection) connection).clearRegionCache();
- TEST_UTIL.deleteTable(tablename);
-
+ deleteTable(TEST_UTIL, tablename);
}
/**
@@ -591,9 +605,9 @@ public class TestHBaseFsck {
assert(h2.getRetCode() >= 0);
}
}
-
+
/**
- * This test makes sure that with 5 retries both parallel instances
+ * This test makes sure that with 5 retries both parallel instances
* of hbck will be completed successfully.
*
* @throws Exception
@@ -607,7 +621,7 @@ public class TestHBaseFsck {
@Override
public HBaseFsck call() throws Exception {
- return doFsck(conf, false);
+ return doFsck(conf, false);
}
}
service = Executors.newFixedThreadPool(2);
@@ -623,7 +637,7 @@ public class TestHBaseFsck {
assertNotNull(h2);
assert(h1.getRetCode() >= 0);
assert(h2.getRetCode() >= 0);
-
+
}
/**
@@ -1224,7 +1238,8 @@ public class TestHBaseFsck {
try {
HTableDescriptor desc = new HTableDescriptor(table);
desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
- admin.createTable(desc);
+ createTable(TEST_UTIL, desc, null);
+
tbl = (HTable) connection.getTable(desc.getTableName());
for (int i = 0; i < 5; i++) {
Put p1 = new Put(("r" + i).getBytes());
@@ -2455,11 +2470,12 @@ public class TestHBaseFsck {
assertNoErrors(hbck);
ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
+ final TableName tableName = TableName.valueOf("foo");
// obtain one lock
- final TableLockManager tableLockManager = TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
- TableLock writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "testCheckTableLocks");
+ final TableLockManager tableLockManager =
+ TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
+ TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
writeLock.acquire();
hbck = doFsck(conf, false);
assertNoErrors(hbck); // should not have expired, no problems
@@ -2474,8 +2490,7 @@ public class TestHBaseFsck {
new Thread() {
@Override
public void run() {
- TableLock readLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "testCheckTableLocks");
+ TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
try {
latch.countDown();
readLock.acquire();
@@ -2509,10 +2524,10 @@ public class TestHBaseFsck {
assertNoErrors(hbck);
// ensure that locks are deleted
- writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "should acquire without blocking");
+ writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
writeLock.acquire(); // this should not block.
writeLock.release(); // release for clean state
+ tableLockManager.tableDeleted(tableName);
}
@Test (timeout=180000)
@@ -2578,7 +2593,7 @@ public class TestHBaseFsck {
HTableDescriptor desc = new HTableDescriptor(table);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
- admin.createTable(desc);
+ createTable(TEST_UTIL, desc, null);
tbl = (HTable) connection.getTable(table, tableExecutorService);
// Mess it up by leaving a hole in the assignment, meta, and hdfs data
@@ -2682,4 +2697,62 @@ public class TestHBaseFsck {
Assert.assertEquals("shouldIgnorePreCheckPermission", true,
hbck.shouldIgnorePreCheckPermission());
}
+
+ public static class MasterSyncObserver extends BaseMasterObserver {
+ volatile CountDownLatch tableCreationLatch = null;
+ volatile CountDownLatch tableDeletionLatch = null;
+
+ @Override
+ public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ // the AccessController test, some times calls only and directly the postCreateTableHandler()
+ if (tableCreationLatch != null) {
+ tableCreationLatch.countDown();
+ }
+ }
+
+ @Override
+ public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName)
+ throws IOException {
+ // the AccessController test, some times calls only and directly the postDeleteTableHandler()
+ if (tableDeletionLatch != null) {
+ tableDeletionLatch.countDown();
+ }
+ }
+ }
+
+ public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
+ byte [][] splitKeys) throws Exception {
+ // NOTE: We need a latch because admin is not sync,
+ // so the postOp coprocessor method may be called after the admin operation returned.
+ MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
+ .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
+ observer.tableCreationLatch = new CountDownLatch(1);
+ if (splitKeys != null) {
+ admin.createTable(htd, splitKeys);
+ } else {
+ admin.createTable(htd);
+ }
+ observer.tableCreationLatch.await();
+ observer.tableCreationLatch = null;
+ testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
+ }
+
+ public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
+ throws Exception {
+ // NOTE: We need a latch because admin is not sync,
+ // so the postOp coprocessor method may be called after the admin operation returned.
+ MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
+ .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
+ observer.tableDeletionLatch = new CountDownLatch(1);
+ try {
+ admin.disableTable(tableName);
+ } catch (Exception e) {
+ LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
+ }
+ admin.deleteTable(tableName);
+ observer.tableDeletionLatch.await();
+ observer.tableDeletionLatch = null;
+ }
}
[2/2] hbase git commit: HBASE-13321 Fix flaky TestHBaseFsck
Posted by mb...@apache.org.
HBASE-13321 Fix flaky TestHBaseFsck
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c305b443
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c305b443
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c305b443
Branch: refs/heads/branch-1
Commit: c305b4436202a2eeafb723f833c059b118f0402b
Parents: 266e883
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Mar 24 16:52:05 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Mar 24 16:52:05 2015 +0000
----------------------------------------------------------------------
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 107 +++++++++++++++----
1 file changed, 89 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c305b443/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 01a74e1..c616b62 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -92,6 +92,10 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
@@ -158,6 +162,9 @@ public class TestHBaseFsck {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ MasterSyncObserver.class.getName());
+
conf.setInt("hbase.regionserver.handler.count", 2);
conf.setInt("hbase.regionserver.metahandler.count", 2);
@@ -180,6 +187,9 @@ public class TestHBaseFsck {
admin = connection.getAdmin();
admin.setBalancerRunning(false, true);
+
+ TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
+ TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
}
@AfterClass
@@ -199,7 +209,10 @@ public class TestHBaseFsck {
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
TableName table = TableName.valueOf("tableBadMetaAssign");
- TEST_UTIL.createTable(table, FAM);
+ HTableDescriptor desc = new HTableDescriptor(table);
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
+ desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
+ createTable(TEST_UTIL, desc, null);
// We created 1 table, should be fine
assertNoErrors(doFsck(conf, false));
@@ -418,7 +431,8 @@ public class TestHBaseFsck {
desc.setRegionReplication(replicaCount);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
- admin.createTable(desc, SPLITS);
+ createTable(TEST_UTIL, desc, SPLITS);
+
tbl = (HTable) connection.getTable(tablename, tableExecutorService);
List<Put> puts = new ArrayList<Put>();
for (byte[] row : ROWKEYS) {
@@ -449,15 +463,14 @@ public class TestHBaseFsck {
* @param tablename
* @throws IOException
*/
- void cleanupTable(TableName tablename) throws IOException {
+ void cleanupTable(TableName tablename) throws Exception {
if (tbl != null) {
tbl.close();
tbl = null;
}
((ClusterConnection) connection).clearRegionCache();
- TEST_UTIL.deleteTable(tablename);
-
+ deleteTable(TEST_UTIL, tablename);
}
/**
@@ -598,9 +611,9 @@ public class TestHBaseFsck {
assert(h2.getRetCode() >= 0);
}
}
-
+
/**
- * This test makes sure that with 5 retries both parallel instances
+ * This test makes sure that with 5 retries both parallel instances
* of hbck will be completed successfully.
*
* @throws Exception
@@ -614,7 +627,7 @@ public class TestHBaseFsck {
@Override
public HBaseFsck call() throws Exception {
- return doFsck(conf, false);
+ return doFsck(conf, false);
}
}
service = Executors.newFixedThreadPool(2);
@@ -630,7 +643,7 @@ public class TestHBaseFsck {
assertNotNull(h2);
assert(h1.getRetCode() >= 0);
assert(h2.getRetCode() >= 0);
-
+
}
/**
@@ -2387,11 +2400,12 @@ public class TestHBaseFsck {
assertNoErrors(hbck);
ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
+ final TableName tableName = TableName.valueOf("foo");
// obtain one lock
- final TableLockManager tableLockManager = TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
- TableLock writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "testCheckTableLocks");
+ final TableLockManager tableLockManager =
+ TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
+ TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
writeLock.acquire();
hbck = doFsck(conf, false);
assertNoErrors(hbck); // should not have expired, no problems
@@ -2406,8 +2420,7 @@ public class TestHBaseFsck {
new Thread() {
@Override
public void run() {
- TableLock readLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "testCheckTableLocks");
+ TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
try {
latch.countDown();
readLock.acquire();
@@ -2441,10 +2454,10 @@ public class TestHBaseFsck {
assertNoErrors(hbck);
// ensure that locks are deleted
- writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
- "should acquire without blocking");
+ writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
writeLock.acquire(); // this should not block.
writeLock.release(); // release for clean state
+ tableLockManager.tableDeleted(tableName);
}
/**
@@ -2559,7 +2572,7 @@ public class TestHBaseFsck {
HTableDescriptor desc = new HTableDescriptor(table);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
- admin.createTable(desc);
+ createTable(TEST_UTIL, desc, null);
tbl = (HTable) connection.getTable(table, tableExecutorService);
// Mess it up by leaving a hole in the assignment, meta, and hdfs data
@@ -2668,7 +2681,7 @@ public class TestHBaseFsck {
try {
HTableDescriptor desc = new HTableDescriptor(table);
desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
- admin.createTable(desc);
+ createTable(TEST_UTIL, desc, null);
tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
for (int i = 0; i < 5; i++) {
Put p1 = new Put(("r" + i).getBytes());
@@ -2718,4 +2731,62 @@ public class TestHBaseFsck {
}
}
+
+ public static class MasterSyncObserver extends BaseMasterObserver {
+ volatile CountDownLatch tableCreationLatch = null;
+ volatile CountDownLatch tableDeletionLatch = null;
+
+ @Override
+ public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ // the AccessController test, some times calls only and directly the postCreateTableHandler()
+ if (tableCreationLatch != null) {
+ tableCreationLatch.countDown();
+ }
+ }
+
+ @Override
+ public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName)
+ throws IOException {
+ // the AccessController test, some times calls only and directly the postDeleteTableHandler()
+ if (tableDeletionLatch != null) {
+ tableDeletionLatch.countDown();
+ }
+ }
+ }
+
+ public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
+ byte [][] splitKeys) throws Exception {
+ // NOTE: We need a latch because admin is not sync,
+ // so the postOp coprocessor method may be called after the admin operation returned.
+ MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
+ .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
+ observer.tableCreationLatch = new CountDownLatch(1);
+ if (splitKeys != null) {
+ admin.createTable(htd, splitKeys);
+ } else {
+ admin.createTable(htd);
+ }
+ observer.tableCreationLatch.await();
+ observer.tableCreationLatch = null;
+ testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
+ }
+
+ public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
+ throws Exception {
+ // NOTE: We need a latch because admin is not sync,
+ // so the postOp coprocessor method may be called after the admin operation returned.
+ MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
+ .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
+ observer.tableDeletionLatch = new CountDownLatch(1);
+ try {
+ admin.disableTable(tableName);
+ } catch (Exception e) {
+ LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
+ }
+ admin.deleteTable(tableName);
+ observer.tableDeletionLatch.await();
+ observer.tableDeletionLatch = null;
+ }
}