You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/05/30 23:51:17 UTC
[06/26] hbase git commit: HBASE-14614 Procedure v2 - Core Assignment
Manager (Matteo Bertozzi) Move to a new AssignmentManager,
one that describes Assignment using a State Machine built on top of
ProcedureV2 facility.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
index 66c5abf..aef67bf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.client;
import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +33,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.ClientTests;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
index 852c5cf..10f466d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
@@ -17,19 +17,24 @@
*/
package org.apache.hadoop.hbase.client;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.Threads;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -37,17 +42,9 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
@Category({MediumTests.class, ClientTests.class})
public class TestSplitOrMergeStatus {
- private static final Log LOG = LogFactory.getLog(TestSplitOrMergeStatus.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static byte [] FAMILY = Bytes.toBytes("testFamily");
@@ -77,7 +74,7 @@ public class TestSplitOrMergeStatus {
TEST_UTIL.loadTable(t, FAMILY, false);
RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName());
- int orignalCount = locator.getAllRegionLocations().size();
+ int originalCount = locator.getAllRegionLocations().size();
Admin admin = TEST_UTIL.getAdmin();
initSwitchStatus(admin);
@@ -85,14 +82,17 @@ public class TestSplitOrMergeStatus {
assertEquals(results.length, 1);
assertTrue(results[0]);
admin.split(t.getName());
- int count = waitOnSplitOrMerge(t).size();
- assertTrue(orignalCount == count);
+ int count = admin.getTableRegions(tableName).size();
+ assertTrue(originalCount == count);
results = admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT);
assertEquals(results.length, 1);
assertFalse(results[0]);
admin.split(t.getName());
- count = waitOnSplitOrMerge(t).size();
- assertTrue(orignalCount<count);
+ while ((count = admin.getTableRegions(tableName).size()) == originalCount) {
+ Threads.sleep(1);;
+ }
+ count = admin.getTableRegions(tableName).size();
+ assertTrue(originalCount < count);
admin.close();
}
@@ -103,33 +103,43 @@ public class TestSplitOrMergeStatus {
Table t = TEST_UTIL.createTable(tableName, FAMILY);
TEST_UTIL.loadTable(t, FAMILY, false);
- RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName());
-
Admin admin = TEST_UTIL.getAdmin();
+ int originalCount = admin.getTableRegions(tableName).size();
initSwitchStatus(admin);
admin.split(t.getName());
- waitOnSplitOrMerge(t); //Split the table to ensure we have two regions at least.
+ int postSplitCount = -1;
+ while ((postSplitCount = admin.getTableRegions(tableName).size()) == originalCount) {
+ Threads.sleep(1);;
+ }
+ assertTrue("originalCount=" + originalCount + ", newCount=" + postSplitCount,
+ originalCount != postSplitCount);
- waitForMergable(admin, tableName);
- int orignalCount = locator.getAllRegionLocations().size();
+ // Merge switch is off so merge should NOT succeed.
boolean[] results = admin.setSplitOrMergeEnabled(false, false, MasterSwitchType.MERGE);
assertEquals(results.length, 1);
assertTrue(results[0]);
List<HRegionInfo> regions = admin.getTableRegions(t.getName());
assertTrue(regions.size() > 1);
- admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
+ Future<?> f = admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
regions.get(1).getEncodedNameAsBytes(), true);
- int count = waitOnSplitOrMerge(t).size();
- assertTrue(orignalCount == count);
+ try {
+ f.get(10, TimeUnit.SECONDS);
+ fail("Should not get here.");
+ } catch (ExecutionException ee) {
+ // Expected.
+ }
+ int count = admin.getTableRegions(tableName).size();
+ assertTrue("newCount=" + postSplitCount + ", count=" + count, postSplitCount == count);
- waitForMergable(admin, tableName);
results = admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.MERGE);
+ regions = admin.getTableRegions(t.getName());
assertEquals(results.length, 1);
assertFalse(results[0]);
- admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
+ f = admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
regions.get(1).getEncodedNameAsBytes(), true);
- count = waitOnSplitOrMerge(t).size();
- assertTrue(orignalCount>count);
+ f.get(10, TimeUnit.SECONDS);
+ count = admin.getTableRegions(tableName).size();
+ assertTrue((postSplitCount / 2 /*Merge*/) == count);
admin.close();
}
@@ -156,47 +166,4 @@ public class TestSplitOrMergeStatus {
assertTrue(admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT));
assertTrue(admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE));
}
-
- private void waitForMergable(Admin admin, TableName t) throws InterruptedException, IOException {
- // Wait for the Regions to be mergeable
- MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();
- int mergeable = 0;
- while (mergeable < 2) {
- Thread.sleep(100);
- admin.majorCompact(t);
- mergeable = 0;
- for (JVMClusterUtil.RegionServerThread regionThread: miniCluster.getRegionServerThreads()) {
- for (Region region: regionThread.getRegionServer().getOnlineRegions(t)) {
- mergeable += ((HRegion)region).isMergeable() ? 1 : 0;
- }
- }
- }
- }
-
- /*
- * Wait on table split. May return because we waited long enough on the split
- * and it didn't happen. Caller should check.
- * @param t
- * @return Map of table regions; caller needs to check table actually split.
- */
- private List<HRegionLocation> waitOnSplitOrMerge(final Table t)
- throws IOException {
- try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) {
- List<HRegionLocation> regions = locator.getAllRegionLocations();
- int originalCount = regions.size();
- for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 10); i++) {
- Thread.currentThread();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- regions = locator.getAllRegionLocations();
- if (regions.size() != originalCount)
- break;
- }
- return regions;
- }
- }
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java
index 3eb65a5..8412e77 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java
@@ -35,29 +35,28 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
-import org.apache.hadoop.hbase.favored.FavoredNodesManager;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
+import org.apache.hadoop.hbase.favored.FavoredNodesManager;
import org.apache.hadoop.hbase.master.LoadBalancer;
-import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
import org.apache.hadoop.hbase.master.balancer.LoadOnlyFavoredStochasticBalancer;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.util.Threads;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -66,6 +65,9 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
@Category({ClientTests.class, MediumTests.class})
public class TestTableFavoredNodes {
@@ -76,7 +78,6 @@ public class TestTableFavoredNodes {
private final static int WAIT_TIMEOUT = 60000;
private final static int SLAVES = 8;
private FavoredNodesManager fnm;
- private RegionStates regionStates;
private Admin admin;
private final byte[][] splitKeys = new byte[][] {Bytes.toBytes(1), Bytes.toBytes(9)};
@@ -101,8 +102,8 @@ public class TestTableFavoredNodes {
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.cleanupTestDir();
TEST_UTIL.shutdownMiniCluster();
+ TEST_UTIL.cleanupTestDir();
}
@Before
@@ -111,8 +112,6 @@ public class TestTableFavoredNodes {
admin = TEST_UTIL.getAdmin();
admin.setBalancerRunning(false, true);
admin.enableCatalogJanitor(false);
- regionStates =
- TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
}
/*
@@ -165,8 +164,9 @@ public class TestTableFavoredNodes {
@Test
public void testSplitTable() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
- TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
+ Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ final int numberOfRegions = admin.getTableRegions(t.getName()).size();
checkIfFavoredNodeInformationIsCorrect(tableName);
@@ -176,13 +176,14 @@ public class TestTableFavoredNodes {
List<ServerName> parentFN = fnm.getFavoredNodes(parent);
assertNotNull("FN should not be null for region: " + parent, parentFN);
+ LOG.info("SPLITTING TABLE");
admin.split(tableName, splitPoint);
TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT);
- waitUntilTableRegionCountReached(tableName, NUM_REGIONS + 1);
+ LOG.info("FINISHED WAITING ON RIT");
+ waitUntilTableRegionCountReached(tableName, numberOfRegions + 1);
- // All regions should have favored nodes
- checkIfFavoredNodeInformationIsCorrect(tableName);
+ // All regions should have favored nodes checkIfFavoredNodeInformationIsCorrect(tableName);
// Get the daughters of parent.
HRegionInfo daughter1 = locator.getRegionLocation(parent.getStartKey(), true).getRegionInfo();
@@ -206,11 +207,18 @@ public class TestTableFavoredNodes {
// Major compact table and run catalog janitor. Parent's FN should be removed
TEST_UTIL.getMiniHBaseCluster().compact(tableName, true);
- assertEquals("Parent region should have been cleaned", 1, admin.runCatalogScan());
+ admin.runCatalogScan();
+ // Catalog cleanup is async. Wait on procedure to finish up.
+ ProcedureTestingUtility.waitAllProcedures(
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
+ // assertEquals("Parent region should have been cleaned", 1, admin.runCatalogScan());
assertNull("Parent FN should be null", fnm.getFavoredNodes(parent));
List<HRegionInfo> regions = admin.getTableRegions(tableName);
-
+ // Split and Table Disable interfere with each other around region replicas
+ // TODO. Meantime pause a few seconds.
+ Threads.sleep(2000);
+ LOG.info("STARTING DELETE");
TEST_UTIL.deleteTable(tableName);
checkNoFNForDeletedTable(regions);
@@ -235,11 +243,12 @@ public class TestTableFavoredNodes {
LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA));
LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB));
+ int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName);
admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(),
regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS);
TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT);
- waitUntilTableRegionCountReached(tableName, NUM_REGIONS - 1);
+ waitUntilTableRegionCountReached(tableName, countOfRegions - 1);
// All regions should have favored nodes
checkIfFavoredNodeInformationIsCorrect(tableName);
@@ -254,6 +263,9 @@ public class TestTableFavoredNodes {
// Major compact table and run catalog janitor. Parent FN should be removed
TEST_UTIL.getMiniHBaseCluster().compact(tableName, true);
assertEquals("Merge parents should have been cleaned", 1, admin.runCatalogScan());
+ // Catalog cleanup is async. Wait on procedure to finish up.
+ ProcedureTestingUtility.waitAllProcedures(
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
assertNull("Parent FN should be null", fnm.getFavoredNodes(regionA));
assertNull("Parent FN should be null", fnm.getFavoredNodes(regionB));
@@ -266,6 +278,7 @@ public class TestTableFavoredNodes {
private void checkNoFNForDeletedTable(List<HRegionInfo> regions) {
for (HRegionInfo region : regions) {
+ LOG.info("Testing if FN data for " + region);
assertNull("FN not null for deleted table's region: " + region, fnm.getFavoredNodes(region));
}
}
@@ -376,8 +389,8 @@ public class TestTableFavoredNodes {
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
- return regionStates.getRegionsOfTable(tableName).size() == numRegions;
+ return MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName) == numRegions;
}
});
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
index 35ed531..8805337 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
@@ -32,7 +32,6 @@ import java.util.NavigableMap;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Put;
@@ -40,6 +39,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher;
import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -84,6 +84,9 @@ public class TestIncrementTimeRange {
public static void setupBeforeClass() throws Exception {
util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
MyObserver.class.getName());
+ // Make general delay zero rather than default. Timing is off in this
+ // test that depends on an evironment edge that is manually moved forward.
+ util.getConfiguration().setInt(RemoteProcedureDispatcher.DISPATCH_DELAY_CONF_KEY, 0);
util.startMiniCluster();
EnvironmentEdgeManager.injectEdge(mee);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
index 1d7c12e..a1b33f7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
@@ -49,6 +49,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Put;
@@ -66,17 +67,19 @@ import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestRule;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@Category({RPCTests.class, SmallTests.class})
-public class TestSimpleRpcScheduler {/*
+public class TestSimpleRpcScheduler {
@Rule
public final TestRule timeout =
CategoryBasedTimeout.builder().withTimeout(this.getClass()).
- withLookingForStuckThread(true).build();*/
+ withLookingForStuckThread(true).build();
private static final Log LOG = LogFactory.getLog(TestSimpleRpcScheduler.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 7f1723c..b6ad2c9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -26,7 +26,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Deque;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index ff6b88e..626e154 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -32,14 +32,17 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.favored.FavoredNodesManager;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.LockInfo;
+import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -374,6 +377,11 @@ public class MockNoopMasterServices implements MasterServices, Server {
}
@Override
+ public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
+ return false;
+ }
+
+ @Override
public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig)
throws ReplicationException {
}
@@ -426,4 +434,16 @@ public class MockNoopMasterServices implements MasterServices, Server {
public LockManager getLockManager() {
return null;
}
+
+ @Override
+ public long dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b, boolean forcible, long nonceGroup,
+ long nonce) throws IOException {
+ return 0;
+ }
+
+ @Override
+ public ProcedureEvent getInitializedEvent() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 67a3f0a..7ac7571 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ClusterConnection;
-import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.locking.EntityLock;
@@ -54,10 +53,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompac
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeRequest;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -70,6 +69,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerIn
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.MergeRegionsRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.MergeRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
@@ -502,13 +503,6 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
- public CloseRegionForSplitOrMergeResponse closeRegionForSplitOrMerge(
- RpcController controller,
- CloseRegionForSplitOrMergeRequest request) throws ServiceException {
- return null;
- }
-
- @Override
public FlushRegionResponse flushRegion(RpcController controller,
FlushRegionRequest request) throws ServiceException {
// TODO Auto-generated method stub
@@ -516,16 +510,6 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
- public long requestRegionSplit(HRegionInfo regionInfo, byte[] splitRow) {
- return -1;
- }
-
- @Override
- public boolean isProcedureFinished(final long procId) {
- return false;
- }
-
- @Override
public SplitRegionResponse splitRegion(RpcController controller,
SplitRegionRequest request) throws ServiceException {
return null;
@@ -738,9 +722,22 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
+ public ExecuteProceduresResponse executeProcedures(RpcController controller,
+ ExecuteProceduresRequest request) throws ServiceException {
+ return null;
+ }
+
+ @Override
public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots(
RpcController controller, GetSpaceQuotaSnapshotsRequest request)
throws ServiceException {
return null;
}
+
+ @Override
+ public MergeRegionsResponse mergeRegions(RpcController controller, MergeRegionsRequest request)
+ throws ServiceException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
index 78b75d5..95c0615 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.MasterTests;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2925bc0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
deleted file mode 100644
index 23e61f6..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
+++ /dev/null
@@ -1,1403 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.zookeeper.KeeperException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-/**
- * This tests AssignmentManager with a testing cluster.
- */
-@SuppressWarnings("deprecation")
-@Category({MasterTests.class, MediumTests.class})
-public class TestAssignmentManagerOnCluster {
- private final static byte[] FAMILY = Bytes.toBytes("FAMILY");
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- final static Configuration conf = TEST_UTIL.getConfiguration();
- private static Admin admin;
-
- @Rule
- public TestName name = new TestName();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- // Using the our load balancer to control region plans
- conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
- MyLoadBalancer.class, LoadBalancer.class);
- conf.setClass(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
- MyRegionObserver.class, RegionObserver.class);
- // Reduce the maximum attempts to speed up the test
- conf.setInt("hbase.assignment.maximum.attempts", 3);
- conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
- conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
-
- TEST_UTIL.startMiniCluster(1, 4, null, MyMaster.class, MyRegionServer.class);
- admin = TEST_UTIL.getAdmin();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- /**
- * This tests restarting meta regionserver
- */
- @Test (timeout=180000)
- public void testRestartMetaRegionServer() throws Exception {
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- boolean stoppedARegionServer = false;
- try {
- HMaster master = cluster.getMaster();
- RegionStates regionStates = master.getAssignmentManager().getRegionStates();
- ServerName metaServerName = regionStates.getRegionServerOfRegion(
- HRegionInfo.FIRST_META_REGIONINFO);
- if (master.getServerName().equals(metaServerName)) {
- // Move meta off master
- metaServerName = cluster.getLiveRegionServerThreads()
- .get(0).getRegionServer().getServerName();
- master.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
- Bytes.toBytes(metaServerName.getServerName()));
- TEST_UTIL.waitUntilNoRegionsInTransition(60000);
- }
- RegionState metaState =
- MetaTableLocator.getMetaRegionState(master.getZooKeeper());
- assertEquals("Meta should be not in transition",
- metaState.getState(), RegionState.State.OPEN);
- assertNotEquals("Meta should be moved off master",
- metaState.getServerName(), master.getServerName());
- assertEquals("Meta should be on the meta server",
- metaState.getServerName(), metaServerName);
- cluster.killRegionServer(metaServerName);
- stoppedARegionServer = true;
- cluster.waitForRegionServerToStop(metaServerName, 60000);
-
- // Wait for SSH to finish
- final ServerName oldServerName = metaServerName;
- final ServerManager serverManager = master.getServerManager();
- TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- return serverManager.isServerDead(oldServerName)
- && !serverManager.areDeadServersInProgress();
- }
- });
-
- TEST_UTIL.waitUntilNoRegionsInTransition(60000);
- // Now, make sure meta is assigned
- assertTrue("Meta should be assigned",
- regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
- // Now, make sure meta is registered in zk
- metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper());
- assertEquals("Meta should be not in transition",
- metaState.getState(), RegionState.State.OPEN);
- assertEquals("Meta should be assigned", metaState.getServerName(),
- regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO));
- assertNotEquals("Meta should be assigned on a different server",
- metaState.getServerName(), metaServerName);
- } finally {
- if (stoppedARegionServer) {
- cluster.startRegionServer();
- }
- }
- }
-
- /**
- * This tests region assignment
- */
- @Test (timeout=60000)
- public void testAssignRegion() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- TEST_UTIL.assignRegion(hri);
-
- RegionStates regionStates = am.getRegionStates();
- ServerName serverName = regionStates.getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
-
- // Region is assigned now. Let's assign it again.
- // Master should not abort, and region should be assigned.
- TEST_UTIL.getAdmin().assign(hri.getRegionName());
- master.getAssignmentManager().waitForAssignment(hri);
- RegionState newState = regionStates.getRegionState(hri);
- assertTrue(newState.isOpened());
- } finally {
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests region assignment on a simulated restarted server
- */
- @Test (timeout=120000)
- public void testAssignRegionOnRestartedServer() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- TEST_UTIL.getMiniHBaseCluster().getConf().setInt("hbase.assignment.maximum.attempts", 20);
- TEST_UTIL.getMiniHBaseCluster().stopMaster(0);
- //restart the master so that conf take into affect
- TEST_UTIL.getMiniHBaseCluster().startMaster();
-
- ServerName deadServer = null;
- HMaster master = null;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- final HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- master = TEST_UTIL.getHBaseCluster().getMaster();
- Set<ServerName> onlineServers = master.getServerManager().getOnlineServers().keySet();
- assertFalse("There should be some servers online", onlineServers.isEmpty());
-
- // Use the first server as the destination server
- ServerName destServer = onlineServers.iterator().next();
-
- // Created faked dead server that is still online in master
- deadServer = ServerName.valueOf(destServer.getHostname(),
- destServer.getPort(), destServer.getStartcode() - 100L);
- master.getServerManager().recordNewServerWithLock(deadServer, ServerLoad.EMPTY_SERVERLOAD);
-
- final AssignmentManager am = master.getAssignmentManager();
- RegionPlan plan = new RegionPlan(hri, null, deadServer);
- am.addPlan(hri.getEncodedName(), plan);
- TEST_UTIL.assignRegion(hri);
-
- TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- return ! am.getRegionStates().isRegionInTransition(hri);
- }
- });
-
- assertFalse("Region should be assigned", am.getRegionStates().isRegionInTransition(hri));
- } finally {
- if (deadServer != null) {
- master.getServerManager().expireServer(deadServer);
- }
-
- TEST_UTIL.deleteTable(tableName);
-
- // reset the value for other tests
- TEST_UTIL.getMiniHBaseCluster().getConf().setInt("hbase.assignment.maximum.attempts", 3);
- ServerName masterServerName = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName();
- TEST_UTIL.getMiniHBaseCluster().stopMaster(masterServerName);
- TEST_UTIL.getMiniHBaseCluster().startMaster();
- }
- }
-
- /**
- * This tests offlining a region
- */
- @Test (timeout=60000)
- public void testOfflineRegion() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HRegionInfo hri = createTableAndGetOneRegion(tableName);
-
- RegionStates regionStates = TEST_UTIL.getHBaseCluster().
- getMaster().getAssignmentManager().getRegionStates();
- ServerName serverName = regionStates.getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- admin.offline(hri.getRegionName());
-
- long timeoutTime = System.currentTimeMillis() + 800;
- while (true) {
- if (regionStates.getRegionByStateOfTable(tableName)
- .get(RegionState.State.OFFLINE).contains(hri))
- break;
- long now = System.currentTimeMillis();
- if (now > timeoutTime) {
- fail("Failed to offline the region in time");
- break;
- }
- Thread.sleep(10);
- }
- RegionState regionState = regionStates.getRegionState(hri);
- assertTrue(regionState.isOffline());
- } finally {
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests moving a region
- */
- @Test (timeout=50000)
- public void testMoveRegion() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HRegionInfo hri = createTableAndGetOneRegion(tableName);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- RegionStates regionStates = master.getAssignmentManager().getRegionStates();
- ServerName serverName = regionStates.getRegionServerOfRegion(hri);
- ServerManager serverManager = master.getServerManager();
- ServerName destServerName = null;
- List<JVMClusterUtil.RegionServerThread> regionServers =
- TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
- for (JVMClusterUtil.RegionServerThread regionServer: regionServers) {
- HRegionServer destServer = regionServer.getRegionServer();
- destServerName = destServer.getServerName();
- if (!destServerName.equals(serverName)
- && serverManager.isServerOnline(destServerName)) {
- break;
- }
- }
- assertTrue(destServerName != null
- && !destServerName.equals(serverName));
- TEST_UTIL.getAdmin().move(hri.getEncodedNameAsBytes(),
- Bytes.toBytes(destServerName.getServerName()));
-
- long timeoutTime = System.currentTimeMillis() + 30000;
- while (true) {
- ServerName sn = regionStates.getRegionServerOfRegion(hri);
- if (sn != null && sn.equals(destServerName)) {
- TEST_UTIL.assertRegionOnServer(hri, sn, 200);
- break;
- }
- long now = System.currentTimeMillis();
- if (now > timeoutTime) {
- fail("Failed to move the region in time: "
- + regionStates.getRegionState(hri));
- }
- regionStates.waitForUpdate(50);
- }
-
- } finally {
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * If a table is deleted, we should not be able to move it anymore.
- * Otherwise, the region will be brought back.
- * @throws Exception
- */
- @Test (timeout=50000)
- public void testMoveRegionOfDeletedTable() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- Admin admin = TEST_UTIL.getAdmin();
- try {
- HRegionInfo hri = createTableAndGetOneRegion(tableName);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- RegionStates regionStates = am.getRegionStates();
- ServerName serverName = regionStates.getRegionServerOfRegion(hri);
- ServerName destServerName = null;
- for (int i = 0; i < 3; i++) {
- HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
- if (!destServer.getServerName().equals(serverName)) {
- destServerName = destServer.getServerName();
- break;
- }
- }
- assertTrue(destServerName != null
- && !destServerName.equals(serverName));
-
- TEST_UTIL.deleteTable(tableName);
-
- try {
- admin.move(hri.getEncodedNameAsBytes(),
- Bytes.toBytes(destServerName.getServerName()));
- fail("We should not find the region");
- } catch (IOException ioe) {
- assertTrue(ioe instanceof UnknownRegionException);
- }
-
- am.balance(new RegionPlan(hri, serverName, destServerName));
- assertFalse("The region should not be in transition",
- regionStates.isRegionInTransition(hri));
- } finally {
- if (admin.tableExists(tableName)) {
- TEST_UTIL.deleteTable(tableName);
- }
- }
- }
-
- HRegionInfo createTableAndGetOneRegion(
- final TableName tableName) throws IOException, InterruptedException {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5);
-
- // wait till the table is assigned
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- long timeoutTime = System.currentTimeMillis() + 1000;
- while (true) {
- List<HRegionInfo> regions = master.getAssignmentManager().
- getRegionStates().getRegionsOfTable(tableName);
- if (regions.size() > 3) {
- return regions.get(2);
- }
- long now = System.currentTimeMillis();
- if (now > timeoutTime) {
- fail("Could not find an online region");
- }
- Thread.sleep(10);
- }
- }
-
- /**
- * This tests assign a region while it's closing.
- */
- @Test (timeout=60000)
- public void testAssignWhileClosing() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- assertTrue(TEST_UTIL.assignRegion(hri));
-
- ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
- MyRegionObserver.preCloseEnabled.set(true);
- am.unassign(hri);
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
-
- MyRegionObserver.preCloseEnabled.set(false);
- am.unassign(hri);
-
- // region is closing now, will be re-assigned automatically.
- // now, let's forcefully assign it again. it should be
- // assigned properly and no double-assignment
- am.assign(hri, true);
-
- // let's check if it's assigned after it's out of transition
- am.waitOnRegionToClearRegionsInTransition(hri);
- assertTrue(am.waitForAssignment(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
- } finally {
- MyRegionObserver.preCloseEnabled.set(false);
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests region close failed
- */
- @Test (timeout=60000)
- public void testCloseFailed() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- assertTrue(TEST_UTIL.assignRegion(hri));
- ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
-
- MyRegionObserver.preCloseEnabled.set(true);
- am.unassign(hri);
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
-
- MyRegionObserver.preCloseEnabled.set(false);
- am.unassign(hri);
-
- // region may still be assigned now since it's closing,
- // let's check if it's assigned after it's out of transition
- am.waitOnRegionToClearRegionsInTransition(hri);
-
- // region should be closed and re-assigned
- assertTrue(am.waitForAssignment(hri));
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- } finally {
- MyRegionObserver.preCloseEnabled.set(false);
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests region open failed
- */
- @Test (timeout=60000)
- public void testOpenFailed() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- MyLoadBalancer.controledRegion = hri;
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- assertFalse(TEST_UTIL.assignRegion(hri));
-
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_OPEN, state.getState());
- // Failed to open since no plan, so it's on no server
- assertNull(state.getServerName());
-
- MyLoadBalancer.controledRegion = null;
- assertTrue(TEST_UTIL.assignRegion(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- } finally {
- MyLoadBalancer.controledRegion = null;
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests round-robin assignment failed due to no bulkplan
- */
- @Test (timeout=60000)
- public void testRoundRobinAssignmentFailed() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = admin.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
-
- // round-robin assignment but balancer cannot find a plan
- // assignment should fail
- MyLoadBalancer.controledRegion = hri;
- // if bulk assignment cannot update region state to online
- // or failed_open this waits until timeout
- assertFalse(TEST_UTIL.assignRegion(hri));
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_OPEN, state.getState());
- // Failed to open since no plan, so it's on no server
- assertNull(state.getServerName());
-
- // try again with valid plan
- MyLoadBalancer.controledRegion = null;
- assertTrue(TEST_UTIL.assignRegion(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- } finally {
- MyLoadBalancer.controledRegion = null;
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests retain assignment failed due to no bulkplan
- */
- @Test (timeout=60000)
- public void testRetainAssignmentFailed() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- MyLoadBalancer.controledRegion = hri;
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
-
- Map<HRegionInfo, ServerName> regions = new HashMap<>();
- ServerName dest = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
- regions.put(hri, dest);
- // retainAssignment but balancer cannot find a plan
- // assignment should fail
- am.assign(regions);
-
- // if retain assignment cannot update region state to online
- // or failed_open this waits until timeout
- assertFalse(am.waitForAssignment(hri));
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_OPEN, state.getState());
- // Failed to open since no plan, so it's on no server
- assertNull(state.getServerName());
-
- // try retainAssigment again with valid plan
- MyLoadBalancer.controledRegion = null;
- am.assign(regions);
- assertTrue(am.waitForAssignment(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
-
- // it retains on same server as specified
- assertEquals(serverName, dest);
- } finally {
- MyLoadBalancer.controledRegion = null;
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests region open failure which is not recoverable
- */
- @Test (timeout=60000)
- public void testOpenFailedUnrecoverable() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- FileSystem fs = FileSystem.get(conf);
- Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
- Path regionDir = new Path(tableDir, hri.getEncodedName());
- // create a file named the same as the region dir to
- // mess up with region opening
- fs.create(regionDir, true);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- assertFalse(TEST_UTIL.assignRegion(hri));
-
- RegionState state = am.getRegionStates().getRegionState(hri);
- assertEquals(RegionState.State.FAILED_OPEN, state.getState());
- // Failed to open due to file system issue. Region state should
- // carry the opening region server so that we can force close it
- // later on before opening it again. See HBASE-9092.
- assertNotNull(state.getServerName());
-
- // remove the blocking file, so that region can be opened
- fs.delete(regionDir, true);
- assertTrue(TEST_UTIL.assignRegion(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- } finally {
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- @Test (timeout=60000)
- public void testSSHWhenDisablingTableRegionsInOpeningOrPendingOpenState() throws Exception {
- final TableName table =
- TableName.valueOf
- ("testSSHWhenDisablingTableRegionsInOpeningOrPendingOpenState");
- AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
- HRegionInfo hri = null;
- ServerName serverName = null;
- try {
- hri = createTableAndGetOneRegion(table);
- serverName = am.getRegionStates().getRegionServerOfRegion(hri);
- ServerName destServerName = null;
- HRegionServer destServer = null;
- for (int i = 0; i < 3; i++) {
- destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
- if (!destServer.getServerName().equals(serverName)) {
- destServerName = destServer.getServerName();
- break;
- }
- }
- am.regionOffline(hri);
- am.getRegionStates().updateRegionState(hri, RegionState.State.PENDING_OPEN, destServerName);
-
- am.getTableStateManager().setTableState(table, TableState.State.DISABLING);
- List<HRegionInfo> toAssignRegions = am.cleanOutCrashedServerReferences(destServerName);
- assertTrue("Regions to be assigned should be empty.", toAssignRegions.isEmpty());
- assertTrue("Regions to be assigned should be empty.", am.getRegionStates()
- .getRegionState(hri).isOffline());
- } finally {
- if (hri != null && serverName != null) {
- am.regionOnline(hri, serverName);
- }
- am.getTableStateManager().setTableState(table, TableState.State.ENABLED);
- TEST_UTIL.getAdmin().disableTable(table);
- TEST_UTIL.deleteTable(table);
- }
- }
-
- /**
- * This tests region close hanging
- */
- @Test (timeout=60000)
- public void testCloseHang() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- assertTrue(TEST_UTIL.assignRegion(hri));
- ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
-
- MyRegionObserver.postCloseEnabled.set(true);
- am.unassign(hri);
-
- // Let region closing move ahead. The region should be closed
- // properly and re-assigned automatically
- MyRegionObserver.postCloseEnabled.set(false);
-
- // region may still be assigned now since it's closing,
- // let's check if it's assigned after it's out of transition
- am.waitOnRegionToClearRegionsInTransition(hri);
-
- // region should be closed and re-assigned
- assertTrue(am.waitForAssignment(hri));
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- } finally {
- MyRegionObserver.postCloseEnabled.set(false);
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * This tests region close racing with open
- */
- @Test (timeout=60000)
- public void testOpenCloseRacing() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
- meta.close();
-
- MyRegionObserver.postOpenEnabled.set(true);
- MyRegionObserver.postOpenCalled = false;
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- // Region will be opened, but it won't complete
- am.assign(hri);
- long end = EnvironmentEdgeManager.currentTime() + 20000;
- // Wait till postOpen is called
- while (!MyRegionObserver.postOpenCalled ) {
- assertFalse("Timed out waiting for postOpen to be called",
- EnvironmentEdgeManager.currentTime() > end);
- Thread.sleep(300);
- }
-
- // Now let's unassign it, it should do nothing
- am.unassign(hri);
- RegionState state = am.getRegionStates().getRegionState(hri);
- ServerName oldServerName = state.getServerName();
- assertTrue(state.isOpening() && oldServerName != null);
-
- // Now the region is stuck in opening
- // Let's forcefully re-assign it to trigger closing/opening
- // racing. This test is to make sure this scenario
- // is handled properly.
- MyRegionObserver.postOpenEnabled.set(false);
- ServerName destServerName = null;
- int numRS = TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size();
- for (int i = 0; i < numRS; i++) {
- HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
- if (!destServer.getServerName().equals(oldServerName)) {
- destServerName = destServer.getServerName();
- break;
- }
- }
- assertNotNull(destServerName);
- assertFalse("Region should be assigned on a new region server",
- oldServerName.equals(destServerName));
- List<HRegionInfo> regions = new ArrayList<>();
- regions.add(hri);
- am.assign(destServerName, regions);
-
- // let's check if it's assigned after it's out of transition
- am.waitOnRegionToClearRegionsInTransition(hri);
- assertTrue(am.waitForAssignment(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 6000);
- } finally {
- MyRegionObserver.postOpenEnabled.set(false);
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * Test force unassign/assign a region hosted on a dead server
- */
- @Test (timeout=60000)
- public void testAssignRacingWithSSH() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- MyMaster master = null;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- // Assign the region
- master = (MyMaster)cluster.getMaster();
- AssignmentManager am = master.getAssignmentManager();
-
- am.assign(hri);
-
- // Hold SSH before killing the hosting server
- master.enableSSH(false);
-
-
- RegionStates regionStates = am.getRegionStates();
- ServerName metaServer = regionStates.getRegionServerOfRegion(
- HRegionInfo.FIRST_META_REGIONINFO);
- while (true) {
- assertTrue(am.waitForAssignment(hri));
- RegionState state = regionStates.getRegionState(hri);
- ServerName oldServerName = state.getServerName();
- if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
- // Kill the hosting server, which doesn't have meta on it.
- cluster.killRegionServer(oldServerName);
- cluster.waitForRegionServerToStop(oldServerName, -1);
- break;
- }
- int i = cluster.getServerWithMeta();
- HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
- oldServerName = rs.getServerName();
- master.move(hri.getEncodedNameAsBytes(),
- Bytes.toBytes(oldServerName.getServerName()));
- }
-
- // You can't assign a dead region before SSH
- am.assign(hri, true);
- RegionState state = regionStates.getRegionState(hri);
- assertTrue(state.isFailedClose());
-
- // You can't unassign a dead region before SSH either
- am.unassign(hri);
- state = regionStates.getRegionState(hri);
- assertTrue(state.isFailedClose());
-
- // Enable SSH so that log can be split
- master.enableSSH(true);
-
- // let's check if it's assigned after it's out of transition.
- // no need to assign it manually, SSH should do it
- am.waitOnRegionToClearRegionsInTransition(hri);
- assertTrue(am.waitForAssignment(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 6000);
- } finally {
- if (master != null) {
- master.enableSSH(true);
- }
- TEST_UTIL.deleteTable(tableName);
- cluster.startRegionServer();
- }
- }
-
- /**
- * Test SSH waiting for extra region server for assignment
- */
- @Test (timeout=300000)
- public void testSSHWaitForServerToAssignRegion() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- boolean startAServer = false;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- HMaster master = cluster.getMaster();
- final ServerManager serverManager = master.getServerManager();
- MyLoadBalancer.countRegionServers = Integer.valueOf(
- serverManager.countOfRegionServers());
- HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
- assertNotNull("First region should be assigned", rs);
- final ServerName serverName = rs.getServerName();
- // Wait till SSH tried to assign regions a several times
- int counter = MyLoadBalancer.counter.get() + 5;
- cluster.killRegionServer(serverName);
- startAServer = true;
- cluster.waitForRegionServerToStop(serverName, -1);
- while (counter > MyLoadBalancer.counter.get()) {
- Thread.sleep(1000);
- }
- cluster.startRegionServer();
- startAServer = false;
- // Wait till the dead server is processed by SSH
- TEST_UTIL.waitFor(120000, 1000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- return serverManager.isServerDead(serverName)
- && !serverManager.areDeadServersInProgress();
- }
- });
- TEST_UTIL.waitUntilNoRegionsInTransition(300000);
-
- rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
- assertTrue("First region should be re-assigned to a different server",
- rs != null && !serverName.equals(rs.getServerName()));
- } finally {
- MyLoadBalancer.countRegionServers = null;
- TEST_UTIL.deleteTable(tableName);
- if (startAServer) {
- cluster.startRegionServer();
- }
- }
- }
-
- /**
- * Test force unassign/assign a region of a disabled table
- */
- @Test (timeout=60000)
- public void testAssignDisabledRegion() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- MyMaster master = null;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- // Assign the region
- master = (MyMaster)cluster.getMaster();
- AssignmentManager am = master.getAssignmentManager();
- RegionStates regionStates = am.getRegionStates();
- assertTrue(TEST_UTIL.assignRegion(hri));
-
- // Disable the table
- admin.disableTable(tableName);
- assertTrue(regionStates.isRegionOffline(hri));
-
- // You can't assign a disabled region
- am.assign(hri, true);
- assertTrue(regionStates.isRegionOffline(hri));
-
- // You can't unassign a disabled region either
- am.unassign(hri);
- assertTrue(regionStates.isRegionOffline(hri));
- } finally {
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * Test offlined region is assigned by SSH
- */
- @Test (timeout=60000)
- public void testAssignOfflinedRegionBySSH() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- MyMaster master = null;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- // Assign the region
- master = (MyMaster)cluster.getMaster();
- AssignmentManager am = master.getAssignmentManager();
- am.assign(hri);
-
- RegionStates regionStates = am.getRegionStates();
- ServerName metaServer = regionStates.getRegionServerOfRegion(
- HRegionInfo.FIRST_META_REGIONINFO);
- ServerName oldServerName = null;
- while (true) {
- assertTrue(am.waitForAssignment(hri));
- RegionState state = regionStates.getRegionState(hri);
- oldServerName = state.getServerName();
- if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
- // Mark the hosting server aborted, but don't actually kill it.
- // It doesn't have meta on it.
- MyRegionServer.abortedServer = oldServerName;
- break;
- }
- int i = cluster.getServerWithMeta();
- HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
- oldServerName = rs.getServerName();
- master.move(hri.getEncodedNameAsBytes(),
- Bytes.toBytes(oldServerName.getServerName()));
- }
-
- // Make sure the region is assigned on the dead server
- assertTrue(regionStates.isRegionOnline(hri));
- assertEquals(oldServerName, regionStates.getRegionServerOfRegion(hri));
-
- // Kill the hosting server, which doesn't have meta on it.
- cluster.killRegionServer(oldServerName);
- cluster.waitForRegionServerToStop(oldServerName, -1);
-
- ServerManager serverManager = master.getServerManager();
- while (!serverManager.isServerDead(oldServerName)
- || serverManager.getDeadServers().areDeadServersInProgress()) {
- Thread.sleep(100);
- }
-
- // Let's check if it's assigned after it's out of transition.
- // no need to assign it manually, SSH should do it
- am.waitOnRegionToClearRegionsInTransition(hri);
- assertTrue(am.waitForAssignment(hri));
-
- ServerName serverName = master.getAssignmentManager().
- getRegionStates().getRegionServerOfRegion(hri);
- TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
- } finally {
- MyRegionServer.abortedServer = null;
- TEST_UTIL.deleteTable(tableName);
- cluster.startRegionServer();
- }
- }
-
- /**
- * Test disabled region is ignored by SSH
- */
- @Test (timeout=60000)
- public void testAssignDisabledRegionBySSH() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
- MyMaster master;
- try {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
-
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri = new HRegionInfo(
- desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
-
- // Assign the region
- master = (MyMaster)cluster.getMaster();
- AssignmentManager am = master.getAssignmentManager();
- am.assign(hri);
-
- RegionStates regionStates = am.getRegionStates();
- ServerName metaServer = regionStates.getRegionServerOfRegion(
- HRegionInfo.FIRST_META_REGIONINFO);
- ServerName oldServerName = null;
- while (true) {
- assertTrue(am.waitForAssignment(hri));
- RegionState state = regionStates.getRegionState(hri);
- oldServerName = state.getServerName();
- if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
- // Mark the hosting server aborted, but don't actually kill it.
- // It doesn't have meta on it.
- MyRegionServer.abortedServer = oldServerName;
- break;
- }
- int i = cluster.getServerWithMeta();
- HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
- oldServerName = rs.getServerName();
- master.move(hri.getEncodedNameAsBytes(),
- Bytes.toBytes(oldServerName.getServerName()));
- }
-
- // Make sure the region is assigned on the dead server
- assertTrue(regionStates.isRegionOnline(hri));
- assertEquals(oldServerName, regionStates.getRegionServerOfRegion(hri));
-
- // Disable the table now.
- master.disableTable(hri.getTable(), HConstants.NO_NONCE, HConstants.NO_NONCE);
-
- // Kill the hosting server, which doesn't have meta on it.
- cluster.killRegionServer(oldServerName);
- cluster.waitForRegionServerToStop(oldServerName, -1);
-
- ServerManager serverManager = master.getServerManager();
- while (!serverManager.isServerDead(oldServerName)
- || serverManager.getDeadServers().areDeadServersInProgress()) {
- Thread.sleep(100);
- }
-
- // Wait till no more RIT, the region should be offline.
- TEST_UTIL.waitUntilNoRegionsInTransition(60000);
- assertTrue(regionStates.isRegionOffline(hri));
- } finally {
- MyRegionServer.abortedServer = null;
- TEST_UTIL.deleteTable(tableName);
- cluster.startRegionServer();
- }
- }
-
- /**
- * Test that region state transition call is idempotent
- */
- @Test(timeout = 60000)
- public void testReportRegionStateTransition() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- try {
- MyRegionServer.simulateRetry = true;
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- admin.createTable(desc);
- Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
- HRegionInfo hri =
- new HRegionInfo(desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
- MetaTableAccessor.addRegionToMeta(meta, hri);
- HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- AssignmentManager am = master.getAssignmentManager();
- TEST_UTIL.assignRegion(hri);
- RegionStates regionStates = am.getRegionStates();
- ServerName serverName = regionStates.getRegionServerOfRegion(hri);
- // Assert the the region is actually open on the server
- TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
- // Closing region should just work fine
- admin.disableTable(tableName);
- assertTrue(regionStates.isRegionOffline(hri));
- List<HRegionInfo> regions = TEST_UTIL.getAdmin().getOnlineRegions(serverName);
- assertTrue(!regions.contains(hri));
- } finally {
- MyRegionServer.simulateRetry = false;
- TEST_UTIL.deleteTable(tableName);
- }
- }
-
- /**
- * Test concurrent updates to meta when meta is not on master
- * @throws Exception
- */
- @Test(timeout = 30000)
- public void testUpdatesRemoteMeta() throws Exception {
- conf.setInt("hbase.regionstatestore.meta.connection", 3);
- final RegionStateStore rss =
- new RegionStateStore(new MyMaster(conf, new ZkCoordinatedStateManager()));
- rss.start();
- // Create 10 threads and make each do 10 puts related to region state update
- Thread[] th = new Thread[10];
- List<String> nameList = new ArrayList<>();
- List<TableName> tableNameList = new ArrayList<>();
- for (int i = 0; i < th.length; i++) {
- th[i] = new Thread() {
- @Override
- public void run() {
- HRegionInfo[] hri = new HRegionInfo[10];
- ServerName serverName = ServerName.valueOf("dummyhost", 1000, 1234);
- for (int i = 0; i < 10; i++) {
- hri[i] = new HRegionInfo(TableName.valueOf(Thread.currentThread().getName() + "_" + i));
- RegionState newState = new RegionState(hri[i], RegionState.State.OPEN, serverName);
- RegionState oldState =
- new RegionState(hri[i], RegionState.State.PENDING_OPEN, serverName);
- rss.updateRegionState(1, newState, oldState);
- }
- }
- };
- th[i].start();
- nameList.add(th[i].getName());
- }
- for (int i = 0; i < th.length; i++) {
- th[i].join();
- }
- // Add all the expected table names in meta to tableNameList
- for (String name : nameList) {
- for (int i = 0; i < 10; i++) {
- tableNameList.add(TableName.valueOf(name + "_" + i));
- }
- }
- List<Result> metaRows = MetaTableAccessor.fullScanRegions(admin.getConnection());
- int count = 0;
- // Check all 100 rows are in meta
- for (Result result : metaRows) {
- if (tableNameList.contains(HRegionInfo.getTable(result.getRow()))) {
- count++;
- if (count == 100) {
- break;
- }
- }
- }
- assertTrue(count == 100);
- rss.stop();
- }
-
- static class MyLoadBalancer extends StochasticLoadBalancer {
- // For this region, if specified, always assign to nowhere
- static volatile HRegionInfo controledRegion = null;
-
- static volatile Integer countRegionServers = null;
- static AtomicInteger counter = new AtomicInteger(0);
-
- @Override
- public ServerName randomAssignment(HRegionInfo regionInfo,
- List<ServerName> servers) throws HBaseIOException {
- if (regionInfo.equals(controledRegion)) {
- return null;
- }
- return super.randomAssignment(regionInfo, servers);
- }
-
- @Override
- public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(
- List<HRegionInfo> regions, List<ServerName> servers) throws HBaseIOException {
- if (countRegionServers != null && services != null) {
- int regionServers = services.getServerManager().countOfRegionServers();
- if (regionServers < countRegionServers.intValue()) {
- // Let's wait till more region servers join in.
- // Before that, fail region assignments.
- counter.incrementAndGet();
- return null;
- }
- }
- if (regions.get(0).equals(controledRegion)) {
- Map<ServerName, List<HRegionInfo>> m = Maps.newHashMap();
- m.put(LoadBalancer.BOGUS_SERVER_NAME, regions);
- return m;
- }
- return super.roundRobinAssignment(regions, servers);
- }
-
- @Override
- public Map<ServerName, List<HRegionInfo>> retainAssignment(
- Map<HRegionInfo, ServerName> regions, List<ServerName> servers) throws HBaseIOException {
- for (HRegionInfo hri : regions.keySet()) {
- if (hri.equals(controledRegion)) {
- Map<ServerName, List<HRegionInfo>> m = Maps.newHashMap();
- m.put(LoadBalancer.BOGUS_SERVER_NAME, Lists.newArrayList(regions.keySet()));
- return m;
- }
- }
- return super.retainAssignment(regions, servers);
- }
- }
-
- public static class MyMaster extends HMaster {
- AtomicBoolean enabled = new AtomicBoolean(true);
-
- public MyMaster(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
- InterruptedException {
- super(conf, cp);
- }
-
- @Override
- public boolean isServerCrashProcessingEnabled() {
- return enabled.get() && super.isServerCrashProcessingEnabled();
- }
-
- public void enableSSH(boolean enabled) {
- this.enabled.set(enabled);
- if (enabled) {
- getServerManager().processQueuedDeadServers();
- }
- }
- }
-
- public static class MyRegionServer extends MiniHBaseClusterRegionServer {
- static volatile ServerName abortedServer = null;
- static volatile boolean simulateRetry = false;
-
- public MyRegionServer(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
- InterruptedException {
- super(conf, cp);
- }
-
- @Override
- public boolean reportRegionStateTransition(TransitionCode code, long openSeqNum,
- HRegionInfo... hris) {
- if (simulateRetry) {
- // Simulate retry by calling the method twice
- super.reportRegionStateTransition(code, openSeqNum, hris);
- return super.reportRegionStateTransition(code, openSeqNum, hris);
- }
- return super.reportRegionStateTransition(code, openSeqNum, hris);
- }
-
- @Override
- public boolean isAborted() {
- return getServerName().equals(abortedServer) || super.isAborted();
- }
- }
-
- public static class MyRegionObserver implements RegionObserver {
- // If enabled, fail all preClose calls
- static AtomicBoolean preCloseEnabled = new AtomicBoolean(false);
-
- // If enabled, stall postClose calls
- static AtomicBoolean postCloseEnabled = new AtomicBoolean(false);
-
- // If enabled, stall postOpen calls
- static AtomicBoolean postOpenEnabled = new AtomicBoolean(false);
-
- // A flag to track if postOpen is called
- static volatile boolean postOpenCalled = false;
-
- @Override
- public void preClose(ObserverContext<RegionCoprocessorEnvironment> c,
- boolean abortRequested) throws IOException {
- if (preCloseEnabled.get()) throw new IOException("fail preClose from coprocessor");
- }
-
- @Override
- public void postClose(ObserverContext<RegionCoprocessorEnvironment> c,
- boolean abortRequested) {
- stallOnFlag(postCloseEnabled);
- }
-
- @Override
- public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
- postOpenCalled = true;
- stallOnFlag(postOpenEnabled);
- }
-
- private void stallOnFlag(final AtomicBoolean flag) {
- try {
- // If enabled, stall
- while (flag.get()) {
- Thread.sleep(1000);
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-}