You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/05/26 23:47:13 UTC
[05/59] [abbrv] hbase git commit: Revert "HBASE-14614 Procedure v2 -
Core Assignment Manager (Matteo Bertozzi)" Revert a mistaken commit!!!
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 59e8fb3..32bce26 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -27,11 +27,8 @@ import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.util.Map;
-import java.util.NavigableMap;
import java.util.SortedMap;
-import java.util.SortedSet;
import java.util.TreeMap;
-import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,141 +36,267 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CategoryBasedTimeout;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaMockingUtil;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
+import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
-import org.apache.hadoop.hbase.master.assignment.MockMasterServices;
-import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
-import org.apache.hadoop.hbase.regionserver.ChunkCreator;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ResultOrException;
import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.ChunkCreator;
import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
-import org.junit.After;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
@Category({MasterTests.class, SmallTests.class})
public class TestCatalogJanitor {
private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
- @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
- withTimeout(this.getClass()).withLookingForStuckThread(true).build();
- @Rule public final TestName name = new TestName();
- private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
- private MockMasterServices masterServices;
- private CatalogJanitor janitor;
+
+ @Rule
+ public TestName name = new TestName();
@BeforeClass
- public static void beforeClass() throws Exception {
+ public static void setup() throws Exception {
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
}
+ /**
+ * Mock MasterServices for tests below.
+ */
+ class MockMasterServices extends MockNoopMasterServices {
+ private final ClusterConnection connection;
+ private final MasterFileSystem mfs;
+ private final AssignmentManager asm;
+ private final ServerManager sm;
+
+ MockMasterServices(final HBaseTestingUtility htu) throws IOException {
+ super(htu.getConfiguration());
+
+ ClientProtos.ClientService.BlockingInterface ri =
+ Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
+ MutateResponse.Builder builder = MutateResponse.newBuilder();
+ builder.setProcessed(true);
+ try {
+ Mockito.when(ri.mutate(
+ (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
+ thenReturn(builder.build());
+ } catch (ServiceException se) {
+ throw ProtobufUtil.handleRemoteException(se);
+ }
+ try {
+ Mockito.when(ri.multi(
+ (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
+ thenAnswer(new Answer<MultiResponse>() {
+ @Override
+ public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
+ return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
+ }
+ });
+ } catch (ServiceException se) {
+ throw ProtobufUtil.getRemoteException(se);
+ }
+ // Mock an ClusterConnection and a AdminProtocol implementation. Have the
+ // ClusterConnection return the HRI. Have the HRI return a few mocked up responses
+ // to make our test work.
+ this.connection =
+ HConnectionTestingUtility.getMockedConnectionAndDecorate(getConfiguration(),
+ Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
+ ServerName.valueOf("example.org,12345,6789"),
+ HRegionInfo.FIRST_META_REGIONINFO);
+ // Set hbase.rootdir into test dir.
+ FileSystem.get(getConfiguration());
+ Path rootdir = FSUtils.getRootDir(getConfiguration());
+ FSUtils.setRootDir(getConfiguration(), rootdir);
+ Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
+
+ this.mfs = new MasterFileSystem(this);
+ this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
+ }
- @Before
- public void setup() throws IOException {
- setRootDirAndCleanIt(HTU, this.name.getMethodName());
- NavigableMap<ServerName, SortedSet<byte []>> regionsToRegionServers =
- new ConcurrentSkipListMap<ServerName, SortedSet<byte []>>();
- this.masterServices =
- new MockMasterServices(HTU.getConfiguration(), regionsToRegionServers);
- this.masterServices.start(10, null);
- this.janitor = new CatalogJanitor(masterServices);
- }
+ @Override
+ public AssignmentManager getAssignmentManager() {
+ return this.asm;
+ }
+
+ @Override
+ public MasterFileSystem getMasterFileSystem() {
+ return this.mfs;
+ }
+
+ @Override
+ public ClusterConnection getConnection() {
+ return this.connection;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return ServerName.valueOf("mockserver.example.org", 1234, -1L);
+ }
+
+ @Override
+ public ServerManager getServerManager() {
+ return this.sm;
+ }
+
+ @Override
+ public CoordinatedStateManager getCoordinatedStateManager() {
+ BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
+ SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
+ Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
+ SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
+ Mockito.when(c.getDetails()).thenReturn(d);
+ return m;
+ }
- @After
- public void teardown() {
- this.janitor.cancel(true);
- this.masterServices.stop("DONE");
+ @Override
+ public TableDescriptors getTableDescriptors() {
+ return new TableDescriptors() {
+ @Override
+ public HTableDescriptor remove(TableName tablename) throws IOException {
+ // noop
+ return null;
+ }
+
+ @Override
+ public Map<String, HTableDescriptor> getAll() throws IOException {
+ // noop
+ return null;
+ }
+
+ @Override public Map<String, HTableDescriptor> getAllDescriptors() throws IOException {
+ // noop
+ return null;
+ }
+
+ @Override
+ public HTableDescriptor get(TableName tablename)
+ throws IOException {
+ return createHTableDescriptor();
+ }
+
+ @Override
+ public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void add(HTableDescriptor htd) throws IOException {
+ // noop
+ }
+
+ @Override
+ public void setCacheOn() throws IOException {
+ }
+
+ @Override
+ public void setCacheOff() throws IOException {
+ }
+ };
+ }
}
- /**
- * Test clearing a split parent.
- */
@Test
public void testCleanParent() throws IOException, InterruptedException {
- TableDescriptor td = createTableDescriptorForCurrentMethod();
- // Create regions.
- HRegionInfo parent =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
- HRegionInfo splita =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
- HRegionInfo splitb =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
- // Test that when both daughter regions are in place, that we do not remove the parent.
- Result r = createResult(parent, splita, splitb);
- // Add a reference under splitA directory so we don't clear out the parent.
- Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
- Path tabledir = FSUtils.getTableDir(rootdir, td.getTableName());
- Path parentdir = new Path(tabledir, parent.getEncodedName());
- Path storedir = HStore.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName());
- Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
- long now = System.currentTimeMillis();
- // Reference name has this format: StoreFile#REF_NAME_PARSER
- Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
- FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
- Path path = ref.write(fs, p);
- assertTrue(fs.exists(path));
- LOG.info("Created reference " + path);
- // Add a parentdir for kicks so can check it gets removed by the catalogjanitor.
- fs.mkdirs(parentdir);
- assertFalse(this.janitor.cleanParent(parent, r));
- ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
- assertTrue(fs.exists(parentdir));
- // Remove the reference file and try again.
- assertTrue(fs.delete(p, true));
- assertTrue(this.janitor.cleanParent(parent, r));
- // Parent cleanup is run async as a procedure. Make sure parentdir is removed.
- ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
- assertTrue(!fs.exists(parentdir));
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ setRootDirAndCleanIt(htu, "testCleanParent");
+ MasterServices services = new MockMasterServices(htu);
+ try {
+ CatalogJanitor janitor = new CatalogJanitor(services);
+ // Create regions.
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
+ htd.addFamily(new HColumnDescriptor("f"));
+ HRegionInfo parent =
+ new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("eee"));
+ HRegionInfo splita =
+ new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("ccc"));
+ HRegionInfo splitb =
+ new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
+ Bytes.toBytes("eee"));
+ // Test that when both daughter regions are in place, that we do not
+ // remove the parent.
+ Result r = createResult(parent, splita, splitb);
+ // Add a reference under splitA directory so we don't clear out the parent.
+ Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path tabledir =
+ FSUtils.getTableDir(rootdir, htd.getTableName());
+ Path storedir = HStore.getStoreHomedir(tabledir, splita,
+ htd.getColumnFamilies()[0].getName());
+ Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
+ long now = System.currentTimeMillis();
+ // Reference name has this format: StoreFile#REF_NAME_PARSER
+ Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
+ FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ Path path = ref.write(fs, p);
+ assertTrue(fs.exists(path));
+ assertFalse(janitor.cleanParent(parent, r));
+ // Remove the reference file and try again.
+ assertTrue(fs.delete(p, true));
+ assertTrue(janitor.cleanParent(parent, r));
+ } finally {
+ services.stop("shutdown");
+ }
}
/**
* Make sure parent gets cleaned up even if daughter is cleaned up before it.
+ * @throws IOException
+ * @throws InterruptedException
*/
@Test
public void testParentCleanedEvenIfDaughterGoneFirst()
throws IOException, InterruptedException {
- parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(),
- Bytes.toBytes("eee"));
+ parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
+ "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
}
/**
* Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it.
+ * @throws IOException
+ * @throws InterruptedException
*/
@Test
public void testLastParentCleanedEvenIfDaughterGoneFirst()
throws IOException, InterruptedException {
- parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(),
- new byte[0]);
- }
-
- /**
- * @return A TableDescriptor with a tableName of current method name and a column
- * family that is MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)
- */
- private TableDescriptor createTableDescriptorForCurrentMethod() {
- return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName())).
- addFamily(new HColumnDescriptor(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)).
- build();
+ parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
+ "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
}
/**
@@ -181,40 +304,50 @@ public class TestCatalogJanitor {
*
* @param rootDir the test case name, used as the HBase testing utility root
* @param lastEndKey the end key of the split parent
+ * @throws IOException
+ * @throws InterruptedException
*/
private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
final String rootDir, final byte[] lastEndKey)
throws IOException, InterruptedException {
- TableDescriptor td = createTableDescriptorForCurrentMethod();
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ setRootDirAndCleanIt(htu, rootDir);
+ MasterServices services = new MockMasterServices(htu);
+ CatalogJanitor janitor = new CatalogJanitor(services);
+ final HTableDescriptor htd = createHTableDescriptor();
+
// Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
- HRegionInfo parent = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey);
+
+ // Parent
+ HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ lastEndKey);
// Sleep a second else the encoded name on these regions comes out
// same for all with same start key and made in same second.
Thread.sleep(1001);
// Daughter a
- HRegionInfo splita =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
+ HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("ccc"));
Thread.sleep(1001);
// Make daughters of daughter a; splitaa and splitab.
- HRegionInfo splitaa =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
- HRegionInfo splitab =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
+ HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("bbb"));
+ HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
+ Bytes.toBytes("ccc"));
// Daughter b
- HRegionInfo splitb =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey);
+ HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
+ lastEndKey);
Thread.sleep(1001);
// Make Daughters of daughterb; splitba and splitbb.
- HRegionInfo splitba =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
- HRegionInfo splitbb =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey);
+ HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
+ Bytes.toBytes("ddd"));
+ HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
+ lastEndKey);
// First test that our Comparator works right up in CatalogJanitor.
- SortedMap<HRegionInfo, Result> regions =
- new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
+ // Just fo kicks.
+ SortedMap<HRegionInfo, Result> regions = new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
// Now make sure that this regions map sorts as we expect it to.
regions.put(parent, createResult(parent, splita, splitb));
regions.put(splitb, createResult(splitb, splitba, splitbb));
@@ -232,9 +365,10 @@ public class TestCatalogJanitor {
index++;
}
- // Now play around with the cleanParent function. Create a ref from splita up to the parent.
+ // Now play around with the cleanParent function. Create a ref from splita
+ // up to the parent.
Path splitaRef =
- createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
+ createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
// Make sure actual super parent sticks around because splita has a ref.
assertFalse(janitor.cleanParent(parent, regions.get(parent)));
@@ -246,13 +380,13 @@ public class TestCatalogJanitor {
// the daughter splita can be split (can't split if still references).
// BUT make the timing such that the daughter gets cleaned up before we
// can get a chance to let go of the parent.
- FileSystem fs = FileSystem.get(HTU.getConfiguration());
+ FileSystem fs = FileSystem.get(htu.getConfiguration());
assertTrue(fs.delete(splitaRef, true));
// Create the refs from daughters of splita.
Path splitaaRef =
- createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false);
+ createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
Path splitabRef =
- createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true);
+ createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
// Test splita. It should stick around because references from splitab, etc.
assertFalse(janitor.cleanParent(splita, regions.get(splita)));
@@ -264,67 +398,74 @@ public class TestCatalogJanitor {
// Super parent should get cleaned up now both splita and splitb are gone.
assertTrue(janitor.cleanParent(parent, regions.get(parent)));
+
+ services.stop("test finished");
+ janitor.cancel(true);
}
/**
* CatalogJanitor.scan() should not clean parent regions if their own
- * parents are still referencing them. This ensures that grandparent regions
+ * parents are still referencing them. This ensures that grandfather regions
* do not point to deleted parent regions.
*/
@Test
public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
- TableDescriptor td = createTableDescriptorForCurrentMethod();
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
+ MasterServices services = new MockMasterServices(htu);
+
+ final HTableDescriptor htd = createHTableDescriptor();
+
// Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
// Parent
- HRegionInfo parent = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"),
- HConstants.EMPTY_BYTE_ARRAY, true);
+ HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ new byte[0], true);
// Sleep a second else the encoded name on these regions comes out
// same for all with same start key and made in same second.
Thread.sleep(1001);
// Daughter a
- HRegionInfo splita =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true);
+ HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("ccc"), true);
Thread.sleep(1001);
-
// Make daughters of daughter a; splitaa and splitab.
- HRegionInfo splitaa =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false);
- HRegionInfo splitab =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
+ HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
+ Bytes.toBytes("bbb"), false);
+ HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
+ Bytes.toBytes("ccc"), false);
// Daughter b
- HRegionInfo splitb =
- new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY);
+ HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
+ new byte[0]);
Thread.sleep(1001);
- // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab.
final Map<HRegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator());
splitParents.put(parent, createResult(parent, splita, splitb));
splita.setOffline(true); //simulate that splita goes offline when it is split
- splitParents.put(splita, createResult(splita, splitaa, splitab));
+ splitParents.put(splita, createResult(splita, splitaa,splitab));
final Map<HRegionInfo, Result> mergedRegions = new TreeMap<>();
- CatalogJanitor spy = spy(this.janitor);
- doReturn(new Triple<>(10, mergedRegions, splitParents)).when(spy).
- getMergedRegionsAndSplitParents();
+ CatalogJanitor janitor = spy(new CatalogJanitor(services));
+ doReturn(new Triple<>(10, mergedRegions, splitParents)).when(janitor)
+ .getMergedRegionsAndSplitParents();
- // Create ref from splita to parent
- LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog());
+ //create ref from splita to parent
Path splitaRef =
- createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
- LOG.info("Created reference " + splitaRef);
+ createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
- // Parent and splita should not be removed because a reference from splita to parent.
- assertEquals(0, spy.scan());
+ //parent and A should not be removed
+ assertEquals(0, janitor.scan());
- // Now delete the ref
- FileSystem fs = FileSystem.get(HTU.getConfiguration());
+ //now delete the ref
+ FileSystem fs = FileSystem.get(htu.getConfiguration());
assertTrue(fs.delete(splitaRef, true));
//now, both parent, and splita can be deleted
- assertEquals(2, spy.scan());
+ assertEquals(2, janitor.scan());
+
+ services.stop("test finished");
+ janitor.cancel(true);
}
/**
@@ -334,7 +475,7 @@ public class TestCatalogJanitor {
@Test
public void testSplitParentFirstComparator() {
SplitParentFirstComparator comp = new SplitParentFirstComparator();
- TableDescriptor td = createTableDescriptorForCurrentMethod();
+ final HTableDescriptor htd = createHTableDescriptor();
/* Region splits:
*
@@ -355,12 +496,15 @@ public class TestCatalogJanitor {
*/
// root region
- HRegionInfo rootRegion = new HRegionInfo(td.getTableName(),
- HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, true);
- HRegionInfo firstRegion = new HRegionInfo(td.getTableName(),
- HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true);
- HRegionInfo lastRegion = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true);
+ HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
+ HConstants.EMPTY_START_ROW,
+ HConstants.EMPTY_END_ROW, true);
+ HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
+ HConstants.EMPTY_START_ROW,
+ Bytes.toBytes("bbb"), true);
+ HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("bbb"),
+ HConstants.EMPTY_END_ROW, true);
assertTrue(comp.compare(rootRegion, rootRegion) == 0);
assertTrue(comp.compare(firstRegion, firstRegion) == 0);
@@ -370,15 +514,19 @@ public class TestCatalogJanitor {
assertTrue(comp.compare(firstRegion, lastRegion) < 0);
//first region split into a, b
- HRegionInfo firstRegiona = new HRegionInfo(td.getTableName(),
- HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true);
- HRegionInfo firstRegionb = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true);
+ HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
+ HConstants.EMPTY_START_ROW,
+ Bytes.toBytes("aaa"), true);
+ HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("aaa"),
+ Bytes.toBytes("bbb"), true);
//last region split into a, b
- HRegionInfo lastRegiona = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true);
- HRegionInfo lastRegionb = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true);
+ HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("bbb"),
+ Bytes.toBytes("ddd"), true);
+ HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("ddd"),
+ HConstants.EMPTY_END_ROW, true);
assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
@@ -401,47 +549,58 @@ public class TestCatalogJanitor {
assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
- HRegionInfo lastRegionaa = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
- HRegionInfo lastRegionab = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false);
+ HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("bbb"),
+ Bytes.toBytes("ccc"), false);
+ HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("ccc"),
+ Bytes.toBytes("ddd"), false);
assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
+
}
@Test
public void testArchiveOldRegion() throws Exception {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ setRootDirAndCleanIt(htu, "testCleanParent");
+ MasterServices services = new MockMasterServices(htu);
+
+ // create the janitor
+ CatalogJanitor janitor = new CatalogJanitor(services);
+
// Create regions.
- TableDescriptor td = createTableDescriptorForCurrentMethod();
- HRegionInfo parent = new HRegionInfo(td.getTableName(),
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
+ htd.addFamily(new HColumnDescriptor("f"));
+ HRegionInfo parent = new HRegionInfo(htd.getTableName(),
Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
- HRegionInfo splita = new HRegionInfo(td.getTableName(),
+ HRegionInfo splita = new HRegionInfo(htd.getTableName(),
Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
- HRegionInfo splitb = new HRegionInfo(td.getTableName(),
- Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
+ HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
+ Bytes.toBytes("ccc"),
+ Bytes.toBytes("eee"));
// Test that when both daughter regions are in place, that we do not
// remove the parent.
Result parentMetaRow = createResult(parent, splita, splitb);
- FileSystem fs = FileSystem.get(HTU.getConfiguration());
- Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
+ FileSystem fs = FileSystem.get(htu.getConfiguration());
+ Path rootdir = services.getMasterFileSystem().getRootDir();
// have to set the root directory since we use it in HFileDisposer to figure out to get to the
// archive directory. Otherwise, it just seems to pick the first root directory it can find (so
// the single test passes, but when the full suite is run, things get borked).
FSUtils.setRootDir(fs.getConf(), rootdir);
- Path tabledir = FSUtils.getTableDir(rootdir, td.getTableName());
- Path storedir = HStore.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
- Path storeArchive =
- HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), parent,
- tabledir, td.getColumnFamilies()[0].getName());
+ Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
+ Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
+ Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
+ tabledir, htd.getColumnFamilies()[0].getName());
LOG.debug("Table dir:" + tabledir);
LOG.debug("Store dir:" + storedir);
LOG.debug("Store archive dir:" + storeArchive);
// add a couple of store files that we can check for
- FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir);
+ FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
// get the current store files for comparison
FileStatus[] storeFiles = fs.listStatus(storedir);
int index = 0;
@@ -454,10 +613,6 @@ public class TestCatalogJanitor {
// do the cleaning of the parent
assertTrue(janitor.cleanParent(parent, parentMetaRow));
- Path parentDir = new Path(tabledir, parent.getEncodedName());
- // Cleanup procedure runs async. Wait till it done.
- ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
- assertTrue(!fs.exists(parentDir));
LOG.debug("Finished cleanup of parent region");
// and now check to make sure that the files have actually been archived
@@ -469,6 +624,8 @@ public class TestCatalogJanitor {
// cleanup
FSUtils.delete(fs, rootdir, true);
+ services.stop("Test finished");
+ janitor.cancel(true);
}
/**
@@ -488,62 +645,69 @@ public class TestCatalogJanitor {
*/
@Test
public void testDuplicateHFileResolution() throws Exception {
- TableDescriptor td = createTableDescriptorForCurrentMethod();
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ setRootDirAndCleanIt(htu, "testCleanParent");
+ MasterServices services = new MockMasterServices(htu);
+
+ // create the janitor
+
+ CatalogJanitor janitor = new CatalogJanitor(services);
// Create regions.
- HRegionInfo parent = new HRegionInfo(td.getTableName(),
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
+ htd.addFamily(new HColumnDescriptor("f"));
+ HRegionInfo parent = new HRegionInfo(htd.getTableName(),
Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
- HRegionInfo splita = new HRegionInfo(td.getTableName(),
+ HRegionInfo splita = new HRegionInfo(htd.getTableName(),
Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
- HRegionInfo splitb = new HRegionInfo(td.getTableName(),
+ HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
// Test that when both daughter regions are in place, that we do not
// remove the parent.
Result r = createResult(parent, splita, splitb);
- FileSystem fs = FileSystem.get(HTU.getConfiguration());
- Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
- // Have to set the root directory since we use it in HFileDisposer to figure out to get to the
+
+ FileSystem fs = FileSystem.get(htu.getConfiguration());
+
+ Path rootdir = services.getMasterFileSystem().getRootDir();
+ // have to set the root directory since we use it in HFileDisposer to figure out to get to the
// archive directory. Otherwise, it just seems to pick the first root directory it can find (so
// the single test passes, but when the full suite is run, things get borked).
FSUtils.setRootDir(fs.getConf(), rootdir);
Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
- Path storedir = HStore.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
+ Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
System.out.println("Old root:" + rootdir);
System.out.println("Old table:" + tabledir);
System.out.println("Old store:" + storedir);
- Path storeArchive =
- HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), parent,
- tabledir, td.getColumnFamilies()[0].getName());
+ Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
+ tabledir, htd.getColumnFamilies()[0].getName());
System.out.println("Old archive:" + storeArchive);
// enable archiving, make sure that files get archived
- addMockStoreFiles(2, this.masterServices, storedir);
+ addMockStoreFiles(2, services, storedir);
// get the current store files for comparison
FileStatus[] storeFiles = fs.listStatus(storedir);
- // Do the cleaning of the parent
+ // do the cleaning of the parent
assertTrue(janitor.cleanParent(parent, r));
- Path parentDir = new Path(tabledir, parent.getEncodedName());
- ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
- assertTrue(!fs.exists(parentDir));
- // And now check to make sure that the files have actually been archived
+ // and now check to make sure that the files have actually been archived
FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
// now add store files with the same names as before to check backup
// enable archiving, make sure that files get archived
- addMockStoreFiles(2, this.masterServices, storedir);
+ addMockStoreFiles(2, services, storedir);
- // Do the cleaning of the parent
+ // do the cleaning of the parent
assertTrue(janitor.cleanParent(parent, r));
- // Cleanup procedure runs async. Wait till it done.
- ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
- assertTrue(!fs.exists(parentDir));
// and now check to make sure that the files have actually been archived
archivedStoreFiles = fs.listStatus(storeArchive);
assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
+
+ // cleanup
+ services.stop("Test finished");
+ janitor.cancel(true);
}
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
@@ -565,7 +729,8 @@ public class TestCatalogJanitor {
return storeFiles;
}
- private String setRootDirAndCleanIt(final HBaseTestingUtility htu, final String subdir)
+ private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
+ final String subdir)
throws IOException {
Path testdir = htu.getDataTestDir(subdir);
FileSystem fs = FileSystem.get(htu.getConfiguration());
@@ -574,14 +739,24 @@ public class TestCatalogJanitor {
return FSUtils.getRootDir(htu.getConfiguration()).toString();
}
+ /**
+ * @param services Master services instance.
+ * @param htd
+ * @param parent
+ * @param daughter
+ * @param midkey
+ * @param top True if we are to write a 'top' reference.
+ * @return Path to reference we created.
+ * @throws IOException
+ */
private Path createReferences(final MasterServices services,
- final TableDescriptor td, final HRegionInfo parent,
+ final HTableDescriptor htd, final HRegionInfo parent,
final HRegionInfo daughter, final byte [] midkey, final boolean top)
throws IOException {
Path rootdir = services.getMasterFileSystem().getRootDir();
Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
Path storedir = HStore.getStoreHomedir(tabledir, daughter,
- td.getColumnFamilies()[0].getName());
+ htd.getColumnFamilies()[0].getName());
Reference ref =
top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
long now = System.currentTimeMillis();
@@ -597,4 +772,30 @@ public class TestCatalogJanitor {
throws IOException {
return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
}
-}
\ No newline at end of file
+
+ private HTableDescriptor createHTableDescriptor() {
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
+ htd.addFamily(new HColumnDescriptor("f"));
+ return htd;
+ }
+
+ private MultiResponse buildMultiResponse(MultiRequest req) {
+ MultiResponse.Builder builder = MultiResponse.newBuilder();
+ RegionActionResult.Builder regionActionResultBuilder =
+ RegionActionResult.newBuilder();
+ ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
+ for (RegionAction regionAction: req.getRegionActionList()) {
+ regionActionResultBuilder.clear();
+ for (ClientProtos.Action action: regionAction.getActionList()) {
+ roeBuilder.clear();
+ roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
+ roeBuilder.setIndex(action.getIndex());
+ regionActionResultBuilder.addResultOrException(roeBuilder.build());
+ }
+ builder.addRegionActionResult(regionActionResultBuilder.build());
+ }
+ return builder.build();
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index fcd2cf9..b78bfd1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -84,7 +84,6 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index d6210b9..eb4ce99 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
index 0073cdf..74f2c91 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java
@@ -34,11 +34,9 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-@Ignore // SimpleLoadBalancer seems borked whether AMv2 or not. Disabling till gets attention.
@Category({MasterTests.class, MediumTests.class})
public class TestMasterBalanceThrottling {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -122,9 +120,8 @@ public class TestMasterBalanceThrottling {
@Override
public void run() {
while (!stop.get()) {
- maxCount.set(Math.max(maxCount.get(),
- master.getAssignmentManager().getRegionStates()
- .getRegionsInTransition().size()));
+ maxCount.set(Math.max(maxCount.get(), master.getAssignmentManager().getRegionStates()
+ .getRegionsInTransitionCount()));
try {
Thread.sleep(10);
} catch (InterruptedException e) {
@@ -139,7 +136,7 @@ public class TestMasterBalanceThrottling {
}
private void unbalance(HMaster master, TableName tableName) throws Exception {
- while (master.getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0) {
+ while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) {
Thread.sleep(100);
}
HRegionServer biasedServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
@@ -147,7 +144,7 @@ public class TestMasterBalanceThrottling {
master.move(regionInfo.getEncodedNameAsBytes(),
Bytes.toBytes(biasedServer.getServerName().getServerName()));
}
- while (master.getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0) {
+ while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) {
Thread.sleep(100);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index 29c0576..f57d6b9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -43,8 +43,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
-import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -57,12 +55,10 @@ import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({FlakeyTests.class, LargeTests.class})
-@Ignore // Needs to be rewritten for AMv2. Uses tricks not ordained when up on AMv2.
public class TestMasterFailover {
private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
@@ -256,22 +252,23 @@ public class TestMasterFailover {
// Put the online region in pending_close. It is actually already opened.
// This is to simulate that the region close RPC is not sent out before failover
RegionState oldState = regionStates.getRegionState(hriOnline);
- RegionState newState = new RegionState(hriOnline, State.CLOSING, oldState.getServerName());
- stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState);
+ RegionState newState = new RegionState(
+ hriOnline, State.PENDING_CLOSE, oldState.getServerName());
+ stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
// Put the offline region in pending_open. It is actually not opened yet.
// This is to simulate that the region open RPC is not sent out before failover
oldState = new RegionState(hriOffline, State.OFFLINE);
- newState = new RegionState(hriOffline, State.OPENING, newState.getServerName());
- stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState);
+ newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
+ stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
HRegionInfo failedClose = new HRegionInfo(offlineTable.getTableName(), null, null);
createRegion(failedClose, rootdir, conf, offlineTable);
MetaTableAccessor.addRegionToMeta(master.getConnection(), failedClose);
- oldState = new RegionState(failedClose, State.CLOSING);
+ oldState = new RegionState(failedClose, State.PENDING_CLOSE);
newState = new RegionState(failedClose, State.FAILED_CLOSE, newState.getServerName());
- stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState);
+ stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
HRegionInfo failedOpen = new HRegionInfo(offlineTable.getTableName(), null, null);
createRegion(failedOpen, rootdir, conf, offlineTable);
@@ -279,9 +276,9 @@ public class TestMasterFailover {
// Simulate a region transitioning to failed open when the region server reports the
// transition as FAILED_OPEN
- oldState = new RegionState(failedOpen, State.OPENING);
+ oldState = new RegionState(failedOpen, State.PENDING_OPEN);
newState = new RegionState(failedOpen, State.FAILED_OPEN, newState.getServerName());
- stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState);
+ stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
HRegionInfo failedOpenNullServer = new HRegionInfo(offlineTable.getTableName(), null, null);
LOG.info("Failed open NUll server " + failedOpenNullServer.getEncodedName());
@@ -292,7 +289,7 @@ public class TestMasterFailover {
// the region
oldState = new RegionState(failedOpenNullServer, State.OFFLINE);
newState = new RegionState(failedOpenNullServer, State.FAILED_OPEN, null);
- stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState);
+ stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
// Stop the master
log("Aborting master");
@@ -381,12 +378,12 @@ public class TestMasterFailover {
assertEquals("hbase:meta should be onlined on RS",
metaState.getState(), State.OPEN);
- // Update meta state as OPENING, then kill master
+ // Update meta state as PENDING_OPEN, then kill master
// that simulates, that RS successfully deployed, but
// RPC was lost right before failure.
// region server should expire (how it can be verified?)
MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(),
- rs.getServerName(), State.OPENING);
+ rs.getServerName(), State.PENDING_OPEN);
Region meta = rs.getFromOnlineRegions(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
rs.removeFromOnlineRegions(meta, null);
((HRegion)meta).close();
@@ -413,12 +410,12 @@ public class TestMasterFailover {
assertEquals("hbase:meta should be onlined on RS",
metaState.getState(), State.OPEN);
- // Update meta state as CLOSING, then kill master
+ // Update meta state as PENDING_CLOSE, then kill master
// that simulates, that RS successfully deployed, but
// RPC was lost right before failure.
// region server should expire (how it can be verified?)
MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(),
- rs.getServerName(), State.CLOSING);
+ rs.getServerName(), State.PENDING_CLOSE);
log("Aborting master");
activeMaster.abort("test-kill");
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
index 8a216c5..0084d44 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -56,13 +55,12 @@ public class TestMasterMetrics {
KeeperException, InterruptedException {
super(conf, cp);
}
-/*
+
@Override
protected void tryRegionServerReport(
long reportStartTime, long reportEndTime) {
// do nothing
}
-*/
}
@BeforeClass
@@ -83,7 +81,7 @@ public class TestMasterMetrics {
}
}
- @Ignore @Test(timeout = 300000)
+ @Test(timeout = 300000)
public void testClusterRequests() throws Exception {
// sending fake request to master to see how metric value has changed
@@ -116,7 +114,7 @@ public class TestMasterMetrics {
master.stopMaster();
}
- @Ignore @Test
+ @Test
public void testDefaultMasterMetrics() throws Exception {
MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
metricsHelper.assertGauge( "numRegionServers", 2, masterSource);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
index 8b0874a..6c737e9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.master;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Collection;
@@ -173,7 +172,7 @@ public class TestMasterOperationsForRegionReplicas {
}
validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
ADMIN.getConnection());
- /* DISABLED!!!!! FOR NOW!!!!
+
// Now shut the whole cluster down, and verify the assignments are kept so that the
// availability constraints are met.
TEST_UTIL.getConfiguration().setBoolean("hbase.master.startup.retainassign", true);
@@ -193,19 +192,17 @@ public class TestMasterOperationsForRegionReplicas {
TEST_UTIL.getMiniHBaseCluster().startRegionServer();
}
- // Check on alter table
+ //check on alter table
ADMIN.disableTable(tableName);
assert(ADMIN.isTableDisabled(tableName));
//increase the replica
desc.setRegionReplication(numReplica + 1);
ADMIN.modifyTable(tableName, desc);
ADMIN.enableTable(tableName);
- LOG.info(ADMIN.getTableDescriptor(tableName).toString());
assert(ADMIN.isTableEnabled(tableName));
List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
- assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" + numReplica,
- regions.size() == numRegions * (numReplica + 1));
+ assert(regions.size() == numRegions * (numReplica + 1));
//decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
ADMIN.disableTable(tableName);
@@ -232,7 +229,6 @@ public class TestMasterOperationsForRegionReplicas {
assert(defaultReplicas.size() == numRegions);
Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
assert(counts.size() == 1 && counts.contains(new Integer(numReplica)));
- */
} finally {
ADMIN.disableTable(tableName);
ADMIN.deleteTable(tableName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
index 23efdb2..b59e6ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
@@ -18,12 +18,15 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+
import java.io.IOException;
import java.io.StringWriter;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -32,23 +35,23 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
-import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
-import org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapperStub;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapperStub;
+import org.apache.hadoop.hbase.tmpl.master.AssignmentManagerStatusTmpl;
+import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* Tests for the master status page and its template.
@@ -87,7 +90,7 @@ public class TestMasterStatusServlet {
// Fake AssignmentManager and RIT
AssignmentManager am = Mockito.mock(AssignmentManager.class);
RegionStates rs = Mockito.mock(RegionStates.class);
- List<RegionState> regionsInTransition = new ArrayList<>();
+ Set<RegionState> regionsInTransition = new HashSet<>();
regionsInTransition.add(new RegionState(FAKE_HRI, RegionState.State.CLOSING, 12345L, FAKE_HOST));
Mockito.doReturn(rs).when(am).getRegionStates();
Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition();
@@ -154,4 +157,45 @@ public class TestMasterStatusServlet {
.setDeadServers(deadServers)
.render(new StringWriter(), master);
}
+
+ @Test
+ public void testAssignmentManagerTruncatedList() throws IOException {
+ AssignmentManager am = Mockito.mock(AssignmentManager.class);
+ RegionStates rs = Mockito.mock(RegionStates.class);
+
+ // Add 100 regions as in-transition
+ TreeSet<RegionState> regionsInTransition = new TreeSet<>(RegionStates.REGION_STATE_COMPARATOR);
+ for (byte i = 0; i < 100; i++) {
+ HRegionInfo hri = new HRegionInfo(FAKE_TABLE.getTableName(),
+ new byte[]{i}, new byte[]{(byte) (i+1)});
+ regionsInTransition.add(
+ new RegionState(hri, RegionState.State.CLOSING, 12345L, FAKE_HOST));
+ }
+ // Add hbase:meta in transition as well
+ regionsInTransition.add(
+ new RegionState(HRegionInfo.FIRST_META_REGIONINFO,
+ RegionState.State.CLOSING, 123L, FAKE_HOST));
+ Mockito.doReturn(rs).when(am).getRegionStates();
+ Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition();
+ Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransitionOrderedByTimestamp();
+
+ // Render to a string
+ StringWriter sw = new StringWriter();
+ new AssignmentManagerStatusTmpl()
+ // NOT IMPLEMENTED!!!! .setLimit(50)
+ .render(sw, am);
+ String result = sw.toString();
+ // Should always include META
+ assertTrue(result.contains(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()));
+
+ /* BROKEN BY HBASE-13839 Fix AssgnmentManagerTmpl.jamon issues (coloring, content etc.) FIX!!
+ // Make sure we only see 50 of them
+ Matcher matcher = Pattern.compile("CLOSING").matcher(result);
+ int count = 0;
+ while (matcher.find()) {
+ count++;
+ }
+ assertEquals(50, count);
+ */
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
index 8641b20..782c400 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
@@ -81,7 +81,7 @@ public class TestMasterWalManager {
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath),
- new SplitLogTask.Owned(inRecoveryServerName).toByteArray(),
+ new SplitLogTask.Owned(inRecoveryServerName, mwm.getLogRecoveryMode()).toByteArray(),
Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
String staleRegionPath = ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, staleRegion);
ZKUtil.createWithParents(zkw, staleRegionPath);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
index 68160df..a845a73 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java
index fe5883b..daf6d43 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java
@@ -35,19 +35,14 @@ public class TestRegionState {
public TestName name = new TestName();
@Test
- public void testSerializeDeserialize() {
- final TableName tableName = TableName.valueOf("testtb");
- for (RegionState.State state: RegionState.State.values()) {
- testSerializeDeserialize(tableName, state);
- }
- }
-
- private void testSerializeDeserialize(final TableName tableName, final RegionState.State state) {
- RegionState state1 = new RegionState(new HRegionInfo(tableName), state);
+ public void test() {
+ RegionState state1 = new RegionState(
+ new HRegionInfo(TableName.valueOf(name.getMethodName())), RegionState.State.OPENING);
ClusterStatusProtos.RegionState protobuf1 = state1.convert();
RegionState state2 = RegionState.convert(protobuf1);
ClusterStatusProtos.RegionState protobuf2 = state1.convert();
- assertEquals("RegionState does not match " + state, state1, state2);
- assertEquals("Protobuf does not match " + state, protobuf1, protobuf2);
+
+ assertEquals(state1, state2);
+ assertEquals(protobuf1, protobuf2);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java
new file mode 100644
index 0000000..17004ec
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.RegionState.State;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import static org.junit.Assert.assertTrue;
+import static junit.framework.Assert.assertFalse;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestRegionStates {
+ @Test (timeout=10000)
+ public void testCanMakeProgressThoughMetaIsDown()
+ throws IOException, InterruptedException, BrokenBarrierException {
+ MasterServices server = mock(MasterServices.class);
+ when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
+ Connection connection = mock(ClusterConnection.class);
+ // Set up a table that gets 'stuck' when we try to fetch a row from the meta table.
+ // It is stuck on a CyclicBarrier latch. We use CyclicBarrier because it will tell us when
+ // thread is waiting on latch.
+ Table metaTable = Mockito.mock(Table.class);
+ final CyclicBarrier latch = new CyclicBarrier(2);
+ when(metaTable.get((Get)Mockito.any())).thenAnswer(new Answer<Result>() {
+ @Override
+ public Result answer(InvocationOnMock invocation) throws Throwable {
+ latch.await();
+ throw new java.net.ConnectException("Connection refused");
+ }
+ });
+ when(connection.getTable(TableName.META_TABLE_NAME)).thenReturn(metaTable);
+ when(server.getConnection()).thenReturn((ClusterConnection)connection);
+ Configuration configuration = mock(Configuration.class);
+ when(server.getConfiguration()).thenReturn(configuration);
+ TableStateManager tsm = mock(TableStateManager.class);
+ ServerManager sm = mock(ServerManager.class);
+ when(sm.isServerOnline(isA(ServerName.class))).thenReturn(true);
+
+ RegionStateStore rss = mock(RegionStateStore.class);
+ final RegionStates regionStates = new RegionStates(server, tsm, sm, rss);
+ final ServerName sn = mockServer("one", 1);
+ regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.SPLITTING_NEW, sn);
+ Thread backgroundThread = new Thread("Get stuck setting server offline") {
+ @Override
+ public void run() {
+ regionStates.serverOffline(sn);
+ }
+ };
+ assertTrue(latch.getNumberWaiting() == 0);
+ backgroundThread.start();
+ while (latch.getNumberWaiting() == 0);
+ // Verify I can do stuff with synchronized RegionStates methods, that I am not locked out.
+ // Below is a call that is synchronized. Can I do it and not block?
+ regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO);
+ // Done. Trip the barrier on the background thread.
+ latch.await();
+ }
+
+ @Test
+ public void testWeDontReturnDrainingServersForOurBalancePlans() throws Exception {
+ MasterServices server = mock(MasterServices.class);
+ when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
+ Configuration configuration = mock(Configuration.class);
+ when(server.getConfiguration()).thenReturn(configuration);
+ TableStateManager tsm = mock(TableStateManager.class);
+ ServerManager sm = mock(ServerManager.class);
+ when(sm.isServerOnline(isA(ServerName.class))).thenReturn(true);
+
+ RegionStateStore rss = mock(RegionStateStore.class);
+ RegionStates regionStates = new RegionStates(server, tsm, sm, rss);
+
+ ServerName one = mockServer("one", 1);
+ ServerName two = mockServer("two", 1);
+ ServerName three = mockServer("three", 1);
+
+ when(sm.getDrainingServersList()).thenReturn(Arrays.asList(three));
+
+ regionStates.regionOnline(createFakeRegion(), one);
+ regionStates.regionOnline(createFakeRegion(), two);
+ regionStates.regionOnline(createFakeRegion(), three);
+
+
+ Map<TableName, Map<ServerName, List<HRegionInfo>>> result =
+ regionStates.getAssignmentsByTable();
+ for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
+ assertFalse(map.keySet().contains(three));
+ }
+ }
+
+ private HRegionInfo createFakeRegion() {
+ HRegionInfo info = mock(HRegionInfo.class);
+ when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
+ return info;
+ }
+
+ private ServerName mockServer(String fakeHost, int fakePort) {
+ ServerName serverName = mock(ServerName.class);
+ when(serverName.getHostname()).thenReturn(fakeHost);
+ when(serverName.getPort()).thenReturn(fakePort);
+ return serverName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 351fca4..7c41c0f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -35,14 +35,12 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -109,7 +107,6 @@ public class TestRestartCluster {
* This tests retaining assignments on a cluster restart
*/
@Test (timeout=300000)
- @Ignore // Does not work in new AMv2 currently.
public void testRetainAssignmentOnRestart() throws Exception {
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
@@ -198,7 +195,7 @@ public class TestRestartCluster {
Threads.sleep(100);
}
- snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getConnection());
+ snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> newRegionToRegionServerMap =
snapshot.getRegionToRegionServerMap();
@@ -207,8 +204,7 @@ public class TestRestartCluster {
if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
ServerName currentServer = entry.getValue();
- LOG.info("Key=" + entry.getKey() + " oldServer=" + oldServer + ", currentServer=" + currentServer);
- assertEquals(entry.getKey().toString(), oldServer.getAddress(), currentServer.getAddress());
+ assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
index 58be83b..ec7ffe6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
@@ -19,10 +19,7 @@
package org.apache.hadoop.hbase.master;
import static org.apache.hadoop.hbase.regionserver.HRegion.warmupHRegion;
-import static org.junit.Assert.assertTrue;
-
import java.io.IOException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,20 +29,21 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
+import org.junit.experimental.categories.Category;
+import org.junit.BeforeClass;
import org.junit.AfterClass;
import org.junit.Before;
-import org.junit.BeforeClass;
+import org.junit.After;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
/**
* Run tests that use the HBase clients; {@link org.apache.hadoop.hbase.client.HTable}.
@@ -160,8 +158,6 @@ public class TestWarmupRegion {
for (int i = 0; i < 10; i++) {
HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(serverid);
byte [] destName = Bytes.toBytes(rs.getServerName().toString());
- assertTrue(destName != null);
- LOG.info("i=" + i );
TEST_UTIL.getMiniHBaseCluster().getMaster().move(info.getEncodedNameAsBytes(), destName);
serverid = (serverid + 1) % 2;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
deleted file mode 100644
index 07b989b..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.assignment;
-
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
-import org.apache.hadoop.hbase.util.Threads;
-
-import static org.junit.Assert.assertEquals;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class AssignmentTestingUtil {
- private static final Log LOG = LogFactory.getLog(AssignmentTestingUtil.class);
-
- private AssignmentTestingUtil() {}
-
- public static void waitForRegionToBeInTransition(final HBaseTestingUtility util,
- final HRegionInfo hri) throws Exception {
- while (!getMaster(util).getAssignmentManager().getRegionStates().isRegionInTransition(hri)) {
- Threads.sleep(10);
- }
- }
-
- public static void waitForRsToBeDead(final HBaseTestingUtility util,
- final ServerName serverName) throws Exception {
- util.waitFor(60000, new ExplainingPredicate<Exception>() {
- @Override
- public boolean evaluate() {
- return getMaster(util).getServerManager().isServerDead(serverName);
- }
-
- @Override
- public String explainFailure() {
- return "Server " + serverName + " is not dead";
- }
- });
- }
-
- public static void stopRs(final HBaseTestingUtility util, final ServerName serverName)
- throws Exception {
- LOG.info("STOP REGION SERVER " + serverName);
- util.getMiniHBaseCluster().stopRegionServer(serverName);
- waitForRsToBeDead(util, serverName);
- }
-
- public static void killRs(final HBaseTestingUtility util, final ServerName serverName)
- throws Exception {
- LOG.info("KILL REGION SERVER " + serverName);
- util.getMiniHBaseCluster().killRegionServer(serverName);
- waitForRsToBeDead(util, serverName);
- }
-
- public static void crashRs(final HBaseTestingUtility util, final ServerName serverName,
- final boolean kill) throws Exception {
- if (kill) {
- killRs(util, serverName);
- } else {
- stopRs(util, serverName);
- }
- }
-
- public static ServerName crashRsWithRegion(final HBaseTestingUtility util,
- final HRegionInfo hri, final boolean kill) throws Exception {
- ServerName serverName = getServerHoldingRegion(util, hri);
- crashRs(util, serverName, kill);
- return serverName;
- }
-
- public static ServerName getServerHoldingRegion(final HBaseTestingUtility util,
- final HRegionInfo hri) throws Exception {
- ServerName serverName = util.getMiniHBaseCluster().getServerHoldingRegion(
- hri.getTable(), hri.getRegionName());
- ServerName amServerName = getMaster(util).getAssignmentManager().getRegionStates()
- .getRegionServerOfRegion(hri);
-
- // Make sure AM and MiniCluster agrees on the Server holding the region
- // and that the server is online.
- assertEquals(amServerName, serverName);
- assertEquals(true, getMaster(util).getServerManager().isServerOnline(serverName));
- return serverName;
- }
-
- public static boolean isServerHoldingMeta(final HBaseTestingUtility util,
- final ServerName serverName) throws Exception {
- for (HRegionInfo hri: getMetaRegions(util)) {
- if (serverName.equals(getServerHoldingRegion(util, hri))) {
- return true;
- }
- }
- return false;
- }
-
- public static Set<HRegionInfo> getMetaRegions(final HBaseTestingUtility util) {
- return getMaster(util).getAssignmentManager().getMetaRegionSet();
- }
-
- private static HMaster getMaster(final HBaseTestingUtility util) {
- return util.getMiniHBaseCluster().getMaster();
- }
-}
\ No newline at end of file