You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/03/23 15:43:07 UTC
[03/21] hbase git commit: HBASE-14614 Procedure v2 - Core Assignment
Manager (Matteo Bertozzi)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
new file mode 100644
index 0000000..fdee307
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.assignment;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.NavigableMap;
+import java.util.SortedSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MasterWalManager;
+import org.apache.hadoop.hbase.master.MockNoopMasterServices;
+import org.apache.hadoop.hbase.master.RegionState.State;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.security.Superusers;
+
+public class MockMasterServices extends MockNoopMasterServices {
+ private final MasterFileSystem fileSystemManager;
+ private final MasterWalManager walManager;
+ private final AssignmentManager assignmentManager;
+
+ private MasterProcedureEnv procedureEnv;
+ private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
+ private ProcedureStore procedureStore;
+
+ private LoadBalancer balancer;
+ private ServerManager serverManager;
+ // Set of regions on a 'server'. Populated externally. Used in below faking 'cluster'.
+ private final NavigableMap<ServerName, SortedSet<byte []>> regionsToRegionServers;
+
+ public MockMasterServices(Configuration conf,
+ NavigableMap<ServerName, SortedSet<byte []>> regionsToRegionServers)
+ throws IOException {
+ super(conf);
+ this.regionsToRegionServers = regionsToRegionServers;
+ Superusers.initialize(conf);
+ this.fileSystemManager = new MasterFileSystem(this);
+ this.walManager = new MasterWalManager(this);
+ this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this)) {
+ public boolean isTableEnabled(final TableName tableName) {
+ return true;
+ }
+
+ public boolean isTableDisabled(final TableName tableName) {
+ return false;
+ }
+
+ @Override
+ protected boolean waitServerReportEvent(ServerName serverName, Procedure proc) {
+ // Make a report with current state of the server 'serverName' before we call wait..
+ SortedSet<byte []> regions = regionsToRegionServers.get(serverName);
+ getAssignmentManager().reportOnlineRegions(serverName, 0,
+ regions == null? new HashSet<byte []>(): regions);
+ return super.waitServerReportEvent(serverName, proc);
+ }
+ };
+ this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
+ this.serverManager = new ServerManager(this);
+ }
+
+ public void start(final int numServes, final RSProcedureDispatcher remoteDispatcher)
+ throws IOException {
+ startProcedureExecutor(remoteDispatcher);
+ assignmentManager.start();
+ for (int i = 0; i < numServes; ++i) {
+ serverManager.regionServerReport(
+ ServerName.valueOf("localhost", 100 + i, 1), ServerLoad.EMPTY_SERVERLOAD);
+ }
+ }
+
+ @Override
+ public void stop(String why) {
+ stopProcedureExecutor();
+ this.assignmentManager.stop();
+ }
+
+ private void startProcedureExecutor(final RSProcedureDispatcher remoteDispatcher)
+ throws IOException {
+ final Configuration conf = getConfiguration();
+ final Path logDir = new Path(fileSystemManager.getRootDir(),
+ MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
+
+ //procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
+ // new MasterProcedureEnv.WALStoreLeaseRecovery(this));
+ procedureStore = new NoopProcedureStore();
+ procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
+
+ procedureEnv = new MasterProcedureEnv(this,
+ remoteDispatcher != null ? remoteDispatcher : new RSProcedureDispatcher(this));
+
+ procedureExecutor = new ProcedureExecutor(conf, procedureEnv, procedureStore,
+ procedureEnv.getProcedureScheduler());
+
+ final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
+ Math.max(Runtime.getRuntime().availableProcessors(),
+ MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
+ final boolean abortOnCorruption = conf.getBoolean(
+ MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
+ MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
+ procedureStore.start(numThreads);
+ procedureExecutor.start(numThreads, abortOnCorruption);
+ procedureEnv.getRemoteDispatcher().start();
+ }
+
+ private void stopProcedureExecutor() {
+ if (procedureEnv != null) {
+ procedureEnv.getRemoteDispatcher().stop();
+ }
+
+ if (procedureExecutor != null) {
+ procedureExecutor.stop();
+ }
+
+ if (procedureStore != null) {
+ procedureStore.stop(isAborted());
+ }
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+
+ @Override
+ public MasterFileSystem getMasterFileSystem() {
+ return fileSystemManager;
+ }
+
+ @Override
+ public MasterWalManager getMasterWalManager() {
+ return walManager;
+ }
+
+ @Override
+ public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return procedureExecutor;
+ }
+
+ @Override
+ public LoadBalancer getLoadBalancer() {
+ return balancer;
+ }
+
+ @Override
+ public ServerManager getServerManager() {
+ return serverManager;
+ }
+
+ @Override
+ public AssignmentManager getAssignmentManager() {
+ return assignmentManager;
+ }
+
+ private static class MockRegionStateStore extends RegionStateStore {
+ public MockRegionStateStore(final MasterServices master) {
+ super(master);
+ }
+
+ public void start() throws IOException {
+ }
+
+ public void stop() {
+ }
+
+ public void updateRegionLocation(final HRegionInfo regionInfo, final State state,
+ final ServerName regionLocation, final ServerName lastHost, final long openSeqNum)
+ throws IOException {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
new file mode 100644
index 0000000..b75f711
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
@@ -0,0 +1,567 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.assignment;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CategoryBasedTimeout;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.RegionState.State;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
+import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
+import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestAssignmentManager {
+ private static final Log LOG = LogFactory.getLog(TestAssignmentManager.class);
+ static {
+ Logger.getLogger(MasterProcedureScheduler.class).setLevel(Level.TRACE);
+ }
+ @Rule public TestName name = new TestName();
+ @Rule public final TestRule timeout =
+ CategoryBasedTimeout.builder().withTimeout(this.getClass()).
+ withLookingForStuckThread(true).build();
+
+ private static final int PROC_NTHREADS = 64;
+ private static final int NREGIONS = 1 * 1000;
+ private static final int NSERVERS = Math.max(1, NREGIONS / 200);
+
+ private HBaseTestingUtility UTIL;
+ private MockRSProcedureDispatcher rsDispatcher;
+ private MockMasterServices master;
+ private AssignmentManager am;
+ private NavigableMap<ServerName, SortedSet<byte []>> regionsToRegionServers =
+ new ConcurrentSkipListMap<ServerName, SortedSet<byte []>>();
+
+ private void setupConfiguration(Configuration conf) throws Exception {
+ FSUtils.setRootDir(conf, UTIL.getDataTestDir());
+ conf.setBoolean(WALProcedureStore.USE_HSYNC_CONF_KEY, false);
+ conf.setInt(WALProcedureStore.SYNC_WAIT_MSEC_CONF_KEY, 10);
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, PROC_NTHREADS);
+ conf.setInt(RSProcedureDispatcher.RS_RPC_STARTUP_WAIT_TIME_CONF_KEY, 1000);
+ conf.setInt(AssignmentManager.ASSIGN_MAX_ATTEMPTS, 5);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ UTIL = new HBaseTestingUtility();
+ setupConfiguration(UTIL.getConfiguration());
+ master = new MockMasterServices(UTIL.getConfiguration(), this.regionsToRegionServers);
+ rsDispatcher = new MockRSProcedureDispatcher(master);
+ master.start(NSERVERS, rsDispatcher);
+ am = master.getAssignmentManager();
+ setUpMeta();
+ }
+
+ private void setUpMeta() throws Exception {
+ rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
+ am.assignMeta(HRegionInfo.FIRST_META_REGIONINFO);
+ am.wakeMetaLoadedEvent();
+ am.setFailoverCleanupDone(true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ master.stop("tearDown");
+ }
+
+ @Test
+ public void testAssignWithGoodExec() throws Exception {
+ testAssign(new GoodRsExecutor());
+ }
+
+ @Test
+ public void testAssignWithRandExec() throws Exception {
+ final TableName tableName = TableName.valueOf("testAssignWithRandExec");
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ rsDispatcher.setMockRsExecutor(new RandRsExecutor());
+
+ AssignProcedure proc = am.createAssignProcedure(hri, false);
+ //waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false, false)));
+ // TODO
+ }
+
+ @Test
+ public void testSocketTimeout() throws Exception {
+ final TableName tableName = TableName.valueOf(this.name.getMethodName());
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 3));
+ waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false)));
+
+ rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 3));
+ waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false)));
+ }
+
+ @Test
+ public void testServerNotYetRunning() throws Exception {
+ testRetriesExhaustedFailure(TableName.valueOf("testServerNotYetRunning"),
+ new ServerNotYetRunningRsExecutor());
+ }
+
+ private void testRetriesExhaustedFailure(final TableName tableName,
+ final MockRSExecutor executor) throws Exception {
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ // Test Assign operation failure
+ rsDispatcher.setMockRsExecutor(executor);
+ try {
+ waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false)));
+ fail("unexpected assign completion");
+ } catch (RetriesExhaustedException e) {
+ // expected exception
+ LOG.info("expected exception from assign operation: " + e.getMessage(), e);
+ }
+
+ // Assign the region (without problems)
+ rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
+ waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false)));
+
+ // Test Unassign operation failure
+ rsDispatcher.setMockRsExecutor(executor);
+ waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false)));
+ }
+
+
+ @Test
+ public void testIOExceptionOnAssignment() throws Exception {
+ testFailedOpen(TableName.valueOf("testExceptionOnAssignment"),
+ new FaultyRsExecutor(new IOException("test fault")));
+ }
+
+ @Test
+ public void testDoNotRetryExceptionOnAssignment() throws Exception {
+ testFailedOpen(TableName.valueOf("testDoNotRetryExceptionOnAssignment"),
+ new FaultyRsExecutor(new DoNotRetryIOException("test do not retry fault")));
+ }
+
+ private void testFailedOpen(final TableName tableName,
+ final MockRSExecutor executor) throws Exception {
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ // Test Assign operation failure
+ rsDispatcher.setMockRsExecutor(executor);
+ try {
+ waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false)));
+ fail("unexpected assign completion");
+ } catch (RetriesExhaustedException e) {
+ // expected exception
+ LOG.info("REGION STATE " + am.getRegionStates().getRegionNode(hri));
+ LOG.info("expected exception from assign operation: " + e.getMessage(), e);
+ assertEquals(true, am.getRegionStates().getRegionState(hri).isFailedOpen());
+ }
+ }
+
+ private void testAssign(final MockRSExecutor executor) throws Exception {
+ testAssign(executor, NREGIONS);
+ }
+
+ private void testAssign(final MockRSExecutor executor, final int nregions) throws Exception {
+ rsDispatcher.setMockRsExecutor(executor);
+
+ AssignProcedure[] assignments = new AssignProcedure[nregions];
+
+ long st = System.currentTimeMillis();
+ bulkSubmit(assignments);
+
+ for (int i = 0; i < assignments.length; ++i) {
+ ProcedureTestingUtility.waitProcedure(
+ master.getMasterProcedureExecutor(), assignments[i]);
+ assertTrue(assignments[i].toString(), assignments[i].isSuccess());
+ }
+ long et = System.currentTimeMillis();
+ float sec = ((et - st) / 1000.0f);
+ LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)",
+ assignments.length, StringUtils.humanTimeDiff(et - st), assignments.length / sec));
+ }
+
+ @Test
+ public void testAssignAnAssignedRegion() throws Exception {
+ final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
+
+ final Future<byte[]> futureA = submitProcedure(am.createAssignProcedure(hri, false));
+
+ // wait first assign
+ waitOnFuture(futureA);
+ am.getRegionStates().isRegionInState(hri, State.OPEN);
+ // Second should be a noop. We should recognize region is already OPEN internally
+ // and skip out doing nothing.
+ // wait second assign
+ final Future<byte[]> futureB = submitProcedure(am.createAssignProcedure(hri, false));
+ waitOnFuture(futureB);
+ am.getRegionStates().isRegionInState(hri, State.OPEN);
+ // TODO: What else can we do to ensure just a noop.
+ }
+
+ @Test
+ public void testUnassignAnUnassignedRegion() throws Exception {
+ final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
+ final HRegionInfo hri = createRegionInfo(tableName, 1);
+
+ rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
+
+ // assign the region first
+ waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false)));
+
+ final Future<byte[]> futureA = submitProcedure(am.createUnassignProcedure(hri, null, false));
+
+ // Wait first unassign.
+ waitOnFuture(futureA);
+ am.getRegionStates().isRegionInState(hri, State.CLOSED);
+ // Second should be a noop. We should recognize region is already CLOSED internally
+ // and skip out doing nothing.
+ final Future<byte[]> futureB = submitProcedure(am.createUnassignProcedure(hri, null, false));
+ waitOnFuture(futureB);
+ // Ensure we are still CLOSED.
+ am.getRegionStates().isRegionInState(hri, State.CLOSED);
+ // TODO: What else can we do to ensure just a noop.
+ }
+
+ private Future<byte[]> submitProcedure(final Procedure proc) {
+ return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc);
+ }
+
+ private byte[] waitOnFuture(final Future<byte[]> future) throws Exception {
+ try {
+ return future.get();
+ } catch (ExecutionException e) {
+ throw (Exception)e.getCause();
+ }
+ }
+
+ // ============================================================================================
+ // Helpers
+ // ============================================================================================
+ private void bulkSubmit(final AssignProcedure[] procs) throws Exception {
+ final Thread[] threads = new Thread[PROC_NTHREADS];
+ for (int i = 0; i < threads.length; ++i) {
+ final int threadId = i;
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ TableName tableName = TableName.valueOf("table-" + threadId);
+ int n = (procs.length / threads.length);
+ int start = threadId * n;
+ int stop = start + n;
+ for (int j = start; j < stop; ++j) {
+ procs[j] = createAndSubmitAssign(tableName, j);
+ }
+ }
+ };
+ threads[i].start();
+ }
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i].join();
+ }
+ for (int i = procs.length - 1; i >= 0 && procs[i] == null; --i) {
+ procs[i] = createAndSubmitAssign(TableName.valueOf("table-sync"), i);
+ }
+ }
+
+ private AssignProcedure createAndSubmitAssign(TableName tableName, int regionId) {
+ HRegionInfo hri = createRegionInfo(tableName, regionId);
+ AssignProcedure proc = am.createAssignProcedure(hri, false);
+ master.getMasterProcedureExecutor().submitProcedure(proc);
+ return proc;
+ }
+
+ private UnassignProcedure createAndSubmitUnassign(TableName tableName, int regionId) {
+ HRegionInfo hri = createRegionInfo(tableName, regionId);
+ UnassignProcedure proc = am.createUnassignProcedure(hri, null, false);
+ master.getMasterProcedureExecutor().submitProcedure(proc);
+ return proc;
+ }
+
+ private HRegionInfo createRegionInfo(final TableName tableName, final long regionId) {
+ return new HRegionInfo(tableName,
+ Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0);
+ }
+
+ private void sendTransitionReport(final ServerName serverName,
+ final RegionInfo regionInfo, final TransitionCode state) throws IOException {
+ ReportRegionStateTransitionRequest.Builder req =
+ ReportRegionStateTransitionRequest.newBuilder();
+ req.setServer(ProtobufUtil.toServerName(serverName));
+ req.addTransition(RegionStateTransition.newBuilder()
+ .addRegionInfo(regionInfo)
+ .setTransitionCode(state)
+ .setOpenSeqNum(1)
+ .build());
+ am.reportRegionStateTransition(req.build());
+ }
+
+ private class NoopRsExecutor implements MockRSExecutor {
+ public ExecuteProceduresResponse sendRequest(ServerName server,
+ ExecuteProceduresRequest request) throws IOException {
+ ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder();
+ if (request.getOpenRegionCount() > 0) {
+ for (OpenRegionRequest req: request.getOpenRegionList()) {
+ OpenRegionResponse.Builder resp = OpenRegionResponse.newBuilder();
+ for (RegionOpenInfo openReq: req.getOpenInfoList()) {
+ RegionOpeningState state = execOpenRegion(server, openReq);
+ if (state != null) {
+ resp.addOpeningState(state);
+ }
+ }
+ builder.addOpenRegion(resp.build());
+ }
+ }
+ if (request.getCloseRegionCount() > 0) {
+ for (CloseRegionRequest req: request.getCloseRegionList()) {
+ CloseRegionResponse resp = execCloseRegion(server,
+ req.getRegion().getValue().toByteArray());
+ if (resp != null) {
+ builder.addCloseRegion(resp);
+ }
+ }
+ }
+ return ExecuteProceduresResponse.newBuilder().build();
+ }
+
+ protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo regionInfo)
+ throws IOException {
+ return null;
+ }
+
+ protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName)
+ throws IOException {
+ return null;
+ }
+ }
+
+ private class GoodRsExecutor extends NoopRsExecutor {
+ @Override
+ protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq)
+ throws IOException {
+ sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED);
+ // Concurrency?
+ // Now update the state of our cluster in regionsToRegionServers.
+ SortedSet<byte []> regions = regionsToRegionServers.get(server);
+ if (regions == null) {
+ regions = new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ regionsToRegionServers.put(server, regions);
+ }
+ HRegionInfo hri = HRegionInfo.convert(openReq.getRegion());
+ if (regions.contains(hri.getRegionName())) {
+ throw new UnsupportedOperationException(hri.getRegionNameAsString());
+ }
+ regions.add(hri.getRegionName());
+ return RegionOpeningState.OPENED;
+ }
+
+ @Override
+ protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName)
+ throws IOException {
+ HRegionInfo hri = am.getRegionInfo(regionName);
+ sendTransitionReport(server, HRegionInfo.convert(hri), TransitionCode.CLOSED);
+ return CloseRegionResponse.newBuilder().setClosed(true).build();
+ }
+ }
+
+ private static class ServerNotYetRunningRsExecutor implements MockRSExecutor {
+ public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req)
+ throws IOException {
+ throw new ServerNotRunningYetException("wait on server startup");
+ }
+ }
+
+ private static class FaultyRsExecutor implements MockRSExecutor {
+ private final IOException exception;
+
+ public FaultyRsExecutor(final IOException exception) {
+ this.exception = exception;
+ }
+
+ public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req)
+ throws IOException {
+ throw exception;
+ }
+ }
+
+ private class SocketTimeoutRsExecutor extends GoodRsExecutor {
+ private final int maxSocketTimeoutRetries;
+ private final int maxServerRetries;
+
+ private ServerName lastServer;
+ private int sockTimeoutRetries;
+ private int serverRetries;
+
+ public SocketTimeoutRsExecutor(int maxSocketTimeoutRetries, int maxServerRetries) {
+ this.maxServerRetries = maxServerRetries;
+ this.maxSocketTimeoutRetries = maxSocketTimeoutRetries;
+ }
+
+ public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req)
+ throws IOException {
+ // SocketTimeoutException should be a temporary problem
+ // unless the server will be declared dead.
+ if (sockTimeoutRetries++ < maxSocketTimeoutRetries) {
+ if (sockTimeoutRetries == 1) assertNotEquals(lastServer, server);
+ lastServer = server;
+ LOG.debug("Socket timeout for server=" + server + " retries=" + sockTimeoutRetries);
+ throw new SocketTimeoutException("simulate socket timeout");
+ } else if (serverRetries++ < maxServerRetries) {
+ LOG.info("Mark server=" + server + " as dead. serverRetries=" + serverRetries);
+ master.getServerManager().moveFromOnlineToDeadServers(server);
+ sockTimeoutRetries = 0;
+ throw new SocketTimeoutException("simulate socket timeout");
+ } else {
+ return super.sendRequest(server, req);
+ }
+ }
+ }
+
+ private class RandRsExecutor extends NoopRsExecutor {
+ private final Random rand = new Random();
+
+ public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req)
+ throws IOException {
+ switch (rand.nextInt(5)) {
+ case 0: throw new ServerNotRunningYetException("wait on server startup");
+ case 1: throw new SocketTimeoutException("simulate socket timeout");
+ case 2: throw new RemoteException("java.io.IOException", "unexpected exception");
+ }
+ return super.sendRequest(server, req);
+ }
+
+ @Override
+ protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq)
+ throws IOException {
+ switch (rand.nextInt(6)) {
+ case 0:
+ return OpenRegionResponse.RegionOpeningState.OPENED;
+ case 1:
+ sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED);
+ return OpenRegionResponse.RegionOpeningState.ALREADY_OPENED;
+ case 2:
+ sendTransitionReport(server, openReq.getRegion(), TransitionCode.FAILED_OPEN);
+ return OpenRegionResponse.RegionOpeningState.FAILED_OPENING;
+ }
+ return null;
+ }
+
+ @Override
+ protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName)
+ throws IOException {
+ CloseRegionResponse.Builder resp = CloseRegionResponse.newBuilder();
+ boolean closed = rand.nextBoolean();
+ if (closed) {
+ HRegionInfo hri = am.getRegionInfo(regionName);
+ sendTransitionReport(server, HRegionInfo.convert(hri), TransitionCode.CLOSED);
+ }
+ resp.setClosed(closed);
+ return resp.build();
+ }
+ }
+
+ private interface MockRSExecutor {
+ ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req)
+ throws IOException;
+ }
+
+ private class MockRSProcedureDispatcher extends RSProcedureDispatcher {
+ private MockRSExecutor mockRsExec;
+
+ public MockRSProcedureDispatcher(final MasterServices master) {
+ super(master);
+ }
+
+ public void setMockRsExecutor(final MockRSExecutor mockRsExec) {
+ this.mockRsExec = mockRsExec;
+ }
+
+ @Override
+ protected void remoteDispatch(ServerName serverName, Set<RemoteProcedure> operations) {
+ submitTask(new MockRemoteCall(serverName, operations));
+ }
+
+ private class MockRemoteCall extends ExecuteProceduresRemoteCall {
+ public MockRemoteCall(final ServerName serverName,
+ final Set<RemoteProcedure> operations) {
+ super(serverName, operations);
+ }
+
+ @Override
+ protected ExecuteProceduresResponse sendRequest(final ServerName serverName,
+ final ExecuteProceduresRequest request) throws IOException {
+ return mockRsExec.sendRequest(serverName, request);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java
new file mode 100644
index 0000000..e4cec45
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.assignment;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+@Category({MasterTests.class, LargeTests.class})
+public class TestAssignmentOnRSCrash {
+ private static final Log LOG = LogFactory.getLog(TestAssignmentOnRSCrash.class);
+
+ private static final TableName TEST_TABLE = TableName.valueOf("testb");
+ private static final String FAMILY_STR = "f";
+ private static final byte[] FAMILY = Bytes.toBytes(FAMILY_STR);
+ private static final int NUM_RS = 3;
+
+ private HBaseTestingUtility UTIL;
+
+ private static void setupConf(Configuration conf) {
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ conf.set("hbase.balancer.tablesOnMaster", "none");
+ }
+
+ @Before
+ public void setup() throws Exception {
+ UTIL = new HBaseTestingUtility();
+
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(NUM_RS);
+
+ UTIL.createTable(TEST_TABLE, new byte[][] { FAMILY }, new byte[][] {
+ Bytes.toBytes("B"), Bytes.toBytes("D"), Bytes.toBytes("F"), Bytes.toBytes("L")
+ });
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test(timeout=30000)
+ public void testKillRsWithUserRegionWithData() throws Exception {
+ testCrashRsWithUserRegion(true, true);
+ }
+
+ @Test(timeout=30000)
+ public void testKillRsWithUserRegionWithoutData() throws Exception {
+ testCrashRsWithUserRegion(true, false);
+ }
+
+ @Test(timeout=30000)
+ public void testStopRsWithUserRegionWithData() throws Exception {
+ testCrashRsWithUserRegion(false, true);
+ }
+
+ @Test(timeout=30000)
+ public void testStopRsWithUserRegionWithoutData() throws Exception {
+ testCrashRsWithUserRegion(false, false);
+ }
+
+ private void testCrashRsWithUserRegion(final boolean kill, final boolean withData)
+ throws Exception {
+ final int NROWS = 100;
+ int nkilled = 0;
+ for (HRegionInfo hri: UTIL.getHBaseAdmin().getTableRegions(TEST_TABLE)) {
+ ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri);
+ if (AssignmentTestingUtil.isServerHoldingMeta(UTIL, serverName)) continue;
+
+ if (withData) {
+ testInsert(hri, NROWS);
+ }
+
+ // wait for regions to enter in transition and then to get out of transition
+ AssignmentTestingUtil.crashRs(UTIL, serverName, kill);
+ AssignmentTestingUtil.waitForRegionToBeInTransition(UTIL, hri);
+ UTIL.waitUntilNoRegionsInTransition();
+
+ if (withData) {
+ assertEquals(NROWS, testGet(hri, NROWS));
+ }
+
+ // region should be moved to another RS
+ assertNotEquals(serverName, AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri));
+
+ if (++nkilled == (NUM_RS - 1)) {
+ break;
+ }
+ }
+ assertTrue("expected RSs to be killed", nkilled > 0);
+ }
+
+ @Test(timeout=60000)
+ public void testKillRsWithMetaRegion() throws Exception {
+ testCrashRsWithMetaRegion(true);
+ }
+
+ @Test(timeout=60000)
+ public void testStopRsWithMetaRegion() throws Exception {
+ testCrashRsWithMetaRegion(false);
+ }
+
+ private void testCrashRsWithMetaRegion(final boolean kill) throws Exception {
+ int nkilled = 0;
+ for (HRegionInfo hri: AssignmentTestingUtil.getMetaRegions(UTIL)) {
+ ServerName serverName = AssignmentTestingUtil.crashRsWithRegion(UTIL, hri, kill);
+
+ // wait for region to enter in transition and then to get out of transition
+ AssignmentTestingUtil.waitForRegionToBeInTransition(UTIL, hri);
+ UTIL.waitUntilNoRegionsInTransition();
+ testGet(hri, 10);
+
+ // region should be moved to another RS
+ assertNotEquals(serverName, AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri));
+
+ if (++nkilled == (NUM_RS - 1)) {
+ break;
+ }
+ }
+ assertTrue("expected RSs to be killed", nkilled > 0);
+ }
+
+ private void testInsert(final HRegionInfo hri, final int nrows) throws IOException {
+ final Table table = UTIL.getConnection().getTable(hri.getTable());
+ for (int i = 0; i < nrows; ++i) {
+ final byte[] row = Bytes.add(hri.getStartKey(), Bytes.toBytes(i));
+ final Put put = new Put(row);
+ put.addColumn(FAMILY, null, row);
+ table.put(put);
+ }
+ }
+
+ public int testGet(final HRegionInfo hri, final int nrows) throws IOException {
+ int nresults = 0;
+ final Table table = UTIL.getConnection().getTable(hri.getTable());
+ for (int i = 0; i < nrows; ++i) {
+ final byte[] row = Bytes.add(hri.getStartKey(), Bytes.toBytes(i));
+ final Result result = table.get(new Get(row));
+ if (result != null && !result.isEmpty() &&
+ Bytes.equals(row, result.getValue(FAMILY, null))) {
+ nresults++;
+ }
+ }
+ return nresults;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
new file mode 100644
index 0000000..51f28af
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.assignment;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestMergeTableRegionsProcedure {
+ private static final Log LOG = LogFactory.getLog(TestMergeTableRegionsProcedure.class);
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static long nonceGroup = HConstants.NO_NONCE;
+ private static long nonce = HConstants.NO_NONCE;
+
+ private static final int initialRegionCount = 4;
+ private final static byte[] FAMILY = Bytes.toBytes("FAMILY");
+ final static Configuration conf = UTIL.getConfiguration();
+ private static Admin admin;
+
+ private static void setupConf(Configuration conf) {
+ // Reduce the maximum attempts to speed up the test
+ conf.setInt("hbase.assignment.maximum.attempts", 3);
+ conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
+ conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
+
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(conf);
+ UTIL.startMiniCluster(1);
+ admin = UTIL.getHBaseAdmin();
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ @Before
+ public void setup() throws Exception {
+ resetProcExecutorTestingKillFlag();
+ nonceGroup =
+ MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
+ nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
+ // Turn off balancer so it doesn't cut in and mess up our placements.
+ UTIL.getHBaseAdmin().setBalancerRunning(false, true);
+ // Turn off the meta scanner so it don't remove parent on us.
+ UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
+ resetProcExecutorTestingKillFlag();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ resetProcExecutorTestingKillFlag();
+ for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
+ LOG.info("Tear down, remove table=" + htd.getTableName());
+ UTIL.deleteTable(htd.getTableName());
+ }
+ }
+
+ private void resetProcExecutorTestingKillFlag() {
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
+ assertTrue("expected executor to be running", procExec.isRunning());
+ }
+
+ /**
+ * This tests two region merges
+ */
+ @Test(timeout=60000)
+ public void testMergeTwoRegions() throws Exception {
+ final TableName tableName = TableName.valueOf("testMergeTwoRegions");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ List<HRegionInfo> tableRegions = createTable(tableName);
+
+ HRegionInfo[] regionsToMerge = new HRegionInfo[2];
+ regionsToMerge[0] = tableRegions.get(0);
+ regionsToMerge[1] = tableRegions.get(1);
+
+ long procId = procExec.submitProcedure(new MergeTableRegionsProcedure(
+ procExec.getEnvironment(), regionsToMerge, true));
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ assertRegionCount(tableName, initialRegionCount - 1);
+ }
+
+ /**
+ * This tests two concurrent region merges
+ */
+ @Test(timeout=60000)
+ public void testMergeRegionsConcurrently() throws Exception {
+ final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ List<HRegionInfo> tableRegions = createTable(tableName);
+
+ HRegionInfo[] regionsToMerge1 = new HRegionInfo[2];
+ HRegionInfo[] regionsToMerge2 = new HRegionInfo[2];
+ regionsToMerge1[0] = tableRegions.get(0);
+ regionsToMerge1[1] = tableRegions.get(1);
+ regionsToMerge2[0] = tableRegions.get(2);
+ regionsToMerge2[1] = tableRegions.get(3);
+
+ long procId1 = procExec.submitProcedure(new MergeTableRegionsProcedure(
+ procExec.getEnvironment(), regionsToMerge1, true));
+ long procId2 = procExec.submitProcedure(new MergeTableRegionsProcedure(
+ procExec.getEnvironment(), regionsToMerge2, true));
+ ProcedureTestingUtility.waitProcedure(procExec, procId1);
+ ProcedureTestingUtility.waitProcedure(procExec, procId2);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
+ assertRegionCount(tableName, initialRegionCount - 2);
+ }
+
+ @Test(timeout=60000)
+ public void testRecoveryAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ List<HRegionInfo> tableRegions = createTable(tableName);
+
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ HRegionInfo[] regionsToMerge = new HRegionInfo[2];
+ regionsToMerge[0] = tableRegions.get(0);
+ regionsToMerge[1] = tableRegions.get(1);
+
+ long procId = procExec.submitProcedure(
+ new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
+
+ // Restart the executor and execute the step twice
+ MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ assertRegionCount(tableName, initialRegionCount - 1);
+ }
+
+ @Test(timeout = 60000)
+ public void testRollbackAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ List<HRegionInfo> tableRegions = createTable(tableName);
+
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ HRegionInfo[] regionsToMerge = new HRegionInfo[2];
+ regionsToMerge[0] = tableRegions.get(0);
+ regionsToMerge[1] = tableRegions.get(1);
+
+ long procId = procExec.submitProcedure(
+ new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
+
+ // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback
+ // NOTE: the 5 (number before MERGE_TABLE_REGIONS_UPDATE_META step) is
+ // hardcoded, so you have to look at this test at least once when you add a new step.
+ int numberOfSteps = 5;
+ MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);
+ }
+
+ private List<HRegionInfo> createTable(final TableName tableName)
+ throws Exception {
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(new HColumnDescriptor(FAMILY));
+ byte[][] splitRows = new byte[initialRegionCount - 1][];
+ for (int i = 0; i < splitRows.length; ++i) {
+ splitRows[i] = Bytes.toBytes(String.format("%d", i));
+ }
+ admin.createTable(desc, splitRows);
+ return assertRegionCount(tableName, initialRegionCount);
+ }
+
+ public List<HRegionInfo> assertRegionCount(final TableName tableName, final int nregions)
+ throws Exception {
+ UTIL.waitUntilNoRegionsInTransition();
+ List<HRegionInfo> tableRegions = admin.getTableRegions(tableName);
+ assertEquals(nregions, tableRegions.size());
+ return tableRegions;
+ }
+
+ private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java
new file mode 100644
index 0000000..1d2481d
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.assignment;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestRegionStates {
+ private static final Log LOG = LogFactory.getLog(TestRegionStates.class);
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static ThreadPoolExecutor threadPool;
+ private static ExecutorCompletionService executorService;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ threadPool = Threads.getBoundedCachedThreadPool(32, 60L, TimeUnit.SECONDS,
+ Threads.newDaemonThreadFactory("ProcedureDispatcher",
+ new UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.warn("Failed thread " + t.getName(), e);
+ }
+ }));
+ executorService = new ExecutorCompletionService(threadPool);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ threadPool.shutdown();
+ }
+
+ @Before
+ public void testSetup() {
+ }
+
+ @After
+ public void testTearDown() throws Exception {
+ while (true) {
+ Future<Object> f = executorService.poll();
+ if (f == null) break;
+ f.get();
+ }
+ }
+
+ private static void waitExecutorService(final int count) throws Exception {
+ for (int i = 0; i < count; ++i) {
+ executorService.take().get();
+ }
+ }
+
+ // ==========================================================================
+ // Regions related
+ // ==========================================================================
+
+ @Test
+ public void testRegionDoubleCreation() throws Exception {
+ // NOTE: HRegionInfo sort by table first, so we are relying on that
+ final TableName TABLE_NAME_A = TableName.valueOf("testOrderedByTableA");
+ final TableName TABLE_NAME_B = TableName.valueOf("testOrderedByTableB");
+ final TableName TABLE_NAME_C = TableName.valueOf("testOrderedByTableC");
+ final RegionStates stateMap = new RegionStates();
+ final int NRUNS = 1000;
+ final int NSMALL_RUNS = 3;
+
+ // add some regions for table B
+ for (int i = 0; i < NRUNS; ++i) {
+ addRegionNode(stateMap, TABLE_NAME_B, i);
+ }
+ // re-add the regions for table B
+ for (int i = 0; i < NRUNS; ++i) {
+ addRegionNode(stateMap, TABLE_NAME_B, i);
+ }
+ waitExecutorService(NRUNS * 2);
+
+ // add two other tables A and C that will be placed before and after table B (sort order)
+ for (int i = 0; i < NSMALL_RUNS; ++i) {
+ addRegionNode(stateMap, TABLE_NAME_A, i);
+ addRegionNode(stateMap, TABLE_NAME_C, i);
+ }
+
+ // check for the list of regions of the 3 tables
+ checkTableRegions(stateMap, TABLE_NAME_A, NSMALL_RUNS);
+ checkTableRegions(stateMap, TABLE_NAME_B, NRUNS);
+ checkTableRegions(stateMap, TABLE_NAME_C, NSMALL_RUNS);
+ }
+
+ private void checkTableRegions(final RegionStates stateMap,
+ final TableName tableName, final int nregions) {
+ List<HRegionInfo> hris = stateMap.getRegionsOfTable(tableName);
+ assertEquals(nregions, hris.size());
+ for (int i = 1; i < hris.size(); ++i) {
+ long a = Bytes.toLong(hris.get(i - 1).getStartKey());
+ long b = Bytes.toLong(hris.get(i + 0).getStartKey());
+ assertEquals(b, a + 1);
+ }
+ }
+
+ private void addRegionNode(final RegionStates stateMap,
+ final TableName tableName, final long regionId) {
+ executorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() {
+ HRegionInfo hri = new HRegionInfo(tableName,
+ Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0);
+ return stateMap.getOrCreateRegionNode(hri);
+ }
+ });
+ }
+
+ private Object createRegionNode(final RegionStates stateMap,
+ final TableName tableName, final long regionId) {
+ return stateMap.getOrCreateRegionNode(createRegionInfo(tableName, regionId));
+ }
+
+ private HRegionInfo createRegionInfo(final TableName tableName, final long regionId) {
+ return new HRegionInfo(tableName,
+ Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0);
+ }
+
+ @Test
+ public void testPerf() throws Exception {
+ final TableName TABLE_NAME = TableName.valueOf("testPerf");
+ final int NRUNS = 1000000; // 1M
+ final RegionStates stateMap = new RegionStates();
+
+ long st = System.currentTimeMillis();
+ for (int i = 0; i < NRUNS; ++i) {
+ final int regionId = i;
+ executorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() {
+ HRegionInfo hri = createRegionInfo(TABLE_NAME, regionId);
+ return stateMap.getOrCreateRegionNode(hri);
+ }
+ });
+ }
+ waitExecutorService(NRUNS);
+ long et = System.currentTimeMillis();
+ LOG.info(String.format("PERF STATEMAP INSERT: %s %s/sec",
+ StringUtils.humanTimeDiff(et - st),
+ StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
+
+ st = System.currentTimeMillis();
+ for (int i = 0; i < NRUNS; ++i) {
+ final int regionId = i;
+ executorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() {
+ HRegionInfo hri = createRegionInfo(TABLE_NAME, regionId);
+ return stateMap.getRegionState(hri);
+ }
+ });
+ }
+
+ waitExecutorService(NRUNS);
+ et = System.currentTimeMillis();
+ LOG.info(String.format("PERF STATEMAP GET: %s %s/sec",
+ StringUtils.humanTimeDiff(et - st),
+ StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
+ }
+
+ @Test
+ public void testPerfSingleThread() {
+ final TableName TABLE_NAME = TableName.valueOf("testPerf");
+ final int NRUNS = 1 * 1000000; // 1M
+
+ final RegionStates stateMap = new RegionStates();
+ long st = System.currentTimeMillis();
+ for (int i = 0; i < NRUNS; ++i) {
+ stateMap.createRegionNode(createRegionInfo(TABLE_NAME, i));
+ }
+ long et = System.currentTimeMillis();
+ LOG.info(String.format("PERF SingleThread: %s %s/sec",
+ StringUtils.humanTimeDiff(et - st),
+ StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
+ }
+
+ // ==========================================================================
+ // Server related
+ // ==========================================================================
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
new file mode 100644
index 0000000..7af9d67
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.assignment;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CategoryBasedTimeout;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.CompactionState;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestSplitTableRegionProcedure {
+ private static final Log LOG = LogFactory.getLog(TestSplitTableRegionProcedure.class);
+ @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()).
+ withLookingForStuckThread(true).build();
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static String ColumnFamilyName1 = "cf1";
+ private static String ColumnFamilyName2 = "cf2";
+
+ private static final int startRowNum = 11;
+ private static final int rowCount = 60;
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static void setupConf(Configuration conf) {
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(3);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ @Before
+ public void setup() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+
+ // Turn off balancer so it doesn't cut in and mess up our placements.
+ UTIL.getAdmin().setBalancerRunning(false, true);
+ // Turn off the meta scanner so it don't remove parent on us.
+ UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ for (HTableDescriptor htd: UTIL.getAdmin().listTables()) {
+ UTIL.deleteTable(htd.getTableName());
+ }
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegion() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionNoStoreFile() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
+ assertTrue(UTIL.countRows(tableName) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionUnevenDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = startRowNum + rowCount / 4;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionEmptyDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = startRowNum + rowCount;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ // Make sure one daughter has 0 rows.
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ assertTrue(UTIL.countRows(tableName) == rowCount);
+ assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = rowCount;
+ deleteData(tableName, splitRowNum);
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ UTIL.getAdmin().majorCompact(tableName);
+ // waiting for the major compaction to complete
+ UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
+ }
+ });
+
+ // Make sure one daughter has 0 rows.
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ final int currentRowCount = splitRowNum - startRowNum;
+ assertTrue(UTIL.countRows(tableName) == currentRowCount);
+ assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testInvalidSplitKey() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table with null split key
+ try {
+ long procId1 = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
+ ProcedureTestingUtility.waitProcedure(procExec, procId1);
+ fail("unexpected procedure start with invalid split-key");
+ } catch (DoNotRetryIOException e) {
+ LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testRollbackAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+
+ // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
+ // rollback
+ // NOTE: the 3 (number before SPLIT_TABLE_REGION_UPDATE_META step) is
+ // hardcoded, so you have to look at this test at least once when you add a new step.
+ int numberOfSteps = 3;
+ MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);
+ // check that we have only 1 region
+ assertEquals(1, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertEquals(1, daughters.size());
+ verifyData(daughters.get(0), startRowNum, rowCount,
+ Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes(ColumnFamilyName2));
+ }
+
+ @Test(timeout=60000)
+ public void testRecoveryAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+
+ // Restart the executor and execute the step twice
+ MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ private void insertData(final TableName tableName) throws IOException, InterruptedException {
+ Table t = UTIL.getConnection().getTable(tableName);
+ Put p;
+ for (int i= 0; i < rowCount / 2; i++) {
+ p = new Put(Bytes.toBytes("" + (startRowNum + i)));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
+ t.put(p);
+ p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1)));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
+ t.put(p);
+ if (i % 5 == 0) {
+ UTIL.getAdmin().flush(tableName);
+ }
+ }
+ }
+
+ private void deleteData(
+ final TableName tableName,
+ final int startDeleteRowNum) throws IOException, InterruptedException {
+ Table t = UTIL.getConnection().getTable(tableName);
+ final int numRows = rowCount + startRowNum - startDeleteRowNum;
+ Delete d;
+ for (int i= startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
+ d = new Delete(Bytes.toBytes("" + i));
+ t.delete(d);
+ if (i % 5 == 0) {
+ UTIL.getAdmin().flush(tableName);
+ }
+ }
+ }
+
+ private void verify(final TableName tableName, final int splitRowNum) throws IOException {
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ LOG.info("Row Count = " + UTIL.countRows(tableName));
+ assertTrue(UTIL.countRows(tableName) == rowCount);
+ int startRow;
+ int numRows;
+ for (int i = 0; i < daughters.size(); i++) {
+ if (Bytes.compareTo(
+ daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) == 0) {
+ startRow = startRowNum; // first region
+ numRows = splitRowNum - startRowNum;
+ } else {
+ startRow = splitRowNum;
+ numRows = rowCount + startRowNum - splitRowNum;
+ }
+ verifyData(
+ daughters.get(i),
+ startRow,
+ numRows,
+ ColumnFamilyName1.getBytes(),
+ ColumnFamilyName2.getBytes());
+ }
+ }
+
+ private void verifyData(
+ final HRegion newReg,
+ final int startRow,
+ final int numRows,
+ final byte[]... families)
+ throws IOException {
+ for (int i = startRow; i < startRow + numRows; i++) {
+ byte[] row = Bytes.toBytes("" + i);
+ Get get = new Get(row);
+ Result result = newReg.get(get);
+ Cell[] raw = result.rawCells();
+ assertEquals(families.length, result.size());
+ for (int j = 0; j < families.length; j++) {
+ assertTrue(CellUtil.matchingRow(raw[j], row));
+ assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
+ }
+ }
+ }
+
+ private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index ce8b0c6..99dcf1d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
import java.util.TreeSet;
+import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -48,7 +49,11 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterMetaBootstrap;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.TableStateManager;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.util.Bytes;
@@ -59,7 +64,45 @@ import org.apache.hadoop.hbase.util.ModifyRegionUtils;
public class MasterProcedureTestingUtility {
private static final Log LOG = LogFactory.getLog(MasterProcedureTestingUtility.class);
- private MasterProcedureTestingUtility() {
+ private MasterProcedureTestingUtility() { }
+
+ public static void restartMasterProcedureExecutor(ProcedureExecutor<MasterProcedureEnv> procExec)
+ throws Exception {
+ final MasterProcedureEnv env = procExec.getEnvironment();
+ final HMaster master = (HMaster)env.getMasterServices();
+ ProcedureTestingUtility.restart(procExec, true, true,
+ // stop services
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final AssignmentManager am = env.getAssignmentManager();
+ // try to simulate a master restart by removing the ServerManager states about seqIDs
+ for (RegionState regionState: am.getRegionStates().getRegionStates()) {
+ env.getMasterServices().getServerManager().removeRegion(regionState.getRegion());
+ }
+ am.stop();
+ master.setInitialized(false);
+ return null;
+ }
+ },
+ // restart services
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final AssignmentManager am = env.getAssignmentManager();
+ am.start();
+ if (true) {
+ MasterMetaBootstrap metaBootstrap = new MasterMetaBootstrap(master,
+ TaskMonitor.get().createStatus("meta"));
+ metaBootstrap.splitMetaLogsBeforeAssignment();
+ metaBootstrap.assignMeta();
+ metaBootstrap.processDeadServers();
+ }
+ am.joinCluster();
+ master.setInitialized(true);
+ return null;
+ }
+ });
}
// ==========================================================================
@@ -295,6 +338,9 @@ public class MasterProcedureTestingUtility {
return put;
}
+ // ==========================================================================
+ // Procedure Helpers
+ // ==========================================================================
public static long generateNonceGroup(final HMaster master) {
return master.getClusterConnection().getNonceGenerator().getNonceGroup();
}
@@ -318,13 +364,6 @@ public class MasterProcedureTestingUtility {
* finish.
* @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long)
*/
- public static void testRecoveryAndDoubleExecution(
- final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
- final int numSteps) throws Exception {
- testRecoveryAndDoubleExecution(procExec, procId, numSteps, true);
- ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
- }
-
private static void testRecoveryAndDoubleExecution(
final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
final int numSteps, final boolean expectExecRunning) throws Exception {
@@ -336,9 +375,9 @@ public class MasterProcedureTestingUtility {
// restart executor/store
// execute step N - save on store
for (int i = 0; i < numSteps; ++i) {
- LOG.info("Restart " + i + " exec state: " + procExec.getProcedure(procId));
+ LOG.info("Restart " + i + " exec state=" + procExec.getProcedure(procId));
ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
- ProcedureTestingUtility.restart(procExec);
+ restartMasterProcedureExecutor(procExec);
ProcedureTestingUtility.waitProcedure(procExec, procId);
}
@@ -366,8 +405,8 @@ public class MasterProcedureTestingUtility {
ProcedureTestingUtility.waitProcedure(procExec, procId);
assertEquals(false, procExec.isRunning());
for (int i = 0; !procExec.isFinished(procId); ++i) {
- LOG.info("Restart " + i + " exec state: " + procExec.getProcedure(procId));
- ProcedureTestingUtility.restart(procExec);
+ LOG.info("Restart " + i + " exec state=" + procExec.getProcedure(procId));
+ restartMasterProcedureExecutor(procExec);
ProcedureTestingUtility.waitProcedure(procExec, procId);
}
assertEquals(true, procExec.isRunning());
@@ -399,7 +438,7 @@ public class MasterProcedureTestingUtility {
for (int i = 0; !procExec.isFinished(procId); ++i) {
LOG.info("Restart " + i + " rollback state: " + procExec.getProcedure(procId));
ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
- ProcedureTestingUtility.restart(procExec);
+ restartMasterProcedureExecutor(procExec);
ProcedureTestingUtility.waitProcedure(procExec, procId);
}
} finally {
@@ -444,7 +483,7 @@ public class MasterProcedureTestingUtility {
try {
ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
LOG.info("Restart and rollback procId=" + procId);
- ProcedureTestingUtility.restart(procExec);
+ restartMasterProcedureExecutor(procExec);
ProcedureTestingUtility.waitProcedure(procExec, procId);
} finally {
assertTrue(procExec.unregisterListener(abortListener));