You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/10/28 04:46:03 UTC
[2/7] hbase git commit: HBASE-14551 Procedure v2 - Reimplement split
(Stephen Yuan Jiang)
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
index 91a5f37..eb9811d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -19,17 +19,15 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.security.PrivilegedAction;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.DroppedSnapshotException;
-import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.util.StringUtils;
import com.google.common.base.Preconditions;
@@ -43,7 +41,6 @@ class SplitRequest implements Runnable {
private final byte[] midKey;
private final HRegionServer server;
private final User user;
- private TableLock tableLock;
SplitRequest(Region region, byte[] midKey, HRegionServer hrs, User user) {
Preconditions.checkNotNull(hrs);
@@ -58,63 +55,48 @@ class SplitRequest implements Runnable {
return "regionName=" + parent + ", midKey=" + Bytes.toStringBinary(midKey);
}
- private void doSplitting(User user) {
+ private void doSplitting() {
boolean success = false;
server.metricsRegionServer.incrSplitRequest();
long startTime = EnvironmentEdgeManager.currentTime();
- SplitTransactionImpl st = new SplitTransactionImpl(parent, midKey);
+
try {
- //acquire a shared read lock on the table, so that table schema modifications
- //do not happen concurrently
- tableLock = server.getTableLockManager().readLock(parent.getTableDesc().getTableName()
- , "SPLIT_REGION:" + parent.getRegionInfo().getRegionNameAsString());
- try {
- tableLock.acquire();
- } catch (IOException ex) {
- tableLock = null;
- throw ex;
+ long procId;
+ if (user != null && user.getUGI() != null) {
+ procId = user.getUGI().doAs (new PrivilegedAction<Long>() {
+ @Override
+ public Long run() {
+ try {
+ return server.requestRegionSplit(parent.getRegionInfo(), midKey);
+ } catch (Exception e) {
+ LOG.error("Failed to complete region split ", e);
+ }
+ return (long)-1;
+ }
+ });
+ } else {
+ procId = server.requestRegionSplit(parent.getRegionInfo(), midKey);
}
- // If prepare does not return true, for some reason -- logged inside in
- // the prepare call -- we are not ready to split just now. Just return.
- if (!st.prepare()) return;
- try {
- st.execute(this.server, this.server, user);
- success = true;
- } catch (Exception e) {
- if (this.server.isStopping() || this.server.isStopped()) {
- LOG.info(
- "Skip rollback/cleanup of failed split of "
- + parent.getRegionInfo().getRegionNameAsString() + " because server is"
- + (this.server.isStopping() ? " stopping" : " stopped"), e);
- return;
- }
- if (e instanceof DroppedSnapshotException) {
- server.abort("Replay of WAL required. Forcing server shutdown", e);
- return;
- }
+ if (procId != -1) {
+ // wait for the split to complete or get interrupted. If the split completes successfully,
+ // the procedure will return true; if the split fails, the procedure would throw exception.
+ //
try {
- LOG.info("Running rollback/cleanup of failed split of " +
- parent.getRegionInfo().getRegionNameAsString() + "; " + e.getMessage(), e);
- if (st.rollback(this.server, this.server)) {
- LOG.info("Successful rollback of failed split of " +
- parent.getRegionInfo().getRegionNameAsString());
- } else {
- this.server.abort("Abort; we got an error after point-of-no-return");
+ while (!(success = server.isProcedureFinished(procId))) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOG.warn("Split region " + parent + " is still in progress. Not waiting...");
+ break;
+ }
}
- } catch (RuntimeException ee) {
- String msg = "Failed rollback of failed split of " +
- parent.getRegionInfo().getRegionNameAsString() + " -- aborting server";
- // If failed rollback, kill this server to avoid having a hole in table.
- LOG.info(msg, ee);
- this.server.abort(msg + " -- Cause: " + ee.getMessage());
+ } catch (IOException e) {
+ LOG.error("Split region " + parent + " failed.", e);
}
- return;
+ } else {
+ LOG.error("Fail to split region " + parent);
}
- } catch (IOException ex) {
- ex = ex instanceof RemoteException ? ((RemoteException) ex).unwrapRemoteException() : ex;
- LOG.error("Split failed " + this, ex);
- server.checkFileSystem();
} finally {
if (this.parent.getCoprocessorHost() != null) {
try {
@@ -124,24 +106,17 @@ class SplitRequest implements Runnable {
io instanceof RemoteException ? ((RemoteException) io).unwrapRemoteException() : io);
}
}
+
+ // Update regionserver metrics with the split transaction total running time
+ server.metricsRegionServer.updateSplitTime(EnvironmentEdgeManager.currentTime() - startTime);
+
if (parent.shouldForceSplit()) {
parent.clearSplit();
}
- releaseTableLock();
- long endTime = EnvironmentEdgeManager.currentTime();
- // Update regionserver metrics with the split transaction total running time
- server.metricsRegionServer.updateSplitTime(endTime - startTime);
+
if (success) {
server.metricsRegionServer.incrSplitSuccess();
- // Log success
- LOG.info("Region split, hbase:meta updated, and report to master. Parent="
- + parent.getRegionInfo().getRegionNameAsString() + ", new regions: "
- + st.getFirstDaughter().getRegionNameAsString() + ", "
- + st.getSecondDaughter().getRegionNameAsString() + ". Split took "
- + StringUtils.formatTimeDiff(EnvironmentEdgeManager.currentTime(), startTime));
}
- // Always log the split transaction journal
- LOG.info("Split transaction journal:\n\t" + StringUtils.join("\n\t", st.getJournal()));
}
}
@@ -152,19 +127,7 @@ class SplitRequest implements Runnable {
this.server.isStopping() + " or stopped=" + this.server.isStopped());
return;
}
- doSplitting(user);
- }
- protected void releaseTableLock() {
- if (this.tableLock != null) {
- try {
- this.tableLock.release();
- } catch (IOException ex) {
- LOG.error("Could not release the table lock (something is really wrong). "
- + "Aborting this server to avoid holding the lock forever.");
- this.server.abort("Abort; we got an error when releasing the table lock "
- + "on " + parent.getRegionInfo().getRegionNameAsString());
- }
- }
+ doSplitting();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 3fc2ef5..831db4b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -1446,6 +1446,14 @@ public class AccessController extends BaseMasterAndRegionObserver
Action.ADMIN, Action.CREATE);
}
+ @Override
+ public void preSplitRegion(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] splitRow) throws IOException {
+ requirePermission(getActiveUser(ctx), "split", tableName, null, null, Action.ADMIN);
+ }
+
/* ---- RegionObserver implementation ---- */
@Override
@@ -1510,19 +1518,6 @@ public class AccessController extends BaseMasterAndRegionObserver
}
@Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException {
- requirePermission(getActiveUser(c), "split", getTableName(c.getEnvironment()), null, null,
- Action.ADMIN);
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> c,
- byte[] splitRow) throws IOException {
- requirePermission(getActiveUser(c), "split", getTableName(c.getEnvironment()), null, null,
- Action.ADMIN);
- }
-
- @Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, final InternalScanner scanner, final ScanType scanType)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index aa53d22..404c9ae 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -307,6 +307,16 @@ public class MockRegionServerServices implements RegionServerServices {
}
@Override
+ public long requestRegionSplit(final HRegionInfo regionInfo, final byte[] splitRow) {
+ return -1;
+ }
+
+ @Override
+ public boolean isProcedureFinished(final long procId) {
+ return false;
+ }
+
+ @Override
public boolean registerService(Service service) {
// TODO Auto-generated method stub
return false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
index b2ef1bd..465853a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
@@ -163,9 +163,7 @@ public class TestCoprocessorInterface {
private boolean postCompactCalled;
private boolean preFlushCalled;
private boolean postFlushCalled;
- private boolean preSplitCalled;
private boolean postSplitCalled;
- private boolean preSplitWithSplitRowCalled;
private ConcurrentMap<String, Object> sharedData;
@Override
@@ -218,16 +216,6 @@ public class TestCoprocessorInterface {
postFlushCalled = true;
}
@Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) {
- preSplitCalled = true;
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> c,
- byte[] splitRow) throws IOException {
- preSplitWithSplitRowCalled = true;
- }
- @Override
public void postSplit(ObserverContext<RegionCoprocessorEnvironment> e, Region l, Region r) {
postSplitCalled = true;
}
@@ -257,7 +245,7 @@ public class TestCoprocessorInterface {
return (preCompactCalled && postCompactCalled);
}
boolean wasSplit() {
- return (preSplitCalled && postSplitCalled && preSplitWithSplitRowCalled);
+ return postSplitCalled;
}
Map<String, Object> getSharedData() {
return sharedData;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 9f2d8f5..c431beb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.MasterSwitchType;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
@@ -1466,6 +1467,44 @@ public class TestMasterObserver {
public void postBalanceRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
String groupName, boolean balancerRan) throws IOException {
}
+
+ @Override
+ public void preSplitRegion(
+ final ObserverContext<MasterCoprocessorEnvironment> c,
+ final TableName tableName,
+ final byte[] splitRow) throws IOException {
+ }
+
+ @Override
+ public void preSplitRegionAction(
+ final ObserverContext<MasterCoprocessorEnvironment> c,
+ final TableName tableName,
+ final byte[] splitRow) throws IOException {
+ }
+
+ @Override
+ public void postCompletedSplitRegionAction(
+ final ObserverContext<MasterCoprocessorEnvironment> c,
+ final HRegionInfo regionInfoA,
+ final HRegionInfo regionInfoB) throws IOException {
+ }
+
+ @Override
+ public void preSplitRegionBeforePONRAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final byte[] splitKey,
+ final List<Mutation> metaEntries) throws IOException {
+ }
+
+ @Override
+ public void preSplitRegionAfterPONRAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
+ }
+
+ @Override
+ public void preRollBackSplitRegionAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
+ }
}
private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 87fb169..2630068 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -287,6 +287,15 @@ public class MockNoopMasterServices implements MasterServices, Server {
}
@Override
+ public long splitRegion(
+ final HRegionInfo regionInfo,
+ final byte[] splitRow,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ return -1;
+ }
+
+ @Override
public TableLockManager getTableLockManager() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 3e430b5..0237f8d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -51,6 +51,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
@@ -382,9 +384,6 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
return builder.build();
}
-
-
-
@Override
public MutateResponse mutate(RpcController controller, MutateRequest request)
throws ServiceException {
@@ -492,6 +491,13 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
+ public CloseRegionForSplitResponse closeRegionForSplit(
+ RpcController controller,
+ CloseRegionForSplitRequest request) throws ServiceException {
+ return null;
+ }
+
+ @Override
public FlushRegionResponse flushRegion(RpcController controller,
FlushRegionRequest request) throws ServiceException {
// TODO Auto-generated method stub
@@ -499,9 +505,18 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
+ public long requestRegionSplit(HRegionInfo regionInfo, byte[] splitRow) {
+ return -1;
+ }
+
+ @Override
+ public boolean isProcedureFinished(final long procId) {
+ return false;
+ }
+
+ @Override
public SplitRegionResponse splitRegion(RpcController controller,
SplitRegionRequest request) throws ServiceException {
- // TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSplitTableRegionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSplitTableRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSplitTableRegionProcedure.java
new file mode 100644
index 0000000..147c354
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSplitTableRegionProcedure.java
@@ -0,0 +1,480 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.CompactionState;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SplitTableRegionState;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestSplitTableRegionProcedure {
+ private static final Log LOG = LogFactory.getLog(TestSplitTableRegionProcedure.class);
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static long nonceGroup = HConstants.NO_NONCE;
+ private static long nonce = HConstants.NO_NONCE;
+
+ private static String ColumnFamilyName1 = "cf1";
+ private static String ColumnFamilyName2 = "cf2";
+
+ private static final int startRowNum = 11;
+ private static final int rowCount = 60;
+
+ private static void setupConf(Configuration conf) {
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(3);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ @Before
+ public void setup() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ nonceGroup =
+ MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
+ nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
+
+ // Turn off balancer so it doesn't cut in and mess up our placements.
+ UTIL.getHBaseAdmin().setBalancerRunning(false, true);
+ // Turn off the meta scanner so it don't remove parent on us.
+ UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
+ LOG.info("Tear down, remove table=" + htd.getTableName());
+ UTIL.deleteTable(htd.getTableName());
+ }
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegion() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegion");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionNoStoreFile() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegionNoStoreFile");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
+ assertTrue(UTIL.countRows(tableName) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionUnevenDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegionUnevenDaughter");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = startRowNum + rowCount / 4;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionEmptyDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegionEmptyDaughter");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = startRowNum + rowCount;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ // Make sure one daughter has 0 rows.
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ assertTrue(UTIL.countRows(tableName) == rowCount);
+ assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegionDeletedRowsDaughter");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ // Split to two daughters with one of them only has 1 row
+ int splitRowNum = rowCount;
+ deleteData(tableName, splitRowNum);
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ UTIL.getHBaseAdmin().majorCompact(tableName);
+ // waiting for the major compaction to complete
+ UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE;
+ }
+ });
+
+ // Make sure one daughter has 0 rows.
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ final int currentRowCount = splitRowNum - startRowNum;
+ assertTrue(UTIL.countRows(tableName) == currentRowCount);
+ assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
+ }
+
+ @Test(timeout=60000)
+ public void testSplitTableRegionTwiceWithSameNonce() throws Exception {
+ final TableName tableName = TableName.valueOf("testSplitTableRegionTwiceWithSameNonce");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table
+ long procId1 = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+ // Split region of the table with the same nonce
+ long procId2 = procExec.submitProcedure(
+ new SplitTableRegionProcedure(
+ procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId1);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
+ // The second proc should succeed too - because it is the same proc.
+ ProcedureTestingUtility.waitProcedure(procExec, procId2);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
+ assertTrue(procId1 == procId2);
+
+ verify(tableName, splitRowNum);
+ }
+
+ @Test(timeout=60000)
+ public void testInvalidSplitKey() throws Exception {
+ final TableName tableName = TableName.valueOf("testInvalidSplitKey");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+
+ // Split region of the table with null split key
+ long procId1 = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), tableName, regions[0], null),
+ nonceGroup,
+ nonce);
+ ProcedureTestingUtility.waitProcedure(procExec, procId1);
+ ProcedureInfo result = procExec.getResult(procId1);
+ assertTrue(result.isFailed());
+ LOG.debug("Split failed with exception: " + result.getExceptionFullMessage());
+ assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 1);
+ }
+
+ @Test(timeout = 600000)
+ public void testRollbackAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+
+ // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
+ // rollback
+ // NOTE: the 5 (number before SPLIT_TABLE_REGION_UPDATE_META step) is
+ // hardcoded, so you have to look at this test at least once when you add a new step.
+ int numberOfSteps = 5;
+ MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
+ procExec,
+ procId,
+ numberOfSteps);
+ }
+
+ @Test(timeout=60000)
+ public void testRecoveryAndDoubleExecution() throws Exception {
+ final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ HRegionInfo [] regions = MasterProcedureTestingUtility.createTable(
+ procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
+ insertData(tableName);
+ int splitRowNum = startRowNum + rowCount / 2;
+ byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+
+ assertTrue("not able to find a splittable region", regions != null);
+ assertTrue("not able to find a splittable region", regions.length == 1);
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), tableName, regions[0], splitKey),
+ nonceGroup,
+ nonce);
+
+ // Restart the executor and execute the step twice
+ int numberOfSteps = SplitTableRegionState.values().length;
+ MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ verify(tableName, splitRowNum);
+ }
+
+ private void insertData(final TableName tableName) throws IOException, InterruptedException {
+ Table t = UTIL.getConnection().getTable(tableName);
+ Put p;
+ for (int i= 0; i < rowCount / 2; i++) {
+ p = new Put(Bytes.toBytes("" + (startRowNum + i)));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
+ t.put(p);
+ p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1)));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
+ p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
+ t.put(p);
+ if (i % 5 == 0) {
+ UTIL.getHBaseAdmin().flush(tableName);
+ }
+ }
+ }
+
+ private void deleteData(
+ final TableName tableName,
+ final int startDeleteRowNum) throws IOException, InterruptedException {
+ Table t = UTIL.getConnection().getTable(tableName);
+ final int numRows = rowCount + startRowNum - startDeleteRowNum;
+ Delete d;
+ for (int i= startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
+ d = new Delete(Bytes.toBytes("" + i));
+ t.delete(d);
+ if (i % 5 == 0) {
+ UTIL.getHBaseAdmin().flush(tableName);
+ }
+ }
+ }
+
+ private void verify(final TableName tableName, final int splitRowNum) throws IOException {
+ List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
+ assertTrue(daughters.size() == 2);
+ LOG.info("Row Count = " + UTIL.countRows(tableName));
+ assertTrue(UTIL.countRows(tableName) == rowCount);
+ int startRow;
+ int numRows;
+ for (int i = 0; i < daughters.size(); i++) {
+ if (Bytes.compareTo(
+ daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) == 0) {
+ startRow = startRowNum; // first region
+ numRows = splitRowNum - startRowNum;
+ } else {
+ startRow = splitRowNum;
+ numRows = rowCount + startRowNum - splitRowNum;
+ }
+ verifyData(
+ daughters.get(i),
+ startRow,
+ numRows,
+ ColumnFamilyName1.getBytes(),
+ ColumnFamilyName2.getBytes());
+ }
+ }
+
+ private void verifyData(
+ final HRegion newReg,
+ final int startRow,
+ final int numRows,
+ final byte[]... families)
+ throws IOException {
+ for (int i = startRow; i < startRow + numRows; i++) {
+ byte[] row = Bytes.toBytes("" + i);
+ Get get = new Get(row);
+ Result result = newReg.get(get);
+ Cell[] raw = result.rawCells();
+ assertEquals(families.length, result.size());
+ for (int j = 0; j < families.length; j++) {
+ assertTrue(CellUtil.matchingRow(raw[j], row));
+ assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
+ }
+ }
+ }
+
+ private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 8c9db88..a25c157 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
@@ -495,7 +494,6 @@ public class TestNamespaceAuditor {
// Make sure no regions have been added.
List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
assertEquals(2, hris.size());
- assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
htable.close();
}
@@ -570,7 +568,6 @@ public class TestNamespaceAuditor {
public static class CustomObserver extends BaseRegionObserver{
volatile CountDownLatch postSplit;
- volatile CountDownLatch preSplitBeforePONR;
volatile CountDownLatch postCompact;
@Override
@@ -586,16 +583,8 @@ public class TestNamespaceAuditor {
}
@Override
- public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
- byte[] splitKey, List<Mutation> metaEntries) throws IOException {
- preSplitBeforePONR.countDown();
- }
-
-
- @Override
public void start(CoprocessorEnvironment e) throws IOException {
postSplit = new CountDownLatch(1);
- preSplitBeforePONR = new CountDownLatch(1);
postCompact = new CountDownLatch(1);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 62d1b49..e2a57eb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -89,92 +89,6 @@ public class TestEndToEndSplitTransaction {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void testMasterOpsWhileSplitting() throws Exception {
- TableName tableName = TableName.valueOf("TestSplit");
- byte[] familyName = Bytes.toBytes("fam");
- try (Table ht = TEST_UTIL.createTable(tableName, familyName)) {
- TEST_UTIL.loadTable(ht, familyName, false);
- }
- HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
- byte[] firstRow = Bytes.toBytes("aaa");
- byte[] splitRow = Bytes.toBytes("lll");
- byte[] lastRow = Bytes.toBytes("zzz");
- try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
- // this will also cache the region
- byte[] regionName = conn.getRegionLocator(tableName).getRegionLocation(splitRow)
- .getRegionInfo().getRegionName();
- Region region = server.getRegion(regionName);
- SplitTransactionImpl split = new SplitTransactionImpl((HRegion) region, splitRow);
- split.prepare();
-
- // 1. phase I
- PairOfSameType<Region> regions = split.createDaughters(server, server, null);
- assertFalse(test(conn, tableName, firstRow, server));
- assertFalse(test(conn, tableName, lastRow, server));
-
- // passing null as services prevents final step
- // 2, most of phase II
- split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
- assertFalse(test(conn, tableName, firstRow, server));
- assertFalse(test(conn, tableName, lastRow, server));
-
- // 3. finish phase II
- // note that this replicates some code from SplitTransaction
- // 2nd daughter first
- server.reportRegionStateTransition(
- RegionServerStatusProtos.RegionStateTransition.TransitionCode.SPLIT,
- region.getRegionInfo(), regions.getFirst().getRegionInfo(), regions.getSecond()
- .getRegionInfo());
-
- // Add to online regions
- server.addToOnlineRegions(regions.getSecond());
- // THIS is the crucial point:
- // the 2nd daughter was added, so querying before the split key should fail.
- assertFalse(test(conn, tableName, firstRow, server));
- // past splitkey is ok.
- assertTrue(test(conn, tableName, lastRow, server));
-
- // Add to online regions
- server.addToOnlineRegions(regions.getFirst());
- assertTrue(test(conn, tableName, firstRow, server));
- assertTrue(test(conn, tableName, lastRow, server));
-
- assertTrue(test(conn, tableName, firstRow, server));
- assertTrue(test(conn, tableName, lastRow, server));
- }
- }
-
- /**
- * attempt to locate the region and perform a get and scan
- * @return True if successful, False otherwise.
- */
- private boolean test(Connection conn, TableName tableName, byte[] row,
- HRegionServer server) {
- // not using HTable to avoid timeouts and retries
- try {
- byte[] regionName = conn.getRegionLocator(tableName).getRegionLocation(row, true)
- .getRegionInfo().getRegionName();
- // get and scan should now succeed without exception
- ClientProtos.GetRequest request =
- RequestConverter.buildGetRequest(regionName, new Get(row));
- server.getRSRpcServices().get(null, request);
- ScanRequest scanRequest = RequestConverter.buildScanRequest(
- regionName, new Scan(row), 1, true);
- try {
- server.getRSRpcServices().scan(
- new HBaseRpcControllerImpl(), scanRequest);
- } catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException e) {
- throw ProtobufUtil.handleRemoteException(e);
- }
- } catch (IOException e) {
- return false;
- } catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException e1) {
- return false;
- }
- return true;
- }
-
/**
* Tests that the client sees meta table changes as atomic during splits
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 085572d..dd79e39 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -31,7 +31,6 @@ import java.io.InterruptedIOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -52,14 +51,11 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -70,10 +66,9 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
-import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterRpcServices;
@@ -95,7 +90,6 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
-import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
@@ -105,13 +99,12 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
- * Like TestSplitTransaction in that we're testing {@link SplitTransactionImpl}
- * only the below tests are against a running cluster where TestSplitTransaction
- * is tests against a bare {@link HRegion}.
+ * The below tests are testing split region against a running cluster
*/
@Category({RegionServerTests.class, LargeTests.class})
@SuppressWarnings("deprecation")
@@ -121,8 +114,6 @@ public class TestSplitTransactionOnCluster {
private Admin admin = null;
private MiniHBaseCluster cluster = null;
private static final int NB_SERVERS = 3;
- private static CountDownLatch latch = new CountDownLatch(1);
- private static volatile boolean secondSplit = false;
static final HBaseTestingUtility TESTING_UTIL =
new HBaseTestingUtility();
@@ -145,21 +136,48 @@ public class TestSplitTransactionOnCluster {
@After
public void tearDown() throws Exception {
this.admin.close();
+ for (HTableDescriptor htd: this.admin.listTables()) {
+ LOG.info("Tear down, remove table=" + htd.getTableName());
+ TESTING_UTIL.deleteTable(htd.getTableName());
+ }
}
private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
throws IOException, InterruptedException {
assertEquals(1, regions.size());
HRegionInfo hri = regions.get(0).getRegionInfo();
- TESTING_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
+ cluster.getMaster().getAssignmentManager()
.waitOnRegionToClearRegionsInTransition(hri, 600000);
return hri;
}
+ private void requestSplitRegion(
+ final HRegionServer rsServer,
+ final Region region,
+ final byte[] midKey) throws IOException {
+ long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0);
+ // wait
+ if (procId != -1) {
+ // wait for the split to complete or get interrupted. If the split completes successfully,
+ // the procedure will return true; if the split fails, the procedure would throw exception.
+ //
+ while (!rsServer.isProcedureFinished(procId)) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new IOException("Split region interrupted.");
+ }
+ }
+ } else {
+ throw new IOException ("Request split region failed.");
+ }
+ }
+
@Test(timeout = 60000)
public void testRITStateForRollback() throws Exception {
final TableName tableName =
TableName.valueOf("testRITStateForRollback");
+ final HMaster master = cluster.getMaster();
try {
// Create table then get the single region for our new table.
Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
@@ -171,22 +189,25 @@ public class TestSplitTransactionOnCluster {
// Turn off balancer so it doesn't cut in and mess up our placements.
this.admin.setBalancerRunning(false, true);
// Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
+ master.setCatalogJanitorEnabled(false);
// find a splittable region
final HRegion region = findSplittableRegion(regions);
assertTrue("not able to find a splittable region", region != null);
- // install region co-processor to fail splits
- region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
- Coprocessor.PRIORITY_USER, region.getBaseConf());
+ // install master co-processor to fail splits
+ master.getMasterCoprocessorHost().load(
+ FailingSplitMasterObserver.class,
+ Coprocessor.PRIORITY_USER,
+ master.getConfiguration());
// split async
this.admin.splitRegion(region.getRegionInfo().getRegionName(), new byte[] {42});
// we have to wait until the SPLITTING state is seen by the master
- FailingSplitRegionObserver observer = (FailingSplitRegionObserver) region
- .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
+ FailingSplitMasterObserver observer =
+ (FailingSplitMasterObserver) master.getMasterCoprocessorHost().findCoprocessor(
+ FailingSplitMasterObserver.class.getName());
assertNotNull(observer);
observer.latch.await();
@@ -194,10 +215,12 @@ public class TestSplitTransactionOnCluster {
cluster.getMaster().getAssignmentManager().waitOnRegionToClearRegionsInTransition(hri, 60000);
} finally {
admin.setBalancerRunning(true, false);
- cluster.getMaster().setCatalogJanitorEnabled(true);
+ master.setCatalogJanitorEnabled(true);
+ abortAndWaitForMaster();
TESTING_UTIL.deleteTable(tableName);
}
}
+
@Test(timeout = 60000)
public void testSplitFailedCompactionAndSplit() throws Exception {
final TableName tableName = TableName.valueOf("testSplitFailedCompactionAndSplit");
@@ -238,80 +261,28 @@ public class TestSplitTransactionOnCluster {
assertTrue(fileNum > store.getStorefiles().size());
// 3, Split
- SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row3"));
- assertTrue(st.prepare());
- st.execute(regionServer, regionServer);
+ requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
assertEquals(2, cluster.getRegions(tableName).size());
}
- public static class FailingSplitRegionObserver extends BaseRegionObserver {
+ public static class FailingSplitMasterObserver extends BaseMasterObserver {
volatile CountDownLatch latch;
@Override
public void start(CoprocessorEnvironment e) throws IOException {
latch = new CountDownLatch(1);
}
@Override
- public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
- byte[] splitKey, List<Mutation> metaEntries) throws IOException {
+ public void preSplitRegionBeforePONRAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final byte[] splitKey,
+ final List<Mutation> metaEntries) throws IOException {
latch.countDown();
throw new IOException("Causing rollback of region split");
}
}
- /**
- * A test that intentionally has master fail the processing of the split message.
- * Tests that after we process server shutdown, the daughters are up on line.
- * @throws IOException
- * @throws InterruptedException
- * @throws ServiceException
- */
- @Test (timeout = 300000) public void testRSSplitDaughtersAreOnlinedAfterShutdownHandling()
- throws IOException, InterruptedException, ServiceException {
- final TableName tableName =
- TableName.valueOf("testRSSplitDaughtersAreOnlinedAfterShutdownHandling");
-
- // Create table then get the single region for our new table.
- Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
- List<HRegion> regions = cluster.getRegions(tableName);
- HRegionInfo hri = getAndCheckSingleTableRegion(regions);
-
- int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
-
- // Turn off balancer so it doesn't cut in and mess up our placements.
- this.admin.setBalancerRunning(false, true);
- // Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
- try {
- // Add a bit of load up into the table so splittable.
- TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
- // Get region pre-split.
- HRegionServer server = cluster.getRegionServer(tableRegionIndex);
- printOutRegions(server, "Initial regions: ");
- int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
- // Now, before we split, set special flag in master, a flag that has
- // it FAIL the processing of split.
- AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
- try {
- // Now try splitting and it should work.
- split(hri, server, regionCount);
- } catch (RegionServerStoppedException rsse) {
- // Expected. The regionserver should crash
- }
-
- waitUntilRegionServerDead();
- awaitDaughters(tableName, 2);
- } finally {
- // Set this flag back.
- AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
- admin.setBalancerRunning(true, false);
- cluster.getMaster().setCatalogJanitorEnabled(true);
- cluster.startRegionServer();
- t.close();
- }
- }
-
- @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
- throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
+ @Test (timeout = 300000)
+ public void testExistingZnodeBlocksSplitAndWeRollback() throws IOException, InterruptedException {
final TableName tableName =
TableName.valueOf("testExistingZnodeBlocksSplitAndWeRollback");
@@ -368,8 +339,9 @@ public class TestSplitTransactionOnCluster {
* @throws IOException
* @throws InterruptedException
*/
- @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
- throws IOException, InterruptedException {
+ @Ignore // TODO: revisit this test when the new AM and SSH is implement
+ @Test (timeout=300000)
+ public void testShutdownFixupWhenDaughterHasSplit()throws IOException, InterruptedException {
final TableName tableName =
TableName.valueOf("testShutdownFixupWhenDaughterHasSplit");
@@ -469,8 +441,8 @@ public class TestSplitTransactionOnCluster {
admin.flush(userTableName);
}
admin.majorCompact(userTableName);
- List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
- .getMaster().getAssignmentManager().getRegionStates()
+ List<HRegionInfo> regionsOfTable =
+ cluster.getMaster().getAssignmentManager().getRegionStates()
.getRegionsOfTable(userTableName);
HRegionInfo hRegionInfo = regionsOfTable.get(0);
Put p = new Put("row6".getBytes());
@@ -484,17 +456,18 @@ public class TestSplitTransactionOnCluster {
table.put(p);
admin.flush(userTableName);
admin.splitRegion(hRegionInfo.getRegionName(), "row7".getBytes());
- regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+ regionsOfTable = cluster.getMaster()
.getAssignmentManager().getRegionStates()
.getRegionsOfTable(userTableName);
while (regionsOfTable.size() != 2) {
Thread.sleep(2000);
- regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
+ regionsOfTable = cluster.getMaster()
.getAssignmentManager().getRegionStates()
.getRegionsOfTable(userTableName);
}
Assert.assertEquals(2, regionsOfTable.size());
+
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
int mainTableCount = 0;
@@ -583,71 +556,6 @@ public class TestSplitTransactionOnCluster {
}
}
- /**
- *
- * While transitioning node from RS_ZK_REGION_SPLITTING to
- * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always
- * fails for the region. HBASE-6088 fixes this scenario.
- * This test case is to test the znode is deleted(if created) or not in roll back.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- */
- @Test(timeout = 60000)
- public void testSplitBeforeSettingSplittingInZK() throws Exception,
- InterruptedException, KeeperException {
- testSplitBeforeSettingSplittingInZKInternals();
- }
-
- @Test(timeout = 60000)
- public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
- final TableName tableName =
- TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
- // Create table then get the single region for our new table.
- Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
- List<HRegion> regions = null;
- try {
- regions = cluster.getRegions(tableName);
- int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
- .getRegionName());
- HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
- insertData(tableName, admin, t);
- // Turn off balancer so it doesn't cut in and mess up our placements.
- admin.setBalancerRunning(false, true);
- // Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
- boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
- tableName);
- assertEquals("The specified table should present.", true, tableExists);
- final HRegion region = findSplittableRegion(regions);
- assertTrue("not able to find a splittable region", region != null);
- SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
- try {
- st.prepare();
- st.createDaughters(regionServer, regionServer, null);
- } catch (IOException e) {
-
- }
- tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
- tableName);
- assertEquals("The specified table should present.", true, tableExists);
- Set<RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
- .getRegionsInTransition();
- assertTrue(rit.size() == 3);
- cluster.getMaster().getAssignmentManager().regionOffline(st.getFirstDaughter());
- cluster.getMaster().getAssignmentManager().regionOffline(st.getSecondDaughter());
- cluster.getMaster().getAssignmentManager().regionOffline(region.getRegionInfo());
- rit = cluster.getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition();
- assertTrue(rit.size() == 0);
- } finally {
- admin.setBalancerRunning(true, false);
- cluster.getMaster().setCatalogJanitorEnabled(true);
- t.close();
- TESTING_UTIL.deleteTable(tableName);
- }
- }
-
@Test
public void testSplitWithRegionReplicas() throws Exception {
final TableName tableName =
@@ -679,10 +587,8 @@ public class TestSplitTransactionOnCluster {
regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
regionServer = cluster.getRegionServer(regionServerIndex);
assertTrue("not able to find a splittable region", region != null);
- SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
try {
- st.prepare();
- st.execute(regionServer, regionServer);
+ requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
} catch (IOException e) {
e.printStackTrace();
fail("Split execution should have succeeded with no exceptions thrown " + e);
@@ -779,10 +685,8 @@ public class TestSplitTransactionOnCluster {
assertTrue("not able to find a splittable region", region != null);
// Now split.
- SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
try {
- st.prepare();
- st.execute(regionServer, regionServer);
+ requestSplitRegion(regionServer, region, Bytes.toBytes("row2"));
} catch (IOException e) {
fail("Split execution should have succeeded with no exceptions thrown");
}
@@ -826,195 +730,6 @@ public class TestSplitTransactionOnCluster {
}
}
- /**
- * Not really restarting the master. Simulate it by clear of new region
- * state since it is not persisted, will be lost after master restarts.
- */
- @Test(timeout = 180000)
- public void testSplitAndRestartingMaster() throws Exception {
- LOG.info("Starting testSplitAndRestartingMaster");
- final TableName tableName = TableName.valueOf("testSplitAndRestartingMaster");
- // Create table then get the single region for our new table.
- createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
- List<HRegion> regions = cluster.getRegions(tableName);
- HRegionInfo hri = getAndCheckSingleTableRegion(regions);
- ensureTableRegionNotOnSameServerAsMeta(admin, hri);
- int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
- .getRegionName());
- HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
- // Turn off balancer so it doesn't cut in and mess up our placements.
- this.admin.setBalancerRunning(false, true);
- // Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
- try {
- MyMasterRpcServices.enabled.set(true);
- // find a splittable region. Refresh the regions list
- regions = cluster.getRegions(tableName);
- final HRegion region = findSplittableRegion(regions);
- assertTrue("not able to find a splittable region", region != null);
-
- // Now split.
- SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
- try {
- st.prepare();
- st.execute(regionServer, regionServer);
- } catch (IOException e) {
- fail("Split execution should have succeeded with no exceptions thrown");
- }
-
- // Postcondition
- List<HRegion> daughters = cluster.getRegions(tableName);
- LOG.info("xxx " + regions.size() + AssignmentManager.TEST_SKIP_SPLIT_HANDLING);
- assertTrue(daughters.size() == 2);
- } finally {
- MyMasterRpcServices.enabled.set(false);
- admin.setBalancerRunning(true, false);
- cluster.getMaster().setCatalogJanitorEnabled(true);
- }
- }
-
- @Test(timeout = 180000)
- public void testSplitHooksBeforeAndAfterPONR() throws Exception {
- TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
- TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
- HColumnDescriptor hcd = new HColumnDescriptor("cf");
-
- HTableDescriptor desc = new HTableDescriptor(firstTable);
- desc.addCoprocessor(MockedRegionObserver.class.getName());
- desc.addFamily(hcd);
- admin.createTable(desc);
- TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
-
- desc = new HTableDescriptor(secondTable);
- desc.addFamily(hcd);
- admin.createTable(desc);
- TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
-
- List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
- List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
-
- // Check that both tables actually have regions.
- if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
- fail("Each table should have at least one region.");
- }
- ServerName serverName = cluster.getServerHoldingRegion(firstTable,
- firstTableRegions.get(0).getRegionInfo().getRegionName());
- admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
- Bytes.toBytes(serverName.getServerName()));
- Table table1 = null;
- Table table2 = null;
- try {
- table1 = TESTING_UTIL.getConnection().getTable(firstTable);
- table2 = TESTING_UTIL.getConnection().getTable(firstTable);
- insertData(firstTable, admin, table1);
- insertData(secondTable, admin, table2);
- admin.split(firstTable, "row2".getBytes());
- firstTableRegions = cluster.getRegions(firstTable);
- while (firstTableRegions.size() != 2) {
- Thread.sleep(1000);
- firstTableRegions = cluster.getRegions(firstTable);
- }
- assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
- secondTableRegions = cluster.getRegions(secondTable);
- assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
- } finally {
- if (table1 != null) {
- table1.close();
- }
- if (table2 != null) {
- table2.close();
- }
- TESTING_UTIL.deleteTable(firstTable);
- TESTING_UTIL.deleteTable(secondTable);
- }
- }
-
- @Test (timeout=300000)
- public void testSSHCleanupDaugtherRegionsOfAbortedSplit() throws Exception {
- TableName table = TableName.valueOf("testSSHCleanupDaugtherRegionsOfAbortedSplit");
- try {
- HTableDescriptor desc = new HTableDescriptor(table);
- desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
- admin.createTable(desc);
- Connection connection = ConnectionFactory.createConnection(cluster.getConfiguration());
- Table hTable = connection.getTable(desc.getTableName());
- for(int i = 1; i < 5; i++) {
- Put p1 = new Put(("r"+i).getBytes());
- p1.addColumn(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
- hTable.put(p1);
- }
- admin.flush(desc.getTableName());
- List<HRegion> regions = cluster.getRegions(desc.getTableName());
- int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
- HRegionServer regionServer = cluster.getRegionServer(serverWith);
- SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
- st.prepare();
- st.stepsBeforePONR(regionServer, regionServer, false);
- Path tableDir =
- FSUtils.getTableDir(cluster.getMaster().getMasterFileSystem().getRootDir(),
- desc.getTableName());
- List<Path> regionDirs =
- FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
- assertEquals(3,regionDirs.size());
- regionServer.kill();
- // Before we check deadServerInProgress, we should ensure server is dead at master side.
- while (!cluster.getMaster().getServerManager().
- getDeadServers().isDeadServer(regionServer.serverName)) {
- Thread.sleep(10);
- }
- // Wait until finish processing of shutdown
- while (cluster.getMaster().getServerManager().areDeadServersInProgress()) {
- Thread.sleep(10);
- }
-
- AssignmentManager am = cluster.getMaster().getAssignmentManager();
- assertEquals(am.getRegionStates().getRegionsInTransition().toString(), 0, am
- .getRegionStates().getRegionsInTransition().size());
- regionDirs =
- FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
- assertEquals(1,regionDirs.size());
- } finally {
- TESTING_UTIL.deleteTable(table);
- }
- }
-
- private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
- final TableName tableName = TableName.valueOf("testSplitBeforeSettingSplittingInZK");
- try {
- // Create table then get the single region for our new table.
- createTableAndWait(tableName, Bytes.toBytes("cf"));
-
- List<HRegion> regions = awaitTableRegions(tableName);
- assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
-
- int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
- .getRegionName());
- HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
- final HRegion region = findSplittableRegion(regions);
- assertTrue("not able to find a splittable region", region != null);
- SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
- @Override
- public PairOfSameType<Region> stepsBeforePONR(final Server server,
- final RegionServerServices services, boolean testing) throws IOException {
- throw new SplittingNodeCreationFailedException ();
- }
- };
- try {
- st.prepare();
- st.execute(regionServer, regionServer);
- } catch (IOException e) {
- // check for the specific instance in case the Split failed due to the
- // existence of the znode in OPENED state.
- // This will at least make the test to fail;
- assertTrue("Should be instance of CreateSplittingNodeFailedException",
- e instanceof SplittingNodeCreationFailedException );
- assertTrue(st.rollback(regionServer, regionServer));
- }
- } finally {
- TESTING_UTIL.deleteTable(tableName);
- }
- }
-
@Test
public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
throws Exception {
@@ -1052,35 +767,6 @@ public class TestSplitTransactionOnCluster {
}
}
- public static class MockedCoordinatedStateManager extends ZkCoordinatedStateManager {
-
- public void initialize(Server server, HRegion region) {
- this.server = server;
- this.watcher = server.getZooKeeper();
- }
- }
-
- public static class MockedSplitTransaction extends SplitTransactionImpl {
-
- private HRegion currentRegion;
- public MockedSplitTransaction(HRegion region, byte[] splitrow) {
- super(region, splitrow);
- this.currentRegion = region;
- }
- @Override
- public boolean rollback(Server server, RegionServerServices services) throws IOException {
- if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
- .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
- if(secondSplit){
- super.rollback(server, services);
- latch.countDown();
- return true;
- }
- }
- return super.rollback(server, services);
- }
- }
-
private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
for (int i = 0; i < 5; ++i) {
for (HRegion r: regions) {
@@ -1118,14 +804,13 @@ public class TestSplitTransactionOnCluster {
private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
throws IOException, InterruptedException {
this.admin.splitRegion(hri.getRegionName());
- for (int i = 0; ProtobufUtil.getOnlineRegions(
- server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
+ for (int i = 0; this.cluster.getRegions(hri.getTable()).size() <= regionCount && i < 60; i++) {
LOG.debug("Waiting on region to split");
- Thread.sleep(100);
+ Thread.sleep(2000);
}
assertFalse("Waited too long for split",
- ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
+ this.cluster.getRegions(hri.getTable()).size() <= regionCount);
}
/**
@@ -1248,14 +933,6 @@ public class TestSplitTransactionOnCluster {
return t;
}
- private static class SplittingNodeCreationFailedException extends IOException {
- private static final long serialVersionUID = 1652404976265623004L;
-
- public SplittingNodeCreationFailedException () {
- super();
- }
- }
-
// Make it public so that JVMClusterUtil can access it.
public static class MyMaster extends HMaster {
public MyMaster(Configuration conf, CoordinatedStateManager cp)
@@ -1297,61 +974,6 @@ public class TestSplitTransactionOnCluster {
}
}
- public static class MockedRegionObserver extends BaseRegionObserver {
- private SplitTransactionImpl st = null;
- private PairOfSameType<Region> daughterRegions = null;
-
- @Override
- public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
- byte[] splitKey, List<Mutation> metaEntries) throws IOException {
- RegionCoprocessorEnvironment environment = ctx.getEnvironment();
- HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
- List<Region> onlineRegions =
- rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
- Region region = onlineRegions.get(0);
- for (Region r : onlineRegions) {
- if (r.getRegionInfo().containsRow(splitKey)) {
- region = r;
- break;
- }
- }
- st = new SplitTransactionImpl((HRegion) region, splitKey);
- if (!st.prepare()) {
- LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
- + " failed. So returning null. ");
- ctx.bypass();
- return;
- }
- ((HRegion)region).forceSplit(splitKey);
- daughterRegions = st.stepsBeforePONR(rs, rs, false);
- HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
- copyOfParent.setOffline(true);
- copyOfParent.setSplit(true);
- // Put for parent
- Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
- MetaTableAccessor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
- daughterRegions.getSecond().getRegionInfo());
- metaEntries.add(putParent);
- // Puts for daughters
- Put putA = MetaTableAccessor.makePutFromRegionInfo(
- daughterRegions.getFirst().getRegionInfo());
- Put putB = MetaTableAccessor.makePutFromRegionInfo(
- daughterRegions.getSecond().getRegionInfo());
- st.addLocation(putA, rs.getServerName(), 1);
- st.addLocation(putB, rs.getServerName(), 1);
- metaEntries.add(putA);
- metaEntries.add(putB);
- }
-
- @Override
- public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
- throws IOException {
- RegionCoprocessorEnvironment environment = ctx.getEnvironment();
- HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
- st.stepsAfterPONR(rs, rs, daughterRegions, null);
- }
- }
-
static class CustomSplitPolicy extends RegionSplitPolicy {
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index ef44693..c4e9f41 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -816,27 +816,15 @@ public class TestAccessController extends SecureTestUtil {
}
@Test (timeout=180000)
- public void testSplit() throws Exception {
- AccessTestAction action = new AccessTestAction() {
- @Override
- public Object run() throws Exception {
- ACCESS_CONTROLLER.preSplit(ObserverContext.createAndPrepare(RCP_ENV, null));
- return null;
- }
- };
-
- verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN);
- verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ,
- USER_GROUP_WRITE, USER_GROUP_CREATE);
- }
-
- @Test (timeout=180000)
public void testSplitWithSplitRow() throws Exception {
+ final TableName tname = TableName.valueOf("testSplitWithSplitRow");
+ createTestTable(tname);
AccessTestAction action = new AccessTestAction() {
@Override
public Object run() throws Exception {
- ACCESS_CONTROLLER.preSplit(
- ObserverContext.createAndPrepare(RCP_ENV, null),
+ ACCESS_CONTROLLER.preSplitRegion(
+ ObserverContext.createAndPrepare(CP_ENV, null),
+ tname,
TEST_ROW);
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java
index 32d54b8..ad35975 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java
@@ -777,6 +777,18 @@ public class TestWithDisabledAuthorization extends SecureTestUtil {
}
}, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
+ // preSplit
+ verifyAllowed(new AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ ACCESS_CONTROLLER.preSplitRegion(
+ ObserverContext.createAndPrepare(CP_ENV, null),
+ TEST_TABLE.getTableName(),
+ Bytes.toBytes("ss"));
+ return null;
+ }
+ }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
+
// preSetUserQuota
verifyAllowed(new AccessTestAction() {
@Override
@@ -873,15 +885,6 @@ public class TestWithDisabledAuthorization extends SecureTestUtil {
}
}, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
- // preSplit
- verifyAllowed(new AccessTestAction() {
- @Override
- public Object run() throws Exception {
- ACCESS_CONTROLLER.preSplit(ObserverContext.createAndPrepare(RCP_ENV, null));
- return null;
- }
- }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
-
// preGetOp
verifyAllowed(new AccessTestAction() {
@Override