You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/07/15 14:07:11 UTC
[4/4] hbase git commit: HBASE-14552 Procedure V2: Reimplement
DispatchMergingRegionHandler (Stephen Yuan Jiang)
HBASE-14552 Procedure V2: Reimplement DispatchMergingRegionHandler (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f04eeecf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f04eeecf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f04eeecf
Branch: refs/heads/master
Commit: f04eeecffc4ec20a2cfc86ef55f6b724ecbc7e5e
Parents: 28d8609
Author: Stephen Yuan Jiang <sy...@gmail.com>
Authored: Fri Jul 15 07:06:53 2016 -0700
Committer: Stephen Yuan Jiang <sy...@gmail.com>
Committed: Fri Jul 15 07:06:53 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 17 +
.../apache/hadoop/hbase/client/HBaseAdmin.java | 104 +-
.../hadoop/hbase/protobuf/ProtobufUtil.java | 8 +-
...MergeRandomAdjacentRegionsOfTableAction.java | 2 +-
.../generated/MasterProcedureProtos.java | 1823 +++++++++++++++---
.../hbase/protobuf/generated/MasterProtos.java | 893 ++++++---
hbase-protocol/src/main/protobuf/Master.proto | 3 +
.../src/main/protobuf/MasterProcedure.proto | 15 +
.../org/apache/hadoop/hbase/master/HMaster.java | 57 +-
.../hadoop/hbase/master/MasterRpcServices.java | 41 +-
.../hadoop/hbase/master/MasterServices.java | 17 +-
.../hadoop/hbase/master/RegionStates.java | 2 +-
.../hadoop/hbase/master/ServerManager.java | 5 +-
.../handler/DispatchMergingRegionHandler.java | 191 --
.../normalizer/MergeNormalizationPlan.java | 2 +-
.../DispatchMergingRegionsProcedure.java | 584 ++++++
.../apache/hadoop/hbase/client/TestAdmin1.java | 7 +-
.../hbase/client/TestHBaseAdminNoCluster.java | 14 -
.../hbase/client/TestSplitOrMergeStatus.java | 4 +-
.../hbase/coprocessor/TestMasterObserver.java | 2 +-
.../coprocessor/TestRegionServerObserver.java | 6 +-
.../hbase/master/MockNoopMasterServices.java | 14 +-
.../hbase/master/TestAssignmentListener.java | 2 +-
.../TestDispatchMergingRegionsProcedure.java | 296 +++
.../TestMasterFailoverWithProcedures.java | 2 -
.../hbase/namespace/TestNamespaceAuditor.java | 8 +-
.../TestRegionMergeTransactionOnCluster.java | 12 +-
.../snapshot/TestFlushSnapshotFromClient.java | 11 +-
.../TestMobFlushSnapshotFromClient.java | 2 -
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 3 +-
30 files changed, 3320 insertions(+), 827 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 51a26bc..34e0a89 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -869,11 +869,28 @@ public interface Admin extends Abortable, Closeable {
* @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent
* regions
* @throws IOException
+ * @deprecated Since 2.0. Will be removed in 3.0. Use
+ * {@link #mergeRegionsAsync(byte[], byte[], boolean)} instead.
*/
+ @Deprecated
void mergeRegions(final byte[] nameOfRegionA, final byte[] nameOfRegionB,
final boolean forcible) throws IOException;
/**
+ * Merge two regions. Asynchronous operation.
+ *
+ * @param nameOfRegionA encoded or full name of region a
+ * @param nameOfRegionB encoded or full name of region b
+ * @param forcible true if do a compulsory merge, otherwise we will only merge
+ * two adjacent regions
+ * @throws IOException
+ */
+ public Future<Void> mergeRegionsAsync(
+ final byte[] nameOfRegionA,
+ final byte[] nameOfRegionB,
+ final boolean forcible) throws IOException;
+
+ /**
* Split a table. Asynchronous operation.
*
* @param tableName table to split
http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index da0de51..d18b8b3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRespon
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
@@ -1527,31 +1528,95 @@ public class HBaseAdmin implements Admin {
}
/**
+ * Merge two regions. Synchronous operation.
+ * Note: It is not feasible to predict the length of merge.
+ * Therefore, this is for internal testing only.
+ * @param nameOfRegionA encoded or full name of region a
+ * @param nameOfRegionB encoded or full name of region b
+ * @param forcible true if do a compulsory merge, otherwise we will only merge
+ * two adjacent regions
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public void mergeRegionsSync(
+ final byte[] nameOfRegionA,
+ final byte[] nameOfRegionB,
+ final boolean forcible) throws IOException {
+ get(
+ mergeRegionsAsync(nameOfRegionA, nameOfRegionB, forcible),
+ syncWaitTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
* Merge two regions. Asynchronous operation.
* @param nameOfRegionA encoded or full name of region a
* @param nameOfRegionB encoded or full name of region b
* @param forcible true if do a compulsory merge, otherwise we will only merge
* two adjacent regions
* @throws IOException
+ * @deprecated Since 2.0. Will be removed in 3.0. Use
+ * {@link #mergeRegionsAsync(byte[], byte[], boolean)} instead.
*/
+ @Deprecated
@Override
public void mergeRegions(final byte[] nameOfRegionA,
final byte[] nameOfRegionB, final boolean forcible)
throws IOException {
+ mergeRegionsAsync(nameOfRegionA, nameOfRegionB, forcible);
+ }
+
+ /**
+ * Merge two regions. Asynchronous operation.
+ * @param nameOfRegionA encoded or full name of region a
+ * @param nameOfRegionB encoded or full name of region b
+ * @param forcible true if do a compulsory merge, otherwise we will only merge
+ * two adjacent regions
+ * @throws IOException
+ */
+ @Override
+ public Future<Void> mergeRegionsAsync(
+ final byte[] nameOfRegionA,
+ final byte[] nameOfRegionB,
+ final boolean forcible) throws IOException {
+
final byte[] encodedNameOfRegionA = isEncodedRegionName(nameOfRegionA) ?
nameOfRegionA : HRegionInfo.encodeRegionName(nameOfRegionA).getBytes();
final byte[] encodedNameOfRegionB = isEncodedRegionName(nameOfRegionB) ?
nameOfRegionB : HRegionInfo.encodeRegionName(nameOfRegionB).getBytes();
+ TableName tableName;
Pair<HRegionInfo, ServerName> pair = getRegion(nameOfRegionA);
- if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID)
- throw new IllegalArgumentException("Can't invoke merge on non-default regions directly");
+
+ if (pair != null) {
+ if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+ throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly");
+ }
+ tableName = pair.getFirst().getTable();
+ } else {
+ throw new UnknownRegionException (
+ "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionA));
+ }
+
pair = getRegion(nameOfRegionB);
- if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID)
- throw new IllegalArgumentException("Can't invoke merge on non-default regions directly");
- executeCallable(new MasterCallable<Void>(getConnection()) {
+ if (pair != null) {
+ if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+ throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly");
+ }
+
+ if (!tableName.equals(pair.getFirst().getTable())) {
+ throw new IllegalArgumentException ("Cannot merge regions from two different tables " +
+ tableName + " and " + pair.getFirst().getTable());
+ }
+ } else {
+ throw new UnknownRegionException (
+ "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionB));
+ }
+
+ DispatchMergingRegionsResponse response =
+ executeCallable(new MasterCallable<DispatchMergingRegionsResponse>(getConnection()) {
@Override
- public Void call(int callTimeout) throws ServiceException {
+ public DispatchMergingRegionsResponse call(int callTimeout) throws ServiceException {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
controller.setCallTimeout(callTimeout);
@@ -1559,13 +1624,36 @@ public class HBaseAdmin implements Admin {
DispatchMergingRegionsRequest request = RequestConverter
.buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
encodedNameOfRegionB, forcible);
- master.dispatchMergingRegions(controller, request);
+ return master.dispatchMergingRegions(controller, request);
} catch (DeserializationException de) {
LOG.error("Could not parse destination server name: " + de);
+ throw new ServiceException(new DoNotRetryIOException(de));
}
- return null;
}
});
+ return new DispatchMergingRegionsFuture(this, tableName, response);
+ }
+
+ private static class DispatchMergingRegionsFuture extends TableFuture<Void> {
+ public DispatchMergingRegionsFuture(
+ final HBaseAdmin admin,
+ final TableName tableName,
+ final DispatchMergingRegionsResponse response) {
+ super(admin, tableName,
+ (response != null && response.hasProcId()) ? response.getProcId() : null);
+ }
+
+ public DispatchMergingRegionsFuture(
+ final HBaseAdmin admin,
+ final TableName tableName,
+ final Long procId) {
+ super(admin, tableName, procId);
+ }
+
+ @Override
+ public String getOperationType() {
+ return "MERGE_REGIONS";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index fecc3c2..c477063 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -42,8 +42,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+
import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier
.RegionSpecifierType.REGION_NAME;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
@@ -159,7 +161,6 @@ import org.apache.hadoop.hbase.quotas.ThrottleType;
import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.TablePermission;
import org.apache.hadoop.hbase.security.access.UserPermission;
@@ -175,6 +176,7 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import com.google.common.collect.ArrayListMultimap;
@@ -1898,12 +1900,12 @@ public final class ProtobufUtil {
public static void mergeRegions(final RpcController controller,
final AdminService.BlockingInterface admin,
final HRegionInfo region_a, final HRegionInfo region_b,
- final boolean forcible, final User user) throws IOException {
+ final boolean forcible, final UserGroupInformation user) throws IOException {
final MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
region_a.getRegionName(), region_b.getRegionName(),forcible);
if (user != null) {
try {
- user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
+ user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
admin.mergeRegions(controller, request);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java
index 8645dc4..03d310e 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java
@@ -65,7 +65,7 @@ public class MergeRandomAdjacentRegionsOfTableAction extends Action {
}
try {
- admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
+ admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
} catch (Exception ex) {
LOG.warn("Merge failed, might be caused by other chaos: " + ex.getMessage());
}