You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/10/28 04:46:08 UTC
[7/7] hbase git commit: HBASE-14551 Procedure v2 - Reimplement split
(Stephen Yuan Jiang)
HBASE-14551 Procedure v2 - Reimplement split (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e108a4f8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e108a4f8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e108a4f8
Branch: refs/heads/master
Commit: e108a4f815ba3a8130f6613109d23a774ac50f16
Parents: 738ff82
Author: Stephen Yuan Jiang <sy...@gmail.com>
Authored: Thu Oct 27 21:45:41 2016 -0700
Committer: Stephen Yuan Jiang <sy...@gmail.com>
Committed: Thu Oct 27 21:45:41 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/client/HBaseAdmin.java | 1 -
.../hbase/shaded/protobuf/ProtobufUtil.java | 46 +
.../hbase/shaded/protobuf/RequestConverter.java | 16 +-
.../shaded/protobuf/ResponseConverter.java | 13 +
.../shaded/protobuf/generated/AdminProtos.java | 1450 ++++++++++--
.../generated/MasterProcedureProtos.java | 2100 ++++++++++++++++--
.../generated/RegionServerStatusProtos.java | 1633 +++++++++++++-
.../src/main/protobuf/Admin.proto | 15 +
.../src/main/protobuf/MasterProcedure.proto | 21 +
.../src/main/protobuf/RegionServerStatus.proto | 27 +
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 40 +-
.../BaseMasterAndRegionObserver.java | 40 +
.../hbase/coprocessor/BaseMasterObserver.java | 40 +
.../hbase/coprocessor/MasterObserver.java | 71 +
.../hbase/coprocessor/RegionObserver.java | 24 +-
.../hadoop/hbase/master/AssignmentManager.java | 36 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 25 +
.../hbase/master/MasterCoprocessorHost.java | 109 +
.../hadoop/hbase/master/MasterRpcServices.java | 18 +
.../hadoop/hbase/master/MasterServices.java | 15 +
.../hadoop/hbase/master/RegionStates.java | 14 +-
.../hadoop/hbase/master/ServerManager.java | 26 +-
.../procedure/SplitTableRegionProcedure.java | 821 +++++++
.../hadoop/hbase/regionserver/HRegion.java | 2 +-
.../hbase/regionserver/HRegionFileSystem.java | 11 +-
.../hbase/regionserver/HRegionServer.java | 116 +
.../hbase/regionserver/RSRpcServices.java | 29 +
.../regionserver/RegionCoprocessorHost.java | 26 +-
.../regionserver/RegionServerServices.java | 10 +
.../hadoop/hbase/regionserver/SplitRequest.java | 115 +-
.../hbase/security/access/AccessController.java | 21 +-
.../hadoop/hbase/MockRegionServerServices.java | 10 +
.../coprocessor/TestCoprocessorInterface.java | 14 +-
.../hbase/coprocessor/TestMasterObserver.java | 39 +
.../hbase/master/MockNoopMasterServices.java | 9 +
.../hadoop/hbase/master/MockRegionServer.java | 23 +-
.../TestSplitTableRegionProcedure.java | 480 ++++
.../hbase/namespace/TestNamespaceAuditor.java | 11 -
.../TestEndToEndSplitTransaction.java | 86 -
.../TestSplitTransactionOnCluster.java | 508 +----
.../security/access/TestAccessController.java | 22 +-
.../access/TestWithDisabledAuthorization.java | 21 +-
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 127 +-
43 files changed, 7151 insertions(+), 1130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 51d07e3..f4008fd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -165,7 +165,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 53101de..94efa37 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -104,6 +104,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -1720,6 +1722,33 @@ public final class ProtobufUtil {
}
/**
+ * A helper to close a region for split
+ * using admin protocol.
+ *
+ * @param controller RPC controller
+ * @param admin Admin service
+ * @param server the RS that hosts the target region
+ * @param parentRegionInfo the target region info
+ * @return true if the region is closed
+ * @throws IOException
+ */
+ public static boolean closeRegionForSplit(
+ final RpcController controller,
+ final AdminService.BlockingInterface admin,
+ final ServerName server,
+ final HRegionInfo parentRegionInfo) throws IOException {
+ CloseRegionForSplitRequest closeRegionForSplitRequest =
+ ProtobufUtil.buildCloseRegionForSplitRequest(server, parentRegionInfo);
+ try {
+ CloseRegionForSplitResponse response =
+ admin.closeRegionForSplit(controller, closeRegionForSplitRequest);
+ return ResponseConverter.isClosed(response);
+ } catch (ServiceException se) {
+ throw getRemoteException(se);
+ }
+ }
+
+ /**
* A helper to warmup a region given a region name
* using admin protocol
*
@@ -3063,6 +3092,23 @@ public final class ProtobufUtil {
}
/**
+ * Create a CloseRegionForSplitRequest for a given region
+ *
+ * @param server the RS server that hosts the region
+ * @param parentRegionInfo the info of the region to close
+ * @return a CloseRegionForSplitRequest
+ */
+ public static CloseRegionForSplitRequest buildCloseRegionForSplitRequest(
+ final ServerName server,
+ final HRegionInfo parentRegionInfo) {
+ CloseRegionForSplitRequest.Builder builder = CloseRegionForSplitRequest.newBuilder();
+ RegionSpecifier parentRegion = RequestConverter.buildRegionSpecifier(
+ RegionSpecifierType.REGION_NAME, parentRegionInfo.getRegionName());
+ builder.setRegion(parentRegion);
+ return builder.build();
+ }
+
+ /**
* Create a CloseRegionRequest for a given encoded region name
*
* @param encodedRegionName the name of the region to close
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index 7da3727..abd1563 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -106,6 +106,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOr
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.SplitTableRegionRequest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
@@ -850,7 +851,7 @@ public final class RequestConverter {
return ubuilder.build();
}
- /**
+ /**
* Create a WarmupRegionRequest for a given region name
*
* @param regionInfo Region we are warming up
@@ -1061,6 +1062,19 @@ public final class RequestConverter {
return builder.build();
}
+ public static SplitTableRegionRequest buildSplitTableRegionRequest(
+ final HRegionInfo regionInfo,
+ final byte[] splitPoint,
+ final long nonceGroup,
+ final long nonce) {
+ SplitTableRegionRequest.Builder builder = SplitTableRegionRequest.newBuilder();
+ builder.setRegionInfo(HRegionInfo.convert(regionInfo));
+ builder.setSplitRow(UnsafeByteOperations.unsafeWrap(splitPoint));
+ builder.setNonceGroup(nonceGroup);
+ builder.setNonce(nonce);
+ return builder.build();
+ }
+
/**
* Create a protocol buffer AssignRegionRequest
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/e108a4f8/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
index dc7b95d..11fc931 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.SingleResponse;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
@@ -252,6 +253,18 @@ public final class ResponseConverter {
}
/**
+ * Check if the region is closed from a CloseRegionForSplitResponse
+ *
+ * @param proto the CloseRegionForSplitResponse
+ * @return the region close state
+ */
+ public static boolean isClosed
+ (final CloseRegionForSplitResponse proto) {
+ if (proto == null || !proto.hasClosed()) return false;
+ return proto.getClosed();
+ }
+
+ /**
* A utility to build a GetServerInfoResponse.
*
* @param serverName