You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/08/14 19:28:06 UTC
[3/3] hbase git commit: HBASE-18497 Add clock type to proto message
for HLC region open/close (revision 4)
HBASE-18497 Add clock type to proto message for HLC region open/close (revision 4)
- Removed overridden methods in HMaster#get/set/updateClock
- Changed parameter name and exception text in ProtobufUtil#toClockType
- Removed unused import of ClockType in TestMasterNoCluster
- Removed unused ServerManager#sendRegionClose(ServerName, HRegionInfo, Long) that recursively called itself
- Added TODO in RSProcedureDispatcher to update clock with top timestamp of both loops
- Added TODO in RequestConverter for removing/merging masterSystemTime
Change-Id: I916e759512d958eaa18c8de8eb0f20dc96adec12
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93c0d71c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93c0d71c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93c0d71c
Branch: refs/heads/HBASE-14070.HLC
Commit: 93c0d71c47ba850b4353872082ddc1ffa76dd30d
Parents: d9a9904
Author: Amit Patel <ia...@gmail.com>
Authored: Wed Aug 2 20:06:09 2017 -0700
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Mon Aug 14 12:24:09 2017 -0700
----------------------------------------------------------------------
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 5 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 69 +-
.../hbase/shaded/protobuf/RequestConverter.java | 29 +-
.../shaded/protobuf/generated/AdminProtos.java | 2151 ++++++++++++------
.../shaded/protobuf/generated/HBaseProtos.java | 303 ++-
.../src/main/protobuf/Admin.proto | 16 +-
.../src/main/protobuf/HBase.proto | 11 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 19 -
.../hadoop/hbase/master/MasterServices.java | 6 -
.../hadoop/hbase/master/ServerManager.java | 17 +-
.../master/procedure/RSProcedureDispatcher.java | 54 +-
.../hbase/regionserver/HRegionServer.java | 8 -
.../hbase/regionserver/RSRpcServices.java | 36 +-
.../regionserver/RegionServerServices.java | 6 -
.../hadoop/hbase/MockRegionServerServices.java | 8 -
.../hbase/master/MockNoopMasterServices.java | 3 -
.../hadoop/hbase/master/MockRegionServer.java | 8 -
.../hbase/master/TestMasterNoCluster.java | 3 +-
18 files changed, 1925 insertions(+), 827 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/93c0d71c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 0cef556..1ae0eec 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -752,12 +752,13 @@ public class RawAsyncHBaseAdmin implements AsyncAdmin {
}
private CompletableFuture<Boolean> closeRegion(HRegionInfo hri, ServerName serverName) {
+ CloseRegionRequest request = ProtobufUtil.buildCloseRegionRequest(serverName,
+ hri.getRegionName());
return this
.<Boolean> newAdminCaller()
.action(
(controller, stub) -> this.<CloseRegionRequest, CloseRegionResponse, Boolean> adminCall(
- controller, stub,
- ProtobufUtil.buildCloseRegionRequest(serverName, hri.getRegionName()),
+ controller, stub, request,
(s, c, req, done) -> s.closeRegion(controller, req, done), resp -> resp.getClosed()))
.serverName(serverName).call();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/93c0d71c/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 4ebba53..407d59f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.ByteBufferCell;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.ClockType;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -154,6 +155,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesP
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ColumnFamilySchema;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NodeTime;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier;
@@ -189,6 +191,7 @@ import org.apache.hadoop.hbase.util.ExceptionUtil;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.Methods;
import org.apache.hadoop.hbase.util.NonceKey;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.ipc.RemoteException;
@@ -1859,11 +1862,11 @@ public final class ProtobufUtil {
*/
public static boolean closeRegion(final RpcController controller,
final AdminService.BlockingInterface admin,
- final ServerName server, final byte[] regionName,
- final ServerName destinationServer, Long masterClockTime) throws IOException {
+ final ServerName server, final byte[] regionName, final ServerName destinationServer,
+ List<Pair<ClockType, Long>> nodeTimes) throws IOException {
CloseRegionRequest closeRegionRequest =
ProtobufUtil.buildCloseRegionRequest(server,
- regionName, destinationServer, masterClockTime);
+ regionName, destinationServer, nodeTimes);
try {
CloseRegionResponse response = admin.closeRegion(controller, closeRegionRequest);
return ResponseConverter.isClosed(response);
@@ -3308,6 +3311,45 @@ public final class ProtobufUtil {
return builder.build();
}
+
+ /**
+ * Convert a protobuf ClockType into a {@link ClockType}
+ *
+ * @param proto the protocol buffer ClockType to convert
+ * @return the converted client ClockType
+ */
+ public static ClockType toClockType(final NodeTime.ClockType proto) {
+ switch (proto) {
+ case SYSTEM:
+ return ClockType.SYSTEM;
+ case SYSTEM_MONOTONIC:
+ return ClockType.SYSTEM_MONOTONIC;
+ case HLC:
+ return ClockType.HLC;
+ default:
+ throw new IllegalArgumentException("Unknown clock type: " + proto);
+ }
+ }
+
+ /**
+ * Convert a {@link ClockType} into a protobuf ClockType
+ *
+ * @param clockType the client ClockType to convert
+ * @return the converted ClockType proto
+ */
+ public static NodeTime.ClockType toClockType(final ClockType clockType) {
+ switch (clockType) {
+ case SYSTEM:
+ return NodeTime.ClockType.SYSTEM;
+ case SYSTEM_MONOTONIC:
+ return NodeTime.ClockType.SYSTEM_MONOTONIC;
+ case HLC:
+ return NodeTime.ClockType.HLC;
+ default:
+ throw new IllegalArgumentException("Unknown clock type: " + clockType);
+ }
+ }
+
/**
* Create a CloseRegionRequest for a given region name
*
@@ -3320,7 +3362,7 @@ public final class ProtobufUtil {
}
public static CloseRegionRequest buildCloseRegionRequest(ServerName server,
- final byte[] regionName, ServerName destinationServer, Long masterClockTime) {
+ final byte[] regionName, ServerName destinationServer, List<Pair<ClockType, Long>> nodeTimes) {
CloseRegionRequest.Builder builder = CloseRegionRequest.newBuilder();
RegionSpecifier region = RequestConverter.buildRegionSpecifier(
RegionSpecifierType.REGION_NAME, regionName);
@@ -3331,8 +3373,12 @@ public final class ProtobufUtil {
if (server != null) {
builder.setServerStartCode(server.getStartcode());
}
- if (masterClockTime != null) {
- builder.setNodeTime(HBaseProtos.NodeTime.newBuilder().setTime(masterClockTime));
+ if (nodeTimes != null) {
+ for (Pair<ClockType, Long> nodeTime : nodeTimes) {
+ builder.addNodeTimesBuilder()
+ .setClockType(ProtobufUtil.toClockType(nodeTime.getFirst()))
+ .setTimestamp(nodeTime.getSecond());
+ }
}
return builder.build();
}
@@ -3341,11 +3387,12 @@ public final class ProtobufUtil {
* Create a CloseRegionRequest for a given encoded region name
*
* @param encodedRegionName the name of the region to close
+ * @param nodeTimes list of clock types and timestamps generated by each clock on master
* @return a CloseRegionRequest
*/
public static CloseRegionRequest
buildCloseRegionRequest(ServerName server, final String encodedRegionName,
- Long masterClockTime) {
+ List<Pair<ClockType, Long>> nodeTimes) {
CloseRegionRequest.Builder builder = CloseRegionRequest.newBuilder();
RegionSpecifier region = RequestConverter.buildRegionSpecifier(
RegionSpecifierType.ENCODED_REGION_NAME,
@@ -3354,8 +3401,12 @@ public final class ProtobufUtil {
if (server != null) {
builder.setServerStartCode(server.getStartcode());
}
- if (masterClockTime != null) {
- builder.setNodeTime(HBaseProtos.NodeTime.newBuilder().setTime(masterClockTime));
+ if (nodeTimes != null) {
+ for (Pair<ClockType, Long> nodeTime : nodeTimes) {
+ builder.addNodeTimesBuilder()
+ .setClockType(ProtobufUtil.toClockType(nodeTime.getFirst()))
+ .setTimestamp(nodeTime.getSecond());
+ }
}
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/93c0d71c/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index 52ce3e9..63ea962 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.CellScannable;
+import org.apache.hadoop.hbase.ClockType;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TimestampType;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Action;
import org.apache.hadoop.hbase.client.Append;
@@ -82,6 +84,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActi
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CompareType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NodeTime;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
@@ -878,12 +881,13 @@ public final class RequestConverter {
* @param server the serverName for the RPC
* @param regionOpenInfos info of a list of regions to open
* @param openForReplay
- * @param masterClockTime timestamp generated by master's clock
+ * @param nodeTimes list clock types and timestamps generated from each clock on master
* @return a protocol buffer OpenRegionRequest
*/
public static OpenRegionRequest
buildOpenRegionRequest(ServerName server, final List<Pair<HRegionInfo,
- List<ServerName>>> regionOpenInfos, Boolean openForReplay, Long masterClockTime) {
+ List<ServerName>>> regionOpenInfos, Boolean openForReplay,
+ List<Pair<ClockType, Long>> nodeTimes) {
OpenRegionRequest.Builder builder = OpenRegionRequest.newBuilder();
for (Pair<HRegionInfo, List<ServerName>> regionOpenInfo: regionOpenInfos) {
builder.addOpenInfo(buildRegionOpenInfo(regionOpenInfo.getFirst(),
@@ -892,9 +896,14 @@ public final class RequestConverter {
if (server != null) {
builder.setServerStartCode(server.getStartcode());
}
- if (masterClockTime != null) {
- builder.setNodeTime(HBaseProtos.NodeTime.newBuilder().setTime(masterClockTime));
+ if (nodeTimes != null) {
+ for (Pair<ClockType, Long> nodeTime : nodeTimes) {
+ builder.addNodeTimesBuilder()
+ .setClockType(ProtobufUtil.toClockType(nodeTime.getFirst()))
+ .setTimestamp(nodeTime.getSecond());
+ }
}
+ // TODO: remove uses of master system time
// send the master's wall clock time as well, so that the RS can refer to it
builder.setMasterSystemTime(EnvironmentEdgeManager.currentTime());
return builder.build();
@@ -907,12 +916,12 @@ public final class RequestConverter {
* @param region the region to open
* @param favoredNodes
* @param openForReplay
- * @param masterClockTime timestamp generated by master's clock
+ * @param nodeTimes list of clock types and timestamps generated by each clock on master
* @return a protocol buffer OpenRegionRequest
*/
public static OpenRegionRequest buildOpenRegionRequest(ServerName server,
final HRegionInfo region, List<ServerName> favoredNodes,
- Boolean openForReplay, Long masterClockTime) {
+ Boolean openForReplay, List<Pair<ClockType, Long>> nodeTimes) {
OpenRegionRequest.Builder builder = OpenRegionRequest.newBuilder();
builder.addOpenInfo(buildRegionOpenInfo(region, favoredNodes,
openForReplay));
@@ -920,8 +929,12 @@ public final class RequestConverter {
builder.setServerStartCode(server.getStartcode());
}
builder.setMasterSystemTime(EnvironmentEdgeManager.currentTime());
- if (masterClockTime != null) {
- builder.setNodeTime(HBaseProtos.NodeTime.newBuilder().setTime(masterClockTime));
+ if (nodeTimes != null) {
+ for (Pair<ClockType, Long> nodeTime : nodeTimes) {
+ builder.addNodeTimesBuilder()
+ .setClockType(ProtobufUtil.toClockType(nodeTime.getFirst()))
+ .setTimestamp(nodeTime.getSecond());
+ }
}
return builder.build();
}