You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/08/29 14:00:34 UTC
(hbase) branch branch-2.5 updated: HBASE-28690 Aborting Active HMaster is not rejecting reportRegionStateTransition if procedure is initialised by next Active master (#6180)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new e15be23a8dc HBASE-28690 Aborting Active HMaster is not rejecting reportRegionStateTransition if procedure is initialised by next Active master (#6180)
e15be23a8dc is described below
commit e15be23a8dc1fd17800d5adcb22dfe329c15c0c5
Author: Umesh <94...@gmail.com>
AuthorDate: Thu Aug 29 19:30:28 2024 +0530
HBASE-28690 Aborting Active HMaster is not rejecting reportRegionStateTransition if procedure is initialised by next Active master (#6180)
Added masterActiveTime as fencing token for remote procedures
Signed-off-by: Duo Zhang <zh...@apache.org>
Reviewed-by: Aman Poonia <am...@gmail.com>
---
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 +++--
.../procedure2/RemoteProcedureDispatcher.java | 10 ++++++-
.../src/main/protobuf/Admin.proto | 6 ++++
.../src/main/protobuf/RegionServerStatus.proto | 5 ++++
.../org/apache/hadoop/hbase/master/HMaster.java | 1 +
.../hadoop/hbase/master/MasterRpcServices.java | 34 ++++++++++++++++++++--
.../apache/hadoop/hbase/master/MasterServices.java | 3 ++
.../master/assignment/CloseRegionProcedure.java | 5 ++--
.../master/assignment/OpenRegionProcedure.java | 5 ++--
.../assignment/RegionRemoteProcedureBase.java | 5 ++--
.../master/procedure/RSProcedureDispatcher.java | 30 ++++++++++---------
.../master/procedure/SplitWALRemoteProcedure.java | 7 +++--
.../SwitchRpcThrottleRemoteProcedure.java | 3 +-
.../ClaimReplicationQueueRemoteProcedure.java | 3 +-
.../master/replication/RefreshPeerProcedure.java | 3 +-
.../hadoop/hbase/regionserver/HRegionServer.java | 15 ++++++----
.../hadoop/hbase/regionserver/RSRpcServices.java | 16 ++++++----
.../hbase/regionserver/RegionServerServices.java | 20 +++++++++++--
.../RemoteProcedureResultReporter.java | 5 ++--
.../hadoop/hbase/regionserver/SplitRequest.java | 2 +-
.../regionserver/handler/AssignRegionHandler.java | 17 +++++++----
.../regionserver/handler/CloseRegionHandler.java | 2 +-
.../regionserver/handler/OpenRegionHandler.java | 7 +++--
.../regionserver/handler/RSProcedureHandler.java | 9 ++++--
.../handler/UnassignRegionHandler.java | 14 ++++++---
.../hbase/master/MockNoopMasterServices.java | 8 +++++
.../procedure/TestServerRemoteProcedure.java | 4 +--
27 files changed, 182 insertions(+), 64 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index cc47c3b2f8b..1de18e030f9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -2986,11 +2986,13 @@ public final class ProtobufUtil {
public static CloseRegionRequest buildCloseRegionRequest(ServerName server, byte[] regionName,
ServerName destinationServer) {
- return buildCloseRegionRequest(server, regionName, destinationServer, -1);
+ // this method is used when we are bypassing active HMaster, so we don't have procId or master
+ // active time.
+ return buildCloseRegionRequest(server, regionName, destinationServer, -1, -1);
}
public static CloseRegionRequest buildCloseRegionRequest(ServerName server, byte[] regionName,
- ServerName destinationServer, long closeProcId) {
+ ServerName destinationServer, long closeProcId, long initiatingMasterActiveTime) {
CloseRegionRequest.Builder builder = CloseRegionRequest.newBuilder();
RegionSpecifier region =
RequestConverter.buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName);
@@ -3001,6 +3003,7 @@ public final class ProtobufUtil {
if (server != null) {
builder.setServerStartCode(server.getStartcode());
}
+ builder.setInitiatingMasterActiveTime(initiatingMasterActiveTime);
builder.setCloseProcId(closeProcId);
return builder.build();
}
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index a1e31a0436e..c89f110cd7c 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -222,14 +222,22 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
*/
public static abstract class RemoteOperation {
private final RemoteProcedure remoteProcedure;
+ // active time of the master that sent this request, used for fencing
+ private final long initiatingMasterActiveTime;
- protected RemoteOperation(final RemoteProcedure remoteProcedure) {
+ protected RemoteOperation(final RemoteProcedure remoteProcedure,
+ long initiatingMasterActiveTime) {
this.remoteProcedure = remoteProcedure;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
}
public RemoteProcedure getRemoteProcedure() {
return remoteProcedure;
}
+
+ public long getInitiatingMasterActiveTime() {
+ return initiatingMasterActiveTime;
+ }
}
/**
diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
index 768c5356969..365101d167c 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
@@ -80,6 +80,8 @@ message OpenRegionRequest {
repeated RegionOpenInfo open_info = 1;
// the intended server for this RPC.
optional uint64 serverStartCode = 2;
+ // Master active time as fencing token
+ optional int64 initiating_master_active_time = 3;
// wall clock time from master
optional uint64 master_system_time = 5;
@@ -122,6 +124,8 @@ message CloseRegionRequest {
// the intended server for this RPC.
optional uint64 serverStartCode = 5;
optional int64 close_proc_id = 6 [default = -1];
+ // Master active time as fencing token
+ optional int64 initiating_master_active_time = 8;
}
message CloseRegionResponse {
@@ -271,6 +275,8 @@ message RemoteProcedureRequest {
required uint64 proc_id = 1;
required string proc_class = 2;
optional bytes proc_data = 3;
+ // Master active time as fencing token
+ optional int64 initiating_master_active_time = 4;
}
message ExecuteProceduresRequest {
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 0137cb1608e..eb11ec3327d 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -97,6 +97,9 @@ message RegionStateTransition {
optional uint64 open_seq_num = 3;
repeated int64 proc_id = 4;
+
+ // Master active time as fencing token
+ optional int64 initiating_master_active_time = 5;
enum TransitionCode {
OPENED = 0;
FAILED_OPEN = 1;
@@ -155,6 +158,8 @@ message RemoteProcedureResult {
}
required Status status = 2;
optional ForeignExceptionMessage error = 3;
+ // Master active time as fencing token
+ optional int64 initiating_master_active_time = 4;
}
message ReportProcedureDoneRequest {
repeated RemoteProcedureResult result = 1;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ae9a322a2d0..8f5bbd760e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3036,6 +3036,7 @@ public class HMaster extends HRegionServer implements MasterServices {
}
/** Returns timestamp in millis when HMaster became the active master. */
+ @Override
public long getMasterActiveTime() {
return masterActiveTime;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index ef83fab4066..c31f116a089 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClusterMetricsBuilder;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.Server;
@@ -69,7 +70,6 @@ import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
import org.apache.hadoop.hbase.ipc.RpcServerFactory;
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
-import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
@@ -327,6 +327,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuo
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RecentLogs;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
@@ -1695,6 +1696,15 @@ public class MasterRpcServices extends RSRpcServices
ReportRegionStateTransitionRequest req) throws ServiceException {
try {
master.checkServiceStarted();
+ for (RegionServerStatusProtos.RegionStateTransition transition : req.getTransitionList()) {
+ long procId =
+ transition.getProcIdCount() > 0 ? transition.getProcId(0) : Procedure.NO_PROC_ID;
+ // -1 is less than any possible MasterActiveCode
+ long initiatingMasterActiveTime = transition.hasInitiatingMasterActiveTime()
+ ? transition.getInitiatingMasterActiveTime()
+ : -1;
+ throwOnOldMaster(procId, initiatingMasterActiveTime);
+ }
return master.getAssignmentManager().reportRegionStateTransition(req);
} catch (IOException ioe) {
throw new ServiceException(ioe);
@@ -2379,8 +2389,14 @@ public class MasterRpcServices extends RSRpcServices
// Check Masters is up and ready for duty before progressing. Remote side will keep trying.
try {
this.master.checkServiceStarted();
- } catch (ServerNotRunningYetException snrye) {
- throw new ServiceException(snrye);
+ for (RemoteProcedureResult result : request.getResultList()) {
+ // -1 is less than any possible MasterActiveCode
+ long initiatingMasterActiveTime =
+ result.hasInitiatingMasterActiveTime() ? result.getInitiatingMasterActiveTime() : -1;
+ throwOnOldMaster(result.getProcId(), initiatingMasterActiveTime);
+ }
+ } catch (IOException ioe) {
+ throw new ServiceException(ioe);
}
request.getResultList().forEach(result -> {
if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) {
@@ -2393,6 +2409,18 @@ public class MasterRpcServices extends RSRpcServices
return ReportProcedureDoneResponse.getDefaultInstance();
}
+ private void throwOnOldMaster(long procId, long initiatingMasterActiveTime)
+ throws MasterNotRunningException {
+ if (initiatingMasterActiveTime > master.getMasterActiveTime()) {
+ // procedure is initiated by new active master but report received on master with older active
+ // time
+ LOG.warn(
+ "Report for procId: {} and initiatingMasterAT {} received on master with activeTime {}",
+ procId, initiatingMasterActiveTime, master.getMasterActiveTime());
+ throw new MasterNotRunningException("Another master is active");
+ }
+ }
+
// HBCK Services
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 97322a0da41..4c53a89c0ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -261,6 +261,9 @@ public interface MasterServices extends Server {
/** Returns true if master is the active one */
boolean isActiveMaster();
+ /** Returns timestamp in millis when this master became the active one. */
+ long getMasterActiveTime();
+
/** Returns true if master is initialized */
boolean isInitialized();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java
index b770d1b16b4..904d14b4c51 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java
@@ -61,8 +61,9 @@ public class CloseRegionProcedure extends RegionRemoteProcedureBase {
}
@Override
- public RemoteOperation newRemoteOperation() {
- return new RegionCloseOperation(this, region, getProcId(), assignCandidate);
+ public RemoteOperation newRemoteOperation(MasterProcedureEnv env) {
+ return new RegionCloseOperation(this, region, getProcId(), assignCandidate,
+ env.getMasterServices().getMasterActiveTime());
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java
index 6116c40d7c8..21218a1177a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java
@@ -57,8 +57,9 @@ public class OpenRegionProcedure extends RegionRemoteProcedureBase {
}
@Override
- public RemoteOperation newRemoteOperation() {
- return new RegionOpenOperation(this, region, getProcId());
+ public RemoteOperation newRemoteOperation(MasterProcedureEnv env) {
+ return new RegionOpenOperation(this, region, getProcId(),
+ env.getMasterServices().getMasterActiveTime());
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index edfed07e588..d1caa209421 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -92,10 +92,11 @@ public abstract class RegionRemoteProcedureBase extends Procedure<MasterProcedur
if (state == RegionRemoteProcedureBaseState.REGION_REMOTE_PROCEDURE_REPORT_SUCCEED) {
return Optional.empty();
}
- return Optional.of(newRemoteOperation());
+ return Optional.of(newRemoteOperation(env));
}
- protected abstract RemoteProcedureDispatcher.RemoteOperation newRemoteOperation();
+ protected abstract RemoteProcedureDispatcher.RemoteOperation
+ newRemoteOperation(MasterProcedureEnv env);
@Override
public void remoteOperationCompleted(MasterProcedureEnv env) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 1241c26ba59..f4e183046be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -427,7 +427,7 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
@Override
public void dispatchServerOperations(MasterProcedureEnv env, List<ServerOperation> operations) {
- operations.stream().map(o -> o.buildRequest()).forEachOrdered(request::addProc);
+ operations.stream().map(ServerOperation::buildRequest).forEachOrdered(request::addProc);
}
// will be overridden in test.
@@ -450,7 +450,9 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
private static OpenRegionRequest buildOpenRegionRequest(final MasterProcedureEnv env,
final ServerName serverName, final List<RegionOpenOperation> operations) {
final OpenRegionRequest.Builder builder = OpenRegionRequest.newBuilder();
- builder.setServerStartCode(serverName.getStartcode());
+ builder.setServerStartCode(serverName.getStartCode());
+ operations.stream().map(RemoteOperation::getInitiatingMasterActiveTime).findAny()
+ .ifPresent(builder::setInitiatingMasterActiveTime);
builder.setMasterSystemTime(EnvironmentEdgeManager.currentTime());
for (RegionOpenOperation op : operations) {
builder.addOpenInfo(op.buildRegionOpenInfoRequest(env));
@@ -473,8 +475,8 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
private final byte[] rsProcData;
public ServerOperation(RemoteProcedure remoteProcedure, long procId, Class<?> rsProcClass,
- byte[] rsProcData) {
- super(remoteProcedure);
+ byte[] rsProcData, long initiatingMasterActiveTime) {
+ super(remoteProcedure, initiatingMasterActiveTime);
this.procId = procId;
this.rsProcClass = rsProcClass;
this.rsProcData = rsProcData;
@@ -482,7 +484,8 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
public RemoteProcedureRequest buildRequest() {
return RemoteProcedureRequest.newBuilder().setProcId(procId)
- .setProcClass(rsProcClass.getName()).setProcData(ByteString.copyFrom(rsProcData)).build();
+ .setProcClass(rsProcClass.getName()).setProcData(ByteString.copyFrom(rsProcData))
+ .setInitiatingMasterActiveTime(getInitiatingMasterActiveTime()).build();
}
}
@@ -490,8 +493,9 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
protected final RegionInfo regionInfo;
protected final long procId;
- protected RegionOperation(RemoteProcedure remoteProcedure, RegionInfo regionInfo, long procId) {
- super(remoteProcedure);
+ protected RegionOperation(RemoteProcedure remoteProcedure, RegionInfo regionInfo, long procId,
+ long initiatingMasterActiveTime) {
+ super(remoteProcedure, initiatingMasterActiveTime);
this.regionInfo = regionInfo;
this.procId = procId;
}
@@ -499,9 +503,9 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
public static class RegionOpenOperation extends RegionOperation {
- public RegionOpenOperation(RemoteProcedure remoteProcedure, RegionInfo regionInfo,
- long procId) {
- super(remoteProcedure, regionInfo, procId);
+ public RegionOpenOperation(RemoteProcedure remoteProcedure, RegionInfo regionInfo, long procId,
+ long initiatingMasterActiveTime) {
+ super(remoteProcedure, regionInfo, procId, initiatingMasterActiveTime);
}
public OpenRegionRequest.RegionOpenInfo
@@ -515,8 +519,8 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
private final ServerName destinationServer;
public RegionCloseOperation(RemoteProcedure remoteProcedure, RegionInfo regionInfo, long procId,
- ServerName destinationServer) {
- super(remoteProcedure, regionInfo, procId);
+ ServerName destinationServer, long initiatingMasterActiveTime) {
+ super(remoteProcedure, regionInfo, procId, initiatingMasterActiveTime);
this.destinationServer = destinationServer;
}
@@ -526,7 +530,7 @@ public class RSProcedureDispatcher extends RemoteProcedureDispatcher<MasterProce
public CloseRegionRequest buildCloseRegionRequest(final ServerName serverName) {
return ProtobufUtil.buildCloseRegionRequest(serverName, regionInfo.getRegionName(),
- getDestinationServer(), procId);
+ getDestinationServer(), procId, getInitiatingMasterActiveTime());
}
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
index 1e6bb78e250..d1a49ebe6ec 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
@@ -97,9 +97,10 @@ public class SplitWALRemoteProcedure extends ServerRemoteProcedure
@Override
public Optional<RemoteProcedureDispatcher.RemoteOperation> remoteCallBuild(MasterProcedureEnv env,
ServerName serverName) {
- return Optional.of(new RSProcedureDispatcher.ServerOperation(this, getProcId(),
- SplitWALCallable.class, MasterProcedureProtos.SplitWALParameter.newBuilder()
- .setWalPath(walPath).build().toByteArray()));
+ return Optional.of(new RSProcedureDispatcher.ServerOperation(
+ this, getProcId(), SplitWALCallable.class, MasterProcedureProtos.SplitWALParameter
+ .newBuilder().setWalPath(walPath).build().toByteArray(),
+ env.getMasterServices().getMasterActiveTime()));
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SwitchRpcThrottleRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SwitchRpcThrottleRemoteProcedure.java
index 668897cd9a4..69fa15e858b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SwitchRpcThrottleRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SwitchRpcThrottleRemoteProcedure.java
@@ -93,7 +93,8 @@ public class SwitchRpcThrottleRemoteProcedure extends ServerRemoteProcedure
SwitchRpcThrottleRemoteCallable.class,
SwitchRpcThrottleRemoteStateData.newBuilder()
.setTargetServer(ProtobufUtil.toServerName(remote))
- .setRpcThrottleEnabled(rpcThrottleEnabled).build().toByteArray()));
+ .setRpcThrottleEnabled(rpcThrottleEnabled).build().toByteArray(),
+ masterProcedureEnv.getMasterServices().getMasterActiveTime()));
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
index d7719a73e65..a6772cc7f8b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
@@ -65,7 +65,8 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
return Optional.of(new ServerOperation(this, getProcId(), ClaimReplicationQueueCallable.class,
ClaimReplicationQueueRemoteParameter.newBuilder()
.setCrashedServer(ProtobufUtil.toServerName(crashedServer)).setQueue(queue).build()
- .toByteArray()));
+ .toByteArray(),
+ env.getMasterServices().getMasterActiveTime()));
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index be5824fe1ed..04fa96558a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -106,7 +106,8 @@ public class RefreshPeerProcedure extends ServerRemoteProcedure
assert targetServer.equals(remote);
return Optional.of(new ServerOperation(this, getProcId(), RefreshPeerCallable.class,
RefreshPeerParameter.newBuilder().setPeerId(peerId).setType(toPeerModificationType(type))
- .setTargetServer(ProtobufUtil.toServerName(remote)).build().toByteArray()));
+ .setTargetServer(ProtobufUtil.toServerName(remote)).build().toByteArray(),
+ env.getMasterServices().getMasterActiveTime()));
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 12c8878fb0d..26d65a68b57 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2471,6 +2471,7 @@ public class HRegionServer extends Thread
HRegion r = context.getRegion();
long openProcId = context.getOpenProcId();
long masterSystemTime = context.getMasterSystemTime();
+ long initiatingMasterActiveTime = context.getInitiatingMasterActiveTime();
rpcServices.checkOpen();
LOG.info("Post open deploy tasks for {}, pid={}, masterSystemTime={}",
r.getRegionInfo().getRegionNameAsString(), openProcId, masterSystemTime);
@@ -2491,7 +2492,7 @@ public class HRegionServer extends Thread
// Notify master
if (
!reportRegionStateTransition(new RegionStateTransitionContext(TransitionCode.OPENED,
- openSeqNum, openProcId, masterSystemTime, r.getRegionInfo()))
+ openSeqNum, openProcId, masterSystemTime, r.getRegionInfo(), initiatingMasterActiveTime))
) {
throw new IOException(
"Failed to report opened region to master: " + r.getRegionInfo().getRegionNameAsString());
@@ -2552,6 +2553,7 @@ public class HRegionServer extends Thread
for (long procId : procIds) {
transition.addProcId(procId);
}
+ transition.setInitiatingMasterActiveTime(context.getInitiatingMasterActiveTime());
return builder.build();
}
@@ -3929,12 +3931,15 @@ public class HRegionServer extends Thread
this.rpcServices, this.rpcServices, new RegionServerRegistry(this));
}
- void executeProcedure(long procId, RSProcedureCallable callable) {
- executorService.submit(new RSProcedureHandler(this, procId, callable));
+ void executeProcedure(long procId, long initiatingMasterActiveTime,
+ RSProcedureCallable callable) {
+ executorService
+ .submit(new RSProcedureHandler(this, procId, initiatingMasterActiveTime, callable));
}
- public void remoteProcedureComplete(long procId, Throwable error) {
- procedureResultReporter.complete(procId, error);
+ public void remoteProcedureComplete(long procId, long initiatingMasterActiveTime,
+ Throwable error) {
+ procedureResultReporter.complete(procId, initiatingMasterActiveTime, error);
}
void reportProcedureDone(ReportProcedureDoneRequest request) throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 584c2baab49..505e5a2b6e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -3874,6 +3874,8 @@ public class RSRpcServices
private void executeOpenRegionProcedures(OpenRegionRequest request,
Map<TableName, TableDescriptor> tdCache) {
long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
+ long initiatingMasterActiveTime =
+ request.hasInitiatingMasterActiveTime() ? request.getInitiatingMasterActiveTime() : -1;
for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
RegionInfo regionInfo = ProtobufUtil.toRegionInfo(regionOpenInfo.getRegion());
TableName tableName = regionInfo.getTable();
@@ -3900,13 +3902,15 @@ public class RSRpcServices
long procId = regionOpenInfo.getOpenProcId();
if (regionServer.submitRegionProcedure(procId)) {
regionServer.executorService.submit(AssignRegionHandler.create(regionServer, regionInfo,
- procId, tableDesc, masterSystemTime));
+ procId, tableDesc, masterSystemTime, initiatingMasterActiveTime));
}
}
}
private void executeCloseRegionProcedures(CloseRegionRequest request) {
String encodedName;
+ long initiatingMasterActiveTime =
+ request.hasInitiatingMasterActiveTime() ? request.getInitiatingMasterActiveTime() : -1;
try {
encodedName = ProtobufUtil.getRegionEncodedName(request.getRegion());
} catch (DoNotRetryIOException e) {
@@ -3917,8 +3921,8 @@ public class RSRpcServices
: null;
long procId = request.getCloseProcId();
if (regionServer.submitRegionProcedure(procId)) {
- regionServer.executorService.submit(
- UnassignRegionHandler.create(regionServer, encodedName, procId, false, destination));
+ regionServer.executorService.submit(UnassignRegionHandler.create(regionServer, encodedName,
+ procId, false, destination, initiatingMasterActiveTime));
}
}
@@ -3930,12 +3934,14 @@ public class RSRpcServices
} catch (Exception e) {
LOG.warn("Failed to instantiating remote procedure {}, pid={}", request.getProcClass(),
request.getProcId(), e);
- regionServer.remoteProcedureComplete(request.getProcId(), e);
+ regionServer.remoteProcedureComplete(request.getProcId(),
+ request.getInitiatingMasterActiveTime(), e);
return;
}
callable.init(request.getProcData().toByteArray(), regionServer);
LOG.debug("Executing remote procedure {}, pid={}", callable.getClass(), request.getProcId());
- regionServer.executeProcedure(request.getProcId(), callable);
+ regionServer.executeProcedure(request.getProcId(), request.getInitiatingMasterActiveTime(),
+ callable);
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 0906a4b44b1..95210f0bdd4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -91,11 +91,14 @@ public interface RegionServerServices extends Server, MutableOnlineRegions, Favo
private final HRegion region;
private final long openProcId;
private final long masterSystemTime;
+ private final long initiatingMasterActiveTime;
- public PostOpenDeployContext(HRegion region, long openProcId, long masterSystemTime) {
+ public PostOpenDeployContext(HRegion region, long openProcId, long masterSystemTime,
+ long initiatingMasterActiveTime) {
this.region = region;
this.openProcId = openProcId;
this.masterSystemTime = masterSystemTime;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
}
public HRegion getRegion() {
@@ -109,6 +112,10 @@ public interface RegionServerServices extends Server, MutableOnlineRegions, Favo
public long getMasterSystemTime() {
return masterSystemTime;
}
+
+ public long getInitiatingMasterActiveTime() {
+ return initiatingMasterActiveTime;
+ }
}
/**
@@ -121,23 +128,26 @@ public interface RegionServerServices extends Server, MutableOnlineRegions, Favo
private final TransitionCode code;
private final long openSeqNum;
private final long masterSystemTime;
+ private final long initiatingMasterActiveTime;
private final long[] procIds;
private final RegionInfo[] hris;
public RegionStateTransitionContext(TransitionCode code, long openSeqNum, long masterSystemTime,
- RegionInfo... hris) {
+ long initiatingMasterActiveTime, RegionInfo... hris) {
this.code = code;
this.openSeqNum = openSeqNum;
this.masterSystemTime = masterSystemTime;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
this.hris = hris;
this.procIds = new long[hris.length];
}
public RegionStateTransitionContext(TransitionCode code, long openSeqNum, long procId,
- long masterSystemTime, RegionInfo hri) {
+ long masterSystemTime, RegionInfo hri, long initiatingMasterActiveTime) {
this.code = code;
this.openSeqNum = openSeqNum;
this.masterSystemTime = masterSystemTime;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
this.hris = new RegionInfo[] { hri };
this.procIds = new long[] { procId };
}
@@ -161,6 +171,10 @@ public interface RegionServerServices extends Server, MutableOnlineRegions, Favo
public long[] getProcIds() {
return procIds;
}
+
+ public long getInitiatingMasterActiveTime() {
+ return initiatingMasterActiveTime;
+ }
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
index 817ecd42ce0..21016fe59dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
@@ -51,8 +51,9 @@ class RemoteProcedureResultReporter extends Thread {
this.server = server;
}
- public void complete(long procId, Throwable error) {
- RemoteProcedureResult.Builder builder = RemoteProcedureResult.newBuilder().setProcId(procId);
+ public void complete(long procId, long initiatingMasterActiveTime, Throwable error) {
+ RemoteProcedureResult.Builder builder = RemoteProcedureResult.newBuilder().setProcId(procId)
+ .setInitiatingMasterActiveTime(initiatingMasterActiveTime);
if (error != null) {
LOG.debug("Failed to complete execution of pid={}", procId, error);
builder.setStatus(RemoteProcedureResult.Status.ERROR).setError(
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
index 20a7e0c9af2..d979a3ac82e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -81,7 +81,7 @@ class SplitRequest implements Runnable {
// are created just to pass the information to the reportRegionStateTransition().
if (
!server.reportRegionStateTransition(new RegionStateTransitionContext(
- TransitionCode.READY_TO_SPLIT, HConstants.NO_SEQNUM, -1, parent, hri_a, hri_b))
+ TransitionCode.READY_TO_SPLIT, HConstants.NO_SEQNUM, -1, -1, parent, hri_a, hri_b))
) {
LOG.error("Unable to ask master to split " + parent.getRegionNameAsString());
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/AssignRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/AssignRegionHandler.java
index a9ab6f502a3..d61f2de30fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/AssignRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/AssignRegionHandler.java
@@ -62,15 +62,20 @@ public class AssignRegionHandler extends EventHandler {
private final long masterSystemTime;
+ // active time of the master that sent this assign request, used for fencing
+ private final long initiatingMasterActiveTime;
+
private final RetryCounter retryCounter;
public AssignRegionHandler(HRegionServer server, RegionInfo regionInfo, long openProcId,
- @Nullable TableDescriptor tableDesc, long masterSystemTime, EventType eventType) {
+ @Nullable TableDescriptor tableDesc, long masterSystemTime, long initiatingMasterActiveTime,
+ EventType eventType) {
super(server, eventType);
this.regionInfo = regionInfo;
this.openProcId = openProcId;
this.tableDesc = tableDesc;
this.masterSystemTime = masterSystemTime;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
this.retryCounter = HandlerUtil.getRetryCounter();
}
@@ -85,7 +90,7 @@ public class AssignRegionHandler extends EventHandler {
rs.getRegionsInTransitionInRS().remove(regionInfo.getEncodedNameAsBytes(), Boolean.TRUE);
if (
!rs.reportRegionStateTransition(new RegionStateTransitionContext(TransitionCode.FAILED_OPEN,
- HConstants.NO_SEQNUM, openProcId, masterSystemTime, regionInfo))
+ HConstants.NO_SEQNUM, openProcId, masterSystemTime, regionInfo, initiatingMasterActiveTime))
) {
throw new IOException(
"Failed to report failed open to master: " + regionInfo.getRegionNameAsString());
@@ -153,7 +158,8 @@ public class AssignRegionHandler extends EventHandler {
}
// From here on out, this is PONR. We can not revert back. The only way to address an
// exception from here on out is to abort the region server.
- rs.postOpenDeployTasks(new PostOpenDeployContext(region, openProcId, masterSystemTime));
+ rs.postOpenDeployTasks(
+ new PostOpenDeployContext(region, openProcId, masterSystemTime, initiatingMasterActiveTime));
rs.addRegion(region);
LOG.info("Opened {}", regionName);
// Cache the open region procedure id after report region transition succeed.
@@ -180,7 +186,8 @@ public class AssignRegionHandler extends EventHandler {
}
public static AssignRegionHandler create(HRegionServer server, RegionInfo regionInfo,
- long openProcId, TableDescriptor tableDesc, long masterSystemTime) {
+ long openProcId, TableDescriptor tableDesc, long masterSystemTime,
+ long initiatingMasterActiveTime) {
EventType eventType;
if (regionInfo.isMetaRegion()) {
eventType = EventType.M_RS_OPEN_META;
@@ -193,6 +200,6 @@ public class AssignRegionHandler extends EventHandler {
eventType = EventType.M_RS_OPEN_REGION;
}
return new AssignRegionHandler(server, regionInfo, openProcId, tableDesc, masterSystemTime,
- eventType);
+ initiatingMasterActiveTime, eventType);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
index 2301b9b8b49..548d5b05902 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
@@ -111,7 +111,7 @@ public class CloseRegionHandler extends EventHandler {
this.rsServices.removeRegion(region, destination);
rsServices.reportRegionStateTransition(new RegionStateTransitionContext(TransitionCode.CLOSED,
- HConstants.NO_SEQNUM, Procedure.NO_PROC_ID, -1, regionInfo));
+ HConstants.NO_SEQNUM, Procedure.NO_PROC_ID, -1, regionInfo, -1));
// Done! Region is closed on this RS
LOG.debug("Closed {}", region.getRegionInfo().getRegionNameAsString());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index 24d47dfbcec..b0e4a7fcf67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -163,8 +163,9 @@ public class OpenRegionHandler extends EventHandler {
cleanupFailedOpen(region);
}
} finally {
- rsServices.reportRegionStateTransition(new RegionStateTransitionContext(
- TransitionCode.FAILED_OPEN, HConstants.NO_SEQNUM, Procedure.NO_PROC_ID, -1, regionInfo));
+ rsServices
+ .reportRegionStateTransition(new RegionStateTransitionContext(TransitionCode.FAILED_OPEN,
+ HConstants.NO_SEQNUM, Procedure.NO_PROC_ID, -1, regionInfo, -1));
}
}
@@ -250,7 +251,7 @@ public class OpenRegionHandler extends EventHandler {
public void run() {
try {
this.services.postOpenDeployTasks(
- new PostOpenDeployContext(region, Procedure.NO_PROC_ID, masterSystemTime));
+ new PostOpenDeployContext(region, Procedure.NO_PROC_ID, masterSystemTime, -1));
} catch (Throwable e) {
String msg = "Exception running postOpenDeployTasks; region="
+ this.region.getRegionInfo().getEncodedName();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
index d3ecc8a51e2..6eacc6b78e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
@@ -35,12 +35,17 @@ public class RSProcedureHandler extends EventHandler {
private final long procId;
+ // active time of the master that sent procedure request, used for fencing
+ private final long initiatingMasterActiveTime;
+
private final RSProcedureCallable callable;
- public RSProcedureHandler(HRegionServer rs, long procId, RSProcedureCallable callable) {
+ public RSProcedureHandler(HRegionServer rs, long procId, long initiatingMasterActiveTime,
+ RSProcedureCallable callable) {
super(rs, callable.getEventType());
this.procId = procId;
this.callable = callable;
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
}
@Override
@@ -53,7 +58,7 @@ public class RSProcedureHandler extends EventHandler {
LOG.error("pid=" + this.procId, t);
error = t;
} finally {
- ((HRegionServer) server).remoteProcedureComplete(procId, error);
+ ((HRegionServer) server).remoteProcedureComplete(procId, initiatingMasterActiveTime, error);
}
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java
index e312df9862d..2c8e2c5716e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java
@@ -63,14 +63,19 @@ public class UnassignRegionHandler extends EventHandler {
private final RetryCounter retryCounter;
+ // active time of the master that sent this unassign request, used for fencing
+ private final long initiatingMasterActiveTime;
+
public UnassignRegionHandler(HRegionServer server, String encodedName, long closeProcId,
- boolean abort, @Nullable ServerName destination, EventType eventType) {
+ boolean abort, @Nullable ServerName destination, EventType eventType,
+ long initiatingMasterActiveTime) {
super(server, eventType);
this.encodedName = encodedName;
this.closeProcId = closeProcId;
this.abort = abort;
this.destination = destination;
this.retryCounter = HandlerUtil.getRetryCounter();
+ this.initiatingMasterActiveTime = initiatingMasterActiveTime;
}
private HRegionServer getServer() {
@@ -139,7 +144,7 @@ public class UnassignRegionHandler extends EventHandler {
}
if (
!rs.reportRegionStateTransition(new RegionStateTransitionContext(TransitionCode.CLOSED,
- HConstants.NO_SEQNUM, closeProcId, -1, region.getRegionInfo()))
+ HConstants.NO_SEQNUM, closeProcId, -1, region.getRegionInfo(), initiatingMasterActiveTime))
) {
throw new IOException("Failed to report close to master: " + regionName);
}
@@ -159,7 +164,8 @@ public class UnassignRegionHandler extends EventHandler {
}
public static UnassignRegionHandler create(HRegionServer server, String encodedName,
- long closeProcId, boolean abort, @Nullable ServerName destination) {
+ long closeProcId, boolean abort, @Nullable ServerName destination,
+ long initiatingMasterActiveTime) {
// Just try our best to determine whether it is for closing meta. It is not the end of the world
// if we put the handler into a wrong executor.
Region region = server.getRegion(encodedName);
@@ -167,6 +173,6 @@ public class UnassignRegionHandler extends EventHandler {
? EventType.M_RS_CLOSE_META
: EventType.M_RS_CLOSE_REGION;
return new UnassignRegionHandler(server, encodedName, closeProcId, abort, destination,
- eventType);
+ eventType, initiatingMasterActiveTime);
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 0471eb0ed9b..864285ee97b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -58,16 +58,19 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.security.access.AccessChecker;
import org.apache.hadoop.hbase.security.access.ZKPermissionWatcher;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
public class MockNoopMasterServices implements MasterServices {
private final Configuration conf;
private final MetricsMaster metricsMaster;
+ private final long masterActiveTime;
public MockNoopMasterServices(final Configuration conf) {
this.conf = conf;
this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(mock(HMaster.class)));
+ this.masterActiveTime = EnvironmentEdgeManager.currentTime();
}
@Override
@@ -316,6 +319,11 @@ public class MockNoopMasterServices implements MasterServices {
return true;
}
+ @Override
+ public long getMasterActiveTime() {
+ return masterActiveTime;
+ }
+
@Override
public boolean isInitialized() {
return false;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java
index 0c7703fd542..901ee0688b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java
@@ -183,8 +183,8 @@ public class TestServerRemoteProcedure {
@Override
public Optional<RemoteProcedureDispatcher.RemoteOperation>
remoteCallBuild(MasterProcedureEnv env, ServerName serverName) {
- return Optional
- .of(new RSProcedureDispatcher.ServerOperation(null, 0L, this.getClass(), new byte[0]));
+ return Optional.of(new RSProcedureDispatcher.ServerOperation(null, 0L, this.getClass(),
+ new byte[0], env.getMasterServices().getMasterActiveTime()));
}
@Override