You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/09/14 03:59:23 UTC
[02/50] [abbrv] hbase git commit: HBASE-18131 Add an hbase shell
command to clear deadserver list in ServerManager
HBASE-18131 Add an hbase shell command to clear deadserver list in ServerManager
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfdbdd20
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfdbdd20
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfdbdd20
Branch: refs/heads/HBASE-18467
Commit: cfdbdd2066bf977c951b74b31c0e41dce152f9ef
Parents: d83af69
Author: Guangxu Cheng <gu...@gmail.com>
Authored: Mon Sep 11 15:56:05 2017 +0800
Committer: tedyu <yu...@gmail.com>
Committed: Tue Sep 12 08:29:16 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 14 +++++
.../apache/hadoop/hbase/client/AsyncAdmin.java | 13 ++++
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 10 ++++
.../hbase/client/ConnectionImplementation.java | 12 ++++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 31 ++++++++++
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 26 ++++++++
.../client/ShortCircuitMasterConnection.java | 16 +++++
.../hbase/shaded/protobuf/ProtobufUtil.java | 11 ++++
.../hbase/shaded/protobuf/RequestConverter.java | 9 +++
.../src/main/protobuf/Master.proto | 23 +++++++
.../hbase/coprocessor/MasterObserver.java | 24 ++++++++
.../apache/hadoop/hbase/master/DeadServer.java | 13 ++++
.../hbase/master/MasterCoprocessorHost.java | 40 +++++++++++++
.../hadoop/hbase/master/MasterRpcServices.java | 63 ++++++++++++++++++++
.../hbase/security/access/AccessController.java | 5 ++
.../hadoop/hbase/master/TestDeadServer.java | 18 ++++++
hbase-shell/src/main/ruby/hbase/admin.rb | 22 +++++++
hbase-shell/src/main/ruby/shell.rb | 2 +
.../ruby/shell/commands/clear_deadservers.rb | 52 ++++++++++++++++
.../ruby/shell/commands/list_deadservers.rb | 43 +++++++++++++
20 files changed, 447 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 9f2084c..397aba0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2265,4 +2265,18 @@ public interface Admin extends Abortable, Closeable {
*/
void clearCompactionQueues(final ServerName sn, final Set<String> queues)
throws IOException, InterruptedException;
+
+ /**
+ * List dead region servers.
+ * @return List of dead region servers.
+ */
+ List<ServerName> listDeadServers() throws IOException;
+
+ /**
+ * Clear dead region servers from master.
+ * @param servers list of dead region servers.
+ * @throws IOException if a remote or network exception occurs
+ * @return List of servers that are not cleared
+ */
+ List<ServerName> clearDeadServers(final List<ServerName> servers) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 995a18f..d7c3389 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -1116,4 +1116,17 @@ public interface AsyncAdmin {
*/
<S, R> CompletableFuture<R> coprocessorService(Function<RpcChannel, S> stubMaker,
CoprocessorCallable<S, R> callable, ServerName serverName);
+
+ /**
+ * List all the dead region servers.
+ * @return - returns a list of dead region servers wrapped by a {@link CompletableFuture}.
+ */
+ CompletableFuture<List<ServerName>> listDeadServers();
+
+ /**
+ * Clear dead region servers from master.
+ * @param servers list of dead region servers.
+ * @return - returns a list of servers that not cleared wrapped by a {@link CompletableFuture}.
+ */
+ CompletableFuture<List<ServerName>> clearDeadServers(final List<ServerName> servers);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index e8e8c95..b92bf35 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -639,4 +639,14 @@ public class AsyncHBaseAdmin implements AsyncAdmin {
CoprocessorCallable<S, R> callable, ServerName serverName) {
return wrap(rawAdmin.coprocessorService(stubMaker, callable, serverName));
}
+
+ @Override
+ public CompletableFuture<List<ServerName>> listDeadServers() {
+ return wrap(rawAdmin.listDeadServers());
+ }
+
+ @Override
+ public CompletableFuture<List<ServerName>> clearDeadServers(List<ServerName> servers) {
+ return wrap(rawAdmin.clearDeadServers(servers));
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index b274371..8e7a261 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -1775,6 +1775,18 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
return stub.getQuotaStates(controller, request);
}
+
+ @Override
+ public MasterProtos.ListDeadServersResponse listDeadServers(RpcController controller,
+ MasterProtos.ListDeadServersRequest request) throws ServiceException {
+ return stub.listDeadServers(controller, request);
+ }
+
+ @Override
+ public MasterProtos.ClearDeadServersResponse clearDeadServers(RpcController controller,
+ MasterProtos.ClearDeadServersRequest request) throws ServiceException {
+ return stub.clearDeadServers(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 5803408..19f6b7e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -113,6 +113,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AbortProce
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest;
@@ -150,6 +151,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsProcedur
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
@@ -4333,4 +4335,33 @@ public class HBaseAdmin implements Admin {
};
ProtobufUtil.call(callable);
}
+
+ @Override
+ public List<ServerName> listDeadServers() throws IOException {
+ return executeCallable(new MasterCallable<List<ServerName>>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ public List<ServerName> rpcCall() throws ServiceException {
+ ListDeadServersRequest req = ListDeadServersRequest.newBuilder().build();
+ return ProtobufUtil.toServerNameList(
+ master.listDeadServers(getRpcController(), req).getServerNameList());
+ }
+ });
+ }
+
+ @Override
+ public List<ServerName> clearDeadServers(final List<ServerName> servers) throws IOException {
+ if (servers == null || servers.size() == 0) {
+ throw new IllegalArgumentException("servers cannot be null or empty");
+ }
+ return executeCallable(new MasterCallable<List<ServerName>>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ protected List<ServerName> rpcCall() throws Exception {
+ ClearDeadServersRequest req = RequestConverter.buildClearDeadServersRequest(servers);
+ return ProtobufUtil.toServerNameList(
+ master.clearDeadServers(getRpcController(), req).getServerNameList());
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 9e6f106..b9949c0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -113,6 +113,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegi
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest;
@@ -169,6 +171,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshot
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
@@ -2900,6 +2904,28 @@ public class RawAsyncHBaseAdmin implements AsyncAdmin {
return future;
}
+ @Override
+ public CompletableFuture<List<ServerName>> listDeadServers() {
+ return this.<List<ServerName>> newMasterCaller()
+ .action((controller, stub) -> this
+ .<ListDeadServersRequest, ListDeadServersResponse, List<ServerName>> call(
+ controller, stub, ListDeadServersRequest.newBuilder().build(),
+ (s, c, req, done) -> s.listDeadServers(c, req, done),
+ (resp) -> ProtobufUtil.toServerNameList(resp.getServerNameList())))
+ .call();
+ }
+
+ @Override
+ public CompletableFuture<List<ServerName>> clearDeadServers(List<ServerName> servers) {
+ return this.<List<ServerName>> newMasterCaller()
+ .action((controller, stub) -> this
+ .<ClearDeadServersRequest, ClearDeadServersResponse, List<ServerName>> call(
+ controller, stub, RequestConverter.buildClearDeadServersRequest(servers),
+ (s, c, req, done) -> s.clearDeadServers(c, req, done),
+ (resp) -> ProtobufUtil.toServerNameList(resp.getServerNameList())))
+ .call();
+ }
+
private <T> ServerRequestCallerBuilder<T> newServerCaller() {
return this.connection.callerFactory.<T> serverRequest()
.rpcTimeout(rpcTimeoutNs, TimeUnit.NANOSECONDS)
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
index 17b21cf..6b7e4b6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegi
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest;
@@ -90,6 +92,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshot
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
@@ -626,6 +630,18 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
}
@Override
+ public ClearDeadServersResponse clearDeadServers(RpcController controller,
+ ClearDeadServersRequest request) throws ServiceException {
+ return stub.clearDeadServers(controller, request);
+ }
+
+ @Override
+ public ListDeadServersResponse listDeadServers(RpcController controller,
+ ListDeadServersRequest request) throws ServiceException {
+ return stub.listDeadServers(controller, request);
+ }
+
+ @Override
public SplitTableRegionResponse splitRegion(RpcController controller, SplitTableRegionRequest request)
throws ServiceException {
return stub.splitRegion(controller, request);
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index b4f23ff..5679d08 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -410,6 +410,17 @@ public final class ProtobufUtil {
}
/**
+ * Convert a list of protocol buffer ServerName to a list of ServerName
+ * @param proto protocol buffer ServerNameList
+ * @return a list of ServerName
+ */
+ public static List<ServerName> toServerNameList(
+ List<HBaseProtos.ServerName> proto) {
+ return proto.stream().map(ProtobufUtil::toServerName)
+ .collect(Collectors.toList());
+ }
+
+ /**
* Get a list of NamespaceDescriptor from ListNamespaceDescriptorsResponse protobuf
* @param proto the ListNamespaceDescriptorsResponse
* @return a list of NamespaceDescriptor
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index 00f051b..d26afcd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteColumnRequest;
@@ -1812,6 +1813,14 @@ public final class RequestConverter {
return builder.build();
}
+ public static ClearDeadServersRequest buildClearDeadServersRequest(List<ServerName> deadServers) {
+ ClearDeadServersRequest.Builder builder = ClearDeadServersRequest.newBuilder();
+ for(ServerName server: deadServers) {
+ builder.addServerName(ProtobufUtil.toServerName(server));
+ }
+ return builder.build();
+ }
+
private static final GetSpaceQuotaRegionSizesRequest GET_SPACE_QUOTA_REGION_SIZES_REQUEST =
GetSpaceQuotaRegionSizesRequest.newBuilder().build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-protocol-shaded/src/main/protobuf/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 44a6a76..6b16bf8 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -622,6 +622,21 @@ message RemoveDrainFromRegionServersRequest {
message RemoveDrainFromRegionServersResponse {
}
+message ListDeadServersRequest {
+}
+
+message ListDeadServersResponse {
+ repeated ServerName server_name = 1;
+}
+
+message ClearDeadServersRequest {
+ repeated ServerName server_name = 1;
+}
+
+message ClearDeadServersResponse {
+ repeated ServerName server_name = 1;
+}
+
service MasterService {
/** Used by the client to get the number of regions that have received the updated schema */
rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@@ -970,4 +985,12 @@ service MasterService {
/** Fetches the Master's view of quotas */
rpc GetQuotaStates(GetQuotaStatesRequest)
returns(GetQuotaStatesResponse);
+
+ /** clear dead servers from master*/
+ rpc ClearDeadServers(ClearDeadServersRequest)
+ returns(ClearDeadServersResponse);
+
+ /** Returns a list of Dead Servers. */
+ rpc ListDeadServers(ListDeadServersRequest)
+ returns(ListDeadServersResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index f1cf49d..cf6cd73 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -1917,4 +1917,28 @@ public interface MasterObserver extends Coprocessor {
*/
default void postLockHeartbeat(ObserverContext<MasterCoprocessorEnvironment> ctx,
LockProcedure proc, boolean keepAlive) throws IOException {}
+
+ /**
+ * Called before list dead region servers.
+ */
+ default void preListDeadServers(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {}
+
+ /**
+ * Called after list dead region servers.
+ */
+ default void postListDeadServers(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {}
+
+ /**
+ * Called before clear dead region servers.
+ */
+ default void preClearDeadServers(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {}
+
+ /**
+ * Called after clear dead region servers.
+ */
+ default void postClearDeadServers(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
index fc86254..23c1705 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
@@ -202,4 +202,17 @@ public class DeadServer {
return o1.getSecond().compareTo(o2.getSecond());
}
};
+
+ /**
+ * remove the specified dead server
+ * @param deadServerName the dead server name
+ * @return true if this server was removed
+ */
+
+ public synchronized boolean removeDeadServer(final ServerName deadServerName) {
+ if (deadServers.remove(deadServerName) == null) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index eaa4f5f..14cebe6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -1879,6 +1879,46 @@ public class MasterCoprocessorHost
});
}
+ public void preListDeadServers() throws IOException {
+ execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
+ @Override
+ public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ oserver.preListDeadServers(ctx);
+ }
+ });
+ }
+
+ public void postListDeadServers() throws IOException {
+ execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
+ @Override
+ public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ oserver.postListDeadServers(ctx);
+ }
+ });
+ }
+
+ public void preClearDeadServers() throws IOException {
+ execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
+ @Override
+ public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ oserver.preClearDeadServers(ctx);
+ }
+ });
+ }
+
+ public void postClearDeadServers() throws IOException {
+ execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
+ @Override
+ public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ oserver.postClearDeadServers(ctx);
+ }
+ });
+ }
+
private static ImmutableHTableDescriptor toImmutableHTableDescriptor(TableDescriptor desc) {
return new ImmutableHTableDescriptor(desc);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 47523a3..eaf9f61 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -104,6 +104,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegi
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest;
@@ -164,6 +166,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshot
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
@@ -2121,4 +2125,63 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public ListDeadServersResponse listDeadServers(RpcController controller,
+ ListDeadServersRequest request) throws ServiceException {
+
+ LOG.debug(master.getClientIdAuditPrefix() + " list dead region servers.");
+ ListDeadServersResponse.Builder response = ListDeadServersResponse.newBuilder();
+ try {
+ master.checkInitialized();
+ if (master.cpHost != null) {
+ master.cpHost.preListDeadServers();
+ }
+
+ Set<ServerName> servers = master.getServerManager().getDeadServers().copyServerNames();
+ for (ServerName server : servers) {
+ response.addServerName(ProtobufUtil.toServerName(server));
+ }
+
+ if (master.cpHost != null) {
+ master.cpHost.postListDeadServers();
+ }
+ } catch (IOException io) {
+ throw new ServiceException(io);
+ }
+
+ return response.build();
+ }
+
+ @Override
+ public ClearDeadServersResponse clearDeadServers(RpcController controller,
+ ClearDeadServersRequest request) throws ServiceException {
+ LOG.debug(master.getClientIdAuditPrefix() + " clear dead region servers.");
+ ClearDeadServersResponse.Builder response = ClearDeadServersResponse.newBuilder();
+ try {
+ master.checkInitialized();
+ if (master.cpHost != null) {
+ master.cpHost.preClearDeadServers();
+ }
+
+ if (master.getServerManager().areDeadServersInProgress()) {
+ LOG.debug("Some dead server is still under processing, won't clear the dead server list");
+ response.addAllServerName(request.getServerNameList());
+ } else {
+ for (HBaseProtos.ServerName pbServer : request.getServerNameList()) {
+ if (!master.getServerManager().getDeadServers()
+ .removeDeadServer(ProtobufUtil.toServerName(pbServer))) {
+ response.addServerName(pbServer);
+ }
+ }
+ }
+
+ if (master.cpHost != null) {
+ master.cpHost.postClearDeadServers();
+ }
+ } catch (IOException io) {
+ throw new ServiceException(io);
+ }
+ return response.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 4d79990..6aeba24 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -1461,6 +1461,11 @@ public class AccessController implements MasterObserver, RegionObserver, RegionS
requirePermission(getActiveUser(ctx), "split", tableName, null, null, Action.ADMIN);
}
+ @Override
+ public void preClearDeadServers(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
+ requirePermission(getActiveUser(ctx), "clearDeadServers", Action.ADMIN);
+ }
+
/* ---- RegionObserver implementation ---- */
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
index fd18b6c..d5f238d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
@@ -151,5 +151,23 @@ public class TestDeadServer {
Assert.assertTrue(d.isEmpty());
}
+ @Test
+ public void testClearDeadServer(){
+ DeadServer d = new DeadServer();
+ d.add(hostname123);
+ d.add(hostname1234);
+ Assert.assertEquals(2, d.size());
+
+ d.removeDeadServer(hostname123);
+ Assert.assertEquals(1, d.size());
+ d.removeDeadServer(hostname1234);
+ Assert.assertTrue(d.isEmpty());
+
+ d.add(hostname1234);
+ Assert.assertFalse(d.removeDeadServer(hostname123_2));
+ Assert.assertEquals(1, d.size());
+ Assert.assertTrue(d.removeDeadServer(hostname1234));
+ Assert.assertTrue(d.isEmpty());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 1dfa0c1..236fac5 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1249,5 +1249,27 @@ module Hbase
end
@admin.clearCompactionQueues(ServerName.valueOf(server_name), queues)
end
+
+ #----------------------------------------------------------------------------------------------
+ # clear dead region servers
+ def list_deadservers
+ @admin.listDeadServers.to_a
+ end
+
+ #----------------------------------------------------------------------------------------------
+ # clear dead region servers
+ def clear_deadservers(dead_servers)
+ # Flatten params array
+ dead_servers = dead_servers.flatten.compact
+ if dead_servers.empty?
+ servers = list_deadservers
+ else
+ servers = java.util.ArrayList.new
+ dead_servers.each do |s|
+ servers.add(ServerName.valueOf(s))
+ end
+ end
+ @admin.clearDeadServers(servers).to_a
+ end
end
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index 469505f..759898b 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -358,6 +358,8 @@ Shell.load_command_group(
splitormerge_switch
splitormerge_enabled
clear_compaction_queues
+ list_deadservers
+ clear_deadservers
],
# TODO: remove older hlog_roll command
aliases: {
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb b/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb
new file mode 100644
index 0000000..c3b5659
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb
@@ -0,0 +1,52 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class ClearDeadservers < Command
+ def help
+ <<-EOF
+ Clear the dead region servers that are never used.
+ Examples:
+ Clear all dead region servers:
+ hbase> clear_deadservers
+ Clear the specified dead region servers:
+ hbase> clear_deadservers 'host187.example.com,60020,1289493121758'
+ or
+ hbase> clear_deadservers 'host187.example.com,60020,1289493121758',
+ 'host188.example.com,60020,1289493121758'
+ EOF
+ end
+
+ def command(*dead_servers)
+ servers = admin.clear_deadservers(dead_servers)
+ if servers.size <= 0
+ formatter.row(['true'])
+ else
+ formatter.row(['Some dead server clear failed'])
+ formatter.row(['SERVERNAME'])
+ servers.each do |server|
+ formatter.row([server.toString])
+ end
+ formatter.footer(servers.size)
+ end
+ end
+ end
+ end
+end
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfdbdd20/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb b/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb
new file mode 100644
index 0000000..be3f0bf
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb
@@ -0,0 +1,43 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class ListDeadservers < Command
+ def help
+ <<-EOF
+ List all dead region servers in hbase
+ Examples:
+ hbase> list_deadservers
+ EOF
+ end
+
+ def command
+ formatter.header(['SERVERNAME'])
+
+ servers = admin.list_deadservers
+ servers.each do |server|
+ formatter.row([server.toString])
+ end
+
+ formatter.footer(servers.size)
+ end
+ end
+ end
+end