You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2020/07/21 22:56:50 UTC
[hbase] branch branch-2 updated: HBASE-24662 Update
DumpClusterStatusAction to notice changes in region server count
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c0d7bfb HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count
c0d7bfb is described below
commit c0d7bfb6f78e131d640936e4580b0a19c45dcaf4
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Tue Jun 30 16:50:10 2020 -0700
HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count
Sometimes running chaos monkey, I've found that we lose accounting of
region servers. I've taken to a manual process of checking the
reported list against a known reference. It occurs to me that
ChaosMonkey has a known reference, and it can do this accounting for
me.
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../chaos/actions/DumpClusterStatusAction.java | 62 +++++++++++++++++++++-
1 file changed, 60 insertions(+), 2 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
index dfb506d..d3ad314 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
@@ -19,6 +19,13 @@
package org.apache.hadoop.hbase.chaos.actions;
import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hbase.ClusterMetrics;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.net.Address;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,18 +35,69 @@ import org.slf4j.LoggerFactory;
public class DumpClusterStatusAction extends Action {
private static final Logger LOG = LoggerFactory.getLogger(DumpClusterStatusAction.class);
- @Override protected Logger getLogger() {
+ private Set<Address> initialRegionServers;
+
+ @Override
+ protected Logger getLogger() {
return LOG;
}
@Override
public void init(ActionContext context) throws IOException {
super.init(context);
+ initialRegionServers = collectKnownRegionServers(initialStatus);
}
@Override
public void perform() throws Exception {
getLogger().debug("Performing action: Dump cluster status");
- getLogger().info("Cluster status\n" + cluster.getClusterMetrics());
+ final ClusterMetrics currentMetrics = cluster.getClusterMetrics();
+ getLogger().info("Cluster status\n{}", currentMetrics);
+ reportMissingRegionServers(currentMetrics);
+ reportNewRegionServers(currentMetrics);
+ }
+
+ /**
+ * Build a set of all the host:port pairs of region servers known to this cluster.
+ */
+ private static Set<Address> collectKnownRegionServers(final ClusterMetrics clusterMetrics) {
+ final Set<Address> regionServers = clusterMetrics.getLiveServerMetrics()
+ .keySet()
+ .stream()
+ .map(ServerName::getAddress)
+ .collect(Collectors.toSet());
+ clusterMetrics.getDeadServerNames()
+ .stream()
+ .map(ServerName::getAddress)
+ .forEach(regionServers::add);
+ return Collections.unmodifiableSet(regionServers);
+ }
+
+ private void reportMissingRegionServers(final ClusterMetrics clusterMetrics) {
+ final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics);
+ final Set<Address> missingRegionServers = new HashSet<>(initialRegionServers);
+ missingRegionServers.removeAll(regionServers);
+ if (!missingRegionServers.isEmpty()) {
+ final StringBuilder stringBuilder = new StringBuilder()
+ .append("region server(s) are missing from this cluster report");
+ missingRegionServers.stream()
+ .sorted()
+ .forEach(address -> stringBuilder.append("\n ").append(address));
+ getLogger().warn(stringBuilder.toString());
+ }
+ }
+
+ private void reportNewRegionServers(final ClusterMetrics clusterMetrics) {
+ final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics);
+ final Set<Address> newRegionServers = new HashSet<>(regionServers);
+ newRegionServers.removeAll(initialRegionServers);
+ if (!newRegionServers.isEmpty()) {
+ final StringBuilder stringBuilder = new StringBuilder()
+ .append("region server(s) are new for this cluster report");
+ newRegionServers.stream()
+ .sorted()
+ .forEach(address -> stringBuilder.append("\n ").append(address));
+ getLogger().warn(stringBuilder.toString());
+ }
}
}