You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2020/07/21 22:56:50 UTC

[hbase] branch branch-2 updated: HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count

This is an automated email from the ASF dual-hosted git repository.

ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
     new c0d7bfb  HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count
c0d7bfb is described below

commit c0d7bfb6f78e131d640936e4580b0a19c45dcaf4
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Tue Jun 30 16:50:10 2020 -0700

    HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count
    
    Sometimes running chaos monkey, I've found that we lose accounting of
    region servers. I've taken to a manual process of checking the
    reported list against a known reference. It occurs to me that
    ChaosMonkey has a known reference, and it can do this accounting for
    me.
    
    Signed-off-by: Viraj Jasani <vj...@apache.org>
---
 .../chaos/actions/DumpClusterStatusAction.java     | 62 +++++++++++++++++++++-
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
index dfb506d..d3ad314 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java
@@ -19,6 +19,13 @@
 package org.apache.hadoop.hbase.chaos.actions;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hbase.ClusterMetrics;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.net.Address;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,18 +35,69 @@ import org.slf4j.LoggerFactory;
 public class DumpClusterStatusAction extends Action {
   private static final Logger LOG = LoggerFactory.getLogger(DumpClusterStatusAction.class);
 
-  @Override protected Logger getLogger() {
+  private Set<Address> initialRegionServers;
+
+  @Override
+  protected Logger getLogger() {
     return LOG;
   }
 
   @Override
   public void init(ActionContext context) throws IOException {
     super.init(context);
+    initialRegionServers = collectKnownRegionServers(initialStatus);
   }
 
   @Override
   public void perform() throws Exception {
     getLogger().debug("Performing action: Dump cluster status");
-    getLogger().info("Cluster status\n" + cluster.getClusterMetrics());
+    final ClusterMetrics currentMetrics = cluster.getClusterMetrics();
+    getLogger().info("Cluster status\n{}", currentMetrics);
+    reportMissingRegionServers(currentMetrics);
+    reportNewRegionServers(currentMetrics);
+  }
+
+  /**
+   * Build a set of all the host:port pairs of region servers known to this cluster.
+   */
+  private static Set<Address> collectKnownRegionServers(final ClusterMetrics clusterMetrics) {
+    final Set<Address> regionServers = clusterMetrics.getLiveServerMetrics()
+      .keySet()
+      .stream()
+      .map(ServerName::getAddress)
+      .collect(Collectors.toSet());
+    clusterMetrics.getDeadServerNames()
+      .stream()
+      .map(ServerName::getAddress)
+      .forEach(regionServers::add);
+    return Collections.unmodifiableSet(regionServers);
+  }
+
+  private void reportMissingRegionServers(final ClusterMetrics clusterMetrics) {
+    final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics);
+    final Set<Address> missingRegionServers = new HashSet<>(initialRegionServers);
+    missingRegionServers.removeAll(regionServers);
+    if (!missingRegionServers.isEmpty()) {
+      final StringBuilder stringBuilder = new StringBuilder()
+        .append("region server(s) are missing from this cluster report");
+      missingRegionServers.stream()
+        .sorted()
+        .forEach(address -> stringBuilder.append("\n  ").append(address));
+      getLogger().warn(stringBuilder.toString());
+    }
+  }
+
+  private void reportNewRegionServers(final ClusterMetrics clusterMetrics) {
+    final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics);
+    final Set<Address> newRegionServers = new HashSet<>(regionServers);
+    newRegionServers.removeAll(initialRegionServers);
+    if (!newRegionServers.isEmpty()) {
+      final StringBuilder stringBuilder = new StringBuilder()
+        .append("region server(s) are new for this cluster report");
+      newRegionServers.stream()
+        .sorted()
+        .forEach(address -> stringBuilder.append("\n  ").append(address));
+      getLogger().warn(stringBuilder.toString());
+    }
   }
 }