You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/04/28 00:52:05 UTC

lucene-solr:jira/solr-10233: Added logging to ChaosMonkey

Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-10233 a342edd9e -> e7d54fa0b


Added logging to ChaosMonkey


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e7d54fa0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e7d54fa0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e7d54fa0

Branch: refs/heads/jira/solr-10233
Commit: e7d54fa0b1e31b01be05c479975da36c53259a96
Parents: a342edd
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Thu Apr 27 17:51:52 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Thu Apr 27 17:51:52 2017 -0700

----------------------------------------------------------------------
 ...keyNothingIsSafeWithPassiveReplicasTest.java |  7 ++-
 ...MonkeySafeLeaderWithPassiveReplicasTest.java |  2 +-
 .../java/org/apache/solr/cloud/ChaosMonkey.java | 46 ++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e7d54fa0/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
index 9daec5b..e351541 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
@@ -197,8 +197,7 @@ public class ChaosMonkeyNothingIsSafeWithPassiveReplicasTest extends AbstractFul
           }
           runLength = runTimes[random().nextInt(runTimes.length - 1)];
         }
-        
-        Thread.sleep(runLength);
+        ChaosMonkey.wait(runLength, DEFAULT_COLLECTION, zkStateReader);
       } finally {
         chaosMonkey.stopTheMonkey();
       }
@@ -291,6 +290,10 @@ public class ChaosMonkeyNothingIsSafeWithPassiveReplicasTest extends AbstractFul
     }
   }
 
+  private void logCollectionStateSummary(String defaultCollection) {
+    
+  }
+
   private Set<String> getAddFails(List<StoppableIndexingThread> threads) {
     Set<String> addFails = new HashSet<String>();
     for (StoppableIndexingThread thread : threads)   {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e7d54fa0/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
index 46dc837..687e7a9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
@@ -156,7 +156,7 @@ public class ChaosMonkeySafeLeaderWithPassiveReplicasTest extends AbstractFullDi
         runLength = runTimes[random().nextInt(runTimes.length - 1)];
       }
       
-      Thread.sleep(runLength);
+      ChaosMonkey.wait(runLength, DEFAULT_COLLECTION, cloudClient.getZkStateReader());
     } finally {
       chaosMonkey.stopTheMonkey();
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e7d54fa0/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
index f42831b..c1f29db 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
@@ -21,10 +21,13 @@ import java.lang.invoke.MethodHandles;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -41,6 +44,7 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.servlet.SolrDispatchFilter;
 import org.apache.solr.update.DirectUpdateHandler2;
 import org.apache.solr.util.RTimer;
+import org.apache.solr.util.TimeOut;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -562,6 +566,10 @@ public class ChaosMonkey {
     log.info("monkey: " + msg);
   }
   
+  public static void monkeyLog(String msg, Object...logParams) {
+    log.info("monkey: " + msg, logParams);
+  }
+  
   public void stopTheMonkey() {
     stop = true;
     try {
@@ -680,4 +688,42 @@ public class ChaosMonkey {
     return true;
   }
 
+  public static void wait(long runLength, String collectionName, ZkStateReader zkStateReader) throws InterruptedException {
+    TimeOut t = new TimeOut(runLength, TimeUnit.MILLISECONDS);
+    while (!t.hasTimedOut()) {
+      Thread.sleep(Math.min(1000, runLength));
+      logCollectionStateSummary(collectionName, zkStateReader);
+    }
+  }
+
+  private static void logCollectionStateSummary(String collectionName, ZkStateReader zkStateReader) {
+    Pattern portPattern = Pattern.compile(".*:([0-9]*).*");
+    DocCollection docCollection = zkStateReader.getClusterState().getCollection(collectionName);
+    if (docCollection == null) {
+      monkeyLog("Could not find collection {}", collectionName);
+    }
+    StringBuilder builder = new StringBuilder();
+    builder.append("Collection status: {");
+    for (Slice slice:docCollection.getSlices()) {
+      builder.append(slice.getName() + ": {");
+      for (Replica replica:slice.getReplicas()) {
+        log.info(replica.toString());
+        java.util.regex.Matcher m = portPattern.matcher(replica.getBaseUrl());
+        m.find();
+        String jettyPort = m.group(1);
+        builder.append(String.format(Locale.ROOT, "%s(%s): {state: %s, type: %s, leader: %s, Live: %s}, ", 
+            replica.getName(), jettyPort, replica.getState(), replica.getType(), (replica.get("leader")!= null), zkStateReader.getClusterState().liveNodesContain(replica.getNodeName())));
+      }
+      if (slice.getReplicas().size() > 0) {
+        builder.setLength(builder.length() - 2);
+      }
+      builder.append("}, ");
+    }
+    if (docCollection.getSlices().size() > 0) {
+      builder.setLength(builder.length() - 2);
+    }
+    builder.append("}");
+    monkeyLog(builder.toString());
+  }
+
 }