You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/07/28 13:28:27 UTC

svn commit: r1366643 [18/19] - in /lucene/dev/branches/lucene3312: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/ dev-tools/maven/lucene/benchmark/ dev-tools/mave...

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Sat Jul 28 11:27:51 2012
@@ -37,6 +37,7 @@ import java.util.concurrent.ThreadPoolEx
 import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util._TestUtil;
 import org.apache.solr.JSONTestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
@@ -46,21 +47,25 @@ import org.apache.solr.client.solrj.impl
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
 import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
 import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.CloudState;
 import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CollectionParams.CollectionAction;
 import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.update.SolrCmdDistributor.Request;
 import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -316,70 +321,216 @@ public class BasicDistributedZkTest exte
     // if there is only one conf set. That and the fact that other tests run first in this
     // env make this pretty fragile
     
-    // create 2 new collections rapid fire
-    
-    ModifiableSolrParams params = new ModifiableSolrParams();
-    params.set("action", CollectionAction.CREATE.toString());
-    params.set("numReplicas", 1);
-    params.set("numShards", 3);
-    String collectionName = "awholynewcollection";
-    params.set("name", collectionName);
-    SolrRequest request = new QueryRequest(params);
-    request.setPath("/admin/collections");
- 
-    clients.get(0).request(request);
+    // create new collections rapid fire
+    Map<String,List<Integer>> collectionInfos = new HashMap<String,List<Integer>>();
+    int cnt = atLeast(3);
+    for (int i = 0; i < cnt; i++) {
+      ModifiableSolrParams params = new ModifiableSolrParams();
+      params.set("action", CollectionAction.CREATE.toString());
+      int numShards = _TestUtil.nextInt(random(), 0, shardCount) + 1;
+      int numReplicas = _TestUtil.nextInt(random(), 0, 5) + 1;
+      params.set("numShards", numShards);
+      params.set("numReplicas", numReplicas);
+      String collectionName = "awholynewcollection_" + i;
+      int clientIndex = random().nextInt(2);
+      List<Integer> list = new ArrayList<Integer>();
+      list.add(numShards);
+      list.add(numReplicas);
+      collectionInfos.put(collectionName, list);
+      params.set("name", collectionName);
+      SolrRequest request = new QueryRequest(params);
+      request.setPath("/admin/collections");
+   
+      clients.get(clientIndex).request(request);
+    }
+    
+    Set<Entry<String,List<Integer>>> collectionInfosEntrySet = collectionInfos.entrySet();
+    for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
+      String collection = entry.getKey();
+      List<Integer> list = entry.getValue();
+      checkForCollection(collection, list.get(0));
+      
+      String url = getUrlFromZk(collection);
+
+      HttpSolrServer collectionClient = new HttpSolrServer(url);
+      
+      // poll for a second - it can take a moment before we are ready to serve
+      waitForNon404(collectionClient);
+    }
     
-    String collectionName2 = "awholynewcollection2";
-    params.set("name", collectionName2);
-    params.set("numShards", 2);
-    clients.get(1).request(request);
+    List<String> collectionNameList = new ArrayList<String>();
+    collectionNameList.addAll(collectionInfos.keySet());
+    String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
     
-    checkForCollection(collectionName, 3);
-    checkForCollection(collectionName2, 2);
+    String url = getUrlFromZk(collectionName);
 
+    HttpSolrServer collectionClient = new HttpSolrServer(url);
+    
+    
     // lets try and use the solrj client to index and retrieve a couple documents
-    SolrInputDocument doc = getDoc(id, 6, i1, -600, tlong, 600, t1,
+    SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
         "humpty dumpy sat on a wall");
-    int which = (doc.getField(id).toString().hashCode() & 0x7fffffff)
-        % clients.size();
-    SolrServer client = clients.get(which);
-    client.add(doc);
+    SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
+        "humpty dumpy3 sat on a walls");
+    SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
+        "humpty dumpy2 sat on a walled");
+
+    collectionClient.add(doc1);
     
-    doc = getDoc(id, 7, i1, 123, tlong, 123, t1,
-        "humpty dumpy had a great fall");
-    which = (doc.getField(id).toString().hashCode() & 0x7fffffff)
-        % clients.size();
-    client = clients.get(which);
-    client.add(doc);
-    doc = getDoc(id, 8, i1, 876, tlong, 876, t1,
-        "all the kings horses and all the kings men");
-    which = (doc.getField(id).toString().hashCode() & 0x7fffffff)
-        % clients.size();
-    client = clients.get(which);
-    client.add(doc);
+    collectionClient.add(doc2);
+
+    collectionClient.add(doc3);
     
-    commit();
+    collectionClient.commit();
+    
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
+    
+    // lets try a collection reload
+    
+    // get core open times
+    Map<String,Long> urlToTimeBefore = new HashMap<String,Long>();
+    collectStartTimes(collectionName, urlToTimeBefore);
+    assertTrue(urlToTimeBefore.size() > 0);
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("action", CollectionAction.RELOAD.toString());
+    params.set("name", collectionName);
+    QueryRequest request = new QueryRequest(params);
+    request.setPath("/admin/collections");
+    
+    // we can use this client because we just want base url
+    clients.get(0).request(request);
+
+    // reloads make take a short while
+    boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore);
+    assertTrue("some core start times did not change on reload", allTimesAreCorrect);
     
     // remove a collection
     params = new ModifiableSolrParams();
     params.set("action", CollectionAction.DELETE.toString());
-    params.set("name", collectionName2);
+    params.set("name", collectionName);
     request = new QueryRequest(params);
     request.setPath("/admin/collections");
  
     clients.get(0).request(request);
     
     // ensure its out of the state
-    checkForMissingCollection(collectionName2);
+    checkForMissingCollection(collectionName);
+
+  }
+
+  private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
+    
     
+    long timeoutAt = System.currentTimeMillis() + 30000;
+
+    boolean allTimesAreCorrect = false;
+    while (System.currentTimeMillis() < timeoutAt) {
+      Map<String,Long> urlToTimeAfter = new HashMap<String,Long>();
+      collectStartTimes(collectionName, urlToTimeAfter);
+      
+      boolean retry = false;
+      Set<Entry<String,Long>> entries = urlToTimeBefore.entrySet();
+      for (Entry<String,Long> entry : entries) {
+        Long beforeTime = entry.getValue();
+        Long afterTime = urlToTimeAfter.get(entry.getKey());
+        assertNotNull(afterTime);
+        if (afterTime <= beforeTime) {
+          retry = true;
+          break;
+        }
+
+      }
+      if (!retry) {
+        allTimesAreCorrect = true;
+        break;
+      }
+    }
+    return allTimesAreCorrect;
+  }
+
+  private void collectStartTimes(String collectionName,
+      Map<String,Long> urlToTime) throws SolrServerException, IOException {
+    Map<String,Map<String,Slice>> collections = solrj.getZkStateReader()
+        .getCloudState().getCollectionStates();
+    if (collections.containsKey(collectionName)) {
+      Map<String,Slice> slices = collections.get(collectionName);
+
+      Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
+      while (it.hasNext()) {
+        Entry<String,Slice> sliceEntry = it.next();
+        Map<String,ZkNodeProps> sliceShards = sliceEntry.getValue().getShards();
+        Iterator<Entry<String,ZkNodeProps>> shardIt = sliceShards.entrySet()
+            .iterator();
+        while (shardIt.hasNext()) {
+          Entry<String,ZkNodeProps> shardEntry = shardIt.next();
+          ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
+          CoreAdminResponse mcr = CoreAdminRequest.getStatus(
+              coreProps.getCoreName(),
+              new HttpSolrServer(coreProps.getBaseUrl()));
+          long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
+          urlToTime.put(coreProps.getCoreUrl(), before);
+        }
+      }
+    } else {
+      throw new IllegalArgumentException("Could not find collection in :"
+          + collections.keySet());
+    }
+  }
+
+  private String getUrlFromZk(String collection) {
+    CloudState cloudState = solrj.getZkStateReader().getCloudState();
+    Map<String,Slice> slices = cloudState.getCollectionStates().get(collection);
+    
+    if (slices == null) {
+      throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
+    }
+    
+    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
+      Slice slice = entry.getValue();
+      Map<String,ZkNodeProps> shards = slice.getShards();
+      Set<Map.Entry<String,ZkNodeProps>> shardEntries = shards.entrySet();
+      for (Map.Entry<String,ZkNodeProps> shardEntry : shardEntries) {
+        final ZkNodeProps node = shardEntry.getValue();
+        if (cloudState.liveNodesContain(node.get(ZkStateReader.NODE_NAME_PROP))) {
+          return new ZkCoreNodeProps(node).getCoreUrl();
+        }
+      }
+    }
+    
+    throw new RuntimeException("Could not find a live node for collection:" + collection);
+  }
+
+  private void waitForNon404(HttpSolrServer collectionClient)
+      throws Exception {
+    
+    long timeoutAt = System.currentTimeMillis() + 30000;
+    
+    while (System.currentTimeMillis() < timeoutAt) {
+      boolean missing = false;
+      try {
+        collectionClient.query(new SolrQuery("*:*"));
+      } catch (SolrException e) {
+        // How do I get the response code!?
+        if (!e.getMessage().contains("(404)")) {
+          throw e;
+        }
+        missing = true;
+      }
+      if (!missing) {
+        return;
+      }
+      Thread.sleep(50);
+    }
     printLayout();
+    fail("Could not find the new collection - 404 : " + collectionClient.getBaseURL());
   }
 
   private void checkForCollection(String collectionName, int expectedSlices)
       throws Exception {
     // check for an expectedSlices new collection - we poll the state
-    long timeoutAt = System.currentTimeMillis() + 30000;
+    long timeoutAt = System.currentTimeMillis() + 60000;
     boolean found = false;
+    boolean sliceMatch = false;
     while (System.currentTimeMillis() < timeoutAt) {
       solrj.getZkStateReader().updateCloudState(true);
       CloudState cloudState = solrj.getZkStateReader().getCloudState();
@@ -389,6 +540,7 @@ public class BasicDistributedZkTest exte
         Map<String,Slice> slices = collections.get(collectionName);
         // did we find expectedSlices slices/shards?
         if (slices.size() == expectedSlices) {
+          sliceMatch = true;
           found = true;
           // also make sure each are active
           Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
@@ -406,15 +558,19 @@ public class BasicDistributedZkTest exte
                 break;
               }
             }
-            
           }
           if (found) break;
         }
       }
-      Thread.sleep(50);
+      Thread.sleep(100);
     }
     if (!found) {
-      fail("Could not find new " + expectedSlices + " slice collection");
+      printLayout();
+      if (!sliceMatch) {
+        fail("Could not find new " + expectedSlices + " slice collection called " + collectionName);
+      } else {
+        fail("Found expected # of slices, but some nodes are not active for collection called " + collectionName);
+      }
     }
   }
   
@@ -432,7 +588,7 @@ public class BasicDistributedZkTest exte
         found = false;
         break;
       }
-      Thread.sleep(50);
+      Thread.sleep(100);
     }
     if (found) {
       fail("Found collection that should be gone " + collectionName);

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java Sat Jul 28 11:27:51 2012
@@ -28,6 +28,7 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.util.RefCounted;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -61,10 +62,12 @@ public class BasicZkTest extends Abstrac
     // test merge factor picked up
     SolrCore core = h.getCore();
 
-    IndexWriter writer = ((DirectUpdateHandler2)core.getUpdateHandler()).getSolrCoreState().getIndexWriter(core);
-
-    assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy)writer.getConfig().getMergePolicy()).getMergeFactor());
-    
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2)core.getUpdateHandler()).getSolrCoreState().getIndexWriter(core);
+    try {
+      assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy)iw.get().getConfig().getMergePolicy()).getMergeFactor());
+    } finally {
+      iw.decref();
+    }
     lrf.args.put(CommonParams.VERSION, "2.2");
     assertQ("test query on empty index", request("qlkciyopsbgzyvkylsjhchghjrdf"),
         "//result[@numFound='0']");

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.cloud.FullSolrCloudTest.CloudJettyRunner;
@@ -35,6 +36,8 @@ import org.apache.solr.core.CoreContaine
 import org.apache.solr.servlet.SolrDispatchFilter;
 import org.apache.zookeeper.KeeperException;
 import org.eclipse.jetty.servlet.FilterHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The monkey can stop random or specific jetties used with SolrCloud.
@@ -44,7 +47,8 @@ import org.eclipse.jetty.servlet.FilterH
  *
  */
 public class ChaosMonkey {
-
+  private static Logger log = LoggerFactory.getLogger(ChaosMonkey.class);
+  
   private static final int CONLOSS_PERCENT = 3; //30%
   private static final int EXPIRE_PERCENT = 4; //40%
   private Map<String,List<CloudJettyRunner>> shardToJetty;
@@ -52,7 +56,6 @@ public class ChaosMonkey {
   private ZkTestServer zkServer;
   private ZkStateReader zkStateReader;
   private String collection;
-  private Random random;
   private volatile boolean stop = false;
   private AtomicInteger stops = new AtomicInteger();
   private AtomicInteger starts = new AtomicInteger();
@@ -63,29 +66,26 @@ public class ChaosMonkey {
   private boolean expireSessions;
   private boolean causeConnectionLoss;
   private boolean aggressivelyKillLeaders;
-  private Map<String,SolrServer> shardToLeaderClient;
   private Map<String,CloudJettyRunner> shardToLeaderJetty;
   private long startTime;
   
   public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader,
       String collection, Map<String,List<CloudJettyRunner>> shardToJetty,
-      Map<String,List<SolrServer>> shardToClient,
-      Map<String,SolrServer> shardToLeaderClient,
-      Map<String,CloudJettyRunner> shardToLeaderJetty, Random random) {
+      Map<String,CloudJettyRunner> shardToLeaderJetty) {
     this.shardToJetty = shardToJetty;
-    this.shardToClient = shardToClient;
-    this.shardToLeaderClient = shardToLeaderClient;
     this.shardToLeaderJetty = shardToLeaderJetty;
     this.zkServer = zkServer;
     this.zkStateReader = zkStateReader;
     this.collection = collection;
-    this.random = random;
-    
+    Random random = LuceneTestCase.random();
     expireSessions = random.nextBoolean();
     causeConnectionLoss = random.nextBoolean();
+    monkeyLog("init - expire sessions:" + expireSessions
+        + " cause connection loss:" + causeConnectionLoss);
   }
   
   public void expireSession(JettySolrRunner jetty) {
+    monkeyLog("expire session for " + jetty.getLocalPort() + " !");
     SolrDispatchFilter solrDispatchFilter = (SolrDispatchFilter) jetty.getDispatchFilter().getFilter();
     if (solrDispatchFilter != null) {
       CoreContainer cores = solrDispatchFilter.getCores();
@@ -99,7 +99,7 @@ public class ChaosMonkey {
   public void expireRandomSession() throws KeeperException, InterruptedException {
     String sliceName = getRandomSlice();
     
-    JettySolrRunner jetty = getRandomJetty(sliceName, aggressivelyKillLeaders);
+    JettySolrRunner jetty = getRandomJetty(sliceName, aggressivelyKillLeaders).jetty;
     if (jetty != null) {
       expireSession(jetty);
       expires.incrementAndGet();
@@ -107,9 +107,10 @@ public class ChaosMonkey {
   }
   
   public void randomConnectionLoss() throws KeeperException, InterruptedException {
-    String sliceName = getRandomSlice();
+    monkeyLog("cause connection loss!");
     
-    JettySolrRunner jetty = getRandomJetty(sliceName, aggressivelyKillLeaders);
+    String sliceName = getRandomSlice();
+    JettySolrRunner jetty = getRandomJetty(sliceName, aggressivelyKillLeaders).jetty;
     if (jetty != null) {
       causeConnectionLoss(jetty);
       connloss.incrementAndGet();
@@ -129,24 +130,30 @@ public class ChaosMonkey {
     }
   }
 
-  public JettySolrRunner stopShard(String slice, int index) throws Exception {
-    JettySolrRunner jetty = shardToJetty.get(slice).get(index).jetty;
-    stopJetty(jetty);
-    return jetty;
+  public CloudJettyRunner stopShard(String slice, int index) throws Exception {
+    CloudJettyRunner cjetty = shardToJetty.get(slice).get(index);
+    stopJetty(cjetty);
+    return cjetty;
   }
 
-  public void stopJetty(JettySolrRunner jetty) throws Exception {
-    stop(jetty);
+  public void stopJetty(CloudJettyRunner cjetty) throws Exception {
+    stop(cjetty.jetty);
     stops.incrementAndGet();
   }
 
-  public void killJetty(JettySolrRunner jetty) throws Exception {
-    kill(jetty);
+  public void killJetty(CloudJettyRunner cjetty) throws Exception {
+    kill(cjetty);
     stops.incrementAndGet();
   }
   
-  public static void stop(JettySolrRunner jetty) throws Exception {
+  public void stopJetty(JettySolrRunner jetty) throws Exception {
+    stops.incrementAndGet();
+    stopJettySolrRunner(jetty);
+  }
+  
+  private static void stopJettySolrRunner(JettySolrRunner jetty) throws Exception {
     
+    monkeyLog("stop shard! " + jetty.getLocalPort());
     // get a clean shutdown so that no dirs are left open...
     FilterHolder fh = jetty.getDispatchFilter();
     if (fh != null) {
@@ -162,7 +169,9 @@ public class ChaosMonkey {
     }
   }
   
-  public static void kill(JettySolrRunner jetty) throws Exception {
+  public static void kill(CloudJettyRunner cjetty) throws Exception {
+    JettySolrRunner jetty = cjetty.jetty;
+    monkeyLog("kill shard! " + jetty.getLocalPort());
     FilterHolder fh = jetty.getDispatchFilter();
     SolrDispatchFilter sdf = null;
     if (fh != null) {
@@ -182,7 +191,7 @@ public class ChaosMonkey {
   public void stopShard(String slice) throws Exception {
     List<CloudJettyRunner> jetties = shardToJetty.get(slice);
     for (CloudJettyRunner jetty : jetties) {
-      stopJetty(jetty.jetty);
+      stopJetty(jetty);
     }
   }
   
@@ -190,7 +199,7 @@ public class ChaosMonkey {
     List<CloudJettyRunner> jetties = shardToJetty.get(slice);
     for (CloudJettyRunner jetty : jetties) {
       if (!jetty.nodeName.equals(shardName)) {
-        stopJetty(jetty.jetty);
+        stopJetty(jetty);
       }
     }
   }
@@ -200,22 +209,22 @@ public class ChaosMonkey {
     return jetty;
   }
   
-  public JettySolrRunner stopRandomShard() throws Exception {
+  public CloudJettyRunner stopRandomShard() throws Exception {
     String sliceName = getRandomSlice();
     
     return stopRandomShard(sliceName);
   }
   
-  public JettySolrRunner stopRandomShard(String slice) throws Exception {
-    JettySolrRunner jetty = getRandomJetty(slice, aggressivelyKillLeaders);
-    if (jetty != null) {
-      stopJetty(jetty);
+  public CloudJettyRunner stopRandomShard(String slice) throws Exception {
+    CloudJettyRunner cjetty = getRandomJetty(slice, aggressivelyKillLeaders);
+    if (cjetty != null) {
+      stopJetty(cjetty);
     }
-    return jetty;
+    return cjetty;
   }
   
   
-  public JettySolrRunner killRandomShard() throws Exception {
+  public CloudJettyRunner killRandomShard() throws Exception {
     // add all the shards to a list
     String sliceName = getRandomSlice();
     
@@ -227,19 +236,19 @@ public class ChaosMonkey {
     
     List<String> sliceKeyList = new ArrayList<String>(slices.size());
     sliceKeyList.addAll(slices.keySet());
-    String sliceName = sliceKeyList.get(random.nextInt(sliceKeyList.size()));
+    String sliceName = sliceKeyList.get(LuceneTestCase.random().nextInt(sliceKeyList.size()));
     return sliceName;
   }
   
-  public JettySolrRunner killRandomShard(String slice) throws Exception {
-    JettySolrRunner jetty = getRandomJetty(slice, aggressivelyKillLeaders);
-    if (jetty != null) {
-      killJetty(jetty);
+  public CloudJettyRunner killRandomShard(String slice) throws Exception {
+    CloudJettyRunner cjetty = getRandomJetty(slice, aggressivelyKillLeaders);
+    if (cjetty != null) {
+      killJetty(cjetty);
     }
-    return jetty;
+    return cjetty;
   }
   
-  public JettySolrRunner getRandomJetty(String slice, boolean aggressivelyKillLeaders) throws KeeperException, InterruptedException {
+  public CloudJettyRunner getRandomJetty(String slice, boolean aggressivelyKillLeaders) throws KeeperException, InterruptedException {
     
 
     int numRunning = 0;
@@ -289,36 +298,40 @@ public class ChaosMonkey {
     
     if (numActive < 2) {
       // we cannot kill anyone
+      monkeyLog("only one active node in shard - monkey cannot kill :(");
       return null;
     }
-    
+    Random random = LuceneTestCase.random();
     int chance = random.nextInt(10);
-    JettySolrRunner jetty;
+    CloudJettyRunner cjetty;
     if (chance <= 5 && aggressivelyKillLeaders) {
       // if killLeader, really aggressively go after leaders
-      jetty = shardToLeaderJetty.get(slice).jetty;
+      cjetty = shardToLeaderJetty.get(slice);
     } else {
       // get random shard
       List<CloudJettyRunner> jetties = shardToJetty.get(slice);
       int index = random.nextInt(jetties.size());
-      jetty = jetties.get(index).jetty;
+      cjetty = jetties.get(index);
       
       ZkNodeProps leader = zkStateReader.getLeaderProps(collection, slice);
       boolean isLeader = leader.get(ZkStateReader.NODE_NAME_PROP).equals(jetties.get(index).nodeName);
       if (!aggressivelyKillLeaders && isLeader) {
         // we don't kill leaders...
+        monkeyLog("abort! I don't kill leaders");
         return null;
       } 
     }
 
-    if (jetty.getLocalPort() == -1) {
+    if (cjetty.jetty.getLocalPort() == -1) {
       // we can't kill the dead
+      monkeyLog("abort! This guy is already dead");
       return null;
     }
     
     //System.out.println("num active:" + numActive + " for " + slice + " sac:" + jetty.getLocalPort());
-    
-    return jetty;
+    monkeyLog("chose a victim! " + cjetty.jetty.getLocalPort());
+  
+    return cjetty;
   }
   
   public SolrServer getRandomClient(String slice) throws KeeperException, InterruptedException {
@@ -327,7 +340,7 @@ public class ChaosMonkey {
 
     // get random shard
     List<SolrServer> clients = shardToClient.get(slice);
-    int index = random.nextInt(clients.size() - 1);
+    int index = LuceneTestCase.random().nextInt(clients.size() - 1);
     SolrServer client = clients.get(index);
 
     return client;
@@ -336,42 +349,27 @@ public class ChaosMonkey {
   // synchronously starts and stops shards randomly, unless there is only one
   // active shard up for a slice or if there is one active and others recovering
   public void startTheMonkey(boolean killLeaders, final int roundPause) {
+    monkeyLog("starting");
     this.aggressivelyKillLeaders = killLeaders;
     startTime = System.currentTimeMillis();
     // TODO: when kill leaders is on, lets kill a higher percentage of leaders
     
     stop = false;
     new Thread() {
-      private List<JettySolrRunner> deadPool = new ArrayList<JettySolrRunner>();
+      private List<CloudJettyRunner> deadPool = new ArrayList<CloudJettyRunner>();
 
       @Override
       public void run() {
         while (!stop) {
           try {
             Thread.sleep(roundPause);
- 
+            Random random = LuceneTestCase.random();
             if (random.nextBoolean()) {
              if (!deadPool.isEmpty()) {
                int index = random.nextInt(deadPool.size());
-               JettySolrRunner jetty = deadPool.get(index);
-               try {
-                 jetty.start();
-               } catch (BindException e) {
-                 jetty.stop();
-                 sleep(2000);
-                 try {
-                   jetty.start();
-                 } catch (BindException e2) {
-                   jetty.stop();
-                   sleep(5000);
-                   try {
-                     jetty.start();
-                   } catch (BindException e3) {
-                     // we coud not get the port
-                     jetty.stop();
-                     continue;
-                   }
-                 }
+               JettySolrRunner jetty = deadPool.get(index).jetty;
+               if (!ChaosMonkey.start(jetty)) {
+                 continue;
                }
                //System.out.println("started on port:" + jetty.getLocalPort());
                deadPool.remove(index);
@@ -391,16 +389,16 @@ public class ChaosMonkey {
               randomConnectionLoss();
             }
             
-            JettySolrRunner jetty;
+            CloudJettyRunner cjetty;
             if (random.nextBoolean()) {
-              jetty = stopRandomShard();
+              cjetty = stopRandomShard();
             } else {
-              jetty = killRandomShard();
+              cjetty = killRandomShard();
             }
-            if (jetty == null) {
+            if (cjetty == null) {
               // we cannot kill
             } else {
-              deadPool.add(jetty);
+              deadPool.add(cjetty);
             }
             
           } catch (InterruptedException e) {
@@ -410,14 +408,18 @@ public class ChaosMonkey {
             e.printStackTrace();
           }
         }
-        
-        System.out.println("I ran for " + (System.currentTimeMillis() - startTime)/1000.0f + "sec. I stopped " + stops + " and I started " + starts
+        monkeyLog("finished");
+        monkeyLog("I ran for " + (System.currentTimeMillis() - startTime)/1000.0f + "sec. I stopped " + stops + " and I started " + starts
             + ". I also expired " + expires.get() + " and caused " + connloss
             + " connection losses");
       }
     }.start();
   }
   
+  public static void monkeyLog(String msg) {
+    log.info("monkey: " + msg);
+  }
+  
   public void stopTheMonkey() {
     stop = true;
   }
@@ -426,4 +428,31 @@ public class ChaosMonkey {
     return starts.get();
   }
 
+  public static void stop(JettySolrRunner jetty) throws Exception {
+    stopJettySolrRunner(jetty);
+  }
+  
+  public static boolean start(JettySolrRunner jetty) throws Exception {
+    try {
+      jetty.start();
+    } catch (BindException e) {
+      jetty.stop();
+      Thread.sleep(2000);
+      try {
+        jetty.start();
+      } catch (BindException e2) {
+        jetty.stop();
+        Thread.sleep(5000);
+        try {
+          jetty.start();
+        } catch (BindException e3) {
+          // we coud not get the port
+          jetty.stop();
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
 }
\ No newline at end of file

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Sat Jul 28 11:27:51 2012
@@ -20,9 +20,9 @@ package org.apache.solr.cloud;
 import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.http.client.HttpClient;
+import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@@ -34,9 +34,15 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+@Slow
 @Ignore("ignore while investigating jenkins fails")
 public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest {
+  public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
+  
+  private static final int BASE_RUN_LENGTH = 180000;
 
   @BeforeClass
   public static void beforeSuperClass() {
@@ -51,7 +57,7 @@ public class ChaosMonkeyNothingIsSafeTes
   public void setUp() throws Exception {
     super.setUp();
     // TODO use @Noisy annotation as we expect lots of exceptions
-    ignoreException(".*");
+    //ignoreException(".*");
     System.setProperty("numShards", Integer.toString(sliceCount));
   }
   
@@ -65,8 +71,8 @@ public class ChaosMonkeyNothingIsSafeTes
   
   public ChaosMonkeyNothingIsSafeTest() {
     super();
-    shardCount = atLeast(3);
     sliceCount = 2;
+    shardCount = 6;
   }
   
   @Override
@@ -81,7 +87,7 @@ public class ChaosMonkeyNothingIsSafeTes
       // as it's not supported for recovery
       // del("*:*");
       
-      List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
+      List<StopableThread> threads = new ArrayList<StopableThread>();
       int threadCount = 1;
       int i = 0;
       for (i = 0; i < threadCount; i++) {
@@ -91,24 +97,33 @@ public class ChaosMonkeyNothingIsSafeTes
         indexThread.start();
       }
       
+      threadCount = 1;
+      i = 0;
+      for (i = 0; i < threadCount; i++) {
+        StopableSearchThread searchThread = new StopableSearchThread();
+        threads.add(searchThread);
+        searchThread.start();
+      }
+      
       FullThrottleStopableIndexingThread ftIndexThread = new FullThrottleStopableIndexingThread(
           clients, i * 50000, true);
       threads.add(ftIndexThread);
       ftIndexThread.start();
       
       chaosMonkey.startTheMonkey(true, 1500);
+      int runLength = atLeast(BASE_RUN_LENGTH);
       try {
-        Thread.sleep(atLeast(6000));
+        Thread.sleep(runLength);
       } finally {
         chaosMonkey.stopTheMonkey();
       }
       
-      for (StopableIndexingThread indexThread : threads) {
+      for (StopableThread indexThread : threads) {
         indexThread.safeStop();
       }
       
       // wait for stop...
-      for (StopableIndexingThread indexThread : threads) {
+      for (StopableThread indexThread : threads) {
         indexThread.join();
       }
       
@@ -122,7 +137,7 @@ public class ChaosMonkeyNothingIsSafeTes
       Thread.sleep(2000);
       
       // wait until there are no recoveries...
-      waitForThingsToLevelOut();
+      waitForThingsToLevelOut(Math.round((runLength / 1000.0f / 5.0f)));
       
       // make sure we again have leaders for each shard
       for (int j = 1; j < sliceCount; j++) {
@@ -154,35 +169,6 @@ public class ChaosMonkeyNothingIsSafeTes
       }
     }
   }
-
-  private void waitForThingsToLevelOut() throws Exception {
-    int cnt = 0;
-    boolean retry = false;
-    do {
-      waitForRecoveriesToFinish(VERBOSE);
-      
-      try {
-        commit();
-      } catch (Exception e) {
-        // we don't care if this commit fails on some nodes
-      }
-      
-      updateMappingsFromZk(jettys, clients);
-      
-      Set<String> theShards = shardToClient.keySet();
-      String failMessage = null;
-      for (String shard : theShards) {
-        failMessage = checkShardConsistency(shard, false);
-      }
-      
-      if (failMessage != null) {
-        retry  = true;
-      }
-      cnt++;
-      if (cnt > 10) break;
-      Thread.sleep(4000);
-    } while (retry);
-  }
   
   // skip the randoms - they can deadlock...
   protected void indexr(Object... fields) throws Exception {

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Sat Jul 28 11:27:51 2012
@@ -19,7 +19,6 @@ package org.apache.solr.cloud;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.common.SolrInputDocument;
@@ -32,6 +31,8 @@ import org.junit.Ignore;
 @Ignore("SOLR-3126")
 public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
   
+  private static final int BASE_RUN_LENGTH = 120000;
+
   @BeforeClass
   public static void beforeSuperClass() {
     
@@ -66,7 +67,7 @@ public class ChaosMonkeySafeLeaderTest e
   public ChaosMonkeySafeLeaderTest() {
     super();
     sliceCount = atLeast(2);
-    shardCount = atLeast(sliceCount);
+    shardCount = atLeast(sliceCount*2);
   }
   
   @Override
@@ -89,8 +90,8 @@ public class ChaosMonkeySafeLeaderTest e
     }
     
     chaosMonkey.startTheMonkey(false, 500);
-    
-    Thread.sleep(atLeast(8000));
+    int runLength = atLeast(BASE_RUN_LENGTH);
+    Thread.sleep(runLength);
     
     chaosMonkey.stopTheMonkey();
     
@@ -109,40 +110,12 @@ public class ChaosMonkeySafeLeaderTest e
     
     // try and wait for any replications and what not to finish...
     
-    waitForThingsToLevelOut();
+    waitForThingsToLevelOut(Math.round((runLength / 1000.0f / 5.0f)));
 
     checkShardConsistency(true, true);
     
     if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
   }
-
-  private void waitForThingsToLevelOut() throws Exception {
-    int cnt = 0;
-    boolean retry = false;
-    do {
-      waitForRecoveriesToFinish(false);
-      
-      commit();
-      
-      updateMappingsFromZk(jettys, clients);
-      
-      Set<String> theShards = shardToClient.keySet();
-      String failMessage = null;
-      for (String shard : theShards) {
-        failMessage = checkShardConsistency(shard, false);
-      }
-      
-      if (failMessage != null) {
-        retry = true;
-      } else {
-        retry = false;
-      }
-      
-      cnt++;
-      if (cnt > 10) break;
-      Thread.sleep(2000);
-    } while (retry);
-  }
   
   // skip the randoms - they can deadlock...
   protected void indexr(Object... fields) throws Exception {

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java Sat Jul 28 11:27:51 2012
@@ -75,8 +75,6 @@ public class CloudStateUpdateTest extend
   @AfterClass
   public static void afterClass() throws InterruptedException {
     System.clearProperty("solrcloud.skip.autorecovery");
-    // wait just a bit for any zk client threads to outlast timeout
-    Thread.sleep(2000);
   }
 
   @Override

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Sat Jul 28 11:27:51 2012
@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -55,6 +56,8 @@ import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * 
@@ -64,6 +67,8 @@ import org.junit.BeforeClass;
  */
 @Slow
 public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
+  static Logger log = LoggerFactory.getLogger(FullSolrCloudTest.class);
+  
   @BeforeClass
   public static void beforeFullSolrCloudTest() {
     // shorten the log output more for this test type
@@ -94,38 +99,58 @@ public class FullSolrCloudTest extends A
   
   protected volatile CloudSolrServer cloudClient;
   
-  protected Map<JettySolrRunner,ZkNodeProps> jettyToInfo = new HashMap<JettySolrRunner,ZkNodeProps>();
-  protected Map<CloudSolrServerClient,ZkNodeProps> clientToInfo = new HashMap<CloudSolrServerClient,ZkNodeProps>();
-  protected Map<String,List<SolrServer>> shardToClient = new HashMap<String,List<SolrServer>>();
+  protected List<CloudJettyRunner> cloudJettys = new ArrayList<CloudJettyRunner>();
   protected Map<String,List<CloudJettyRunner>> shardToJetty = new HashMap<String,List<CloudJettyRunner>>();
   private AtomicInteger jettyIntCntr = new AtomicInteger(0);
   protected ChaosMonkey chaosMonkey;
   protected volatile ZkStateReader zkStateReader;
   
-  private Map<String,SolrServer> shardToLeaderClient = new HashMap<String,SolrServer>();
-  private Map<String,CloudJettyRunner> shardToLeaderJetty = new HashMap<String,CloudJettyRunner>();
+  protected Map<String,CloudJettyRunner> shardToLeaderJetty = new HashMap<String,CloudJettyRunner>();
   
-  class CloudJettyRunner {
+  static class CloudJettyRunner {
     JettySolrRunner jetty;
     String nodeName;
     String coreNodeName;
+    String url;
+    CloudSolrServerClient client;
+    public ZkNodeProps info;
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((url == null) ? 0 : url.hashCode());
+      return result;
+    }
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null) return false;
+      if (getClass() != obj.getClass()) return false;
+      CloudJettyRunner other = (CloudJettyRunner) obj;
+      if (url == null) {
+        if (other.url != null) return false;
+      } else if (!url.equals(other.url)) return false;
+      return true;
+    }
   }
   
   static class CloudSolrServerClient {
-    SolrServer client;
+    SolrServer solrClient;
     String shardName;
+    int port;
+    public ZkNodeProps info;
     
     public CloudSolrServerClient() {}
     
     public CloudSolrServerClient(SolrServer client) {
-      this.client = client;
+      this.solrClient = client;
     }
     
     @Override
     public int hashCode() {
       final int prime = 31;
       int result = 1;
-      result = prime * result + ((client == null) ? 0 : client.hashCode());
+      result = prime * result + ((solrClient == null) ? 0 : solrClient.hashCode());
       return result;
     }
     
@@ -135,9 +160,9 @@ public class FullSolrCloudTest extends A
       if (obj == null) return false;
       if (getClass() != obj.getClass()) return false;
       CloudSolrServerClient other = (CloudSolrServerClient) obj;
-      if (client == null) {
-        if (other.client != null) return false;
-      } else if (!client.equals(other.client)) return false;
+      if (solrClient == null) {
+        if (other.solrClient != null) return false;
+      } else if (!solrClient.equals(other.solrClient)) return false;
       return true;
     }
     
@@ -184,8 +209,8 @@ public class FullSolrCloudTest extends A
       }
       
       chaosMonkey = new ChaosMonkey(zkServer, zkStateReader,
-          DEFAULT_COLLECTION, shardToJetty, shardToClient, shardToLeaderClient,
-          shardToLeaderJetty, random());
+          DEFAULT_COLLECTION, shardToJetty,
+          shardToLeaderJetty);
     }
     
     // wait until shards have started registering...
@@ -330,9 +355,8 @@ public class FullSolrCloudTest extends A
   protected void updateMappingsFromZk(List<JettySolrRunner> jettys,
       List<SolrServer> clients) throws Exception {
     zkStateReader.updateCloudState(true);
-    shardToClient.clear();
+    cloudJettys.clear();
     shardToJetty.clear();
-    jettyToInfo.clear();
     
     CloudState cloudState = zkStateReader.getCloudState();
     Map<String,Slice> slices = cloudState.getSlices(DEFAULT_COLLECTION);
@@ -342,6 +366,7 @@ public class FullSolrCloudTest extends A
           + DEFAULT_COLLECTION + " in " + cloudState.getCollections());
     }
     
+    List<CloudSolrServerClient> theClients = new ArrayList<CloudSolrServerClient>();
     for (SolrServer client : clients) {
       // find info for this client in zk 
       nextClient:
@@ -354,36 +379,23 @@ public class FullSolrCloudTest extends A
           
           if (shard.getKey().contains(":" + port + "_")) {
             CloudSolrServerClient csc = new CloudSolrServerClient();
-            csc.client = client;
+            csc.solrClient = client;
+            csc.port = port;
             csc.shardName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
-            boolean isLeader = shard.getValue().containsKey(
-                ZkStateReader.LEADER_PROP);
-            clientToInfo.put(csc, shard.getValue());
-            List<SolrServer> list = shardToClient.get(slice.getKey());
-            if (list == null) {
-              list = new ArrayList<SolrServer>();
-              shardToClient.put(slice.getKey(), list);
-            }
-            list.add(client);
+            csc.info = shard.getValue();
+            
+            theClients .add(csc);
             
-            if (isLeader) {
-              shardToLeaderClient.put(slice.getKey(), client);
-            }
             break nextClient;
           }
         }
       }
     }
-    
-    for (Map.Entry<String,Slice> slice : slices.entrySet()) {
-      // check that things look right
-      assertEquals(slice.getValue().getShards().size(), shardToClient.get(slice.getKey()).size());
-    }
  
     for (JettySolrRunner jetty : jettys) {
       int port = jetty.getLocalPort();
       if (port == -1) {
-        continue; // If we cannot get the port, this jetty is down
+        throw new RuntimeException("Cannot find the port for jetty");
       }
       
       nextJetty:
@@ -391,7 +403,6 @@ public class FullSolrCloudTest extends A
         Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
         for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
           if (shard.getKey().contains(":" + port + "_")) {
-            jettyToInfo.put(jetty, shard.getValue());
             List<CloudJettyRunner> list = shardToJetty.get(slice.getKey());
             if (list == null) {
               list = new ArrayList<CloudJettyRunner>();
@@ -401,12 +412,16 @@ public class FullSolrCloudTest extends A
                 ZkStateReader.LEADER_PROP);
             CloudJettyRunner cjr = new CloudJettyRunner();
             cjr.jetty = jetty;
+            cjr.info = shard.getValue();
             cjr.nodeName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
             cjr.coreNodeName = shard.getKey();
+            cjr.url = shard.getValue().get(ZkStateReader.BASE_URL_PROP) + "/" + shard.getValue().get(ZkStateReader.CORE_NAME_PROP);
+            cjr.client = findClientByPort(port, theClients);
             list.add(cjr);
             if (isLeader) {
               shardToLeaderJetty.put(slice.getKey(), cjr);
             }
+            cloudJettys.add(cjr);
             break nextJetty;
           }
         }
@@ -421,10 +436,19 @@ public class FullSolrCloudTest extends A
       List<CloudJettyRunner> jetties = shardToJetty.get(slice.getKey());
       assertNotNull("Test setup problem: We found no jetties for shard: " + slice.getKey()
           + " just:" + shardToJetty.keySet(), jetties);
-      assertTrue(jetties.size() > 0);
+      assertEquals(slice.getValue().getShards().size(), jetties.size());
     }
   }
   
+  private CloudSolrServerClient findClientByPort(int port, List<CloudSolrServerClient> theClients) {
+    for (CloudSolrServerClient client : theClients) {
+      if (client.port == port) {
+        return client;
+      }
+    }
+    throw new IllegalArgumentException("Client with the give port does not exist:" + port);
+  }
+
   @Override
   protected void setDistributedParams(ModifiableSolrParams params) {
     
@@ -503,11 +527,15 @@ public class FullSolrCloudTest extends A
   
   protected void del(String q) throws Exception {
     controlClient.deleteByQuery(q);
+    cloudClient.deleteByQuery(q);
+
+    /***
     for (SolrServer client : clients) {
       UpdateRequest ureq = new UpdateRequest();
       // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
       ureq.deleteByQuery(q).process(client);
     }
+     ***/
   }// serial commit...
   
   /*
@@ -538,6 +566,9 @@ public class FullSolrCloudTest extends A
       
       indexAbunchOfDocs();
       
+      // check again 
+      waitForRecoveriesToFinish(false);
+      
       commit();
       
       assertDocCounts(VERBOSE);
@@ -638,9 +669,9 @@ public class FullSolrCloudTest extends A
     
     // new server should be part of first shard
     // how many docs are on the new shard?
-    for (SolrServer client : shardToClient.get("shard1")) {
+    for (CloudJettyRunner cjetty : shardToJetty.get("shard1")) {
       if (VERBOSE) System.err.println("total:"
-          + client.query(new SolrQuery("*:*")).getResults().getNumFound());
+          + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
     }
     
     checkShardConsistency("shard1");
@@ -653,31 +684,43 @@ public class FullSolrCloudTest extends A
     super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose);
   }
   
+  protected void waitForRecoveriesToFinish(boolean verbose, int timeoutSeconds)
+      throws Exception {
+    super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose, true, timeoutSeconds);
+  }
+  
   private void brindDownShardIndexSomeDocsAndRecover() throws Exception {
     SolrQuery query = new SolrQuery("*:*");
     query.set("distrib", false);
     
     commit();
     
-    long deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
-    System.err.println("dsc:" + deadShardCount);
-    
+    long deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+        .query(query).getResults().getNumFound();
+
     query("q", "*:*", "sort", "n_tl1 desc");
     
     // kill a shard
-    JettySolrRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
+    CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
     cloudClient.connect();
-    int tries = 0;
-    while (cloudClient.getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
-      if (tries++ == 60) {
-        fail("Shard still reported as live in zk");
-      }
-      Thread.sleep(1000);
+
+    // we are careful to make sure the downed node is no longer in the state,
+    // because on some systems (especially freebsd w/ blackhole enabled), trying
+    // to talk to a downed node causes grief
+    Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
+    jetties.addAll(shardToJetty.get(SHARD2));
+    jetties.remove(deadShard);
+    
+    for (CloudJettyRunner cjetty : jetties) {
+      waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
+          .getFilter()).getCores().getZkController().getZkStateReader(),
+          deadShard);
     }
-	
+    waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
+
     // ensure shard is dead
     try {
-      index_specific(shardToClient.get(SHARD2).get(0), id, 999, i1, 107, t1,
+      index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1,
           "specific doc!");
       fail("This server should be down and this update should have failed");
     } catch (SolrServerException e) {
@@ -692,21 +735,11 @@ public class FullSolrCloudTest extends A
     // System.out.println("clouddocs:" + cloudClientDocs);
     
     // try to index to a living shard at shard2
-    
-    // we are careful to make sure the downed node is no longer in the state,
-    // because on some systems (especially freebsd w/ blackhole enabled), trying
-    // to talk to a downed node causes grief
-    tries = 0;
-    while (((SolrDispatchFilter) shardToJetty.get(SHARD2).get(1).jetty.getDispatchFilter().getFilter()).getCores().getZkController().getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
-      if (tries++ == 60) {
-        fail("Shard still reported as live in zk");
-      }
-      Thread.sleep(1000);
-    }
+
 	
     long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
     
-    index_specific(shardToClient.get(SHARD2).get(1), id, 1000, i1, 108, t1,
+    index_specific(shardToJetty.get(SHARD2).get(1).client.solrClient, id, 1000, i1, 108, t1,
         "specific doc!");
     
     commit();
@@ -764,21 +797,29 @@ public class FullSolrCloudTest extends A
     // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
     
     // this should trigger a recovery phase on deadShard
-    deadShard.start(true);
+    ChaosMonkey.start(deadShard.jetty);
     
     // make sure we have published we are recovering
     Thread.sleep(1500);
     
     waitForRecoveriesToFinish(false);
     
-    deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
+    deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+        .query(query).getResults().getNumFound();
     // if we properly recovered, we should now have the couple missing docs that
     // came in while shard was down
     checkShardConsistency(true, false);
     
     
     // recover over 100 docs so we do more than just peer sync (replicate recovery)
-    deadShard = chaosMonkey.stopShard(SHARD2, 0);
+    chaosMonkey.stopJetty(deadShard);
+    
+    for (CloudJettyRunner cjetty : jetties) {
+      waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
+          .getFilter()).getCores().getZkController().getZkStateReader(),
+          deadShard);
+    }
+    waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
     
     for (int i = 0; i < 226; i++) {
       doc = new SolrInputDocument();
@@ -791,7 +832,9 @@ public class FullSolrCloudTest extends A
     }
     commit();
     
-    deadShard.start(true);
+    Thread.sleep(1500);
+    
+    ChaosMonkey.start(deadShard.jetty);
     
     // make sure we have published we are recovering
     Thread.sleep(1500);
@@ -996,10 +1039,10 @@ public class FullSolrCloudTest extends A
   protected String checkShardConsistency(String shard, boolean verbose)
       throws Exception {
     
-    List<SolrServer> solrClients = shardToClient.get(shard);
-    if (solrClients == null) {
+    List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
+    if (solrJetties == null) {
       throw new RuntimeException("shard not found:" + shard + " keys:"
-          + shardToClient.keySet());
+          + shardToJetty.keySet());
     }
     long num = -1;
     long lastNum = -1;
@@ -1011,22 +1054,26 @@ public class FullSolrCloudTest extends A
         "The client count does not match up with the shard count for slice:"
             + shard,
         zkStateReader.getCloudState().getSlice(DEFAULT_COLLECTION, shard)
-            .getShards().size(), solrClients.size());
+            .getShards().size(), solrJetties.size());
 
     SolrServer lastClient = null;
-    for (SolrServer client : solrClients) {
-      ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(client));
+    for (CloudJettyRunner cjetty : solrJetties) {
+      ZkNodeProps props = cjetty.info;
       if (verbose) System.err.println("client" + cnt++);
       if (verbose) System.err.println("PROPS:" + props);
       
       try {
         SolrQuery query = new SolrQuery("*:*");
         query.set("distrib", false);
-        num = client.query(query).getResults().getNumFound();
+        num = cjetty.client.solrClient.query(query).getResults().getNumFound();
       } catch (SolrServerException e) {
         if (verbose) System.err.println("error contacting client: "
             + e.getMessage() + "\n");
         continue;
+      } catch (SolrException e) {
+        if (verbose) System.err.println("error contacting client: "
+            + e.getMessage() + "\n");
+        continue;
       }
       
       boolean live = false;
@@ -1043,7 +1090,7 @@ public class FullSolrCloudTest extends A
       if (active && live) {
         if (lastNum > -1 && lastNum != num && failMessage == null) {
           failMessage = shard + " is not consistent.  Got " + lastNum + " from " + lastClient + "lastClient"
-              + " and got " + num + " from " + client;
+              + " and got " + num + " from " + cjetty.url;
 
           if (verbose || true) {
             System.err.println("######" + failMessage);
@@ -1054,14 +1101,14 @@ public class FullSolrCloudTest extends A
             query.set("sort","id asc");
 
             SolrDocumentList lst1 = lastClient.query(query).getResults();
-            SolrDocumentList lst2 = client.query(query).getResults();
+            SolrDocumentList lst2 = cjetty.client.solrClient.query(query).getResults();
 
-            showDiff(lst1, lst2, lastClient.toString(), client.toString());
+            showDiff(lst1, lst2, lastClient.toString(), cjetty.client.solrClient.toString());
           }
 
         }
         lastNum = num;
-        lastClient = client;
+        lastClient = cjetty.client.solrClient;
       }
     }
     return failMessage;
@@ -1108,7 +1155,7 @@ public class FullSolrCloudTest extends A
     
     updateMappingsFromZk(jettys, clients);
     
-    Set<String> theShards = shardToClient.keySet();
+    Set<String> theShards = shardToJetty.keySet();
     String failMessage = null;
     for (String shard : theShards) {
       String shardFailMessage = checkShardConsistency(shard, verbose);
@@ -1123,15 +1170,15 @@ public class FullSolrCloudTest extends A
     
     if (checkVsControl) {
       // now check that the right # are on each shard
-      theShards = shardToClient.keySet();
+      theShards = shardToJetty.keySet();
       int cnt = 0;
       for (String s : theShards) {
-        int times = shardToClient.get(s).size();
+        int times = shardToJetty.get(s).size();
         for (int i = 0; i < times; i++) {
           try {
-            SolrServer client = shardToClient.get(s).get(i);
-            ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(
-                client));
+            CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
+            ZkNodeProps props = cjetty.info;
+            SolrServer client = cjetty.client.solrClient;
             boolean active = props.get(ZkStateReader.STATE_PROP).equals(
                 ZkStateReader.ACTIVE);
             if (active) {
@@ -1163,9 +1210,10 @@ public class FullSolrCloudTest extends A
   }
   
   private SolrServer getClient(String nodeName) {
-    for (CloudSolrServerClient client : clientToInfo.keySet()) {
+    for (CloudJettyRunner cjetty : cloudJettys) {
+      CloudSolrServerClient client = cjetty.client;
       if (client.shardName.equals(nodeName)) {
-        return client.client;
+        return client.solrClient;
       }
     }
     return null;
@@ -1199,12 +1247,13 @@ public class FullSolrCloudTest extends A
           + DEFAULT_COLLECTION + " in " + cloudState.getCollections());
     }
     
-    for (SolrServer client : clients) {
+    for (CloudJettyRunner cjetty : cloudJettys) {
+      CloudSolrServerClient client = cjetty.client;
       for (Map.Entry<String,Slice> slice : slices.entrySet()) {
         Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
         for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
           String shardName = new URI(
-              ((HttpSolrServer) client).getBaseURL()).getPort()
+              ((HttpSolrServer) client.solrClient).getBaseURL()).getPort()
               + "_solr_";
           if (verbose && shard.getKey().endsWith(shardName)) {
             System.err.println("shard:" + slice.getKey());
@@ -1214,12 +1263,14 @@ public class FullSolrCloudTest extends A
       }
       
       long count = 0;
-      String currentState = clientToInfo.get(new CloudSolrServerClient(client))
-          .get(ZkStateReader.STATE_PROP);
-      if (currentState != null && currentState.equals(ZkStateReader.ACTIVE)) {
+      String currentState = cjetty.info.get(ZkStateReader.STATE_PROP);
+      if (currentState != null
+          && currentState.equals(ZkStateReader.ACTIVE)
+          && zkStateReader.getCloudState().liveNodesContain(
+              cjetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
         SolrQuery query = new SolrQuery("*:*");
         query.set("distrib", false);
-        count = client.query(query).getResults().getNumFound();
+        count = client.solrClient.query(query).getResults().getNumFound();
       }
       
       if (verbose) System.err.println("client docs:" + count + "\n\n");
@@ -1242,7 +1293,14 @@ public class FullSolrCloudTest extends A
     return rsp;
   }
   
-  class StopableIndexingThread extends Thread {
+  abstract class StopableThread extends Thread {
+    public StopableThread(String name) {
+      super(name);
+    }
+    public abstract void safeStop();
+  }
+  
+  class StopableIndexingThread extends StopableThread {
     private volatile boolean stop = false;
     protected final int startI;
     protected final List<Integer> deletes = new ArrayList<Integer>();
@@ -1274,6 +1332,10 @@ public class FullSolrCloudTest extends A
           } catch (Exception e) {
             System.err.println("REQUEST FAILED:");
             e.printStackTrace();
+            if (e instanceof SolrServerException) {
+              System.err.println("ROOT CAUSE:");
+              ((SolrServerException) e).getRootCause().printStackTrace();
+            }
             fails.incrementAndGet();
           }
         }
@@ -1285,6 +1347,10 @@ public class FullSolrCloudTest extends A
         } catch (Exception e) {
           System.err.println("REQUEST FAILED:");
           e.printStackTrace();
+          if (e instanceof SolrServerException) {
+            System.err.println("ROOT CAUSE:");
+            ((SolrServerException) e).getRootCause().printStackTrace();
+          }
           fails.incrementAndGet();
         }
         
@@ -1308,6 +1374,85 @@ public class FullSolrCloudTest extends A
     
   };
   
+  class StopableSearchThread extends StopableThread {
+    private volatile boolean stop = false;
+    protected final AtomicInteger fails = new AtomicInteger();
+    private String[] QUERIES = new String[] {"to come","their country","aid","co*"};
+    
+    public StopableSearchThread() {
+      super("StopableSearchThread");
+      setDaemon(true);
+    }
+    
+    @Override
+    public void run() {
+      Random random = random();
+      int numSearches = 0;
+      
+      while (true && !stop) {
+        numSearches++;
+        try {
+          //to come to the aid of their country.
+          cloudClient.query(new SolrQuery(QUERIES[random.nextInt(QUERIES.length)]));
+        } catch (Exception e) {
+          System.err.println("QUERY REQUEST FAILED:");
+          e.printStackTrace();
+          if (e instanceof SolrServerException) {
+            System.err.println("ROOT CAUSE:");
+            ((SolrServerException) e).getRootCause().printStackTrace();
+          }
+          fails.incrementAndGet();
+        }
+        try {
+          Thread.sleep(random.nextInt(4000) + 300);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+        }
+      }
+      
+      System.err.println("num searches done:" + numSearches + " with " + fails + " fails");
+    }
+    
+    public void safeStop() {
+      stop = true;
+    }
+    
+    public int getFails() {
+      return fails.get();
+    }
+    
+  };
+  
+  protected void waitForThingsToLevelOut(int waitForRecTimeSeconds) throws Exception {
+    log.info("Wait for recoveries to finish - wait " + waitForRecTimeSeconds + " for each attempt");
+    int cnt = 0;
+    boolean retry = false;
+    do {
+      waitForRecoveriesToFinish(VERBOSE, waitForRecTimeSeconds);
+      
+      try {
+        commit();
+      } catch (Exception e) {
+        // we don't care if this commit fails on some nodes
+      }
+      
+      updateMappingsFromZk(jettys, clients);
+      
+      Set<String> theShards = shardToJetty.keySet();
+      String failMessage = null;
+      for (String shard : theShards) {
+        failMessage = checkShardConsistency(shard, false);
+      }
+      
+      if (failMessage != null) {
+        retry  = true;
+      }
+      cnt++;
+      if (cnt > 2) break;
+      Thread.sleep(4000);
+    } while (retry);
+  }
+  
   @Override
   @After
   public void tearDown() throws Exception {
@@ -1352,7 +1497,7 @@ public class FullSolrCloudTest extends A
           + DEFAULT_COLLECTION;
       HttpSolrServer s = new HttpSolrServer(url);
       s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
-      s.setSoTimeout(15000);
+      s.setSoTimeout(20000);
       s.setDefaultMaxConnectionsPerHost(100);
       s.setMaxTotalConnections(100);
       return s;
@@ -1360,4 +1505,16 @@ public class FullSolrCloudTest extends A
       throw new RuntimeException(ex);
     }
   }
+  
+  protected void waitToSeeNotLive(ZkStateReader zkStateReader,
+      CloudJettyRunner cjetty) throws InterruptedException {
+    int tries = 0;
+    while (zkStateReader.getCloudState()
+        .liveNodesContain(cjetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
+      if (tries++ == 120) {
+        fail("Shard still reported as live in zk");
+      }
+      Thread.sleep(1000);
+    }
+  }
 }

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Sat Jul 28 11:27:51 2012
@@ -191,7 +191,7 @@ public class LeaderElectionIntegrationTe
       int newLeaderPort = getLeaderPort(leader);
       int retry = 0;
       while (leaderPort == newLeaderPort) {
-        if (retry++ == 20) {
+        if (retry++ == 60) {
           break;
         }
         Thread.sleep(1000);

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Sat Jul 28 11:27:51 2012
@@ -26,6 +26,7 @@ import org.apache.solr.client.solrj.embe
 import org.apache.solr.common.SolrInputDocument;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -71,7 +72,7 @@ public class RecoveryZkTest extends Full
     Thread.sleep(atLeast(2000));   
     
     // bring shard replica down
-    JettySolrRunner replica = chaosMonkey.stopShard("shard1", 1);
+    JettySolrRunner replica = chaosMonkey.stopShard("shard1", 1).jetty;
 
     
     // wait a moment - lets allow some docs to be indexed so replication time is non 0
@@ -99,8 +100,8 @@ public class RecoveryZkTest extends Full
     checkShardConsistency("shard1", false); 
     SolrQuery query = new SolrQuery("*:*");
     query.setParam("distrib", "false");
-    long client1Docs = shardToClient.get("shard1").get(0).query(query).getResults().getNumFound();
-    long client2Docs = shardToClient.get("shard1").get(1).query(query).getResults().getNumFound();
+    long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();
+    long client2Docs = shardToJetty.get("shard1").get(1).client.solrClient.query(query).getResults().getNumFound();
     
     assertTrue(client1Docs > 0);
     assertEquals(client1Docs, client2Docs);

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java Sat Jul 28 11:27:51 2012
@@ -48,7 +48,8 @@ public class TestMultiCoreConfBootstrap 
   
   @AfterClass
   public static void afterClass() {
-
+    zkServer = null;
+    zkDir = null;
   }
   
   @Override

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java Sat Jul 28 11:27:51 2012
@@ -20,8 +20,8 @@ package org.apache.solr.core;
 import junit.framework.Assert;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.analysis.KeywordTokenizerFactory;
-import org.apache.solr.analysis.NGramFilterFactory;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.ngram.NGramFilterFactory;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.handler.admin.LukeRequestHandler;
 import org.apache.solr.handler.component.FacetComponent;

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,7 @@ import org.apache.solr.handler.component
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.junit.Test;
 
@@ -49,6 +50,8 @@ public class SolrCoreTest extends SolrTe
   public void testRemoveThenAddDefaultCore() throws Exception {
     final CoreContainer cores = h.getCoreContainer();
     SolrCore core = cores.getCore("");
+
+    IndexSchema schema = h.getCore().getSchema();
     assertEquals(COLLECTION1, cores.getDefaultCoreName());
     
     cores.remove("");
@@ -57,7 +60,7 @@ public class SolrCoreTest extends SolrTe
     
     
     SolrCore newCore = new SolrCore(COLLECTION1, dataDir + File.separator
-        + "datadir2", new SolrConfig("solr/collection1", "solrconfig.xml", null), h.getCore().getSchema(),
+        + "datadir2", new SolrConfig("solr/collection1", "solrconfig.xml", null), schema,
         new CoreDescriptor(cores, COLLECTION1, "solr/collection1"));
     
     cores.register(newCore, false);

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestBadConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestBadConfig.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestBadConfig.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestBadConfig.java Sat Jul 28 11:27:51 2012
@@ -17,31 +17,36 @@
 
 package org.apache.solr.core;
 
-import org.apache.solr.util.AbstractSolrTestCase;
+import javax.script.ScriptEngineManager;
 
-public class TestBadConfig extends AbstractSolrTestCase {
+import org.junit.Assume;
 
-  @Override
-  public String getSchemaFile() { return "schema.xml"; }
-  @Override
-  public String getSolrConfigFile() { return "bad_solrconfig.xml"; }
-
-  @Override
-  public void setUp() throws Exception {
-    ignoreException("unset.sys.property");
-    try {
-      super.setUp();
-      fail("Exception should have been thrown");
-    } catch (Exception e) {
-      assertTrue(e.getMessage().contains("unset.sys.property"));
-    } finally {
-      resetExceptionIgnores();
-    }
+public class TestBadConfig extends AbstractBadConfigTestBase {
+
+  public void testUnsetSysProperty() throws Exception {
+    assertConfigs("bad_solrconfig.xml","schema.xml","unset.sys.property");
+  }
+
+  public void testBogusScriptEngine() throws Exception {
+    // sanity check
+    Assume.assumeTrue(null == (new ScriptEngineManager()).getEngineByName("giberish"));
+                      
+    assertConfigs("bad-solrconfig-bogus-scriptengine-name.xml",
+                  "schema.xml","giberish");
   }
-    
 
-  public void testNothing() {
-    // Empty test case as the real test is that the initialization of the TestHarness fails
-    assertTrue(true);
+  public void testMissingScriptFile() throws Exception {
+    // sanity check
+    Assume.assumeNotNull((new ScriptEngineManager()).getEngineByExtension("js"));
+    assertConfigs("bad-solrconfig-missing-scriptfile.xml",
+                  "schema.xml","a-file-name-that-does-not-exist.js");
   }
-}
\ No newline at end of file
+
+  public void testInvalidScriptFile() throws Exception {
+    // sanity check
+    Assume.assumeNotNull((new ScriptEngineManager()).getEngineByName("javascript"));
+    assertConfigs("bad-solrconfig-invalid-scriptfile.xml",
+                  "schema.xml","currency.xml");
+  }
+
+}

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java Sat Jul 28 11:27:51 2012
@@ -29,7 +29,7 @@ public class TestCodecSupport extends So
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    initCore("solrconfig-basic.xml", "schema_codec.xml");
+    initCore("solrconfig_codec.xml", "schema_codec.xml");
   }
 
   public void testPostingsFormats() {

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestConfig.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestConfig.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestConfig.java Sat Jul 28 11:27:51 2012
@@ -23,6 +23,7 @@ import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.handler.admin.ShowFileRequestHandler;
 import org.apache.solr.update.DirectUpdateHandler2;
 import org.apache.solr.update.SolrIndexConfig;
+import org.apache.solr.util.RefCounted;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.w3c.dom.Node;
@@ -115,8 +116,15 @@ public class TestConfig extends SolrTest
 
   @Test
   public void testTermIndexInterval() throws Exception {
-    IndexWriter writer = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
-    int interval = writer.getConfig().getTermIndexInterval();
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2) h.getCore()
+        .getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
+    int interval = 0;
+    try {
+      IndexWriter writer = iw.get();
+      interval = writer.getConfig().getTermIndexInterval();
+    } finally {
+      iw.decref();
+    }
     assertEquals(256, interval);
   }
 

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInject.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInject.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInject.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInject.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.LogByteSizeMergePolicy;
 import org.apache.solr.update.DirectUpdateHandler2;
 import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.RefCounted;
 
 public class TestPropInject extends AbstractSolrTestCase {
   @Override
@@ -37,14 +38,25 @@ public class TestPropInject extends Abst
   }
 
   public void testMergePolicy() throws Exception {
-    IndexWriter writer = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
-    LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getConfig().getMergePolicy();
+
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
+    LogByteSizeMergePolicy mp;
+    try {
+     mp = (LogByteSizeMergePolicy)iw.get().getConfig().getMergePolicy();
+    } finally {
+      iw.decref();
+    }
     assertEquals(64.0, mp.getMaxMergeMB(), 0);
   }
   
   public void testProps() throws Exception {
-    IndexWriter writer = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
-    ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getConfig().getMergeScheduler();
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
+    ConcurrentMergeScheduler cms;
+    try {
+      cms = (ConcurrentMergeScheduler)iw.get().getConfig().getMergeScheduler();
+    } finally {
+      iw.decref();
+    }
     assertEquals(2, cms.getMaxThreadCount());
   }
 }

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java Sat Jul 28 11:27:51 2012
@@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.LogByteSizeMergePolicy;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.util.RefCounted;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -33,15 +34,27 @@ public class TestPropInjectDefaults exte
 
   @Test
   public void testMergePolicyDefaults() throws Exception {
-    IndexWriter writer = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
-    LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getConfig().getMergePolicy();
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2) h.getCore()
+        .getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
+    LogByteSizeMergePolicy mp;
+    try {
+      mp = (LogByteSizeMergePolicy) iw.get().getConfig().getMergePolicy();
+    } finally {
+      iw.decref();
+    }
     assertEquals(32.0, mp.getMaxMergeMB(), 0);
   }
   
   @Test
   public void testPropsDefaults() throws Exception {
-    IndexWriter writer = ((DirectUpdateHandler2)h.getCore().getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
-    ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getConfig().getMergeScheduler();
+    RefCounted<IndexWriter> iw = ((DirectUpdateHandler2) h.getCore()
+        .getUpdateHandler()).getSolrCoreState().getIndexWriter(h.getCore());
+    ConcurrentMergeScheduler cms;
+    try {
+      cms = (ConcurrentMergeScheduler) iw.get().getConfig().getMergeScheduler();
+    } finally {
+      iw.decref();
+    }
     assertEquals(4, cms.getMaxThreadCount());
   }
 

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java Sat Jul 28 11:27:51 2012
@@ -39,6 +39,7 @@ import javax.xml.xpath.XPathFactory;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 import org.apache.solr.core.SolrXMLSerializer.SolrCoreXMLDef;
 import org.apache.solr.core.SolrXMLSerializer.SolrXMLDef;
 import org.junit.Test;
@@ -81,11 +82,12 @@ public class TestSolrXMLSerializer exten
     assertResults(((StringWriter) w).getBuffer().toString().getBytes("UTF-8"));
     
     // again with default file
-    File tmpFile = File.createTempFile("solr", ".xml", TEMP_DIR);
+    File tmpFile = _TestUtil.getTempDir("solr.xml");
     
     serializer.persistFile(tmpFile, solrXMLDef);
 
     assertResults(FileUtils.readFileToString(tmpFile, "UTF-8").getBytes("UTF-8"));
+    tmpFile.delete();
   }
 
   private void assertResults(byte[] bytes)

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Sat Jul 28 11:27:51 2012
@@ -57,6 +57,7 @@ import org.apache.solr.common.util.Simpl
 import org.apache.solr.util.AbstractSolrTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 
 /**
  * Test for ReplicationHandler
@@ -116,6 +117,9 @@ public class TestReplicationHandler exte
     slaveJetty.stop();
     master.tearDown();
     slave.tearDown();
+    masterJetty = slaveJetty = null;
+    master = slave = null;
+    masterClient = slaveClient = null;
   }
 
   private static JettySolrRunner createJetty(SolrInstance instance) throws Exception {
@@ -884,7 +888,7 @@ public class TestReplicationHandler exte
           break;
         }
         Thread.sleep(200);
-        if(waitCnt == 10) {
+        if(waitCnt == 20) {
           fail("Backup success not detected:" + checkStatus.response);
         }
         waitCnt++;

Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java Sat Jul 28 11:27:51 2012
@@ -20,16 +20,20 @@ package org.apache.solr.handler.admin;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.SolrTestCaseJ4;
 
+import java.util.Map;
 import java.io.File;
 import java.io.IOException;
 
 import javax.xml.xpath.XPathExpressionException;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.solr.SolrTestCaseJ4;
+
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.xml.sax.SAXException;
@@ -94,7 +98,49 @@ public class CoreAdminHandlerTest extend
        ,"/solr/cores/core[@name='props']/property[@name='hoss' and @value='man']"
        ,"/solr/cores/core[@name='props']/property[@name='foo' and @value='baz']"
        );
-    
+
+    // attempt to create a bogus core and confirm failure
+    try {
+      resp = new SolrQueryResponse();
+      admin.handleRequestBody
+        (req(CoreAdminParams.ACTION, 
+             CoreAdminParams.CoreAdminAction.CREATE.toString(),
+             CoreAdminParams.NAME, "bogus_dir_core",
+             CoreAdminParams.INSTANCE_DIR, "dir_does_not_exist_127896"),
+         resp);
+      fail("bogus collection created ok");
+    } catch (SolrException e) {
+      // :NOOP:
+      // :TODO: CoreAdminHandler's exception messages are terrible, otherwise we could asert something useful here
+    }
+
+    // check specificly for status of the failed core name
+    resp = new SolrQueryResponse();
+    admin.handleRequestBody
+      (req(CoreAdminParams.ACTION, 
+           CoreAdminParams.CoreAdminAction.STATUS.toString(),
+           CoreAdminParams.CORE, "bogus_dir_core"),
+         resp);
+    Map<String,Exception> failures = 
+      (Map<String,Exception>) resp.getValues().get("initFailures");
+    assertNotNull("core failures is null", failures);
+
+    NamedList<Object> status = 
+      (NamedList<Object>)resp.getValues().get("status");
+    assertNotNull("core status is null", status);
+
+    assertEquals("wrong number of core failures", 1, failures.size());
+    Exception fail = failures.get("bogus_dir_core");
+    assertNotNull("null failure for test core", fail);
+    assertTrue("init failure doesn't mention problem: " + fail.getMessage(),
+               0 < fail.getMessage().indexOf("dir_does_not_exist"));
+
+    assertEquals("bogus_dir_core status isn't empty",
+                 0, ((NamedList)status.get("bogus_dir_core")).size());
+
+               
+    // :TODO: because of SOLR-3665 we can't ask for status from all cores
+
   }