You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC

svn commit: r1454862 [43/49] - in /lucene/dev/branches/lucene4258: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/ dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml Sun Mar 10 15:04:57 2013
@@ -17,13 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-
-Config for testing spellcheck component
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <requestHandler name="standard"

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml Sun Mar 10 15:04:57 2013
@@ -17,11 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <!-- see TestConfig.testLib() -->
@@ -52,7 +47,7 @@
 
   <admin>
     <defaultQuery>solr</defaultQuery>
-    <gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
+    <gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
   </admin>
 
   <!-- test getting system property -->

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Sun Mar 10 15:04:57 2013
@@ -18,10 +18,21 @@
 -->
 
 <config>
+  <jmx />
+
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
-  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+  
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
+    <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
+    <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
+  </directoryFactory>
+  
   <dataDir>${solr.data.dir:}</dataDir>
 
+  <indexConfig>
+    <lockType>${solr.lock.type:native}</lockType>
+  </indexConfig>
+
   <!-- an update processor the explicitly excludes distrib to test
        clean errors when people attempt atomic updates w/o it
   -->
@@ -38,15 +49,71 @@
       <str name="omitHeader">true</str>
     </lst>
   </requestHandler>
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
 
   <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
 
   <updateHandler class="solr.DirectUpdateHandler2">
     <updateLog>
-      <!-- <str name="dir">/tmp/solr/</str> -->
+      <str name="dir">${solr.ulog.dir:}</str>
     </updateLog>
   </updateHandler>
 
+  <updateRequestProcessorChain name="dedupe">
+    <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
+      <bool name="enabled">true</bool>
+      <bool name="overwriteDupes">true</bool>
+      <str name="fields">v_t,t_field</str>
+      <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
+    </processor>
+    <processor class="solr.RunUpdateProcessorFactory" />
+  </updateRequestProcessorChain>
+  <updateRequestProcessorChain name="stored_sig">
+    <!-- this chain is valid even though the signature field is not
+         indexed, because we are not asking for dups to be overwritten
+      -->
+    <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
+      <bool name="enabled">true</bool>
+      <str name="signatureField">non_indexed_signature_sS</str>
+      <bool name="overwriteDupes">false</bool>
+      <str name="fields">v_t,t_field</str>
+      <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
+    </processor>
+    <processor class="solr.RunUpdateProcessorFactory" />
+  </updateRequestProcessorChain>
+
   <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
 
+  <updateRequestProcessorChain name="distrib-dup-test-chain-explicit">
+    <!-- explicit test using processors before and after distrib -->
+    <processor class="solr.RegexReplaceProcessorFactory">
+      <str name="fieldName">regex_dup_A_s</str>
+      <str name="pattern">x</str>
+      <str name="replacement">x_x</str>
+    </processor>
+    <processor class="solr.DistributedUpdateProcessorFactory" />
+    <processor class="solr.RegexReplaceProcessorFactory">
+      <str name="fieldName">regex_dup_B_s</str>
+      <str name="pattern">x</str>
+      <str name="replacement">x_x</str>
+    </processor>
+    <processor class="solr.RunUpdateProcessorFactory" />
+  </updateRequestProcessorChain>  
+
+  <updateRequestProcessorChain name="distrib-dup-test-chain-implicit">
+    <!-- implicit test w/o distrib declared-->
+    <processor class="solr.RegexReplaceProcessorFactory">
+      <str name="fieldName">regex_dup_A_s</str>
+      <str name="pattern">x</str>
+      <str name="replacement">x_x</str>
+    </processor>
+    <processor class="solr.RegexReplaceProcessorFactory">
+      <str name="fieldName">regex_dup_B_s</str>
+      <str name="pattern">x</str>
+      <str name="replacement">x_x</str>
+    </processor>
+    <processor class="solr.RunUpdateProcessorFactory" />
+  </updateRequestProcessorChain>  
+
 </config>

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Sun Mar 10 15:04:57 2013
@@ -97,6 +97,10 @@
     <updateLog enable="${enable.update.log:false}">
   	  <str name="dir">${solr.ulog.dir:}</str>
     </updateLog> 
+    
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
 
   </updateHandler>
 
@@ -486,7 +490,7 @@
 
   <admin>
     <defaultQuery>solr</defaultQuery>
-    <gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
+    <gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
   </admin>
 
   <!-- test getting system property -->

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml Sun Mar 10 15:04:57 2013
@@ -19,8 +19,6 @@
      Striped down solrconfig used by SampleTest to demonstrate picking any
      config filename you want.
 
-     $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
   -->
 
 <config>
@@ -59,7 +57,7 @@
 
   <admin>
     <defaultQuery>solr</defaultQuery>
-    <gettableFiles>solrconfig.xml scheam.xml</gettableFiles>
+    <gettableFiles>solrconfig.xml schema.xml</gettableFiles>
   </admin>
 
 

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml Sun Mar 10 15:04:57 2013
@@ -19,8 +19,6 @@
      Striped down schema used by SampleTest to demonstrate picking any
      schema filename you want.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
   -->
 
 <schema name="test" version="1.0">

Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml Sun Mar 10 15:04:57 2013
@@ -29,15 +29,14 @@
     If 'null' (or absent), cores will not be manageable via request handler
   -->
   <cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}" 
-         hostContext="${hostContext:solr}" zkClientTimeout="30000" numShards="${numShards:3}" shareSchema="${shareSchema:false}" 
-         distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:30000}">
-    <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
+         hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" numShards="${numShards:3}" shareSchema="${shareSchema:false}" 
+         distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:120000}">
+    <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"
+          coreNodeName="${coreNodeName:}"/>
     <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
-      <int name="socketTimeout">${socketTimeout:30000}</int>
+      <int name="socketTimeout">${socketTimeout:120000}</int>
       <int name="connTimeout">${connTimeout:15000}</int>
     </shardHandlerFactory>
   </cores>
   
-    
-
 </solr>

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Sun Mar 10 15:04:57 2013
@@ -252,6 +252,100 @@ public class BasicFunctionalityTest exte
             );
   }
 
+
+  /**
+   * verify that delete by query works with the QParser framework and
+   * pure negative queries
+   */
+  public void testNonTrivialDeleteByQuery() throws Exception {
+    clearIndex();
+    
+    // setup
+    assertU( add(doc("id","101", "text", "red apple" )) );
+    assertU( add(doc("id","102", "text", "purple grape" )) );
+    assertU( add(doc("id","103", "text", "green grape" )) );
+    assertU( add(doc("id","104", "text", "green pear" )) );
+    assertU( add(doc("id","105", "text", "yellow banana" )) );
+    assertU( add(doc("id","106", "text", "red cherry" )) );
+
+    // sanity checks
+    assertU(commit());
+    assertQ(req("id:[100 TO 110]")
+            ,"//*[@numFound='6']"
+            );
+    assertQ(req("*:*")
+            ,"//*[@numFound='6']"
+            );
+    assertQ(req("text:red")
+            ,"//*[@numFound='2']"
+            );
+    assertQ(req("-text:red")
+            ,"//*[@numFound='4']"
+            );
+    assertQ(req("text:grape")
+            ,"//*[@numFound='2']"
+            );
+    assertQ(req("-text:grape")
+            ,"//*[@numFound='4']"
+            );
+    assertQ(req("-text:red -text:grape")
+            ,"//*[@numFound='2']"
+            );
+    assertQ(req("{!lucene q.op=AND df=text}grape green")
+            ,"//*[@numFound='1']"
+            ,"//int[@name='id'][.='103']"
+             );
+    assertQ(req("-_val_:\"{!lucene q.op=AND df=text}grape green\"")
+            ,"//*[@numFound='5']"
+            ,"//int[@name='id'][.='101']"
+            ,"//int[@name='id'][.='102']"
+            ,"//int[@name='id'][.='104']"
+            ,"//int[@name='id'][.='105']"
+            ,"//int[@name='id'][.='106']"
+            );
+
+    // tests
+
+    assertU(delQ("-*:*")); // NOOP
+    assertU(commit());
+    assertQ(req("*:*")
+            ,"//*[@numFound='6']"
+            );
+
+    assertU(delQ("-text:grape -text:red"));
+    assertU(commit());
+    assertQ(req("*:*")
+            ,"//*[@numFound='4']"
+            ,"//int[@name='id'][.='101']"
+            ,"//int[@name='id'][.='102']"
+            ,"//int[@name='id'][.='103']"
+            ,"//int[@name='id'][.='106']"
+            );
+
+    assertU(delQ("{!term f=id}106"));
+    assertU(commit());
+    assertQ(req("*:*")
+            ,"//*[@numFound='3']"
+            ,"//int[@name='id'][.='101']"
+            ,"//int[@name='id'][.='102']"
+            ,"//int[@name='id'][.='103']"
+            );
+
+    assertU(delQ("-_val_:\"{!lucene q.op=AND df=text}grape green\""));
+    assertU(commit());
+    assertQ(req("*:*")
+            ,"//*[@numFound='1']"
+            ,"//int[@name='id'][.='103']"
+            );
+
+    assertU(delQ("-text:doesnotexist"));
+    assertU(commit());
+    assertQ(req("*:*")
+            ,"//*[@numFound='0']"
+            );
+
+  }
+
   @Test
   public void testHTMLStrip() {
     assertU(add(doc("id","200", "HTMLwhitetok","&#65;&#66;&#67;")));

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Sun Mar 10 15:04:57 2013
@@ -360,6 +360,29 @@ public class TestGroupingSearch extends 
     );
   }
 
+  @Test
+  public void testGroupingGroupedBasedFacetingWithTaggedFilter() throws Exception {
+    assertU(add(doc("id", "1", "cat_sI", "a", "bday", "2012-11-20T00:00:00Z")));
+    assertU(add(doc("id", "2", "cat_sI", "b", "bday", "2012-11-21T00:00:00Z")));
+    assertU(add(doc("id", "3", "cat_sI", "a", "bday", "2012-11-20T00:00:00Z")));
+    assertU(add(doc("id", "4", "cat_sI", "b", "bday", "2013-01-15T00:00:00Z")));
+    assertU(add(doc("id", "5", "cat_sI", "a", "bday", "2013-01-14T00:00:00Z")));
+    assertU(commit());
+
+    // Facet counts based on groups
+    SolrQueryRequest req = req("q", "*:*", "rows", "1", "group", "true", "group.field", "cat_sI",
+        "sort", "cat_sI asc", "fl", "id", "fq", "{!tag=chk}bday:[2012-12-18T00:00:00Z TO 2013-01-17T23:59:59Z]",
+        "facet", "true", "group.truncate", "true", "group.sort", "bday desc",
+        "facet.query", "{!ex=chk key=LW1}bday:[2013-01-11T00:00:00Z TO 2013-01-17T23:59:59Z]",
+        "facet.query", "{!ex=chk key=LM1}bday:[2012-12-18T00:00:00Z TO 2013-01-17T23:59:59Z]",
+        "facet.query", "{!ex=chk key=LM3}bday:[2012-10-18T00:00:00Z TO 2013-01-17T23:59:59Z]");
+    assertJQ(
+        req,
+        "/grouped=={'cat_sI':{'matches':2,'groups':[{'groupValue':'a','doclist':{'numFound':1,'start':0,'docs':[{'id':'5'}]}}]}}",
+        "/facet_counts=={'facet_queries':{'LW1':2,'LM1':2,'LM3':2},'facet_fields':{},'facet_dates':{},'facet_ranges':{}}"
+    );
+  }
+
   static String f = "foo_i";
   static String f2 = "foo2_i";
 

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Sun Mar 10 15:04:57 2013
@@ -16,7 +16,6 @@
  */
 package org.apache.solr;
 
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.util.AbstractSolrTestCase;
@@ -37,9 +36,6 @@ import java.util.Properties;
  *
  * @since solr 1.4
  */
-@ThreadLeakFilters(defaultFilters = true, filters = {
-    SolrIgnoredThreadsFilter.class
-})
 public class TestSolrCoreProperties extends LuceneTestCase {
   private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
   JettySolrRunner solrJetty;

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java Sun Mar 10 15:04:57 2013
@@ -16,6 +16,8 @@
  */
 package org.apache.solr;
 
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.schema.DateField;
 import org.apache.solr.schema.FieldType;
@@ -25,6 +27,7 @@ import org.junit.After;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.io.StringReader;
 import java.text.SimpleDateFormat;
 import java.util.Locale;
 import java.util.TimeZone;
@@ -47,6 +50,38 @@ public class TestTrie extends SolrTestCa
     clearIndex();
     super.tearDown();
   }
+  
+  @Test
+  public void testTokenizer() throws Exception {
+    FieldType type = h.getCore().getSchema().getFieldType("tint");
+    assertTrue(type instanceof TrieField);
+    
+    String value = String.valueOf(random().nextInt());
+    TokenStream ts = type.getAnalyzer().tokenStream("dummy", new StringReader(value));
+    OffsetAttribute ofsAtt = ts.addAttribute(OffsetAttribute.class);
+    ts.reset();
+    int count = 0;
+    while (ts.incrementToken()) {
+      count++;
+      assertEquals(0, ofsAtt.startOffset());
+      assertEquals(value.length(), ofsAtt.endOffset());
+    }
+    final int precStep = ((TrieField) type).getPrecisionStep();
+    assertEquals( (32 + precStep - 1) / precStep, count);
+    ts.end();
+    assertEquals(value.length(), ofsAtt.startOffset());
+    assertEquals(value.length(), ofsAtt.endOffset());
+    ts.close();
+    
+    // Test empty one:
+    ts = type.getAnalyzer().tokenStream("dummy", new StringReader(""));
+    ts.reset();
+    assertFalse(ts.incrementToken());
+    ts.end();
+    assertEquals(0, ofsAtt.startOffset());
+    assertEquals(0, ofsAtt.endOffset());
+    ts.close();    
+  }
 
   @Test
   public void testTrieIntRangeSearch() throws Exception {

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Sun Mar 10 15:04:57 2013
@@ -17,13 +17,25 @@ package org.apache.solr.cloud;
  * limitations under the License.
  */
 
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
+import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrException;
@@ -32,7 +44,11 @@ import org.apache.solr.common.cloud.ZkNo
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.ReplicationHandler;
 import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.AbstractSolrTestCase;
 import org.junit.BeforeClass;
 
 /**
@@ -40,15 +56,21 @@ import org.junit.BeforeClass;
  * work as expected.
  */
 public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
+  private static final String ONE_NODE_COLLECTION = "onenodecollection";
+
   @BeforeClass
   public static void beforeThisClass2() throws Exception {
-    // TODO: we use an fs based dir because something
-    // like a ram dir will not recover correctly right now
-    // because tran log will still exist on restart and ram
-    // dir will not persist - perhaps translog can empty on
-    // start if using an EphemeralDirectoryFactory 
-    useFactory(null);
+
+  }
+  
+  public BasicDistributedZk2Test() {
+    super();
+    fixShardCount = true;
+    
+    sliceCount = 2;
+    shardCount = 3;
   }
+  
   /*
    * (non-Javadoc)
    * 
@@ -64,6 +86,8 @@ public class BasicDistributedZk2Test ext
       handle.put("QTime", SKIPVAL);
       handle.put("timestamp", SKIPVAL);
       
+      testNodeWithoutCollectionForwarding();
+     
       indexr(id, 1, i1, 100, tlong, 100, t1,
           "now is the time for all good men", "foo_f", 1.414f, "foo_b", "true",
           "foo_d", 1.414d);
@@ -139,6 +163,52 @@ public class BasicDistributedZk2Test ext
     
   }
   
+  private void testNodeWithoutCollectionForwarding() throws Exception,
+      SolrServerException, IOException {
+    try {
+      final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
+          0,
+          ((HttpSolrServer) clients.get(0)).getBaseURL().length()
+              - DEFAULT_COLLECTION.length() - 1);
+      HttpSolrServer server = new HttpSolrServer(baseUrl);
+      server.setConnectionTimeout(15000);
+      server.setSoTimeout(30000);
+      Create createCmd = new Create();
+      createCmd.setRoles("none");
+      createCmd.setCoreName(ONE_NODE_COLLECTION + "core");
+      createCmd.setCollection(ONE_NODE_COLLECTION);
+      createCmd.setNumShards(1);
+      createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+          + ONE_NODE_COLLECTION);
+      server.request(createCmd);
+    } catch (Exception e) {
+      e.printStackTrace();
+      //fail
+    }
+    
+    waitForRecoveriesToFinish(ONE_NODE_COLLECTION, cloudClient.getZkStateReader(), false);
+    
+    final String baseUrl2 = ((HttpSolrServer) clients.get(1)).getBaseURL().substring(
+        0,
+        ((HttpSolrServer) clients.get(1)).getBaseURL().length()
+            - DEFAULT_COLLECTION.length() - 1);
+    HttpSolrServer qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection" + "core");
+    
+    // add a doc
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "1");
+    qclient.add(doc);
+    qclient.commit();
+    
+    SolrQuery query = new SolrQuery("*:*");
+    QueryResponse results = qclient.query(query);
+    assertEquals(1, results.getResults().getNumFound());
+    
+    qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection");
+    results = qclient.query(query);
+    assertEquals(1, results.getResults().getNumFound());
+  }
+  
   private long testUpdateAndDelete() throws Exception {
     long docId = 99999999L;
     indexr("id", docId, t1, "originalcontent");
@@ -187,17 +257,17 @@ public class BasicDistributedZk2Test ext
     
     int oldLiveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
     
-    assertEquals(5, oldLiveNodes);
+    assertEquals(4, oldLiveNodes);
     
     // kill a shard
-    CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
+    CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
 
 
     // we are careful to make sure the downed node is no longer in the state,
     // because on some systems (especially freebsd w/ blackhole enabled), trying
     // to talk to a downed node causes grief
     Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
-    jetties.addAll(shardToJetty.get(SHARD2));
+    jetties.addAll(shardToJetty.get(SHARD1));
     jetties.remove(deadShard);
     
     // ensure shard is dead
@@ -211,8 +281,6 @@ public class BasicDistributedZk2Test ext
     
     commit();
     
-    printLayout();
-    
     query("q", "*:*", "sort", "n_tl1 desc");
     
     // long cloudClientDocs = cloudClient.query(new
@@ -224,7 +292,8 @@ public class BasicDistributedZk2Test ext
   
     long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
     
-    index_specific(shardToJetty.get(SHARD2).get(1).client.solrClient, id, 1000, i1, 108, t1,
+    cloudClient.getZkStateReader().getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 15000);
+    index_specific(shardToJetty.get(SHARD1).get(1).client.solrClient, id, 1000, i1, 108, t1,
         "specific doc!");
     
     commit();
@@ -297,7 +366,7 @@ public class BasicDistributedZk2Test ext
     
     waitForRecoveriesToFinish(false);
     
-    deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+    deadShardCount = shardToJetty.get(SHARD1).get(0).client.solrClient
         .query(query).getResults().getNumFound();
     // if we properly recovered, we should now have the couple missing docs that
     // came in while shard was down
@@ -332,6 +401,100 @@ public class BasicDistributedZk2Test ext
     waitForRecoveriesToFinish(false);
     
     checkShardConsistency(true, false);
+    
+    // try a backup command
+    final HttpSolrServer client = (HttpSolrServer) shardToJetty.get(SHARD2).get(0).client.solrClient;
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("qt", "/replication");
+    params.set("command", "backup");
+
+    QueryRequest request = new QueryRequest(params);
+    NamedList<Object> results = client.request(request );
+    
+    checkForBackupSuccess(client);
+    
+  }
+  private void checkForBackupSuccess(final HttpSolrServer client)
+      throws InterruptedException, IOException {
+    class CheckStatus extends Thread {
+      volatile String fail = null;
+      volatile String response = null;
+      volatile boolean success = false;
+      final Pattern p = Pattern
+          .compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+      
+      CheckStatus() {}
+      
+      @Override
+      public void run() {
+        String masterUrl = client.getBaseURL() + "/replication?command="
+            + ReplicationHandler.CMD_DETAILS;
+        URL url;
+        InputStream stream = null;
+        try {
+          url = new URL(masterUrl);
+          stream = url.openStream();
+          response = IOUtils.toString(stream, "UTF-8");
+          if (response.contains("<str name=\"status\">success</str>")) {
+            Matcher m = p.matcher(response);
+            if (!m.find()) {
+              fail("could not find the completed timestamp in response.");
+            }
+            
+            success = true;
+          }
+          stream.close();
+        } catch (Exception e) {
+          e.printStackTrace();
+          fail = e.getMessage();
+        } finally {
+          IOUtils.closeQuietly(stream);
+        }
+        
+      };
+    }
+    ;
+    SolrCore core = ((SolrDispatchFilter) shardToJetty.get(SHARD2).get(0).jetty
+        .getDispatchFilter().getFilter()).getCores().getCore("collection1");
+    String ddir;
+    try {
+      ddir = core.getDataDir(); 
+    } finally {
+      core.close();
+    }
+    File dataDir = new File(ddir);
+    
+    int waitCnt = 0;
+    CheckStatus checkStatus = new CheckStatus();
+    while (true) {
+      checkStatus.run();
+      if (checkStatus.fail != null) {
+        fail(checkStatus.fail);
+      }
+      if (checkStatus.success) {
+        break;
+      }
+      Thread.sleep(200);
+      if (waitCnt == 20) {
+        fail("Backup success not detected:" + checkStatus.response);
+      }
+      waitCnt++;
+    }
+    
+    File[] files = dataDir.listFiles(new FilenameFilter() {
+      
+      @Override
+      public boolean accept(File dir, String name) {
+        if (name.startsWith("snapshot")) {
+          return true;
+        }
+        return false;
+      }
+    });
+    assertEquals(Arrays.asList(files).toString(), 1, files.length);
+    File snapDir = files[0];
+    
+    AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
   }
   
   private void addNewReplica() throws Exception {

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Sun Mar 10 15:04:57 2013
@@ -25,7 +25,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
@@ -54,8 +53,6 @@ import org.apache.solr.common.SolrDocume
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
@@ -79,7 +76,7 @@ import org.junit.BeforeClass;
 public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
   
   private static final String DEFAULT_COLLECTION = "collection1";
-  private static final boolean DEBUG = false;
+  protected static final boolean DEBUG = false;
   String t1="a_t";
   String i1="a_si";
   String nint = "n_i";
@@ -111,9 +108,6 @@ public class BasicDistributedZkTest exte
   
   @BeforeClass
   public static void beforeThisClass2() throws Exception {
-    // TODO: we use an fs based dir because something
-    // like a ram dir will not recover correctly right now
-    useFactory(null);
   }
   
   @Before
@@ -328,335 +322,26 @@ public class BasicDistributedZkTest exte
     testMultipleCollections();
     testANewCollectionInOneInstance();
     testSearchByCollectionName();
+    testUpdateByCollectionName();
     testANewCollectionInOneInstanceWithManualShardAssignement();
     testNumberOfCommitsWithCommitAfterAdd();
 
     testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
     testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
 
-    testCoreUnloadAndLeaders();
-    testUnloadLotsOfCores();
     testStopAndStartCoresInOneInstance();
-    testUnloadShardAndCollection();
     // Thread.sleep(10000000000L);
     if (DEBUG) {
       super.printLayout();
     }
   }
-
-  private void testUnloadShardAndCollection() throws Exception{
-    // create one leader and one replica
-    
-    Create createCmd = new Create();
-    createCmd.setCoreName("test_unload_shard_and_collection_1");
-    String collection = "test_unload_shard_and_collection";
-    createCmd.setCollection(collection);
-    String coreDataDir = dataDir.getAbsolutePath() + File.separator
-        + System.currentTimeMillis() + collection + "1";
-    createCmd.setDataDir(coreDataDir);
-    createCmd.setNumShards(2);
-    
-    SolrServer client = clients.get(0);
-    String url1 = getBaseUrl(client);
-    HttpSolrServer server = new HttpSolrServer(url1);
-    server.setConnectionTimeout(15000);
-    server.setSoTimeout(30000);
-    server.request(createCmd);
-    
-    createCmd = new Create();
-    createCmd.setCoreName("test_unload_shard_and_collection_2");
-    collection = "test_unload_shard_and_collection";
-    createCmd.setCollection(collection);
-    coreDataDir = dataDir.getAbsolutePath() + File.separator
-        + System.currentTimeMillis() + collection + "2";
-    createCmd.setDataDir(coreDataDir);
-    
-    server.request(createCmd);
-
-    // now unload one of the two
-    Unload unloadCmd = new Unload(false);
-    unloadCmd.setCoreName("test_unload_shard_and_collection_2");
-    server.request(unloadCmd);
-    
-    // there should be only one shard
-    Slice shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
-    long timeoutAt = System.currentTimeMillis() + 30000;
-    while (shard2 != null) {
-      if (System.currentTimeMillis() > timeoutAt) {
-        printLayout();
-        fail("Still found shard");
-      }
-      
-      Thread.sleep(50);
-      shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
-    }
-
-    Slice shard1 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard1");
-    assertNotNull(shard1);
-    assertTrue(getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection));
-    
-    // now unload one of the other
-    unloadCmd = new Unload(false);
-    unloadCmd.setCoreName("test_unload_shard_and_collection_1");
-    server.request(unloadCmd);
-    
-    //printLayout();
-    // the collection should be gone
-    timeoutAt = System.currentTimeMillis() + 30000;
-    while (getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection)) {
-      if (System.currentTimeMillis() > timeoutAt) {
-        printLayout();
-        fail("Still found collection");
-      }
-      
-      Thread.sleep(50);
-    }
-    
-  }
-
-  /**
-   * @throws Exception on any problem
-   */
-  private void testCoreUnloadAndLeaders() throws Exception {
-    // create a new collection collection
-    SolrServer client = clients.get(0);
-    String url1 = getBaseUrl(client);
-    HttpSolrServer server = new HttpSolrServer(url1);
-    server.setConnectionTimeout(15000);
-    server.setSoTimeout(30000);
-    
-    Create createCmd = new Create();
-    createCmd.setCoreName("unloadcollection1");
-    createCmd.setCollection("unloadcollection");
-    createCmd.setNumShards(1);
-    String core1DataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_1n";
-    createCmd.setDataDir(core1DataDir);
-    server.request(createCmd);
-    
-    ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
-    
-    zkStateReader.updateClusterState(true);
-
-    int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
-    assertEquals(1, slices);
-    
-    client = clients.get(1);
-    String url2 = getBaseUrl(client);
-    server = new HttpSolrServer(url2);
-    
-    createCmd = new Create();
-    createCmd.setCoreName("unloadcollection2");
-    createCmd.setCollection("unloadcollection");
-    String core2dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_2n";
-    createCmd.setDataDir(core2dataDir);
-    server.request(createCmd);
-    
-    zkStateReader.updateClusterState(true);
-    slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
-    assertEquals(1, slices);
-    
-    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-    
-    ZkCoreNodeProps leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
-    
-    Random random = random();
-    HttpSolrServer collectionClient;
-    if (random.nextBoolean()) {
-      collectionClient = new HttpSolrServer(leaderProps.getCoreUrl());
-      // lets try and use the solrj client to index and retrieve a couple
-      // documents
-      SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
-          "humpty dumpy sat on a wall");
-      SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
-          "humpty dumpy3 sat on a walls");
-      SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
-          "humpty dumpy2 sat on a walled");
-      collectionClient.add(doc1);
-      collectionClient.add(doc2);
-      collectionClient.add(doc3);
-      collectionClient.commit();
-    }
-
-    // create another replica for our collection
-    client = clients.get(2);
-    String url3 = getBaseUrl(client);
-    server = new HttpSolrServer(url3);
-    
-    createCmd = new Create();
-    createCmd.setCoreName("unloadcollection3");
-    createCmd.setCollection("unloadcollection");
-    String core3dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_3n";
-    createCmd.setDataDir(core3dataDir);
-    server.request(createCmd);
-    
-    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-    
-    // so that we start with some versions when we reload...
-    DirectUpdateHandler2.commitOnClose = false;
-    
-    HttpSolrServer addClient = new HttpSolrServer(url3 + "/unloadcollection3");
-    // add a few docs
-    for (int x = 20; x < 100; x++) {
-      SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1,
-          "humpty dumpy sat on a wall");
-      addClient.add(doc1);
-    }
-
-    // don't commit so they remain in the tran log
-    //collectionClient.commit();
-    
-    // unload the leader
-    collectionClient = new HttpSolrServer(leaderProps.getBaseUrl());
-    
-    Unload unloadCmd = new Unload(false);
-    unloadCmd.setCoreName(leaderProps.getCoreName());
-    ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
-
-    collectionClient.request(unloadCmd);
-
-//    Thread.currentThread().sleep(500);
-//    printLayout();
-    
-    int tries = 50;
-    while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
-      Thread.sleep(100);
-      if (tries-- == 0) {
-        fail("Leader never changed");
-      }
-    }
-    
-    // ensure there is a leader
-    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-    
-    addClient = new HttpSolrServer(url2 + "/unloadcollection2");
-    // add a few docs while the leader is down
-    for (int x = 101; x < 200; x++) {
-      SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1,
-          "humpty dumpy sat on a wall");
-      addClient.add(doc1);
-    }
-    
-    
-    // create another replica for our collection
-    client = clients.get(3);
-    String url4 = getBaseUrl(client);
-    server = new HttpSolrServer(url4);
-    
-    createCmd = new Create();
-    createCmd.setCoreName("unloadcollection4");
-    createCmd.setCollection("unloadcollection");
-    String core4dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_4n";
-    createCmd.setDataDir(core4dataDir);
-    server.request(createCmd);
-    
-    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-    
-    // unload the leader again
-    leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
-    collectionClient = new HttpSolrServer(leaderProps.getBaseUrl());
-    
-    unloadCmd = new Unload(false);
-    unloadCmd.setCoreName(leaderProps.getCoreName());
-    p = (ModifiableSolrParams) unloadCmd.getParams();
-    collectionClient.request(unloadCmd);
-    
-    tries = 50;
-    while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
-      Thread.sleep(100);
-      if (tries-- == 0) {
-        fail("Leader never changed");
-      }
-    }
-    
-    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-    
-    
-    // set this back
-    DirectUpdateHandler2.commitOnClose = true;
-    
-    // bring the downed leader back as replica
-    server = new HttpSolrServer(leaderProps.getBaseUrl());
-    
-    createCmd = new Create();
-    createCmd.setCoreName(leaderProps.getCoreName());
-    createCmd.setCollection("unloadcollection");
-    createCmd.setDataDir(core1DataDir);
-    server.request(createCmd);
-
-    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-    
-    
-    server = new HttpSolrServer(url1 + "/unloadcollection");
-   // System.out.println(server.query(new SolrQuery("*:*")).getResults().getNumFound());
-    server = new HttpSolrServer(url2 + "/unloadcollection");
-    server.commit();
-    SolrQuery q = new SolrQuery("*:*");
-    q.set("distrib", false);
-    long found1 = server.query(q).getResults().getNumFound();
-    server = new HttpSolrServer(url3 + "/unloadcollection");
-    server.commit();
-    q = new SolrQuery("*:*");
-    q.set("distrib", false);
-    long found3 = server.query(q).getResults().getNumFound();
-    server = new HttpSolrServer(url4 + "/unloadcollection");
-    server.commit();
-    q = new SolrQuery("*:*");
-    q.set("distrib", false);
-    long found4 = server.query(q).getResults().getNumFound();
-    
-    // all 3 shards should now have the same number of docs
-    assertEquals(found1, found3);
-    assertEquals(found3, found4);
-    
-  }
-  
-  private void testUnloadLotsOfCores() throws Exception {
-    SolrServer client = clients.get(2);
-    String url3 = getBaseUrl(client);
-    final HttpSolrServer server = new HttpSolrServer(url3);
-    server.setConnectionTimeout(15000);
-    server.setSoTimeout(60000);
-    ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
-        5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-        new DefaultSolrThreadFactory("testExecutor"));
-    int cnt = random().nextInt(12) + 1;
-    
-    // create the cores
-    createCores(server, executor, "multiunload", 2, cnt);
-    
-    executor.shutdown();
-    executor.awaitTermination(120, TimeUnit.SECONDS);
-    executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
-        TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-        new DefaultSolrThreadFactory("testExecutor"));
-    for (int j = 0; j < cnt; j++) {
-      final int freezeJ = j;
-      executor.execute(new Runnable() {
-        @Override
-        public void run() {
-          Unload unloadCmd = new Unload(true);
-          unloadCmd.setCoreName("multiunload" + freezeJ);
-          try {
-            server.request(unloadCmd);
-          } catch (SolrServerException e) {
-            throw new RuntimeException(e);
-          } catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        }
-      });
-      Thread.sleep(random().nextInt(50));
-    }
-    executor.shutdown();
-    executor.awaitTermination(120, TimeUnit.SECONDS);
-  }
   
   private void testStopAndStartCoresInOneInstance() throws Exception {
     SolrServer client = clients.get(0);
     String url3 = getBaseUrl(client);
     final HttpSolrServer server = new HttpSolrServer(url3);
     server.setConnectionTimeout(15000);
-    server.setSoTimeout(30000);
+    server.setSoTimeout(60000);
     ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
         5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
         new DefaultSolrThreadFactory("testExecutor"));
@@ -685,7 +370,7 @@ public class BasicDistributedZkTest exte
 
   }
 
-  private void createCores(final HttpSolrServer server,
+  protected void createCores(final HttpSolrServer server,
       ThreadPoolExecutor executor, final String collection, final int numShards, int cnt) {
     for (int i = 0; i < cnt; i++) {
       final int freezeI = i;
@@ -713,7 +398,7 @@ public class BasicDistributedZkTest exte
     }
   }
 
-  private String getBaseUrl(SolrServer client) {
+  protected String getBaseUrl(SolrServer client) {
     String url2 = ((HttpSolrServer) client).getBaseURL()
         .substring(
             0,
@@ -753,7 +438,7 @@ public class BasicDistributedZkTest exte
     }
   }
   
-  private ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
+  protected ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
     ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
     ZkNodeProps leader = clusterState.getLeader(collection, slice);
     if (leader == null) {
@@ -761,89 +446,7 @@ public class BasicDistributedZkTest exte
     }
     return new ZkCoreNodeProps(leader);
   }
-
-  private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
-    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-    
-    int expectedSlices = numShardsNumReplicaList.get(0);
-    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
-    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
-    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-    
-      Map<String,DocCollection> collections = clusterState
-          .getCollectionStates();
-      if (collections.containsKey(collectionName)) {
-        Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
-        // did we find expectedSlices slices/shards?
-      if (slices.size() != expectedSlices) {
-        return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
-      }
-      int totalShards = 0;
-      for (String sliceName : slices.keySet()) {
-        for (Replica replica : slices.get(sliceName).getReplicas()) {
-          if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
-            return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
-          }
-        }
-        totalShards += slices.get(sliceName).getReplicas().size();
-      }
-      if (totalShards != expectedTotalShards) {
-        return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards; 
-        }
-      return null;
-    } else {
-      return "Could not find new collection " + collectionName;
-    }
-  }
   
-  private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
-      throws Exception {
-    // check for an expectedSlices new collection - we poll the state
-    long timeoutAt = System.currentTimeMillis() + 120000;
-    boolean success = false;
-    String checkResult = "Didnt get to perform a single check";
-    while (System.currentTimeMillis() < timeoutAt) {
-      checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
-      if (checkResult == null) {
-        success = true;
-        break;
-      }
-      Thread.sleep(500);
-    }
-    if (!success) {
-      super.printLayout();
-      fail(checkResult);
-      }
-    }
-
-  private void checkCollectionIsNotCreated(String collectionName)
-    throws Exception {
-    // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
-    Thread.sleep(10000);
-    assertFalse(collectionName + " not supposed to exist", getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collectionName));
-  }
-  
-  private void checkForMissingCollection(String collectionName)
-      throws Exception {
-    // check for a  collection - we poll the state
-    long timeoutAt = System.currentTimeMillis() + 30000;
-    boolean found = true;
-    while (System.currentTimeMillis() < timeoutAt) {
-      getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
-      ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-      Map<String,DocCollection> collections = clusterState
-          .getCollectionStates();
-      if (!collections.containsKey(collectionName)) {
-        found = false;
-        break;
-      }
-      Thread.sleep(100);
-    }
-    if (found) {
-      fail("Found collection that should be gone " + collectionName);
-    }
-  }
-
   /**
    * Expects a RegexReplaceProcessorFactories in the chain which will
    * "double up" the values in two (stored) string fields.
@@ -1045,6 +648,8 @@ public class BasicDistributedZkTest exte
     
     // now test that unloading a core gets us a new leader
     HttpSolrServer server = new HttpSolrServer(baseUrl);
+    server.setConnectionTimeout(15000);
+    server.setSoTimeout(30000);
     Unload unloadCmd = new Unload(true);
     unloadCmd.setCoreName(props.getCoreName());
     
@@ -1077,6 +682,21 @@ public class BasicDistributedZkTest exte
     long oneDocs = client1.query(query).getResults().getNumFound();
     assertEquals(3, oneDocs);
   }
+  
+  private void testUpdateByCollectionName() throws SolrServerException, IOException {
+    log.info("### STARTING testUpdateByCollectionName");
+    SolrServer client = clients.get(0);
+    final String baseUrl = ((HttpSolrServer) client).getBaseURL().substring(
+        0,
+        ((HttpSolrServer) client).getBaseURL().length()
+            - DEFAULT_COLLECTION.length() - 1);
+    
+    // the cores each have different names, but if we add the collection name to the url
+    // we should get mapped to the right core
+    // test hitting an update url
+    SolrServer client1 = createNewSolrServer(oneInstanceCollection, baseUrl);
+    client1.commit();
+  }
 
   private void testANewCollectionInOneInstance() throws Exception {
     log.info("### STARTING testANewCollectionInOneInstance");
@@ -1317,7 +937,7 @@ public class BasicDistributedZkTest exte
   }
 
   volatile CloudSolrServer commondCloudSolrServer;
-  private CloudSolrServer getCommonCloudSolrServer() {
+  protected CloudSolrServer getCommonCloudSolrServer() {
     if (commondCloudSolrServer == null) {
       synchronized(this) {
         try {

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Sun Mar 10 15:04:57 2013
@@ -21,8 +21,10 @@ import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.http.client.HttpClient;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.apache.lucene.util.LuceneTestCase.Slow;
+
+import org.apache.http.client.HttpClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@@ -30,27 +32,39 @@ import org.apache.solr.client.solrj.impl
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.Diagnostics;
+import org.apache.solr.update.SolrCmdDistributor;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @Slow
-@Ignore("ignore while investigating jenkins fails")
+@BadApple
 public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
   public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
   
-  private static final int BASE_RUN_LENGTH = 60000;
+  private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
 
   @BeforeClass
   public static void beforeSuperClass() {
+    SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
+      @Override
+      public void call(Object... data) {
+        SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
+        if (sreq.exception == null) return;
+        if (sreq.exception.getMessage().contains("Timeout")) {
+          Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+        }
+      }
+    };
   }
   
   @AfterClass
   public static void afterSuperClass() {
+    SolrCmdDistributor.testing_errorHook = null;
   }
   
   @Before
@@ -60,6 +74,7 @@ public class ChaosMonkeyNothingIsSafeTes
     // can help to hide this when testing and looking at logs
     //ignoreException("shard update error");
     System.setProperty("numShards", Integer.toString(sliceCount));
+    useFactory("solr.StandardDirectoryFactory");
   }
   
   @Override
@@ -72,8 +87,8 @@ public class ChaosMonkeyNothingIsSafeTes
   
   public ChaosMonkeyNothingIsSafeTest() {
     super();
-    sliceCount = 1;
-    shardCount = 7;
+    sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "2"));
+    shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "7"));
   }
   
   @Override
@@ -100,7 +115,7 @@ public class ChaosMonkeyNothingIsSafeTes
       int i = 0;
       for (i = 0; i < threadCount; i++) {
         StopableIndexingThread indexThread = new StopableIndexingThread(
-            i * 50000, true);
+            (i+1) * 50000, true);
         threads.add(indexThread);
         indexThread.start();
       }
@@ -117,14 +132,21 @@ public class ChaosMonkeyNothingIsSafeTes
       boolean runFullThrottle = random().nextBoolean();
       if (runFullThrottle) {
         FullThrottleStopableIndexingThread ftIndexThread = new FullThrottleStopableIndexingThread(
-            clients, i * 50000, true);
+            clients, (i+1) * 50000, true);
         threads.add(ftIndexThread);
         ftIndexThread.start();
       }
       
       chaosMonkey.startTheMonkey(true, 10000);
-      //int runLength = atLeast(BASE_RUN_LENGTH);
-      int runLength = BASE_RUN_LENGTH;
+
+      long runLength;
+      if (RUN_LENGTH != -1) {
+        runLength = RUN_LENGTH;
+      } else {
+        int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000};
+        runLength = runTimes[random().nextInt(runTimes.length - 1)];
+      }
+      
       try {
         Thread.sleep(runLength);
       } finally {
@@ -140,12 +162,13 @@ public class ChaosMonkeyNothingIsSafeTes
         indexThread.join();
       }
       
-       // we expect full throttle fails, but not cloud client...
-       for (StopableThread indexThread : threads) {
-         if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
-           //assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
-         }
-       }
+       // we expect full throttle fails, but cloud client should not easily fail
+       // but it's allowed to fail and sometimes does, so commented out for now
+//       for (StopableThread indexThread : threads) {
+//         if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
+//           assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
+//         }
+//       }
       
       // try and wait for any replications and what not to finish...
       
@@ -156,7 +179,7 @@ public class ChaosMonkeyNothingIsSafeTes
       
       // make sure we again have leaders for each shard
       for (int j = 1; j < sliceCount; j++) {
-        zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
+        zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
       }
       
       commit();
@@ -307,12 +330,5 @@ public class ChaosMonkeyNothingIsSafeTes
     SolrInputDocument doc = getDoc(fields);
     indexDoc(doc);
   }
-
-  SolrInputDocument getDoc(Object... fields) throws Exception {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    addFields(doc, "rnd_b", true);
-    return doc;
-  }
   
 }

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Sun Mar 10 15:04:57 2013
@@ -20,30 +20,41 @@ package org.apache.solr.cloud;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.Diagnostics;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.servlet.SolrDispatchFilter;
 import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.update.SolrCmdDistributor;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 
-@Ignore("SOLR-3126")
-public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
+@Slow
+public class  ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
   
-  private static final int BASE_RUN_LENGTH = 120000;
+  private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
 
   @BeforeClass
   public static void beforeSuperClass() {
-
+    SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
+      @Override
+      public void call(Object... data) {
+        SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
+        if (sreq.exception == null) return;
+        if (sreq.exception.getMessage().contains("Timeout")) {
+          Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+        }
+      }
+    };
   }
   
   @AfterClass
   public static void afterSuperClass() {
-    
+    SolrCmdDistributor.testing_errorHook = null;
   }
   
   @Before
@@ -66,8 +77,8 @@ public class ChaosMonkeySafeLeaderTest e
   
   public ChaosMonkeySafeLeaderTest() {
     super();
-    sliceCount = 3;//atLeast(2);
-    shardCount = 12;//atLeast(sliceCount*2);
+    sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "3"));
+    shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "12"));
   }
   
   @Override
@@ -85,16 +96,24 @@ public class ChaosMonkeySafeLeaderTest e
     List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
     int threadCount = 2;
     for (int i = 0; i < threadCount; i++) {
-      StopableIndexingThread indexThread = new StopableIndexingThread(i * 50000, true);
+      StopableIndexingThread indexThread = new StopableIndexingThread(10000 + i*50000, true);
       threads.add(indexThread);
       indexThread.start();
     }
     
     chaosMonkey.startTheMonkey(false, 500);
-    int runLength = atLeast(BASE_RUN_LENGTH);
-    Thread.sleep(runLength);
-    
-    chaosMonkey.stopTheMonkey();
+    long runLength;
+    if (RUN_LENGTH != -1) {
+      runLength = RUN_LENGTH;
+    } else {
+      int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000};
+      runLength = runTimes[random().nextInt(runTimes.length - 1)];
+    }
+    try {
+      Thread.sleep(runLength);
+    } finally {
+      chaosMonkey.stopTheMonkey();
+    }
     
     for (StopableIndexingThread indexThread : threads) {
       indexThread.safeStop();
@@ -113,7 +132,7 @@ public class ChaosMonkeySafeLeaderTest e
 
     Thread.sleep(2000);
 
-    waitForThingsToLevelOut(Integer.MAX_VALUE); //Math.round((runLength / 1000.0f / 3.0f)));
+    waitForThingsToLevelOut(180000);
 
     checkShardConsistency(true, true);
     

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Sun Mar 10 15:04:57 2013
@@ -254,7 +254,7 @@ public class ClusterStateUpdateTest exte
     System.clearProperty("solrcloud.update.delay");
   }
   
-  private void printLayout(String zkHost) throws Exception {
+  static void printLayout(String zkHost) throws Exception {
     SolrZkClient zkClient = new SolrZkClient(
         zkHost, AbstractZkTestCase.TIMEOUT);
     zkClient.printLayoutToStdOut();

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Sun Mar 10 15:04:57 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.net.MalformedURLException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -43,8 +44,6 @@ import javax.management.ObjectName;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.lucene.util._TestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -91,9 +90,7 @@ public class CollectionsAPIDistributedZk
   
   @BeforeClass
   public static void beforeThisClass2() throws Exception {
-    // TODO: we use an fs based dir because something
-    // like a ram dir will not recover correctly right now
-    useFactory(null);
+
   }
   
   @Before
@@ -103,6 +100,10 @@ public class CollectionsAPIDistributedZk
     System.setProperty("numShards", Integer.toString(sliceCount));
     System.setProperty("solr.xml.persist", "true");
   }
+  
+  protected String getSolrXml() {
+    return "solr-no-core.xml";
+  }
 
   
   public CollectionsAPIDistributedZkTest() {
@@ -112,6 +113,7 @@ public class CollectionsAPIDistributedZk
     shardCount = 4;
     completionService = new ExecutorCompletionService<Request>(executor);
     pending = new HashSet<Future<Request>>();
+    checkCreatedVsState = false;
     
   }
   
@@ -134,29 +136,57 @@ public class CollectionsAPIDistributedZk
   
   @Override
   public void doTest() throws Exception {
-    // setLoggingLevel(null);
-
-    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
-    // make sure we have leaders for each shard
-    for (int j = 1; j < sliceCount; j++) {
-      zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
-    }      // make sure we again have leaders for each shard
-    
-    waitForRecoveriesToFinish(false);
-    
-    del("*:*");
-
-    // would be better if these where all separate tests - but much, much
-    // slower
 
+    testNodesUsedByCreate();
     testCollectionsAPI();
 
-    // Thread.sleep(10000000000L);
     if (DEBUG) {
       super.printLayout();
     }
   }
 
+  private void testNodesUsedByCreate() throws Exception {
+    // we can use this client because we just want base url
+    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
+    
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("action", CollectionAction.CREATE.toString());
+
+    params.set("numShards", 2);
+    params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, 2);
+    String collectionName = "nodes_used_collection";
+
+    params.set("name", collectionName);
+    QueryRequest request = new QueryRequest(params);
+    request.setPath("/admin/collections");
+    createNewSolrServer("", baseUrl).request(request);
+    
+    List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
+    numShardsNumReplicaList.add(2);
+    numShardsNumReplicaList.add(2);
+    checkForCollection("nodes_used_collection", numShardsNumReplicaList , null);
+    
+    List<String> createNodeList = new ArrayList<String>();
+
+    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState()
+        .getLiveNodes();
+    
+    for (String node : liveNodes) {
+      createNodeList.add(node);
+    }
+    
+    DocCollection col = cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
+    Collection<Slice> slices = col.getSlices();
+    for (Slice slice : slices) {
+      Collection<Replica> replicas = slice.getReplicas();
+      for (Replica replica : replicas) {
+        createNodeList.remove(replica.getNodeName());
+      }
+    }
+    assertEquals(createNodeList.toString(), 1, createNodeList.size());
+
+  }
+  
   private void testCollectionsAPI() throws Exception {
  
     // TODO: fragile - because we dont pass collection.confName, it will only
@@ -262,10 +292,7 @@ public class CollectionsAPIDistributedZk
     request.setPath("/admin/collections");
     
     // we can use this client because we just want base url
-    final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
-        0,
-        ((HttpSolrServer) clients.get(0)).getBaseURL().length()
-            - DEFAULT_COLLECTION.length() - 1);
+    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
     
     createNewSolrServer("", baseUrl).request(request);
 
@@ -374,38 +401,6 @@ public class CollectionsAPIDistributedZk
     checkNoTwoShardsUseTheSameIndexDir();
   }
 
-
-  protected void createCollection(Map<String,List<Integer>> collectionInfos,
-      String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
-    ModifiableSolrParams params = new ModifiableSolrParams();
-    params.set("action", CollectionAction.CREATE.toString());
-
-    params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
-    params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
-    params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
-    if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
-
-    int clientIndex = random().nextInt(2);
-    List<Integer> list = new ArrayList<Integer>();
-    list.add(numShards);
-    list.add(numReplicas);
-    collectionInfos.put(collectionName, list);
-    params.set("name", collectionName);
-    SolrRequest request = new QueryRequest(params);
-    request.setPath("/admin/collections");
-  
-    if (client == null) {
-      final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
-          0,
-          ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
-              - DEFAULT_COLLECTION.length() - 1);
-      
-      createNewSolrServer("", baseUrl).request(request);
-    } else {
-      client.request(request);
-    }
-  }
-
   private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
     
     
@@ -514,60 +509,6 @@ public class CollectionsAPIDistributedZk
     fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
   }
 
-  private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
-    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-    
-    int expectedSlices = numShardsNumReplicaList.get(0);
-    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
-    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
-    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-    
-      Map<String,DocCollection> collections = clusterState
-          .getCollectionStates();
-      if (collections.containsKey(collectionName)) {
-        Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
-        // did we find expectedSlices slices/shards?
-      if (slices.size() != expectedSlices) {
-        return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
-      }
-      int totalShards = 0;
-      for (String sliceName : slices.keySet()) {
-        for (Replica replica : slices.get(sliceName).getReplicas()) {
-          if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
-            return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
-          }
-        }
-        totalShards += slices.get(sliceName).getReplicas().size();
-      }
-      if (totalShards != expectedTotalShards) {
-        return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards; 
-        }
-      return null;
-    } else {
-      return "Could not find new collection " + collectionName;
-    }
-  }
-  
-  private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
-      throws Exception {
-    // check for an expectedSlices new collection - we poll the state
-    long timeoutAt = System.currentTimeMillis() + 120000;
-    boolean success = false;
-    String checkResult = "Didnt get to perform a single check";
-    while (System.currentTimeMillis() < timeoutAt) {
-      checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
-      if (checkResult == null) {
-        success = true;
-        break;
-      }
-      Thread.sleep(500);
-    }
-    if (!success) {
-      super.printLayout();
-      fail(checkResult);
-      }
-    }
-
   private void checkCollectionIsNotCreated(String collectionName)
     throws Exception {
     // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
@@ -609,65 +550,39 @@ public class CollectionsAPIDistributedZk
         Object value;
         Object indexDir;
         Object name;
+
         try {
-          if (((value = server.getAttribute(mbean, "category")) != null && value.toString().equals(Category.CORE.toString())) &&
-              ((value = server.getAttribute(mbean, "source")) != null && value.toString().contains(SolrCore.class.getSimpleName())) &&
-              ((indexDir = server.getAttribute(mbean, "indexDir")) != null) &&
-              ((name = server.getAttribute(mbean, "name")) != null)) {
-              if (!indexDirToShardNamesMap.containsKey(indexDir.toString())) {
-                indexDirToShardNamesMap.put(indexDir.toString(), new HashSet<String>());
-              }
-              indexDirToShardNamesMap.get(indexDir.toString()).add(name.toString());
+          if (((value = server.getAttribute(mbean, "category")) != null && value
+              .toString().equals(Category.CORE.toString()))
+              && ((indexDir = server.getAttribute(mbean, "coreName")) != null)
+              && ((indexDir = server.getAttribute(mbean, "indexDir")) != null)
+              && ((name = server.getAttribute(mbean, "name")) != null)) {
+            if (!indexDirToShardNamesMap.containsKey(indexDir.toString())) {
+              indexDirToShardNamesMap.put(indexDir.toString(),
+                  new HashSet<String>());
+            }
+            indexDirToShardNamesMap.get(indexDir.toString()).add(
+                name.toString());
           }
         } catch (Exception e) {
-          // ignore, just continue - probably a "category" or "source" attribute not found
+          // ignore, just continue - probably a "category" or "source" attribute
+          // not found
         }
       }
     }
     
-    assertTrue("Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " + SolrCore.class.getSimpleName(), indexDirToShardNamesMap.size() > 0);
-    for (Entry<String, Set<String>> entry : indexDirToShardNamesMap.entrySet()) {
+    assertTrue(
+        "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of "
+            + SolrCore.class.getSimpleName() + " : " + indexDirToShardNamesMap,
+        indexDirToShardNamesMap.size() > 0);
+    for (Entry<String,Set<String>> entry : indexDirToShardNamesMap.entrySet()) {
       if (entry.getValue().size() > 1) {
-        fail("We have shards using the same indexDir. E.g. shards " + entry.getValue().toString() + " all use indexDir " + entry.getKey());
+        fail("We have shards using the same indexDir. E.g. shards "
+            + entry.getValue().toString() + " all use indexDir "
+            + entry.getKey());
       }
     }
-    
-  }
-  
-  protected SolrInputDocument getDoc(Object... fields) throws Exception {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    return doc;
-  }
-  
-  protected SolrServer createNewSolrServer(String collection, String baseUrl) {
-    try {
-      // setup the server...
-      HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
-      s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
-      s.setDefaultMaxConnectionsPerHost(100);
-      s.setMaxTotalConnections(100);
-      return s;
-    }
-    catch (Exception ex) {
-      throw new RuntimeException(ex);
-    }
-  }
 
-  volatile CloudSolrServer commondCloudSolrServer;
-  private CloudSolrServer getCommonCloudSolrServer() {
-    if (commondCloudSolrServer == null) {
-      synchronized(this) {
-        try {
-          commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
-          commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
-          commondCloudSolrServer.connect();
-        } catch (MalformedURLException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-    return commondCloudSolrServer;
   }
 
   @Override
@@ -686,9 +601,6 @@ public class CollectionsAPIDistributedZk
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
-    if (commondCloudSolrServer != null) {
-      commondCloudSolrServer.shutdown();
-    }
     System.clearProperty("numShards");
     System.clearProperty("zkHost");
     System.clearProperty("solr.xml.persist");

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Sun Mar 10 15:04:57 2013
@@ -19,10 +19,11 @@ package org.apache.solr.cloud;
 
 import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
 
 import java.util.ArrayList;
@@ -36,6 +37,8 @@ import java.util.Queue;
 import java.util.Set;
 
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.cloud.DistributedQueue.QueueEvent;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkNodeProps;
@@ -43,11 +46,13 @@ import org.apache.solr.common.cloud.ZkSt
 import org.apache.solr.common.params.CoreAdminParams;
 import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.handler.component.ShardRequest;
 import org.apache.solr.handler.component.ShardResponse;
 import org.easymock.Capture;
+import org.easymock.EasyMock;
 import org.easymock.IAnswer;
 import org.eclipse.jetty.util.BlockingArrayQueue;
 import org.junit.After;
@@ -71,12 +76,12 @@ public class OverseerCollectionProcessor
   private OverseerCollectionProcessorToBeTested underTest;
   
   private Thread thread;
-  private Queue<byte[]> queue = new BlockingArrayQueue<byte[]>();
+  private Queue<QueueEvent> queue = new BlockingArrayQueue<QueueEvent>();
   
   private class OverseerCollectionProcessorToBeTested extends
       OverseerCollectionProcessor {
     
-    private boolean lastProcessMessageResult = true;
+    private SolrResponse lastProcessMessageResult;
     
     public OverseerCollectionProcessorToBeTested(ZkStateReader zkStateReader,
         String myId, ShardHandler shardHandler, String adminPath,
@@ -85,7 +90,7 @@ public class OverseerCollectionProcessor
     }
     
     @Override
-    protected boolean processMessage(ZkNodeProps message, String operation) {
+    protected SolrResponse processMessage(ZkNodeProps message, String operation) {
       lastProcessMessageResult = super.processMessage(message, operation);
       return lastProcessMessageResult;
     }
@@ -147,11 +152,12 @@ public class OverseerCollectionProcessor
       }
     }).anyTimes();
     
-    workQueueMock.remove();
+    workQueueMock.remove(anyObject(QueueEvent.class));
     expectLastCall().andAnswer(new IAnswer<Object>() {
       @Override
       public Object answer() throws Throwable {
-        return queue.poll();
+        queue.remove((QueueEvent)EasyMock.getCurrentArguments()[0]);
+        return null;
       }
     }).anyTimes();
     
@@ -273,7 +279,8 @@ public class OverseerCollectionProcessor
           OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
           maxShardsPerNode.toString());
     }
-    queue.add(ZkStateReader.toJSON(props));
+    QueueEvent qe = new QueueEvent("id", ZkStateReader.toJSON(props), null);
+    queue.add(qe);
   }
   
   protected void verifySubmitCaptures(List<SubmitCapture> submitCaptures,
@@ -443,7 +450,9 @@ public class OverseerCollectionProcessor
     
     waitForEmptyQueue(10000);
     
-    assertEquals(collectionExceptedToBeCreated, underTest.lastProcessMessageResult);
+    if (collectionExceptedToBeCreated) {
+      assertNotNull(underTest.lastProcessMessageResult.getResponse().toString(), underTest.lastProcessMessageResult);
+    }
     verify(shardHandlerMock);
     
     if (collectionExceptedToBeCreated) {

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Sun Mar 10 15:04:57 2013
@@ -50,30 +50,34 @@ public class RecoveryZkTest extends Abst
     
     // start a couple indexing threads
     
-    indexThread = new StopableIndexingThread(0, true);
+    int[] maxDocList = new int[] {300, 700, 1200, 1350, 5000, 15000};
+    
+    int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
+    
+    indexThread = new StopableIndexingThread(0, true, maxDoc);
     indexThread.start();
     
-    indexThread2 = new StopableIndexingThread(10000, true);
+    indexThread2 = new StopableIndexingThread(10000, true, maxDoc);
     
     indexThread2.start();
 
     // give some time to index...
-    Thread.sleep(atLeast(2000));   
-    
+    int[] waitTimes = new int[] {200, 2000, 3000};
+    Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
+     
     // bring shard replica down
     JettySolrRunner replica = chaosMonkey.stopShard("shard1", 1).jetty;
 
     
     // wait a moment - lets allow some docs to be indexed so replication time is non 0
-    Thread.sleep(atLeast(2000));
+    Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
     
     // bring shard replica up
     replica.start();
     
     // make sure replication can start
-    Thread.sleep(1500);
+    Thread.sleep(3000);
     ZkStateReader zkStateReader = cloudClient.getZkStateReader();
-    waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
     
     // stop indexing threads
     indexThread.safeStop();
@@ -86,12 +90,10 @@ public class RecoveryZkTest extends Abst
   
     waitForThingsToLevelOut(30);
     
-    Thread.sleep(1000);
+    Thread.sleep(2000);
     
     waitForThingsToLevelOut(30);
     
-    Thread.sleep(1000);
-    
     waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
 
     // test that leader and replica have same doc count

Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Sun Mar 10 15:04:57 2013
@@ -41,6 +41,7 @@ import org.junit.Ignore;
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 
@@ -117,6 +118,7 @@ public class ShardRoutingTest extends Ab
 
       doHashingTest();
       doTestNumRequests();
+      doAtomicUpdate();
 
       testFinished = true;
     } finally {
@@ -130,6 +132,7 @@ public class ShardRoutingTest extends Ab
 
 
   private void doHashingTest() throws Exception {
+    log.info("### STARTING doHashingTest");
     assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
     String shardKeys = ShardParams.SHARD_KEYS;
     // for now,  we know how ranges will be distributed to shards.
@@ -203,7 +206,12 @@ public class ShardRoutingTest extends Ab
   }
 
 
+
+
+
+
   public void doTestNumRequests() throws Exception {
+    log.info("### STARTING doTestNumRequests");
 
     List<CloudJettyRunner> runners = shardToJetty.get(bucket1);
     CloudJettyRunner leader = shardToLeaderJetty.get(bucket1);
@@ -257,8 +265,23 @@ public class ShardRoutingTest extends Ab
     assertEquals(5, nEnd - nStart);   // original + 2 phase distrib search * 2 shards.
   }
 
+  public void doAtomicUpdate() throws Exception {
+    log.info("### STARTING doAtomicUpdate");
+    int nClients = clients.size();
+    assertEquals(8, nClients);
+
+    int expectedVal = 0;
+    for (SolrServer client : clients) {
+      client.add(sdoc("id", "b!doc", "foo_i", map("inc",1)));
+      expectedVal++;
+
+      QueryResponse rsp = client.query(params("qt","/get", "id","b!doc"));
+      Object val = ((Map)rsp.getResponse().get("doc")).get("foo_i");
+      assertEquals((Integer)expectedVal, val);
+    }
+  }
 
-  long getNumRequests() {
+    long getNumRequests() {
     long n = controlJetty.getDebugFilter().getTotalRequests();
     for (JettySolrRunner jetty : jettys) {
       n += jetty.getDebugFilter().getTotalRequests();