You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC
svn commit: r1454862 [43/49] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml Sun Mar 10 15:04:57 2013
@@ -17,13 +17,6 @@
limitations under the License.
-->
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
- $Source$
- $Name$
-
-Config for testing spellcheck component
- -->
-
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<requestHandler name="standard"
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml Sun Mar 10 15:04:57 2013
@@ -17,11 +17,6 @@
limitations under the License.
-->
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
- $Source$
- $Name$
- -->
-
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<!-- see TestConfig.testLib() -->
@@ -52,7 +47,7 @@
<admin>
<defaultQuery>solr</defaultQuery>
- <gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
+ <gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
</admin>
<!-- test getting system property -->
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Sun Mar 10 15:04:57 2013
@@ -18,10 +18,21 @@
-->
<config>
+ <jmx />
+
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
+ <!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
+ </directoryFactory>
+
<dataDir>${solr.data.dir:}</dataDir>
+ <indexConfig>
+ <lockType>${solr.lock.type:native}</lockType>
+ </indexConfig>
+
<!-- an update processor the explicitly excludes distrib to test
clean errors when people attempt atomic updates w/o it
-->
@@ -38,15 +49,71 @@
<str name="omitHeader">true</str>
</lst>
</requestHandler>
+
+ <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
<updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
- <!-- <str name="dir">/tmp/solr/</str> -->
+ <str name="dir">${solr.ulog.dir:}</str>
</updateLog>
</updateHandler>
+ <updateRequestProcessorChain name="dedupe">
+ <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
+ <bool name="enabled">true</bool>
+ <bool name="overwriteDupes">true</bool>
+ <str name="fields">v_t,t_field</str>
+ <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+ <updateRequestProcessorChain name="stored_sig">
+ <!-- this chain is valid even though the signature field is not
+ indexed, because we are not asking for dups to be overwritten
+ -->
+ <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
+ <bool name="enabled">true</bool>
+ <str name="signatureField">non_indexed_signature_sS</str>
+ <bool name="overwriteDupes">false</bool>
+ <str name="fields">v_t,t_field</str>
+ <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
<requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+ <updateRequestProcessorChain name="distrib-dup-test-chain-explicit">
+ <!-- explicit test using processors before and after distrib -->
+ <processor class="solr.RegexReplaceProcessorFactory">
+ <str name="fieldName">regex_dup_A_s</str>
+ <str name="pattern">x</str>
+ <str name="replacement">x_x</str>
+ </processor>
+ <processor class="solr.DistributedUpdateProcessorFactory" />
+ <processor class="solr.RegexReplaceProcessorFactory">
+ <str name="fieldName">regex_dup_B_s</str>
+ <str name="pattern">x</str>
+ <str name="replacement">x_x</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="distrib-dup-test-chain-implicit">
+ <!-- implicit test w/o distrib declared-->
+ <processor class="solr.RegexReplaceProcessorFactory">
+ <str name="fieldName">regex_dup_A_s</str>
+ <str name="pattern">x</str>
+ <str name="replacement">x_x</str>
+ </processor>
+ <processor class="solr.RegexReplaceProcessorFactory">
+ <str name="fieldName">regex_dup_B_s</str>
+ <str name="pattern">x</str>
+ <str name="replacement">x_x</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
</config>
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Sun Mar 10 15:04:57 2013
@@ -97,6 +97,10 @@
<updateLog enable="${enable.update.log:false}">
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
+
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
</updateHandler>
@@ -486,7 +490,7 @@
<admin>
<defaultQuery>solr</defaultQuery>
- <gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
+ <gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
</admin>
<!-- test getting system property -->
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-config.xml Sun Mar 10 15:04:57 2013
@@ -19,8 +19,6 @@
Striped down solrconfig used by SampleTest to demonstrate picking any
config filename you want.
- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
- $Source$
-->
<config>
@@ -59,7 +57,7 @@
<admin>
<defaultQuery>solr</defaultQuery>
- <gettableFiles>solrconfig.xml scheam.xml</gettableFiles>
+ <gettableFiles>solrconfig.xml schema.xml</gettableFiles>
</admin>
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/crazy-path-to-schema.xml Sun Mar 10 15:04:57 2013
@@ -19,8 +19,6 @@
Striped down schema used by SampleTest to demonstrate picking any
schema filename you want.
- $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
- $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-->
<schema name="test" version="1.0">
Modified: lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test-files/solr/solr.xml Sun Mar 10 15:04:57 2013
@@ -29,15 +29,14 @@
If 'null' (or absent), cores will not be manageable via request handler
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
- hostContext="${hostContext:solr}" zkClientTimeout="30000" numShards="${numShards:3}" shareSchema="${shareSchema:false}"
- distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:30000}">
- <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
+ hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" numShards="${numShards:3}" shareSchema="${shareSchema:false}"
+ distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:120000}">
+ <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"
+ coreNodeName="${coreNodeName:}"/>
<shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
- <int name="socketTimeout">${socketTimeout:30000}</int>
+ <int name="socketTimeout">${socketTimeout:120000}</int>
<int name="connTimeout">${connTimeout:15000}</int>
</shardHandlerFactory>
</cores>
-
-
</solr>
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Sun Mar 10 15:04:57 2013
@@ -252,6 +252,100 @@ public class BasicFunctionalityTest exte
);
}
+
+ /**
+ * verify that delete by query works with the QParser framework and
+ * pure negative queries
+ */
+ public void testNonTrivialDeleteByQuery() throws Exception {
+ clearIndex();
+
+ // setup
+ assertU( add(doc("id","101", "text", "red apple" )) );
+ assertU( add(doc("id","102", "text", "purple grape" )) );
+ assertU( add(doc("id","103", "text", "green grape" )) );
+ assertU( add(doc("id","104", "text", "green pear" )) );
+ assertU( add(doc("id","105", "text", "yellow banana" )) );
+ assertU( add(doc("id","106", "text", "red cherry" )) );
+
+ // sanity checks
+ assertU(commit());
+ assertQ(req("id:[100 TO 110]")
+ ,"//*[@numFound='6']"
+ );
+ assertQ(req("*:*")
+ ,"//*[@numFound='6']"
+ );
+ assertQ(req("text:red")
+ ,"//*[@numFound='2']"
+ );
+ assertQ(req("-text:red")
+ ,"//*[@numFound='4']"
+ );
+ assertQ(req("text:grape")
+ ,"//*[@numFound='2']"
+ );
+ assertQ(req("-text:grape")
+ ,"//*[@numFound='4']"
+ );
+ assertQ(req("-text:red -text:grape")
+ ,"//*[@numFound='2']"
+ );
+ assertQ(req("{!lucene q.op=AND df=text}grape green")
+ ,"//*[@numFound='1']"
+ ,"//int[@name='id'][.='103']"
+ );
+ assertQ(req("-_val_:\"{!lucene q.op=AND df=text}grape green\"")
+ ,"//*[@numFound='5']"
+ ,"//int[@name='id'][.='101']"
+ ,"//int[@name='id'][.='102']"
+ ,"//int[@name='id'][.='104']"
+ ,"//int[@name='id'][.='105']"
+ ,"//int[@name='id'][.='106']"
+ );
+
+ // tests
+
+ assertU(delQ("-*:*")); // NOOP
+ assertU(commit());
+ assertQ(req("*:*")
+ ,"//*[@numFound='6']"
+ );
+
+ assertU(delQ("-text:grape -text:red"));
+ assertU(commit());
+ assertQ(req("*:*")
+ ,"//*[@numFound='4']"
+ ,"//int[@name='id'][.='101']"
+ ,"//int[@name='id'][.='102']"
+ ,"//int[@name='id'][.='103']"
+ ,"//int[@name='id'][.='106']"
+ );
+
+ assertU(delQ("{!term f=id}106"));
+ assertU(commit());
+ assertQ(req("*:*")
+ ,"//*[@numFound='3']"
+ ,"//int[@name='id'][.='101']"
+ ,"//int[@name='id'][.='102']"
+ ,"//int[@name='id'][.='103']"
+ );
+
+ assertU(delQ("-_val_:\"{!lucene q.op=AND df=text}grape green\""));
+ assertU(commit());
+ assertQ(req("*:*")
+ ,"//*[@numFound='1']"
+ ,"//int[@name='id'][.='103']"
+ );
+
+ assertU(delQ("-text:doesnotexist"));
+ assertU(commit());
+ assertQ(req("*:*")
+ ,"//*[@numFound='0']"
+ );
+
+ }
+
@Test
public void testHTMLStrip() {
assertU(add(doc("id","200", "HTMLwhitetok","ABC")));
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Sun Mar 10 15:04:57 2013
@@ -360,6 +360,29 @@ public class TestGroupingSearch extends
);
}
+ @Test
+ public void testGroupingGroupedBasedFacetingWithTaggedFilter() throws Exception {
+ assertU(add(doc("id", "1", "cat_sI", "a", "bday", "2012-11-20T00:00:00Z")));
+ assertU(add(doc("id", "2", "cat_sI", "b", "bday", "2012-11-21T00:00:00Z")));
+ assertU(add(doc("id", "3", "cat_sI", "a", "bday", "2012-11-20T00:00:00Z")));
+ assertU(add(doc("id", "4", "cat_sI", "b", "bday", "2013-01-15T00:00:00Z")));
+ assertU(add(doc("id", "5", "cat_sI", "a", "bday", "2013-01-14T00:00:00Z")));
+ assertU(commit());
+
+ // Facet counts based on groups
+ SolrQueryRequest req = req("q", "*:*", "rows", "1", "group", "true", "group.field", "cat_sI",
+ "sort", "cat_sI asc", "fl", "id", "fq", "{!tag=chk}bday:[2012-12-18T00:00:00Z TO 2013-01-17T23:59:59Z]",
+ "facet", "true", "group.truncate", "true", "group.sort", "bday desc",
+ "facet.query", "{!ex=chk key=LW1}bday:[2013-01-11T00:00:00Z TO 2013-01-17T23:59:59Z]",
+ "facet.query", "{!ex=chk key=LM1}bday:[2012-12-18T00:00:00Z TO 2013-01-17T23:59:59Z]",
+ "facet.query", "{!ex=chk key=LM3}bday:[2012-10-18T00:00:00Z TO 2013-01-17T23:59:59Z]");
+ assertJQ(
+ req,
+ "/grouped=={'cat_sI':{'matches':2,'groups':[{'groupValue':'a','doclist':{'numFound':1,'start':0,'docs':[{'id':'5'}]}}]}}",
+ "/facet_counts=={'facet_queries':{'LW1':2,'LM1':2,'LM3':2},'facet_fields':{},'facet_dates':{},'facet_ranges':{}}"
+ );
+ }
+
static String f = "foo_i";
static String f2 = "foo2_i";
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Sun Mar 10 15:04:57 2013
@@ -16,7 +16,6 @@
*/
package org.apache.solr;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.util.AbstractSolrTestCase;
@@ -37,9 +36,6 @@ import java.util.Properties;
*
* @since solr 1.4
*/
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class
-})
public class TestSolrCoreProperties extends LuceneTestCase {
private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
JettySolrRunner solrJetty;
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/TestTrie.java Sun Mar 10 15:04:57 2013
@@ -16,6 +16,8 @@
*/
package org.apache.solr;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.DateField;
import org.apache.solr.schema.FieldType;
@@ -25,6 +27,7 @@ import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.TimeZone;
@@ -47,6 +50,38 @@ public class TestTrie extends SolrTestCa
clearIndex();
super.tearDown();
}
+
+ @Test
+ public void testTokenizer() throws Exception {
+ FieldType type = h.getCore().getSchema().getFieldType("tint");
+ assertTrue(type instanceof TrieField);
+
+ String value = String.valueOf(random().nextInt());
+ TokenStream ts = type.getAnalyzer().tokenStream("dummy", new StringReader(value));
+ OffsetAttribute ofsAtt = ts.addAttribute(OffsetAttribute.class);
+ ts.reset();
+ int count = 0;
+ while (ts.incrementToken()) {
+ count++;
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ }
+ final int precStep = ((TrieField) type).getPrecisionStep();
+ assertEquals( (32 + precStep - 1) / precStep, count);
+ ts.end();
+ assertEquals(value.length(), ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ ts.close();
+
+ // Test empty one:
+ ts = type.getAnalyzer().tokenStream("dummy", new StringReader(""));
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(0, ofsAtt.endOffset());
+ ts.close();
+ }
@Test
public void testTrieIntRangeSearch() throws Exception {
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Sun Mar 10 15:04:57 2013
@@ -17,13 +17,25 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
@@ -32,7 +44,11 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
/**
@@ -40,15 +56,21 @@ import org.junit.BeforeClass;
* work as expected.
*/
public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
+ private static final String ONE_NODE_COLLECTION = "onenodecollection";
+
@BeforeClass
public static void beforeThisClass2() throws Exception {
- // TODO: we use an fs based dir because something
- // like a ram dir will not recover correctly right now
- // because tran log will still exist on restart and ram
- // dir will not persist - perhaps translog can empty on
- // start if using an EphemeralDirectoryFactory
- useFactory(null);
+
+ }
+
+ public BasicDistributedZk2Test() {
+ super();
+ fixShardCount = true;
+
+ sliceCount = 2;
+ shardCount = 3;
}
+
/*
* (non-Javadoc)
*
@@ -64,6 +86,8 @@ public class BasicDistributedZk2Test ext
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
+ testNodeWithoutCollectionForwarding();
+
indexr(id, 1, i1, 100, tlong, 100, t1,
"now is the time for all good men", "foo_f", 1.414f, "foo_b", "true",
"foo_d", 1.414d);
@@ -139,6 +163,52 @@ public class BasicDistributedZk2Test ext
}
+ private void testNodeWithoutCollectionForwarding() throws Exception,
+ SolrServerException, IOException {
+ try {
+ final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
+ 0,
+ ((HttpSolrServer) clients.get(0)).getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
+ HttpSolrServer server = new HttpSolrServer(baseUrl);
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
+ Create createCmd = new Create();
+ createCmd.setRoles("none");
+ createCmd.setCoreName(ONE_NODE_COLLECTION + "core");
+ createCmd.setCollection(ONE_NODE_COLLECTION);
+ createCmd.setNumShards(1);
+ createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ + ONE_NODE_COLLECTION);
+ server.request(createCmd);
+ } catch (Exception e) {
+ e.printStackTrace();
+ //fail
+ }
+
+ waitForRecoveriesToFinish(ONE_NODE_COLLECTION, cloudClient.getZkStateReader(), false);
+
+ final String baseUrl2 = ((HttpSolrServer) clients.get(1)).getBaseURL().substring(
+ 0,
+ ((HttpSolrServer) clients.get(1)).getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
+ HttpSolrServer qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection" + "core");
+
+ // add a doc
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "1");
+ qclient.add(doc);
+ qclient.commit();
+
+ SolrQuery query = new SolrQuery("*:*");
+ QueryResponse results = qclient.query(query);
+ assertEquals(1, results.getResults().getNumFound());
+
+ qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection");
+ results = qclient.query(query);
+ assertEquals(1, results.getResults().getNumFound());
+ }
+
private long testUpdateAndDelete() throws Exception {
long docId = 99999999L;
indexr("id", docId, t1, "originalcontent");
@@ -187,17 +257,17 @@ public class BasicDistributedZk2Test ext
int oldLiveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
- assertEquals(5, oldLiveNodes);
+ assertEquals(4, oldLiveNodes);
// kill a shard
- CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
+ CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
// we are careful to make sure the downed node is no longer in the state,
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
- jetties.addAll(shardToJetty.get(SHARD2));
+ jetties.addAll(shardToJetty.get(SHARD1));
jetties.remove(deadShard);
// ensure shard is dead
@@ -211,8 +281,6 @@ public class BasicDistributedZk2Test ext
commit();
- printLayout();
-
query("q", "*:*", "sort", "n_tl1 desc");
// long cloudClientDocs = cloudClient.query(new
@@ -224,7 +292,8 @@ public class BasicDistributedZk2Test ext
long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- index_specific(shardToJetty.get(SHARD2).get(1).client.solrClient, id, 1000, i1, 108, t1,
+ cloudClient.getZkStateReader().getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 15000);
+ index_specific(shardToJetty.get(SHARD1).get(1).client.solrClient, id, 1000, i1, 108, t1,
"specific doc!");
commit();
@@ -297,7 +366,7 @@ public class BasicDistributedZk2Test ext
waitForRecoveriesToFinish(false);
- deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+ deadShardCount = shardToJetty.get(SHARD1).get(0).client.solrClient
.query(query).getResults().getNumFound();
// if we properly recovered, we should now have the couple missing docs that
// came in while shard was down
@@ -332,6 +401,100 @@ public class BasicDistributedZk2Test ext
waitForRecoveriesToFinish(false);
checkShardConsistency(true, false);
+
+ // try a backup command
+ final HttpSolrServer client = (HttpSolrServer) shardToJetty.get(SHARD2).get(0).client.solrClient;
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/replication");
+ params.set("command", "backup");
+
+ QueryRequest request = new QueryRequest(params);
+ NamedList<Object> results = client.request(request );
+
+ checkForBackupSuccess(client);
+
+ }
+ private void checkForBackupSuccess(final HttpSolrServer client)
+ throws InterruptedException, IOException {
+ class CheckStatus extends Thread {
+ volatile String fail = null;
+ volatile String response = null;
+ volatile boolean success = false;
+ final Pattern p = Pattern
+ .compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+
+ CheckStatus() {}
+
+ @Override
+ public void run() {
+ String masterUrl = client.getBaseURL() + "/replication?command="
+ + ReplicationHandler.CMD_DETAILS;
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ response = IOUtils.toString(stream, "UTF-8");
+ if (response.contains("<str name=\"status\">success</str>")) {
+ Matcher m = p.matcher(response);
+ if (!m.find()) {
+ fail("could not find the completed timestamp in response.");
+ }
+
+ success = true;
+ }
+ stream.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+
+ };
+ }
+ ;
+ SolrCore core = ((SolrDispatchFilter) shardToJetty.get(SHARD2).get(0).jetty
+ .getDispatchFilter().getFilter()).getCores().getCore("collection1");
+ String ddir;
+ try {
+ ddir = core.getDataDir();
+ } finally {
+ core.close();
+ }
+ File dataDir = new File(ddir);
+
+ int waitCnt = 0;
+ CheckStatus checkStatus = new CheckStatus();
+ while (true) {
+ checkStatus.run();
+ if (checkStatus.fail != null) {
+ fail(checkStatus.fail);
+ }
+ if (checkStatus.success) {
+ break;
+ }
+ Thread.sleep(200);
+ if (waitCnt == 20) {
+ fail("Backup success not detected:" + checkStatus.response);
+ }
+ waitCnt++;
+ }
+
+ File[] files = dataDir.listFiles(new FilenameFilter() {
+
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.startsWith("snapshot")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(Arrays.asList(files).toString(), 1, files.length);
+ File snapDir = files[0];
+
+ AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
}
private void addNewReplica() throws Exception {
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Sun Mar 10 15:04:57 2013
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@@ -54,8 +53,6 @@ import org.apache.solr.common.SolrDocume
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -79,7 +76,7 @@ import org.junit.BeforeClass;
public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
private static final String DEFAULT_COLLECTION = "collection1";
- private static final boolean DEBUG = false;
+ protected static final boolean DEBUG = false;
String t1="a_t";
String i1="a_si";
String nint = "n_i";
@@ -111,9 +108,6 @@ public class BasicDistributedZkTest exte
@BeforeClass
public static void beforeThisClass2() throws Exception {
- // TODO: we use an fs based dir because something
- // like a ram dir will not recover correctly right now
- useFactory(null);
}
@Before
@@ -328,335 +322,26 @@ public class BasicDistributedZkTest exte
testMultipleCollections();
testANewCollectionInOneInstance();
testSearchByCollectionName();
+ testUpdateByCollectionName();
testANewCollectionInOneInstanceWithManualShardAssignement();
testNumberOfCommitsWithCommitAfterAdd();
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
- testCoreUnloadAndLeaders();
- testUnloadLotsOfCores();
testStopAndStartCoresInOneInstance();
- testUnloadShardAndCollection();
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
-
- private void testUnloadShardAndCollection() throws Exception{
- // create one leader and one replica
-
- Create createCmd = new Create();
- createCmd.setCoreName("test_unload_shard_and_collection_1");
- String collection = "test_unload_shard_and_collection";
- createCmd.setCollection(collection);
- String coreDataDir = dataDir.getAbsolutePath() + File.separator
- + System.currentTimeMillis() + collection + "1";
- createCmd.setDataDir(coreDataDir);
- createCmd.setNumShards(2);
-
- SolrServer client = clients.get(0);
- String url1 = getBaseUrl(client);
- HttpSolrServer server = new HttpSolrServer(url1);
- server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
- server.request(createCmd);
-
- createCmd = new Create();
- createCmd.setCoreName("test_unload_shard_and_collection_2");
- collection = "test_unload_shard_and_collection";
- createCmd.setCollection(collection);
- coreDataDir = dataDir.getAbsolutePath() + File.separator
- + System.currentTimeMillis() + collection + "2";
- createCmd.setDataDir(coreDataDir);
-
- server.request(createCmd);
-
- // now unload one of the two
- Unload unloadCmd = new Unload(false);
- unloadCmd.setCoreName("test_unload_shard_and_collection_2");
- server.request(unloadCmd);
-
- // there should be only one shard
- Slice shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
- long timeoutAt = System.currentTimeMillis() + 30000;
- while (shard2 != null) {
- if (System.currentTimeMillis() > timeoutAt) {
- printLayout();
- fail("Still found shard");
- }
-
- Thread.sleep(50);
- shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
- }
-
- Slice shard1 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard1");
- assertNotNull(shard1);
- assertTrue(getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection));
-
- // now unload one of the other
- unloadCmd = new Unload(false);
- unloadCmd.setCoreName("test_unload_shard_and_collection_1");
- server.request(unloadCmd);
-
- //printLayout();
- // the collection should be gone
- timeoutAt = System.currentTimeMillis() + 30000;
- while (getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection)) {
- if (System.currentTimeMillis() > timeoutAt) {
- printLayout();
- fail("Still found collection");
- }
-
- Thread.sleep(50);
- }
-
- }
-
- /**
- * @throws Exception on any problem
- */
- private void testCoreUnloadAndLeaders() throws Exception {
- // create a new collection collection
- SolrServer client = clients.get(0);
- String url1 = getBaseUrl(client);
- HttpSolrServer server = new HttpSolrServer(url1);
- server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
-
- Create createCmd = new Create();
- createCmd.setCoreName("unloadcollection1");
- createCmd.setCollection("unloadcollection");
- createCmd.setNumShards(1);
- String core1DataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_1n";
- createCmd.setDataDir(core1DataDir);
- server.request(createCmd);
-
- ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
-
- zkStateReader.updateClusterState(true);
-
- int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
- assertEquals(1, slices);
-
- client = clients.get(1);
- String url2 = getBaseUrl(client);
- server = new HttpSolrServer(url2);
-
- createCmd = new Create();
- createCmd.setCoreName("unloadcollection2");
- createCmd.setCollection("unloadcollection");
- String core2dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_2n";
- createCmd.setDataDir(core2dataDir);
- server.request(createCmd);
-
- zkStateReader.updateClusterState(true);
- slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
- assertEquals(1, slices);
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- ZkCoreNodeProps leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
-
- Random random = random();
- HttpSolrServer collectionClient;
- if (random.nextBoolean()) {
- collectionClient = new HttpSolrServer(leaderProps.getCoreUrl());
- // lets try and use the solrj client to index and retrieve a couple
- // documents
- SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
- "humpty dumpy sat on a wall");
- SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
- "humpty dumpy3 sat on a walls");
- SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
- "humpty dumpy2 sat on a walled");
- collectionClient.add(doc1);
- collectionClient.add(doc2);
- collectionClient.add(doc3);
- collectionClient.commit();
- }
-
- // create another replica for our collection
- client = clients.get(2);
- String url3 = getBaseUrl(client);
- server = new HttpSolrServer(url3);
-
- createCmd = new Create();
- createCmd.setCoreName("unloadcollection3");
- createCmd.setCollection("unloadcollection");
- String core3dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_3n";
- createCmd.setDataDir(core3dataDir);
- server.request(createCmd);
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- // so that we start with some versions when we reload...
- DirectUpdateHandler2.commitOnClose = false;
-
- HttpSolrServer addClient = new HttpSolrServer(url3 + "/unloadcollection3");
- // add a few docs
- for (int x = 20; x < 100; x++) {
- SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1,
- "humpty dumpy sat on a wall");
- addClient.add(doc1);
- }
-
- // don't commit so they remain in the tran log
- //collectionClient.commit();
-
- // unload the leader
- collectionClient = new HttpSolrServer(leaderProps.getBaseUrl());
-
- Unload unloadCmd = new Unload(false);
- unloadCmd.setCoreName(leaderProps.getCoreName());
- ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
-
- collectionClient.request(unloadCmd);
-
-// Thread.currentThread().sleep(500);
-// printLayout();
-
- int tries = 50;
- while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("Leader never changed");
- }
- }
-
- // ensure there is a leader
- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-
- addClient = new HttpSolrServer(url2 + "/unloadcollection2");
- // add a few docs while the leader is down
- for (int x = 101; x < 200; x++) {
- SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1,
- "humpty dumpy sat on a wall");
- addClient.add(doc1);
- }
-
-
- // create another replica for our collection
- client = clients.get(3);
- String url4 = getBaseUrl(client);
- server = new HttpSolrServer(url4);
-
- createCmd = new Create();
- createCmd.setCoreName("unloadcollection4");
- createCmd.setCollection("unloadcollection");
- String core4dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_4n";
- createCmd.setDataDir(core4dataDir);
- server.request(createCmd);
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
- // unload the leader again
- leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
- collectionClient = new HttpSolrServer(leaderProps.getBaseUrl());
-
- unloadCmd = new Unload(false);
- unloadCmd.setCoreName(leaderProps.getCoreName());
- p = (ModifiableSolrParams) unloadCmd.getParams();
- collectionClient.request(unloadCmd);
-
- tries = 50;
- while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("Leader never changed");
- }
- }
-
- zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
-
-
- // set this back
- DirectUpdateHandler2.commitOnClose = true;
-
- // bring the downed leader back as replica
- server = new HttpSolrServer(leaderProps.getBaseUrl());
-
- createCmd = new Create();
- createCmd.setCoreName(leaderProps.getCoreName());
- createCmd.setCollection("unloadcollection");
- createCmd.setDataDir(core1DataDir);
- server.request(createCmd);
-
- waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
-
-
- server = new HttpSolrServer(url1 + "/unloadcollection");
- // System.out.println(server.query(new SolrQuery("*:*")).getResults().getNumFound());
- server = new HttpSolrServer(url2 + "/unloadcollection");
- server.commit();
- SolrQuery q = new SolrQuery("*:*");
- q.set("distrib", false);
- long found1 = server.query(q).getResults().getNumFound();
- server = new HttpSolrServer(url3 + "/unloadcollection");
- server.commit();
- q = new SolrQuery("*:*");
- q.set("distrib", false);
- long found3 = server.query(q).getResults().getNumFound();
- server = new HttpSolrServer(url4 + "/unloadcollection");
- server.commit();
- q = new SolrQuery("*:*");
- q.set("distrib", false);
- long found4 = server.query(q).getResults().getNumFound();
-
- // all 3 shards should now have the same number of docs
- assertEquals(found1, found3);
- assertEquals(found3, found4);
-
- }
-
- private void testUnloadLotsOfCores() throws Exception {
- SolrServer client = clients.get(2);
- String url3 = getBaseUrl(client);
- final HttpSolrServer server = new HttpSolrServer(url3);
- server.setConnectionTimeout(15000);
- server.setSoTimeout(60000);
- ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
- 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new DefaultSolrThreadFactory("testExecutor"));
- int cnt = random().nextInt(12) + 1;
-
- // create the cores
- createCores(server, executor, "multiunload", 2, cnt);
-
- executor.shutdown();
- executor.awaitTermination(120, TimeUnit.SECONDS);
- executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new DefaultSolrThreadFactory("testExecutor"));
- for (int j = 0; j < cnt; j++) {
- final int freezeJ = j;
- executor.execute(new Runnable() {
- @Override
- public void run() {
- Unload unloadCmd = new Unload(true);
- unloadCmd.setCoreName("multiunload" + freezeJ);
- try {
- server.request(unloadCmd);
- } catch (SolrServerException e) {
- throw new RuntimeException(e);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- });
- Thread.sleep(random().nextInt(50));
- }
- executor.shutdown();
- executor.awaitTermination(120, TimeUnit.SECONDS);
- }
private void testStopAndStartCoresInOneInstance() throws Exception {
SolrServer client = clients.get(0);
String url3 = getBaseUrl(client);
final HttpSolrServer server = new HttpSolrServer(url3);
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
@@ -685,7 +370,7 @@ public class BasicDistributedZkTest exte
}
- private void createCores(final HttpSolrServer server,
+ protected void createCores(final HttpSolrServer server,
ThreadPoolExecutor executor, final String collection, final int numShards, int cnt) {
for (int i = 0; i < cnt; i++) {
final int freezeI = i;
@@ -713,7 +398,7 @@ public class BasicDistributedZkTest exte
}
}
- private String getBaseUrl(SolrServer client) {
+ protected String getBaseUrl(SolrServer client) {
String url2 = ((HttpSolrServer) client).getBaseURL()
.substring(
0,
@@ -753,7 +438,7 @@ public class BasicDistributedZkTest exte
}
}
- private ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
+ protected ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
ZkNodeProps leader = clusterState.getLeader(collection, slice);
if (leader == null) {
@@ -761,89 +446,7 @@ public class BasicDistributedZkTest exte
}
return new ZkCoreNodeProps(leader);
}
-
- private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-
- int expectedSlices = numShardsNumReplicaList.get(0);
- // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
- int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
- int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-
- Map<String,DocCollection> collections = clusterState
- .getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
- // did we find expectedSlices slices/shards?
- if (slices.size() != expectedSlices) {
- return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
- }
- int totalShards = 0;
- for (String sliceName : slices.keySet()) {
- for (Replica replica : slices.get(sliceName).getReplicas()) {
- if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
- }
- }
- totalShards += slices.get(sliceName).getReplicas().size();
- }
- if (totalShards != expectedTotalShards) {
- return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
- }
- return null;
- } else {
- return "Could not find new collection " + collectionName;
- }
- }
- private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
- throws Exception {
- // check for an expectedSlices new collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 120000;
- boolean success = false;
- String checkResult = "Didnt get to perform a single check";
- while (System.currentTimeMillis() < timeoutAt) {
- checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
- if (checkResult == null) {
- success = true;
- break;
- }
- Thread.sleep(500);
- }
- if (!success) {
- super.printLayout();
- fail(checkResult);
- }
- }
-
- private void checkCollectionIsNotCreated(String collectionName)
- throws Exception {
- // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
- Thread.sleep(10000);
- assertFalse(collectionName + " not supposed to exist", getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collectionName));
- }
-
- private void checkForMissingCollection(String collectionName)
- throws Exception {
- // check for a collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 30000;
- boolean found = true;
- while (System.currentTimeMillis() < timeoutAt) {
- getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
- Map<String,DocCollection> collections = clusterState
- .getCollectionStates();
- if (!collections.containsKey(collectionName)) {
- found = false;
- break;
- }
- Thread.sleep(100);
- }
- if (found) {
- fail("Found collection that should be gone " + collectionName);
- }
- }
-
/**
* Expects a RegexReplaceProcessorFactories in the chain which will
* "double up" the values in two (stored) string fields.
@@ -1045,6 +648,8 @@ public class BasicDistributedZkTest exte
// now test that unloading a core gets us a new leader
HttpSolrServer server = new HttpSolrServer(baseUrl);
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Unload unloadCmd = new Unload(true);
unloadCmd.setCoreName(props.getCoreName());
@@ -1077,6 +682,21 @@ public class BasicDistributedZkTest exte
long oneDocs = client1.query(query).getResults().getNumFound();
assertEquals(3, oneDocs);
}
+
+ private void testUpdateByCollectionName() throws SolrServerException, IOException {
+ log.info("### STARTING testUpdateByCollectionName");
+ SolrServer client = clients.get(0);
+ final String baseUrl = ((HttpSolrServer) client).getBaseURL().substring(
+ 0,
+ ((HttpSolrServer) client).getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
+
+ // the cores each have different names, but if we add the collection name to the url
+ // we should get mapped to the right core
+ // test hitting an update url
+ SolrServer client1 = createNewSolrServer(oneInstanceCollection, baseUrl);
+ client1.commit();
+ }
private void testANewCollectionInOneInstance() throws Exception {
log.info("### STARTING testANewCollectionInOneInstance");
@@ -1317,7 +937,7 @@ public class BasicDistributedZkTest exte
}
volatile CloudSolrServer commondCloudSolrServer;
- private CloudSolrServer getCommonCloudSolrServer() {
+ protected CloudSolrServer getCommonCloudSolrServer() {
if (commondCloudSolrServer == null) {
synchronized(this) {
try {
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Sun Mar 10 15:04:57 2013
@@ -21,8 +21,10 @@ import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.http.client.HttpClient;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
+
+import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@@ -30,27 +32,39 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.Diagnostics;
+import org.apache.solr.update.SolrCmdDistributor;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Slow
-@Ignore("ignore while investigating jenkins fails")
+@BadApple
public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
- private static final int BASE_RUN_LENGTH = 60000;
+ private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
@BeforeClass
public static void beforeSuperClass() {
+ SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
+ @Override
+ public void call(Object... data) {
+ SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
+ if (sreq.exception == null) return;
+ if (sreq.exception.getMessage().contains("Timeout")) {
+ Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+ }
+ }
+ };
}
@AfterClass
public static void afterSuperClass() {
+ SolrCmdDistributor.testing_errorHook = null;
}
@Before
@@ -60,6 +74,7 @@ public class ChaosMonkeyNothingIsSafeTes
// can help to hide this when testing and looking at logs
//ignoreException("shard update error");
System.setProperty("numShards", Integer.toString(sliceCount));
+ useFactory("solr.StandardDirectoryFactory");
}
@Override
@@ -72,8 +87,8 @@ public class ChaosMonkeyNothingIsSafeTes
public ChaosMonkeyNothingIsSafeTest() {
super();
- sliceCount = 1;
- shardCount = 7;
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "2"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "7"));
}
@Override
@@ -100,7 +115,7 @@ public class ChaosMonkeyNothingIsSafeTes
int i = 0;
for (i = 0; i < threadCount; i++) {
StopableIndexingThread indexThread = new StopableIndexingThread(
- i * 50000, true);
+ (i+1) * 50000, true);
threads.add(indexThread);
indexThread.start();
}
@@ -117,14 +132,21 @@ public class ChaosMonkeyNothingIsSafeTes
boolean runFullThrottle = random().nextBoolean();
if (runFullThrottle) {
FullThrottleStopableIndexingThread ftIndexThread = new FullThrottleStopableIndexingThread(
- clients, i * 50000, true);
+ clients, (i+1) * 50000, true);
threads.add(ftIndexThread);
ftIndexThread.start();
}
chaosMonkey.startTheMonkey(true, 10000);
- //int runLength = atLeast(BASE_RUN_LENGTH);
- int runLength = BASE_RUN_LENGTH;
+
+ long runLength;
+ if (RUN_LENGTH != -1) {
+ runLength = RUN_LENGTH;
+ } else {
+ int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000};
+ runLength = runTimes[random().nextInt(runTimes.length - 1)];
+ }
+
try {
Thread.sleep(runLength);
} finally {
@@ -140,12 +162,13 @@ public class ChaosMonkeyNothingIsSafeTes
indexThread.join();
}
- // we expect full throttle fails, but not cloud client...
- for (StopableThread indexThread : threads) {
- if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
- //assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
- }
- }
+ // we expect full throttle fails, but cloud client should not easily fail
+ // but it's allowed to fail and sometimes does, so commented out for now
+// for (StopableThread indexThread : threads) {
+// if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
+// assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
+// }
+// }
// try and wait for any replications and what not to finish...
@@ -156,7 +179,7 @@ public class ChaosMonkeyNothingIsSafeTes
// make sure we again have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
}
commit();
@@ -307,12 +330,5 @@ public class ChaosMonkeyNothingIsSafeTes
SolrInputDocument doc = getDoc(fields);
indexDoc(doc);
}
-
- SolrInputDocument getDoc(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- return doc;
- }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Sun Mar 10 15:04:57 2013
@@ -20,30 +20,41 @@ package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.Diagnostics;
import org.apache.solr.core.SolrCore;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.update.SolrCmdDistributor;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
-@Ignore("SOLR-3126")
-public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
+@Slow
+public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
- private static final int BASE_RUN_LENGTH = 120000;
+ private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
@BeforeClass
public static void beforeSuperClass() {
-
+ SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
+ @Override
+ public void call(Object... data) {
+ SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
+ if (sreq.exception == null) return;
+ if (sreq.exception.getMessage().contains("Timeout")) {
+ Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+ }
+ }
+ };
}
@AfterClass
public static void afterSuperClass() {
-
+ SolrCmdDistributor.testing_errorHook = null;
}
@Before
@@ -66,8 +77,8 @@ public class ChaosMonkeySafeLeaderTest e
public ChaosMonkeySafeLeaderTest() {
super();
- sliceCount = 3;//atLeast(2);
- shardCount = 12;//atLeast(sliceCount*2);
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "3"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "12"));
}
@Override
@@ -85,16 +96,24 @@ public class ChaosMonkeySafeLeaderTest e
List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
int threadCount = 2;
for (int i = 0; i < threadCount; i++) {
- StopableIndexingThread indexThread = new StopableIndexingThread(i * 50000, true);
+ StopableIndexingThread indexThread = new StopableIndexingThread(10000 + i*50000, true);
threads.add(indexThread);
indexThread.start();
}
chaosMonkey.startTheMonkey(false, 500);
- int runLength = atLeast(BASE_RUN_LENGTH);
- Thread.sleep(runLength);
-
- chaosMonkey.stopTheMonkey();
+ long runLength;
+ if (RUN_LENGTH != -1) {
+ runLength = RUN_LENGTH;
+ } else {
+ int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000};
+ runLength = runTimes[random().nextInt(runTimes.length - 1)];
+ }
+ try {
+ Thread.sleep(runLength);
+ } finally {
+ chaosMonkey.stopTheMonkey();
+ }
for (StopableIndexingThread indexThread : threads) {
indexThread.safeStop();
@@ -113,7 +132,7 @@ public class ChaosMonkeySafeLeaderTest e
Thread.sleep(2000);
- waitForThingsToLevelOut(Integer.MAX_VALUE); //Math.round((runLength / 1000.0f / 3.0f)));
+ waitForThingsToLevelOut(180000);
checkShardConsistency(true, true);
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Sun Mar 10 15:04:57 2013
@@ -254,7 +254,7 @@ public class ClusterStateUpdateTest exte
System.clearProperty("solrcloud.update.delay");
}
- private void printLayout(String zkHost) throws Exception {
+ static void printLayout(String zkHost) throws Exception {
SolrZkClient zkClient = new SolrZkClient(
zkHost, AbstractZkTestCase.TIMEOUT);
zkClient.printLayoutToStdOut();
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Sun Mar 10 15:04:57 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.MalformedURLException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -43,8 +44,6 @@ import javax.management.ObjectName;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util._TestUtil;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -91,9 +90,7 @@ public class CollectionsAPIDistributedZk
@BeforeClass
public static void beforeThisClass2() throws Exception {
- // TODO: we use an fs based dir because something
- // like a ram dir will not recover correctly right now
- useFactory(null);
+
}
@Before
@@ -103,6 +100,10 @@ public class CollectionsAPIDistributedZk
System.setProperty("numShards", Integer.toString(sliceCount));
System.setProperty("solr.xml.persist", "true");
}
+
+ protected String getSolrXml() {
+ return "solr-no-core.xml";
+ }
public CollectionsAPIDistributedZkTest() {
@@ -112,6 +113,7 @@ public class CollectionsAPIDistributedZk
shardCount = 4;
completionService = new ExecutorCompletionService<Request>(executor);
pending = new HashSet<Future<Request>>();
+ checkCreatedVsState = false;
}
@@ -134,29 +136,57 @@ public class CollectionsAPIDistributedZk
@Override
public void doTest() throws Exception {
- // setLoggingLevel(null);
-
- ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- // make sure we have leaders for each shard
- for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
- } // make sure we again have leaders for each shard
-
- waitForRecoveriesToFinish(false);
-
- del("*:*");
-
- // would be better if these where all separate tests - but much, much
- // slower
+ testNodesUsedByCreate();
testCollectionsAPI();
- // Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
+ private void testNodesUsedByCreate() throws Exception {
+ // we can use this client because we just want base url
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.CREATE.toString());
+
+ params.set("numShards", 2);
+ params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, 2);
+ String collectionName = "nodes_used_collection";
+
+ params.set("name", collectionName);
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ createNewSolrServer("", baseUrl).request(request);
+
+ List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
+ numShardsNumReplicaList.add(2);
+ numShardsNumReplicaList.add(2);
+ checkForCollection("nodes_used_collection", numShardsNumReplicaList , null);
+
+ List<String> createNodeList = new ArrayList<String>();
+
+ Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState()
+ .getLiveNodes();
+
+ for (String node : liveNodes) {
+ createNodeList.add(node);
+ }
+
+ DocCollection col = cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
+ Collection<Slice> slices = col.getSlices();
+ for (Slice slice : slices) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ createNodeList.remove(replica.getNodeName());
+ }
+ }
+ assertEquals(createNodeList.toString(), 1, createNodeList.size());
+
+ }
+
private void testCollectionsAPI() throws Exception {
// TODO: fragile - because we dont pass collection.confName, it will only
@@ -262,10 +292,7 @@ public class CollectionsAPIDistributedZk
request.setPath("/admin/collections");
// we can use this client because we just want base url
- final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(0)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
createNewSolrServer("", baseUrl).request(request);
@@ -374,38 +401,6 @@ public class CollectionsAPIDistributedZk
checkNoTwoShardsUseTheSameIndexDir();
}
-
- protected void createCollection(Map<String,List<Integer>> collectionInfos,
- String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
-
- params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
- params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
- if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
-
- int clientIndex = random().nextInt(2);
- List<Integer> list = new ArrayList<Integer>();
- list.add(numShards);
- list.add(numReplicas);
- collectionInfos.put(collectionName, list);
- params.set("name", collectionName);
- SolrRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- if (client == null) {
- final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
- } else {
- client.request(request);
- }
- }
-
private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
@@ -514,60 +509,6 @@ public class CollectionsAPIDistributedZk
fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
- private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-
- int expectedSlices = numShardsNumReplicaList.get(0);
- // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
- int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
- int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-
- Map<String,DocCollection> collections = clusterState
- .getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
- // did we find expectedSlices slices/shards?
- if (slices.size() != expectedSlices) {
- return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
- }
- int totalShards = 0;
- for (String sliceName : slices.keySet()) {
- for (Replica replica : slices.get(sliceName).getReplicas()) {
- if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
- }
- }
- totalShards += slices.get(sliceName).getReplicas().size();
- }
- if (totalShards != expectedTotalShards) {
- return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
- }
- return null;
- } else {
- return "Could not find new collection " + collectionName;
- }
- }
-
- private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
- throws Exception {
- // check for an expectedSlices new collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 120000;
- boolean success = false;
- String checkResult = "Didnt get to perform a single check";
- while (System.currentTimeMillis() < timeoutAt) {
- checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
- if (checkResult == null) {
- success = true;
- break;
- }
- Thread.sleep(500);
- }
- if (!success) {
- super.printLayout();
- fail(checkResult);
- }
- }
-
private void checkCollectionIsNotCreated(String collectionName)
throws Exception {
// TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
@@ -609,65 +550,39 @@ public class CollectionsAPIDistributedZk
Object value;
Object indexDir;
Object name;
+
try {
- if (((value = server.getAttribute(mbean, "category")) != null && value.toString().equals(Category.CORE.toString())) &&
- ((value = server.getAttribute(mbean, "source")) != null && value.toString().contains(SolrCore.class.getSimpleName())) &&
- ((indexDir = server.getAttribute(mbean, "indexDir")) != null) &&
- ((name = server.getAttribute(mbean, "name")) != null)) {
- if (!indexDirToShardNamesMap.containsKey(indexDir.toString())) {
- indexDirToShardNamesMap.put(indexDir.toString(), new HashSet<String>());
- }
- indexDirToShardNamesMap.get(indexDir.toString()).add(name.toString());
+ if (((value = server.getAttribute(mbean, "category")) != null && value
+ .toString().equals(Category.CORE.toString()))
+ && ((indexDir = server.getAttribute(mbean, "coreName")) != null)
+ && ((indexDir = server.getAttribute(mbean, "indexDir")) != null)
+ && ((name = server.getAttribute(mbean, "name")) != null)) {
+ if (!indexDirToShardNamesMap.containsKey(indexDir.toString())) {
+ indexDirToShardNamesMap.put(indexDir.toString(),
+ new HashSet<String>());
+ }
+ indexDirToShardNamesMap.get(indexDir.toString()).add(
+ name.toString());
}
} catch (Exception e) {
- // ignore, just continue - probably a "category" or "source" attribute not found
+ // ignore, just continue - probably a "category" or "source" attribute
+ // not found
}
}
}
- assertTrue("Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " + SolrCore.class.getSimpleName(), indexDirToShardNamesMap.size() > 0);
- for (Entry<String, Set<String>> entry : indexDirToShardNamesMap.entrySet()) {
+ assertTrue(
+ "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of "
+ + SolrCore.class.getSimpleName() + " : " + indexDirToShardNamesMap,
+ indexDirToShardNamesMap.size() > 0);
+ for (Entry<String,Set<String>> entry : indexDirToShardNamesMap.entrySet()) {
if (entry.getValue().size() > 1) {
- fail("We have shards using the same indexDir. E.g. shards " + entry.getValue().toString() + " all use indexDir " + entry.getKey());
+ fail("We have shards using the same indexDir. E.g. shards "
+ + entry.getValue().toString() + " all use indexDir "
+ + entry.getKey());
}
}
-
- }
-
- protected SolrInputDocument getDoc(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- return doc;
- }
-
- protected SolrServer createNewSolrServer(String collection, String baseUrl) {
- try {
- // setup the server...
- HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
- s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
- s.setDefaultMaxConnectionsPerHost(100);
- s.setMaxTotalConnections(100);
- return s;
- }
- catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
- volatile CloudSolrServer commondCloudSolrServer;
- private CloudSolrServer getCommonCloudSolrServer() {
- if (commondCloudSolrServer == null) {
- synchronized(this) {
- try {
- commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
- commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
- commondCloudSolrServer.connect();
- } catch (MalformedURLException e) {
- throw new RuntimeException(e);
- }
- }
- }
- return commondCloudSolrServer;
}
@Override
@@ -686,9 +601,6 @@ public class CollectionsAPIDistributedZk
@Override
public void tearDown() throws Exception {
super.tearDown();
- if (commondCloudSolrServer != null) {
- commondCloudSolrServer.shutdown();
- }
System.clearProperty("numShards");
System.clearProperty("zkHost");
System.clearProperty("solr.xml.persist");
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Sun Mar 10 15:04:57 2013
@@ -19,10 +19,11 @@ package org.apache.solr.cloud;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
import java.util.ArrayList;
@@ -36,6 +37,8 @@ import java.util.Queue;
import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.cloud.DistributedQueue.QueueEvent;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -43,11 +46,13 @@ import org.apache.solr.common.cloud.ZkSt
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
import org.easymock.Capture;
+import org.easymock.EasyMock;
import org.easymock.IAnswer;
import org.eclipse.jetty.util.BlockingArrayQueue;
import org.junit.After;
@@ -71,12 +76,12 @@ public class OverseerCollectionProcessor
private OverseerCollectionProcessorToBeTested underTest;
private Thread thread;
- private Queue<byte[]> queue = new BlockingArrayQueue<byte[]>();
+ private Queue<QueueEvent> queue = new BlockingArrayQueue<QueueEvent>();
private class OverseerCollectionProcessorToBeTested extends
OverseerCollectionProcessor {
- private boolean lastProcessMessageResult = true;
+ private SolrResponse lastProcessMessageResult;
public OverseerCollectionProcessorToBeTested(ZkStateReader zkStateReader,
String myId, ShardHandler shardHandler, String adminPath,
@@ -85,7 +90,7 @@ public class OverseerCollectionProcessor
}
@Override
- protected boolean processMessage(ZkNodeProps message, String operation) {
+ protected SolrResponse processMessage(ZkNodeProps message, String operation) {
lastProcessMessageResult = super.processMessage(message, operation);
return lastProcessMessageResult;
}
@@ -147,11 +152,12 @@ public class OverseerCollectionProcessor
}
}).anyTimes();
- workQueueMock.remove();
+ workQueueMock.remove(anyObject(QueueEvent.class));
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
- return queue.poll();
+ queue.remove((QueueEvent)EasyMock.getCurrentArguments()[0]);
+ return null;
}
}).anyTimes();
@@ -273,7 +279,8 @@ public class OverseerCollectionProcessor
OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
maxShardsPerNode.toString());
}
- queue.add(ZkStateReader.toJSON(props));
+ QueueEvent qe = new QueueEvent("id", ZkStateReader.toJSON(props), null);
+ queue.add(qe);
}
protected void verifySubmitCaptures(List<SubmitCapture> submitCaptures,
@@ -443,7 +450,9 @@ public class OverseerCollectionProcessor
waitForEmptyQueue(10000);
- assertEquals(collectionExceptedToBeCreated, underTest.lastProcessMessageResult);
+ if (collectionExceptedToBeCreated) {
+ assertNotNull(underTest.lastProcessMessageResult.getResponse().toString(), underTest.lastProcessMessageResult);
+ }
verify(shardHandlerMock);
if (collectionExceptedToBeCreated) {
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Sun Mar 10 15:04:57 2013
@@ -50,30 +50,34 @@ public class RecoveryZkTest extends Abst
// start a couple indexing threads
- indexThread = new StopableIndexingThread(0, true);
+ int[] maxDocList = new int[] {300, 700, 1200, 1350, 5000, 15000};
+
+ int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
+
+ indexThread = new StopableIndexingThread(0, true, maxDoc);
indexThread.start();
- indexThread2 = new StopableIndexingThread(10000, true);
+ indexThread2 = new StopableIndexingThread(10000, true, maxDoc);
indexThread2.start();
// give some time to index...
- Thread.sleep(atLeast(2000));
-
+ int[] waitTimes = new int[] {200, 2000, 3000};
+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
+
// bring shard replica down
JettySolrRunner replica = chaosMonkey.stopShard("shard1", 1).jetty;
// wait a moment - lets allow some docs to be indexed so replication time is non 0
- Thread.sleep(atLeast(2000));
+ Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
// bring shard replica up
replica.start();
// make sure replication can start
- Thread.sleep(1500);
+ Thread.sleep(3000);
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
// stop indexing threads
indexThread.safeStop();
@@ -86,12 +90,10 @@ public class RecoveryZkTest extends Abst
waitForThingsToLevelOut(30);
- Thread.sleep(1000);
+ Thread.sleep(2000);
waitForThingsToLevelOut(30);
- Thread.sleep(1000);
-
waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
// test that leader and replica have same doc count
Modified: lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Sun Mar 10 15:04:57 2013
@@ -41,6 +41,7 @@ import org.junit.Ignore;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
@@ -117,6 +118,7 @@ public class ShardRoutingTest extends Ab
doHashingTest();
doTestNumRequests();
+ doAtomicUpdate();
testFinished = true;
} finally {
@@ -130,6 +132,7 @@ public class ShardRoutingTest extends Ab
private void doHashingTest() throws Exception {
+ log.info("### STARTING doHashingTest");
assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
String shardKeys = ShardParams.SHARD_KEYS;
// for now, we know how ranges will be distributed to shards.
@@ -203,7 +206,12 @@ public class ShardRoutingTest extends Ab
}
+
+
+
+
public void doTestNumRequests() throws Exception {
+ log.info("### STARTING doTestNumRequests");
List<CloudJettyRunner> runners = shardToJetty.get(bucket1);
CloudJettyRunner leader = shardToLeaderJetty.get(bucket1);
@@ -257,8 +265,23 @@ public class ShardRoutingTest extends Ab
assertEquals(5, nEnd - nStart); // original + 2 phase distrib search * 2 shards.
}
+ public void doAtomicUpdate() throws Exception {
+ log.info("### STARTING doAtomicUpdate");
+ int nClients = clients.size();
+ assertEquals(8, nClients);
+
+ int expectedVal = 0;
+ for (SolrServer client : clients) {
+ client.add(sdoc("id", "b!doc", "foo_i", map("inc",1)));
+ expectedVal++;
+
+ QueryResponse rsp = client.query(params("qt","/get", "id","b!doc"));
+ Object val = ((Map)rsp.getResponse().get("doc")).get("foo_i");
+ assertEquals((Integer)expectedVal, val);
+ }
+ }
- long getNumRequests() {
+ long getNumRequests() {
long n = controlJetty.getDebugFilter().getTotalRequests();
for (JettySolrRunner jetty : jettys) {
n += jetty.getDebugFilter().getTotalRequests();