You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/10/21 20:58:44 UTC
svn commit: r1534320 [33/39] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/lucene/expressions/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/lucene/ dev-tools/maven/lucene/expressions/...
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml Mon Oct 21 18:58:24 2013
@@ -22,7 +22,7 @@
<fieldType name="string_standard" class="solr.StrField" postingsFormat="Lucene41"/>
<fieldType name="string_disk" class="solr.StrField" docValuesFormat="Disk" />
- <fieldType name="string_memory" class="solr.StrField" docValuesFormat="Lucene42" />
+ <fieldType name="string_memory" class="solr.StrField" docValuesFormat="Lucene45" />
<fieldType name="string" class="solr.StrField" />
@@ -41,8 +41,8 @@
<dynamicField name="*_pulsing" type="string_pulsing" indexed="true" stored="true"/>
<dynamicField name="*_standard" type="string_standard" indexed="true" stored="true"/>
- <dynamicField name="*_disk" type="string_disk" indexed="false" stored="false" docValues="true" default="" />
- <dynamicField name="*_memory" type="string_memory" indexed="false" stored="false" docValues="true" default="" />
+ <dynamicField name="*_disk" type="string_disk" indexed="false" stored="false" docValues="true" />
+ <dynamicField name="*_memory" type="string_memory" indexed="false" stored="false" docValues="true" />
</fields>
<defaultSearchField>string_f</defaultSearchField>
<uniqueKey>string_f</uniqueKey>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml Mon Oct 21 18:58:24 2013
@@ -31,11 +31,8 @@
solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <indexConfig>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- <lockType>single</lockType>
- </indexConfig>
-
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml Mon Oct 21 18:58:24 2013
@@ -23,8 +23,29 @@
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<schemaFactory class="ManagedIndexSchemaFactory">
- <bool name="mutable">false</bool>
+ <bool name="mutable">${managed.schema.mutable}</bool>
<str name="managedSchemaResourceName">managed-schema</str>
</schemaFactory>
+ <codecFactory class="solr.SchemaCodecFactory"/>
+
+ <updateHandler>
+ <updateLog enable="${enable.update.log}">
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <requestHandler name="standard" class="solr.StandardRequestHandler">
+ <bool name="httpCaching">true</bool>
+ </requestHandler>
+
+ <requestHandler name="/get" class="solr.RealTimeGetHandler">
+ <lst name="defaults">
+ <str name="omitHeader">true</str>
+ </lst>
+ </requestHandler>
+ <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+ <requestHandler name="/update" class="solr.UpdateRequestHandler"/>
+ <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />
+
</config>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Mon Oct 21 18:58:24 2013
@@ -65,6 +65,24 @@
<str name="queryAnalyzerFieldType">phrase_suggest</str>
</searchComponent>
+ <!-- AnalyzingInfixLookup suggest component (default)-->
+ <searchComponent class="solr.SpellCheckComponent" name="infix_suggest_analyzing">
+ <lst name="spellchecker">
+ <str name="name">infix_suggest_analyzing</str>
+ <str name="classname">org.apache.solr.spelling.suggest.Suggester</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.AnalyzingInfixLookupFactory</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <str name="suggestAnalyzerFieldType">text</str>
+
+ <str name="sourceLocation">analyzingInfixSuggest.txt</str>
+ </lst>
+
+ <!-- specify a fieldtype using keywordtokenizer + lowercase + cleanup -->
+ <str name="queryAnalyzerFieldType">phrase_suggest</str>
+ </searchComponent>
+
<!-- FuzzyLookup suggest component (default)-->
<searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing">
<lst name="spellchecker">
@@ -183,7 +201,20 @@
</arr>
</requestHandler>
- <!-- Fuzzy analyzing handler with 1 max edit -->
+ <!-- Infix analyzing handler (default) -->
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/infix_suggest_analyzing">
+ <lst name="defaults">
+ <str name="spellcheck">true</str>
+ <str name="spellcheck.dictionary">infix_suggest_analyzing</str>
+ <str name="spellcheck.collate">false</str>
+ <!-- NOTE: if this is false, results are alpha-ordered, not by weight! -->
+ <str name="spellcheck.onlyMorePopular">true</str>
+ </lst>
+ <arr name="components">
+ <str>infix_suggest_analyzing</str>
+ </arr>
+ </requestHandler>
+
<requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing">
<lst name="defaults">
<str name="spellcheck">true</str>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml Mon Oct 21 18:58:24 2013
@@ -21,42 +21,15 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
- <!-- The DirectoryFactory to use for indexes.
- solr.StandardDirectoryFactory, the default, is filesystem based.
- solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <indexConfig>
- <lockType>single</lockType>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
-
- <updateHandler class="solr.DirectUpdateHandler2">
- </updateHandler>
-
- <requestHandler name="standard" class="solr.StandardRequestHandler">
- <bool name="httpCaching">true</bool>
- </requestHandler>
-
- <!-- test query parameter defaults -->
- <requestHandler name="defaults" class="solr.StandardRequestHandler">
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+ <requestHandler name="/select" class="solr.StandardRequestHandler">
+ <lst name="defaults">
+ <str name="p1">${foo.foo1}</str>
+ <str name="p2">${foo.foo2}</str>
+ </lst>
</requestHandler>
- <tag1>${foo.foo1}</tag1>
- <tag2>${foo.foo2}</tag2>
-
- <!-- test query parameter defaults -->
- <requestHandler name="lazy" class="solr.StandardRequestHandler" startup="lazy">
- </requestHandler>
-
- <requestHandler name="/update" class="solr.UpdateRequestHandler" />
-
- <!-- enable streaming for testing... -->
- <requestDispatcher handleSelect="true">
- <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048"/>
- <httpCaching lastModifiedFrom="openTime" etagSeed="Solr" never304="false">
- <cacheControl>max-age=30, public</cacheControl>
- </httpCaching>
- </requestDispatcher>
</config>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml Mon Oct 21 18:58:24 2013
@@ -39,10 +39,14 @@
<lib dir="../../lib-dirs/c" regex="c1" />
<lib path="../../lib-dirs/d/d1/" />
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
-
<!-- see TestConfig.testJavaProperty -->
<propTest attr1="${solr.test.sys.prop1}-$${literal}"
attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>
+ <!-- see TestConfig.testDisableRequetsHandler -->
+ <requestHandler name="disabled" class="solr.StandardRequestHandler" enable="false"/>
+ <requestHandler name="enabled" class="solr.StandardRequestHandler" enable="true"/>
+
+
+
</config>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Mon Oct 21 18:58:24 2013
@@ -32,10 +32,7 @@
<dataDir>${solr.data.dir:}</dataDir>
- <indexConfig>
- <lockType>${solr.lock.type:native}</lockType>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
- </indexConfig>
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<!-- an update processor the explicitly excludes distrib to test
clean errors when people attempt atomic updates w/o it
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml Mon Oct 21 18:58:24 2013
@@ -429,6 +429,12 @@
</processor>
</updateRequestProcessorChain>
+ <updateRequestProcessorChain name="uniq-values">
+ <processor class="solr.UniqFieldsUpdateProcessorFactory">
+ <str name="fieldRegex">uniq_.*</str>
+ </processor>
+ </updateRequestProcessorChain>
+
<updateRequestProcessorChain name="pre-analyzed-simple">
<processor class="solr.PreAnalyzedUpdateProcessorFactory">
<str name="fieldName">subject</str>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml Mon Oct 21 18:58:24 2013
@@ -26,7 +26,21 @@
<requestHandler name="standard" class="solr.StandardRequestHandler"></requestHandler>
<indexConfig>
- <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ <!-- we don't use solrconfig.snippet.randomindexconfig.xml here
+ because we explicitly test that a mergedSegmentWarmer works,
+ but we can still test some of the other randomized indexConfig
+ settings
+ -->
<mergedSegmentWarmer class="org.apache.lucene.index.SimpleMergedSegmentWarmer"/>
+ <mergePolicy class="org.apache.solr.util.RandomMergePolicy" />
+
+ <useCompoundFile>${useCompoundFile}</useCompoundFile>
+ <maxBufferedDocs>${solr.tests.maxBufferedDocs}</maxBufferedDocs>
+ <maxIndexingThreads>${solr.tests.maxIndexingThreads}</maxIndexingThreads>
+ <ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
+ <mergeScheduler class="${solr.tests.mergeScheduler}" />
+ <writeLockTimeout>1000</writeLockTimeout>
+ <commitLockTimeout>10000</commitLockTimeout>
+ <lockType>single</lockType>
</indexConfig>
</config>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml Mon Oct 21 18:58:24 2013
@@ -35,6 +35,7 @@ A solrconfig.xml snippet containing inde
<ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
<mergeScheduler class="${solr.tests.mergeScheduler}" />
+ <nrtMode>${solr.tests.nrtMode:true}</nrtMode>
<writeLockTimeout>1000</writeLockTimeout>
<commitLockTimeout>10000</commitLockTimeout>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Mon Oct 21 18:58:24 2013
@@ -45,6 +45,8 @@
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
+ <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
+ <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
</directoryFactory>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
@@ -518,11 +520,11 @@
</updateRequestProcessorChain>
<updateRequestProcessorChain name="uniq-fields">
<processor class="org.apache.solr.update.processor.UniqFieldsUpdateProcessorFactory">
- <lst name="fields">
+ <arr name="fieldName">
<str>uniq</str>
<str>uniq2</str>
<str>uniq3</str>
- </lst>
+ </arr>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/solr.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/solr.xml Mon Oct 21 18:58:24 2013
@@ -29,8 +29,8 @@
If 'null' (or absent), cores will not be manageable via request handler
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
- hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" numShards="${numShards:3}" shareSchema="${shareSchema:false}"
- genericCoreNodeNames="${genericCoreNodeNames:true}"
+ hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" shareSchema="${shareSchema:false}"
+ genericCoreNodeNames="${genericCoreNodeNames:true}" leaderVoteWait="0"
distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:120000}">
<core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"
coreNodeName="${coreNodeName:}"/>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java Mon Oct 21 18:58:24 2013
@@ -39,7 +39,7 @@ import org.junit.Test;
* to the indexed facet results as if it were just another faceting method.
*/
@Slow
-@SuppressCodecs({"Lucene40", "Lucene41"})
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"})
public class TestRandomDVFaceting extends SolrTestCaseJ4 {
@BeforeClass
@@ -162,6 +162,8 @@ public class TestRandomDVFaceting extend
SchemaField sf = req.getSchema().getField(ftype.fname);
boolean multiValued = sf.getType().multiValuedFieldCache();
+ boolean indexed = sf.indexed();
+ boolean numeric = sf.getType().getNumericType() != null;
int offset = 0;
if (rand.nextInt(100) < 20) {
@@ -179,8 +181,21 @@ public class TestRandomDVFaceting extend
params.add("facet.limit", Integer.toString(limit));
}
- if (rand.nextBoolean()) {
- params.add("facet.sort", rand.nextBoolean() ? "index" : "count");
+ // the following two situations cannot work for unindexed single-valued numerics:
+ // (currently none of the dv fields in this test config)
+ // facet.sort = index
+ // facet.minCount = 0
+ if (!numeric || sf.multiValued()) {
+ if (rand.nextBoolean()) {
+ params.add("facet.sort", rand.nextBoolean() ? "index" : "count");
+ }
+
+ if (rand.nextInt(100) < 10) {
+ params.add("facet.mincount", Integer.toString(rand.nextInt(5)));
+ }
+ } else {
+ params.add("facet.sort", "count");
+ params.add("facet.mincount", Integer.toString(1+rand.nextInt(5)));
}
if ((ftype.vals instanceof SVal) && rand.nextInt(100) < 20) {
@@ -192,10 +207,6 @@ public class TestRandomDVFaceting extend
params.add("facet.prefix", prefix);
}
- if (rand.nextInt(100) < 10) {
- params.add("facet.mincount", Integer.toString(rand.nextInt(5)));
- }
-
if (rand.nextInt(100) < 20) {
params.add("facet.missing", "true");
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomFaceting.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomFaceting.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestRandomFaceting.java Mon Oct 21 18:58:24 2013
@@ -59,8 +59,11 @@ public class TestRandomFaceting extends
types.add(new FldType("small_f",ONE_ONE, new FVal(-4,5)));
types.add(new FldType("small_d",ONE_ONE, new FVal(-4,5)));
types.add(new FldType("foo_i",ZERO_ONE, new IRange(-2,indexSize)));
- types.add(new FldType("small_s",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
- types.add(new FldType("small2_s",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
+ types.add(new FldType("rare_s1",new IValsPercent(95,0,5,1), new SVal('a','b',1,5)));
+ types.add(new FldType("str_s1",ZERO_ONE, new SVal('a','z',1,2)));
+ types.add(new FldType("long_s1",ZERO_ONE, new SVal('a','b',1,5)));
+ types.add(new FldType("small_s1",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
+ types.add(new FldType("small2_s1",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
types.add(new FldType("small2_ss",ZERO_TWO, new SVal('a',(char)('c'+indexSize/3),1,1)));
types.add(new FldType("small3_ss",new IRange(0,25), new SVal('A','z',1,1)));
types.add(new FldType("small_i",ZERO_ONE, new IRange(-2,5+indexSize/3)));
@@ -70,7 +73,7 @@ public class TestRandomFaceting extends
types.add(new FldType("missing_i",new IRange(0,0), new IRange(0,100)));
types.add(new FldType("missing_is",new IRange(0,0), new IRange(0,100)));
- types.add(new FldType("missing_s",new IRange(0,0), new SVal('a','b',1,1)));
+ types.add(new FldType("missing_s1",new IRange(0,0), new SVal('a','b',1,1)));
types.add(new FldType("missing_ss",new IRange(0,0), new SVal('a','b',1,1)));
// TODO: doubles, multi-floats, ints with precisionStep>0, booleans
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Mon Oct 21 18:58:24 2013
@@ -17,14 +17,19 @@
package org.apache.solr;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.util.AbstractSolrTestCase;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
-import org.apache.solr.client.solrj.SolrServer;
-import org.apache.solr.client.solrj.SolrServerException;
+//import org.apache.lucene.util.LuceneTestCase;
+//import org.apache.solr.util.AbstractSolrTestCase;
+//import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+//import org.apache.solr.client.solrj.impl.HttpSolrServer;
+//import org.apache.solr.client.solrj.SolrServer;
+//import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import org.apache.commons.io.FileUtils;
+
+import org.junit.BeforeClass;
import java.io.*;
import java.util.Properties;
@@ -36,96 +41,49 @@ import java.util.Properties;
*
* @since solr 1.4
*/
-public class TestSolrCoreProperties extends LuceneTestCase {
- private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
- JettySolrRunner solrJetty;
- SolrServer client;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- setUpMe();
- System.setProperty("solr.data.dir", getDataDir());
- System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
-
- solrJetty = new JettySolrRunner(getHomeDir(), "/solr", 0);
-
- solrJetty.start();
- String url = "http://127.0.0.1:" + solrJetty.getLocalPort() + "/solr";
- client = new HttpSolrServer(url);
-
- }
-
- @Override
- public void tearDown() throws Exception {
- solrJetty.stop();
- System.clearProperty("tests.shardhandler.randomSeed");
- AbstractSolrTestCase.recurseDelete(homeDir);
- super.tearDown();
- }
-
- public void testSimple() throws SolrServerException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("q", "*:*");
- QueryResponse res = client.query(params);
- assertEquals(0, res.getResults().getNumFound());
- }
-
-
- File homeDir;
- File confDir;
- File dataDir;
-
- /**
- * if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is
- * on localhost at the specified port.
- */
-
-
- public String getHomeDir() {
- return homeDir.toString();
- }
-
- public String getSchemaFile() {
- return CONF_DIR + "schema-replication1.xml";
- }
-
- public String getConfDir() {
- return confDir.toString();
- }
-
- public String getDataDir() {
- return dataDir.toString();
- }
-
- public String getSolrConfigFile() {
- return CONF_DIR + "solrconfig-solcoreproperties.xml";
- }
-
- public void setUpMe() throws Exception {
-
- homeDir = new File(TEMP_DIR,
- getClass().getName() + "-" + System.currentTimeMillis());
-
-
- dataDir = new File(homeDir + "/collection1", "data");
- confDir = new File(homeDir + "/collection1", "conf");
+public class TestSolrCoreProperties extends SolrJettyTestBase {
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ File homeDir = new File(TEMP_DIR,
+ "solrtest-TestSolrCoreProperties-" + System.currentTimeMillis());
+ File collDir = new File(homeDir, "collection1");
+ File dataDir = new File(collDir, "data");
+ File confDir = new File(collDir, "conf");
homeDir.mkdirs();
+ collDir.mkdirs();
dataDir.mkdirs();
confDir.mkdirs();
- File f = new File(confDir, "solrconfig.xml");
- IOUtils.copy(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
+ FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(homeDir, "solr.xml"));
+ String src_dir = TEST_HOME() + "/collection1/conf";
+ FileUtils.copyFile(new File(src_dir, "schema-tiny.xml"),
+ new File(confDir, "schema.xml"));
+ FileUtils.copyFile(new File(src_dir, "solrconfig-solcoreproperties.xml"),
+ new File(confDir, "solrconfig.xml"));
+ FileUtils.copyFile(new File(src_dir, "solrconfig.snippet.randomindexconfig.xml"),
+ new File(confDir, "solrconfig.snippet.randomindexconfig.xml"));
- f = new File(confDir, "schema.xml");
- IOUtils.copy(SolrTestCaseJ4.getFile(getSchemaFile()), f);
Properties p = new Properties();
p.setProperty("foo.foo1", "f1");
p.setProperty("foo.foo2", "f2");
- Writer fos = new OutputStreamWriter(new FileOutputStream(confDir + File.separator + "solrcore.properties"), IOUtils.CHARSET_UTF_8);
+ Writer fos = new OutputStreamWriter(new FileOutputStream(new File(confDir, "solrcore.properties")), IOUtils.CHARSET_UTF_8);
p.store(fos, null);
IOUtils.close(fos);
+
+ createJetty(homeDir.getAbsolutePath(), null, null);
+ }
+
+ public void testSimple() throws Exception {
+ SolrParams params = params("q", "*:*",
+ "echoParams", "all");
+ QueryResponse res = getSolrServer().query(params);
+ assertEquals(0, res.getResults().getNumFound());
+
+ NamedList echoedParams = (NamedList) res.getHeader().get("params");
+ assertEquals("f1", echoedParams.get("p1"));
+ assertEquals("f2", echoedParams.get("p2"));
}
+
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestTrie.java Mon Oct 21 18:58:24 2013
@@ -16,8 +16,6 @@
*/
package org.apache.solr;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.DateField;
import org.apache.solr.schema.FieldType;
@@ -49,38 +47,6 @@ public class TestTrie extends SolrTestCa
clearIndex();
super.tearDown();
}
-
- @Test
- public void testTokenizer() throws Exception {
- FieldType type = h.getCore().getLatestSchema().getFieldType("tint");
- assertTrue(type instanceof TrieField);
-
- String value = String.valueOf(random().nextInt());
- TokenStream ts = type.getAnalyzer().tokenStream("dummy", value);
- OffsetAttribute ofsAtt = ts.addAttribute(OffsetAttribute.class);
- ts.reset();
- int count = 0;
- while (ts.incrementToken()) {
- count++;
- assertEquals(0, ofsAtt.startOffset());
- assertEquals(value.length(), ofsAtt.endOffset());
- }
- final int precStep = ((TrieField) type).getPrecisionStep();
- assertEquals( (32 + precStep - 1) / precStep, count);
- ts.end();
- assertEquals(value.length(), ofsAtt.startOffset());
- assertEquals(value.length(), ofsAtt.endOffset());
- ts.close();
-
- // Test empty one:
- ts = type.getAnalyzer().tokenStream("dummy", "");
- ts.reset();
- assertFalse(ts.incrementToken());
- ts.end();
- assertEquals(0, ofsAtt.startOffset());
- assertEquals(0, ofsAtt.endOffset());
- ts.close();
- }
@Test
public void testTrieIntRangeSearch() throws Exception {
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java Mon Oct 21 18:58:24 2013
@@ -153,6 +153,7 @@ public class AliasIntegrationTest extend
// search with new cloud client
CloudSolrServer cloudSolrServer = new CloudSolrServer(zkServer.getZkAddress(), random().nextBoolean());
+ cloudSolrServer.setParallelUpdates(random().nextBoolean());
query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudSolrServer.query(query);
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Mon Oct 21 18:58:24 2013
@@ -168,7 +168,7 @@ public class BasicDistributedZk2Test ext
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
HttpSolrServer server = new HttpSolrServer(baseUrl);
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
Create createCmd = new Create();
createCmd.setRoles("none");
createCmd.setCoreName(ONE_NODE_COLLECTION + "core");
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Mon Oct 21 18:58:24 2013
@@ -65,7 +65,6 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.SolrCmdDistributor.Request;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -105,8 +104,8 @@ public class BasicDistributedZkTest exte
Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- CompletionService<Request> completionService;
- Set<Future<Request>> pending;
+ CompletionService<Object> completionService;
+ Set<Future<Object>> pending;
@BeforeClass
public static void beforeThisClass2() throws Exception {
@@ -126,8 +125,8 @@ public class BasicDistributedZkTest exte
sliceCount = 2;
shardCount = 4;
- completionService = new ExecutorCompletionService<Request>(executor);
- pending = new HashSet<Future<Request>>();
+ completionService = new ExecutorCompletionService<Object>(executor);
+ pending = new HashSet<Future<Object>>();
}
@@ -717,7 +716,7 @@ public class BasicDistributedZkTest exte
SolrServerException, IOException {
HttpSolrServer server = new HttpSolrServer(solrServer.getBaseURL());
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/admin/mbeans?key=updateHandler&stats=true");
// use generic request to avoid extra processing of queries
@@ -749,7 +748,7 @@ public class BasicDistributedZkTest exte
while (pending != null && pending.size() > 0) {
- Future<Request> future = completionService.take();
+ Future<Object> future = completionService.take();
pending.remove(future);
}
@@ -810,7 +809,7 @@ public class BasicDistributedZkTest exte
// now test that unloading a core gets us a new leader
HttpSolrServer server = new HttpSolrServer(baseUrl);
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
Unload unloadCmd = new Unload(true);
unloadCmd.setCoreName(props.getCoreName());
@@ -875,7 +874,7 @@ public class BasicDistributedZkTest exte
while (pending != null && pending.size() > 0) {
- Future<Request> future = completionService.take();
+ Future<Object> future = completionService.take();
if (future == null) return;
pending.remove(future);
}
@@ -924,11 +923,11 @@ public class BasicDistributedZkTest exte
Callable call = new Callable() {
@Override
public Object call() {
- HttpSolrServer server;
+ HttpSolrServer server = null;
try {
server = new HttpSolrServer(baseUrl);
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
Create createCmd = new Create();
createCmd.setRoles("none");
createCmd.setCoreName(collection + num);
@@ -945,6 +944,10 @@ public class BasicDistributedZkTest exte
} catch (Exception e) {
e.printStackTrace();
//fail
+ } finally {
+ if (server != null) {
+ server.shutdown();
+ }
}
return null;
}
@@ -964,7 +967,7 @@ public class BasicDistributedZkTest exte
while (pending != null && pending.size() > 0) {
- Future<Request> future = completionService.take();
+ Future<Object> future = completionService.take();
if (future == null) return;
pending.remove(future);
}
@@ -1053,11 +1056,11 @@ public class BasicDistributedZkTest exte
Callable call = new Callable() {
@Override
public Object call() {
- HttpSolrServer server;
+ HttpSolrServer server = null;
try {
server = new HttpSolrServer(baseUrl);
server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
+ server.setSoTimeout(60000);
Create createCmd = new Create();
createCmd.setCoreName(collection);
createCmd.setDataDir(getDataDir(dataDir.getAbsolutePath() + File.separator
@@ -1068,6 +1071,10 @@ public class BasicDistributedZkTest exte
} catch (Exception e) {
e.printStackTrace();
//fails
+ } finally {
+ if (server != null) {
+ server.shutdown();
+ }
}
return null;
}
@@ -1077,7 +1084,7 @@ public class BasicDistributedZkTest exte
pending.add(completionService.submit(call));
while (pending != null && pending.size() > 0) {
- Future<Request> future = completionService.take();
+ Future<Object> future = completionService.take();
if (future == null) return;
pending.remove(future);
}
@@ -1088,7 +1095,7 @@ public class BasicDistributedZkTest exte
try {
// setup the server...
HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
- s.setSoTimeout(30000);
+ s.setSoTimeout(120000);
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
@@ -1104,6 +1111,7 @@ public class BasicDistributedZkTest exte
synchronized(this) {
try {
commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress(), random().nextBoolean());
+ commondCloudSolrServer.setParallelUpdates(random().nextBoolean());
commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
commondCloudSolrServer.getLbServer().setConnectionTimeout(15000);
commondCloudSolrServer.getLbServer().setSoTimeout(30000);
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Mon Oct 21 18:58:24 2013
@@ -21,12 +21,12 @@ import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.http.client.HttpClient;
import org.apache.lucene.util.LuceneTestCase.Slow;
-
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -52,10 +52,10 @@ public class ChaosMonkeyNothingIsSafeTes
SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
@Override
public void call(Object... data) {
- SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
- if (sreq.exception == null) return;
- if (sreq.exception.getMessage().contains("Timeout")) {
- Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+ Exception e = (Exception) data[0];
+ if (e == null) return;
+ if (e.getMessage().contains("Timeout")) {
+ Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + e.getMessage());
}
}
};
@@ -205,6 +205,29 @@ public class ChaosMonkeyNothingIsSafeTes
if (VERBOSE) System.out.println("control docs:"
+ controlClient.query(new SolrQuery("*:*")).getResults()
.getNumFound() + "\n\n");
+
+ // try and make a collection to make sure the overseer has survived the expiration and session loss
+
+ // sometimes we restart zookeeper as well
+ if (random().nextBoolean()) {
+ zkServer.shutdown();
+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
+ zkServer.run();
+ }
+
+ CloudSolrServer client = createCloudClient("collection1");
+ try {
+ createCollection(null, "testcollection",
+ 1, 1, 1, client, null, "conf1");
+
+ } finally {
+ client.shutdown();
+ }
+ List<Integer> numShardsNumReplicas = new ArrayList<Integer>(2);
+ numShardsNumReplicas.add(1);
+ numShardsNumReplicas.add(1);
+ checkForCollection("testcollection",numShardsNumReplicas, null);
+
testsSuccesful = true;
} finally {
if (!testsSuccesful) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Mon Oct 21 18:58:24 2013
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.Diagnostics;
import org.apache.solr.core.SolrCore;
@@ -43,10 +44,10 @@ public class ChaosMonkeySafeLeaderTest
SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() {
@Override
public void call(Object... data) {
- SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data[1];
- if (sreq.exception == null) return;
- if (sreq.exception.getMessage().contains("Timeout")) {
- Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + sreq.exception.getMessage());
+ Exception e = (Exception) data[0];
+ if (e == null) return;
+ if (e.getMessage().contains("Timeout")) {
+ Diagnostics.logThreadDumps("REQUESTING THREAD DUMP DUE TO TIMEOUT: " + e.getMessage());
}
}
};
@@ -137,6 +138,28 @@ public class ChaosMonkeySafeLeaderTest
checkShardConsistency(true, true);
if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
+
+ // try and make a collection to make sure the overseer has survived the expiration and session loss
+
+ // sometimes we restart zookeeper as well
+ if (random().nextBoolean()) {
+ zkServer.shutdown();
+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
+ zkServer.run();
+ }
+
+ CloudSolrServer client = createCloudClient("collection1");
+ try {
+ createCollection(null, "testcollection",
+ 1, 1, 1, client, null, "conf1");
+
+ } finally {
+ client.shutdown();
+ }
+ List<Integer> numShardsNumReplicas = new ArrayList<Integer>(2);
+ numShardsNumReplicas.add(1);
+ numShardsNumReplicas.add(1);
+ checkForCollection("testcollection",numShardsNumReplicas, null);
}
private void randomlyEnableAutoSoftCommit() {
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java Mon Oct 21 18:58:24 2013
@@ -135,7 +135,7 @@ public class ChaosMonkeyShardSplitTest e
killerThread.start();
killCounter.incrementAndGet();
- splitShard(SHARD1);
+ splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null, null);
log.info("Layout after split: \n");
printLayout();
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Mon Oct 21 18:58:24 2013
@@ -17,6 +17,7 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
@@ -36,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -64,19 +66,31 @@ public class ClusterStateUpdateTest exte
private File dataDir3;
private File dataDir4;
-
+
+
+ private static final File solrHomeDirectory = new File(TEMP_DIR, "ZkControllerTest");
+
@BeforeClass
- public static void beforeClass() {
+ public static void beforeClass() throws IOException {
System.setProperty("solrcloud.skip.autorecovery", "true");
System.setProperty("genericCoreNodeNames", "false");
+ if (solrHomeDirectory.exists()) {
+ FileUtils.deleteDirectory(solrHomeDirectory);
+ }
+ copyMinFullSetup(solrHomeDirectory);
+
}
-
+
@AfterClass
- public static void afterClass() throws InterruptedException {
+ public static void afterClass() throws InterruptedException, IOException {
System.clearProperty("solrcloud.skip.autorecovery");
System.clearProperty("genericCoreNodeNames");
+ if (solrHomeDirectory.exists()) {
+ FileUtils.deleteDirectory(solrHomeDirectory);
+ }
}
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -111,19 +125,19 @@ public class ClusterStateUpdateTest exte
System.setProperty("solr.solr.home", TEST_HOME());
System.setProperty("hostPort", "1661");
System.setProperty("solr.data.dir", ClusterStateUpdateTest.this.dataDir1.getAbsolutePath());
- container1 = new CoreContainer();
+ container1 = new CoreContainer(solrHomeDirectory.getAbsolutePath());
container1.load();
System.clearProperty("hostPort");
System.setProperty("hostPort", "1662");
System.setProperty("solr.data.dir", ClusterStateUpdateTest.this.dataDir2.getAbsolutePath());
- container2 = new CoreContainer();
+ container2 = new CoreContainer(solrHomeDirectory.getAbsolutePath());
container2.load();
System.clearProperty("hostPort");
System.setProperty("hostPort", "1663");
System.setProperty("solr.data.dir", ClusterStateUpdateTest.this.dataDir3.getAbsolutePath());
- container3 = new CoreContainer();
+ container3 = new CoreContainer(solrHomeDirectory.getAbsolutePath());
container3.load();
System.clearProperty("hostPort");
System.clearProperty("solr.solr.home");
@@ -222,7 +236,7 @@ public class ClusterStateUpdateTest exte
System.setProperty("hostPort", "1662");
System.setProperty("solr.data.dir", ClusterStateUpdateTest.this.dataDir2.getAbsolutePath());
- container2 = new CoreContainer();
+ container2 = new CoreContainer(solrHomeDirectory.getAbsolutePath());
container2.load();
System.clearProperty("hostPort");
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Mon Oct 21 18:58:24 2013
@@ -17,14 +17,42 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.MBeanServer;
+import javax.management.MBeanServerFactory;
+import javax.management.ObjectName;
+
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util._TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
import org.apache.solr.client.solrj.request.QueryRequest;
@@ -37,6 +65,7 @@ import org.apache.solr.common.cloud.Clus
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -45,40 +74,16 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean.Category;
+import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.SolrCmdDistributor.Request;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.junit.Before;
import org.junit.BeforeClass;
-import javax.management.MBeanServer;
-import javax.management.MBeanServerFactory;
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
-
/**
* Tests the Cloud Collections API.
*/
@@ -92,8 +97,12 @@ public class CollectionsAPIDistributedZk
Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- CompletionService<Request> completionService;
- Set<Future<Request>> pending;
+ CompletionService<Object> completionService;
+ Set<Future<Object>> pending;
+
+ // we randomly use a second config set rather than just one
+ private boolean secondConfigSet = random().nextBoolean();
+ private boolean oldStyleSolrXml = false;
@BeforeClass
public static void beforeThisClass2() throws Exception {
@@ -104,12 +113,49 @@ public class CollectionsAPIDistributedZk
@Override
public void setUp() throws Exception {
super.setUp();
+
+ useJettyDataDir = false;
+
+ oldStyleSolrXml = random().nextBoolean();
+ if (oldStyleSolrXml) {
+ System.err.println("Using old style solr.xml");
+ } else {
+ System.err.println("Using new style solr.xml");
+ }
+ if (secondConfigSet ) {
+ String zkHost = zkServer.getZkHost();
+ String zkAddress = zkServer.getZkAddress();
+ SolrZkClient zkClient = new SolrZkClient(zkHost, AbstractZkTestCase.TIMEOUT);
+ zkClient.makePath("/solr", false, true);
+ zkClient.close();
+
+ zkClient = new SolrZkClient(zkAddress, AbstractZkTestCase.TIMEOUT);
+
+ File solrhome = new File(TEST_HOME());
+
+ // for now, always upload the config and schema to the canonical names
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "solrconfig.xml", "solrconfig.xml");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "schema.xml", "schema.xml");
+
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "stopwords.txt");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "protwords.txt");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "currency.xml");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "open-exchange-rates.json");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "mapping-ISOLatin1Accent.txt");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "old_synonyms.txt");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "synonyms.txt");
+ AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "elevate.xml");
+ zkClient.close();
+ }
+
System.setProperty("numShards", Integer.toString(sliceCount));
System.setProperty("solr.xml.persist", "true");
}
protected String getSolrXml() {
- return "solr-no-core.xml";
+ // test old style and new style solr.xml
+ return oldStyleSolrXml ? "solr-no-core-old-style.xml" : "solr-no-core.xml";
}
@@ -118,8 +164,8 @@ public class CollectionsAPIDistributedZk
sliceCount = 2;
shardCount = 4;
- completionService = new ExecutorCompletionService<Request>(executor);
- pending = new HashSet<Future<Request>>();
+ completionService = new ExecutorCompletionService<Object>(executor);
+ pending = new HashSet<Future<Object>>();
checkCreatedVsState = false;
}
@@ -147,12 +193,48 @@ public class CollectionsAPIDistributedZk
testCollectionsAPI();
testErrorHandling();
deletePartiallyCreatedCollection();
+ deleteCollectionRemovesStaleZkCollectionsNode();
+
+ // last
deleteCollectionWithDownNodes();
if (DEBUG) {
super.printLayout();
}
}
+ private void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception {
+
+ // we can use this client because we just want base url
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
+
+ String collectionName = "out_of_sync_collection";
+
+ List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
+ numShardsNumReplicaList.add(2);
+ numShardsNumReplicaList.add(1);
+
+
+ cloudClient.getZkStateReader().getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true);
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.DELETE.toString());
+ params.set("name", collectionName);
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ try {
+ NamedList<Object> resp = createNewSolrServer("", baseUrl)
+ .request(request);
+ fail("Expected to fail, because collection is not in clusterstate");
+ } catch (RemoteSolrException e) {
+
+ }
+
+ checkForMissingCollection(collectionName);
+
+ assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
+
+ }
+
private void deletePartiallyCreatedCollection() throws Exception {
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
String collectionName = "halfdeletedcollection";
@@ -163,6 +245,9 @@ public class CollectionsAPIDistributedZk
+ System.currentTimeMillis() + "halfcollection" + "_hdn";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(2);
+ if (secondConfigSet) {
+ createCmd.setCollectionConfigName("conf1");
+ }
createNewSolrServer("", baseUrl).request(createCmd);
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -182,6 +267,9 @@ public class CollectionsAPIDistributedZk
params.set("numShards", 2);
request = new QueryRequest(params);
request.setPath("/admin/collections");
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
resp = createNewSolrServer("", baseUrl).request(request);
}
@@ -189,8 +277,13 @@ public class CollectionsAPIDistributedZk
private void deleteCollectionWithDownNodes() throws Exception {
String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
// now try to remove a collection when a couple of it's nodes are down
- createCollection(null, "halfdeletedcollection2", 3, 2, 6,
- createNewSolrServer("", baseUrl), null);
+ if (secondConfigSet) {
+ createCollection(null, "halfdeletedcollection2", 3, 2, 6,
+ createNewSolrServer("", baseUrl), null, "conf2");
+ } else {
+ createCollection(null, "halfdeletedcollection2", 3, 2, 6,
+ createNewSolrServer("", baseUrl), null);
+ }
waitForRecoveriesToFinish("halfdeletedcollection2", false);
@@ -244,6 +337,9 @@ public class CollectionsAPIDistributedZk
collectionName = "collection";
// No Name
// params.set("name", collectionName);
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@@ -261,6 +357,9 @@ public class CollectionsAPIDistributedZk
collectionName = "collection";
params.set("name", collectionName);
params.set("numShards", 2);
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
params.set(REPLICATION_FACTOR, 10);
request = new QueryRequest(params);
request.setPath("/admin/collections");
@@ -279,6 +378,9 @@ public class CollectionsAPIDistributedZk
collectionName = "acollection";
params.set("name", collectionName);
params.set(REPLICATION_FACTOR, 10);
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@@ -297,6 +399,9 @@ public class CollectionsAPIDistributedZk
params.set("name", collectionName);
params.set(REPLICATION_FACTOR, 10);
params.set("numShards", 0);
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@@ -319,6 +424,9 @@ public class CollectionsAPIDistributedZk
+ System.currentTimeMillis() + "halfcollection" + "_3n";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(1);
+ if (secondConfigSet) {
+ createCmd.setCollectionConfigName("conf1");
+ }
createNewSolrServer("", baseUrl).request(createCmd);
createCmd = new Create();
@@ -328,6 +436,9 @@ public class CollectionsAPIDistributedZk
+ System.currentTimeMillis() + "halfcollection" + "_3n";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(1);
+ if (secondConfigSet) {
+ createCmd.setCollectionConfigName("conf1");
+ }
createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(1))).request(createCmd);
params = new ModifiableSolrParams();
@@ -337,6 +448,10 @@ public class CollectionsAPIDistributedZk
params.set("numShards", 2);
params.set("wt", "xml");
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
+
String nn1 = ((SolrDispatchFilter) jettys.get(0).getDispatchFilter().getFilter()).getCores().getZkController().getNodeName();
String nn2 = ((SolrDispatchFilter) jettys.get(1).getDispatchFilter().getFilter()).getCores().getZkController().getNodeName();
@@ -349,6 +464,9 @@ public class CollectionsAPIDistributedZk
SimpleOrderedMap success = (SimpleOrderedMap) resp.get("success");
SimpleOrderedMap failure = (SimpleOrderedMap) resp.get("failure");
+ assertNotNull(resp.toString(), success);
+ assertNotNull(resp.toString(), failure);
+
String val1 = success.getVal(0).toString();
String val2 = failure.getVal(0).toString();
assertTrue(val1.contains("SolrException") || val2.contains("SolrException"));
@@ -366,6 +484,11 @@ public class CollectionsAPIDistributedZk
String collectionName = "nodes_used_collection";
params.set("name", collectionName);
+
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
+
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
@@ -435,9 +558,13 @@ public class CollectionsAPIDistributedZk
// and use this CloudSolrServer to create the collection
client = createCloudClient("awholynewcollection_" + i);
}
-
- createCollection(collectionInfos, "awholynewcollection_" + i,
- numShards, replicationFactor, maxShardsPerNode, client, null);
+ if (secondConfigSet) {
+ createCollection(collectionInfos, "awholynewcollection_" + i,
+ numShards, replicationFactor, maxShardsPerNode, client, null, "conf2");
+ } else {
+ createCollection(collectionInfos, "awholynewcollection_" + i,
+ numShards, replicationFactor, maxShardsPerNode, client, null);
+ }
} finally {
if (client != null) client.shutdown();
}
@@ -456,11 +583,60 @@ public class CollectionsAPIDistributedZk
// poll for a second - it can take a moment before we are ready to serve
waitForNon403or404or503(collectionClient);
}
+
+ // sometimes we restart one of the jetty nodes
+ if (random().nextBoolean()) {
+ JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size()));
+ ChaosMonkey.stop(jetty);
+ ChaosMonkey.start(jetty);
+
+ for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
+ String collection = entry.getKey();
+ List<Integer> list = entry.getValue();
+ checkForCollection(collection, list, null);
+
+ String url = getUrlFromZk(collection);
+
+ HttpSolrServer collectionClient = new HttpSolrServer(url);
+
+ // poll for a second - it can take a moment before we are ready to serve
+ waitForNon403or404or503(collectionClient);
+ }
+ }
+
+ // sometimes we restart zookeeper
+ if (random().nextBoolean()) {
+ zkServer.shutdown();
+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
+ zkServer.run();
+ }
+
+ // sometimes we cause a connection loss - sometimes it will hit the overseer
+ if (random().nextBoolean()) {
+ JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size()));
+ ChaosMonkey.causeConnectionLoss(jetty);
+ }
+
ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
+
+ if (secondConfigSet) {
+ // let's see if they are using the second config set
+ byte[] data = zkStateReader.getZkClient()
+ .getData(
+ ZkStateReader.COLLECTIONS_ZKNODE + "/" + "awholynewcollection_"
+ + j, null, null, true);
+ assertNotNull(data);
+ ZkNodeProps props = ZkNodeProps.load(data);
+ String configName = props.getStr(ZkController.CONFIGNAME_PROP);
+ assertEquals("conf2", configName);
+
+ }
}
+ checkInstanceDirs(jettys.get(0));
+
List<String> collectionNameList = new ArrayList<String>();
collectionNameList.addAll(collectionInfos.keySet());
String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
@@ -550,6 +726,9 @@ public class CollectionsAPIDistributedZk
collectionName = "acollectionafterbaddelete";
params.set("name", collectionName);
+ if (secondConfigSet) {
+ params.set("collection.configName", "conf1");
+ }
request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
@@ -580,7 +759,8 @@ public class CollectionsAPIDistributedZk
try {
exp = false;
try {
- createCollection(collectionInfos, "awholynewcollection_" + cnt, numShards, replicationFactor, maxShardsPerNode, client, null);
+ createCollection(collectionInfos, "awholynewcollection_" + cnt,
+ numShards, replicationFactor, maxShardsPerNode, client, null, "conf1");
} catch (SolrException e) {
exp = true;
}
@@ -610,7 +790,7 @@ public class CollectionsAPIDistributedZk
collectionInfos = new HashMap<String,List<Integer>>();
client = createCloudClient("awholynewcollection_" + (cnt+1));
try {
- createCollection(collectionInfos, "awholynewcollection_" + (cnt+1), numShards, replicationFactor, maxShardsPerNode, client, StrUtils.join(createNodeList, ','));
+ createCollection(collectionInfos, "awholynewcollection_" + (cnt+1), numShards, replicationFactor, maxShardsPerNode, client, StrUtils.join(createNodeList, ','), "conf1");
} finally {
client.shutdown();
}
@@ -619,6 +799,26 @@ public class CollectionsAPIDistributedZk
checkNoTwoShardsUseTheSameIndexDir();
}
+ private void checkInstanceDirs(JettySolrRunner jetty) {
+ CoreContainer cores = ((SolrDispatchFilter) jetty.getDispatchFilter()
+ .getFilter()).getCores();
+ Collection<SolrCore> theCores = cores.getCores();
+ for (SolrCore core : theCores) {
+ if (!oldStyleSolrXml) {
+ // look for core props file
+ assertTrue("Could not find expected core.properties file",
+ new File((String) core.getStatistics().get("instanceDir"),
+ "core.properties").exists());
+ }
+
+ assertEquals(
+ new File(SolrResourceLoader.normalizeDir(jetty.getSolrHome() + File.separator
+ + core.getName())).getAbsolutePath(),
+ new File(SolrResourceLoader.normalizeDir((String) core.getStatistics().get(
+ "instanceDir"))).getAbsolutePath());
+ }
+ }
+
private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
@@ -665,9 +865,13 @@ public class CollectionsAPIDistributedZk
while (shardIt.hasNext()) {
Entry<String,Replica> shardEntry = shardIt.next();
ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
- CoreAdminResponse mcr = CoreAdminRequest.getStatus(
- coreProps.getCoreName(),
- new HttpSolrServer(coreProps.getBaseUrl()));
+ HttpSolrServer server = new HttpSolrServer(coreProps.getBaseUrl());
+ CoreAdminResponse mcr;
+ try {
+ mcr = CoreAdminRequest.getStatus(coreProps.getCoreName(), server);
+ } finally {
+ server.shutdown();
+ }
long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
urlToTime.put(coreProps.getCoreUrl(), before);
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java Mon Oct 21 18:58:24 2013
@@ -17,6 +17,27 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
+import static org.apache.solr.common.params.ShardParams._ROUTE_;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util._TestUtil;
@@ -32,7 +53,6 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -42,31 +62,10 @@ import org.apache.solr.common.cloud.ZkSt
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.SolrCmdDistributor.Request;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.junit.Before;
import org.junit.BeforeClass;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
-import static org.apache.solr.common.params.ShardParams._ROUTE_;
-
/**
* Tests the Custom Sharding API.
*/
@@ -80,8 +79,8 @@ public class CustomCollectionTest extend
Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- CompletionService<Request> completionService;
- Set<Future<Request>> pending;
+ CompletionService<Object> completionService;
+ Set<Future<Object>> pending;
@BeforeClass
public static void beforeThisClass2() throws Exception {
@@ -106,8 +105,8 @@ public class CustomCollectionTest extend
sliceCount = 2;
shardCount = 4;
- completionService = new ExecutorCompletionService<Request>(executor);
- pending = new HashSet<Future<Request>>();
+ completionService = new ExecutorCompletionService<Object>(executor);
+ pending = new HashSet<Future<Object>>();
checkCreatedVsState = false;
}
@@ -132,6 +131,8 @@ public class CustomCollectionTest extend
@Override
public void doTest() throws Exception {
testCustomCollectionsAPI();
+ testRouteFieldForHashRouter();
+ testCreateShardRepFactor();
if (DEBUG) {
super.printLayout();
}
@@ -180,8 +181,8 @@ public class CustomCollectionTest extend
client = createCloudClient(COLL_PREFIX + i);
}
- Map<String, Object> props = OverseerCollectionProcessor.asMap(
- ROUTER, ImplicitDocRouter.NAME,
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ "router.name", ImplicitDocRouter.NAME,
REPLICATION_FACTOR, replicationFactor,
MAX_SHARDS_PER_NODE, maxShardsPerNode,
SHARDS_PROP,"a,b,c");
@@ -198,7 +199,7 @@ public class CustomCollectionTest extend
List<Integer> list = entry.getValue();
checkForCollection(collection, list, null);
- String url = getUrlFromZk(collection);
+ String url = getUrlFromZk(getCommonCloudSolrServer().getZkStateReader().getClusterState(), collection);
HttpSolrServer collectionClient = new HttpSolrServer(url);
@@ -213,9 +214,11 @@ public class CustomCollectionTest extend
ClusterState clusterState = zkStateReader.getClusterState();
DocCollection coll = clusterState.getCollection(COLL_PREFIX + 0);
- assertEquals("implicit", coll.getStr(ROUTER));
+ assertEquals("implicit", ((Map)coll.get(ROUTER)).get("name") );
assertNotNull(coll.getStr(REPLICATION_FACTOR));
assertNotNull(coll.getStr(MAX_SHARDS_PER_NODE));
+ assertNull("A shard of a Collection configured with implicit router must have null range",
+ coll.getSlice("a").getRange());
List<String> collectionNameList = new ArrayList<String>();
collectionNameList.addAll(collectionInfos.keySet());
@@ -223,7 +226,7 @@ public class CustomCollectionTest extend
String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
- String url = getUrlFromZk(collectionName);
+ String url = getUrlFromZk(getCommonCloudSolrServer().getZkStateReader().getClusterState(), collectionName);
HttpSolrServer collectionClient = new HttpSolrServer(url);
@@ -242,8 +245,8 @@ public class CustomCollectionTest extend
collectionClient.commit();
assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","b")).getResults().getNumFound());
- assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","a")).getResults().getNumFound());
+ assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"b")).getResults().getNumFound());
+ assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"a")).getResults().getNumFound());
collectionClient.deleteByQuery("*:*");
collectionClient.commit(true,true);
@@ -263,8 +266,8 @@ public class CustomCollectionTest extend
collectionClient.request(up);
assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","a")).getResults().getNumFound());
- assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","c")).getResults().getNumFound());
+ assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"a")).getResults().getNumFound());
+ assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"c")).getResults().getNumFound());
//Testing CREATESHARD
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -292,7 +295,7 @@ public class CustomCollectionTest extend
collectionClient.add(getDoc(id, 66, i1, -600, tlong, 600, t1,
"humpty dumpy sat on a wall", _ROUTE_,"x"));
collectionClient.commit();
- assertEquals(1, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","x")).getResults().getNumFound());
+ assertEquals(1, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"x")).getResults().getNumFound());
int numShards = 4;
@@ -305,12 +308,12 @@ public class CustomCollectionTest extend
String shard_fld = "shard_s";
try {
client = createCloudClient(null);
- Map<String, Object> props = OverseerCollectionProcessor.asMap(
- ROUTER, ImplicitDocRouter.NAME,
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ "router.name", ImplicitDocRouter.NAME,
REPLICATION_FACTOR, replicationFactor,
MAX_SHARDS_PER_NODE, maxShardsPerNode,
SHARDS_PROP,"a,b,c,d",
- DocRouter.ROUTE_FIELD, shard_fld);
+ "router.field", shard_fld);
collectionName = COLL_PREFIX + "withShardField";
createCollection(collectionInfos, collectionName,props,client);
@@ -322,7 +325,7 @@ public class CustomCollectionTest extend
checkForCollection(collectionName, list, null);
- url = getUrlFromZk(collectionName);
+ url = getUrlFromZk(getCommonCloudSolrServer().getZkStateReader().getClusterState(), collectionName);
collectionClient = new HttpSolrServer(url);
@@ -349,18 +352,123 @@ public class CustomCollectionTest extend
collectionClient.commit();
assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
- assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","b")).getResults().getNumFound());
+ assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"b")).getResults().getNumFound());
+ //TODO debug the following case
+ assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_, "a")).getResults().getNumFound());
+
+
+ }
+
+ private void testRouteFieldForHashRouter()throws Exception{
+ String collectionName = "routeFieldColl";
+ int numShards = 4;
+ int replicationFactor = 2;
+ int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrServer()
+ .getZkStateReader().getClusterState().getLiveNodes().size())) + 1;
+
+ HashMap<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
+ CloudSolrServer client = null;
+ String shard_fld = "shard_s";
+ try {
+ client = createCloudClient(null);
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ REPLICATION_FACTOR, replicationFactor,
+ MAX_SHARDS_PER_NODE, maxShardsPerNode,
+ NUM_SLICES,numShards,
+ "router.field", shard_fld);
+
+ createCollection(collectionInfos, collectionName,props,client);
+ } finally {
+ if (client != null) client.shutdown();
+ }
+
+ List<Integer> list = collectionInfos.get(collectionName);
+ checkForCollection(collectionName, list, null);
+
+
+ String url = getUrlFromZk(getCommonCloudSolrServer().getZkStateReader().getClusterState(), collectionName);
+
+ HttpSolrServer collectionClient = new HttpSolrServer(url);
+
+ // poll for a second - it can take a moment before we are ready to serve
+ waitForNon403or404or503(collectionClient);
+
+
+ collectionClient = new HttpSolrServer(url);
+
+
+ // lets try and use the solrj client to index a couple documents
+
+ collectionClient.add(getDoc(id, 6, i1, -600, tlong, 600, t1,
+ "humpty dumpy sat on a wall", shard_fld,"a"));
+
+ collectionClient.add(getDoc(id, 7, i1, -600, tlong, 600, t1,
+ "humpty dumpy3 sat on a walls", shard_fld,"a"));
+
+ collectionClient.add(getDoc(id, 8, i1, -600, tlong, 600, t1,
+ "humpty dumpy2 sat on a walled", shard_fld,"a"));
+
+ collectionClient.commit();
+
+ assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
//TODO debug the following case
- assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys", "a")).getResults().getNumFound());
+ assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_, "a")).getResults().getNumFound());
+
+ collectionClient.deleteByQuery("*:*");
+ collectionClient.commit();
+ collectionClient.add (getDoc( id,100,shard_fld, "b!doc1"));
+ collectionClient.commit();
+ assertEquals(1, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_, "b!")).getResults().getNumFound());
}
+ private void testCreateShardRepFactor() throws Exception {
+ String collectionName = "testCreateShardRepFactor";
+ HashMap<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
+ CloudSolrServer client = null;
+ try {
+ client = createCloudClient(null);
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ REPLICATION_FACTOR, 1,
+ MAX_SHARDS_PER_NODE, 5,
+ NUM_SLICES, 2,
+ "shards", "a,b",
+ "router.name", "implicit");
+ createCollection(collectionInfos, collectionName, props, client);
+ } finally {
+ if (client != null) client.shutdown();
+ }
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
+ waitForRecoveriesToFinish(collectionName, zkStateReader, false);
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.CREATESHARD.toString());
+ params.set("collection", collectionName);
+ params.set("shard", "x");
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
+
+ waitForRecoveriesToFinish(collectionName, zkStateReader, false);
+
+ int replicaCount = 0;
+ int attempts = 0;
+ while (true) {
+ if (attempts > 30) fail("Not enough active replicas in the shard 'x'");
+ zkStateReader.updateClusterState(true);
+ attempts++;
+ replicaCount = zkStateReader.getClusterState().getSlice(collectionName, "x").getReplicas().size();
+ if (replicaCount >= 1) break;
+ Thread.sleep(500);
+ }
+
+ assertEquals("CREATESHARD API created more than replicationFactor number of replicas", 1, replicaCount);
+ }
- private String getUrlFromZk(String collection) {
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+ public static String getUrlFromZk(ClusterState clusterState, String collection) {
Map<String,Slice> slices = clusterState.getCollectionStates().get(collection).getSlicesMap();
if (slices == null) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java Mon Oct 21 18:58:24 2013
@@ -101,18 +101,18 @@ public class DeleteShardTest extends Abs
deleteShard(SHARD1);
- confirmShardDeletion();
+ confirmShardDeletion(SHARD1);
}
- protected void confirmShardDeletion() throws SolrServerException, KeeperException,
+ protected void confirmShardDeletion(String shard) throws SolrServerException, KeeperException,
InterruptedException {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- ClusterState clusterState = null;
+ ClusterState clusterState = zkStateReader.getClusterState();
int counter = 10;
while (counter-- > 0) {
zkStateReader.updateClusterState(true);
clusterState = zkStateReader.getClusterState();
- if (clusterState.getSlice("collection1", SHARD1) == null) {
+ if (clusterState.getSlice("collection1", shard) == null) {
break;
}
Thread.sleep(1000);
@@ -138,7 +138,7 @@ public class DeleteShardTest extends Abs
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
baseServer.setConnectionTimeout(15000);
- baseServer.setSoTimeout((int) (CollectionsHandler.DEFAULT_ZK_TIMEOUT));
+ baseServer.setSoTimeout(60000);
baseServer.request(request);
}